diff options
Diffstat (limited to 'contrib/llvm/lib/Target/X86/X86MCInstLower.cpp')
-rw-r--r-- | contrib/llvm/lib/Target/X86/X86MCInstLower.cpp | 264 |
1 files changed, 158 insertions, 106 deletions
diff --git a/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp b/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp index 906e342..feeb2fd 100644 --- a/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -16,6 +16,7 @@ #include "X86RegisterInfo.h" #include "X86ShuffleDecodeConstantPool.h" #include "InstPrinter/X86ATTInstPrinter.h" +#include "InstPrinter/X86InstComments.h" #include "MCTargetDesc/X86BaseInfo.h" #include "Utils/X86ShuffleDecode.h" #include "llvm/ADT/Optional.h" @@ -41,6 +42,7 @@ #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCSymbolELF.h" #include "llvm/MC/MCSectionELF.h" +#include "llvm/MC/MCSectionMachO.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/ELF.h" #include "llvm/Target/TargetLoweringObjectFile.h" @@ -68,9 +70,6 @@ public: private: MachineModuleInfoMachO &getMachOMMI() const; - Mangler *getMang() const { - return AsmPrinter.Mang; - } }; } // end anonymous namespace @@ -499,18 +498,13 @@ ReSimplify: break; } - // TAILJMPd, TAILJMPd64 - Lower to the correct jump instructions. - case X86::TAILJMPr: + // TAILJMPd, TAILJMPd64 - Lower to the correct jump instruction. + { unsigned Opcode; + case X86::TAILJMPr: Opcode = X86::JMP32r; goto SetTailJmpOpcode; case X86::TAILJMPd: - case X86::TAILJMPd64: { - unsigned Opcode; - switch (OutMI.getOpcode()) { - default: llvm_unreachable("Invalid opcode"); - case X86::TAILJMPr: Opcode = X86::JMP32r; break; - case X86::TAILJMPd: - case X86::TAILJMPd64: Opcode = X86::JMP_1; break; - } + case X86::TAILJMPd64: Opcode = X86::JMP_1; goto SetTailJmpOpcode; + SetTailJmpOpcode: MCOperand Saved = OutMI.getOperand(0); OutMI = MCInst(); OutMI.setOpcode(Opcode); @@ -979,8 +973,7 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, PatchPointOpers opers(&MI); unsigned ScratchIdx = opers.getNextScratchIdx(); unsigned EncodedBytes = 0; - const MachineOperand &CalleeMO = - opers.getMetaOper(PatchPointOpers::TargetPos); + const MachineOperand &CalleeMO = opers.getCallTarget(); // Check for null target. If target is non-null (i.e. is non-zero or is // symbolic) then emit a call. @@ -1016,7 +1009,7 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, } // Emit padding. - unsigned NumBytes = opers.getMetaOper(PatchPointOpers::NBytesPos).getImm(); + unsigned NumBytes = opers.getNumPatchBytes(); assert(NumBytes >= EncodedBytes && "Patchpoint can't request size less than the length of a call."); @@ -1024,22 +1017,12 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, getSubtargetInfo()); } -void X86AsmPrinter::recordSled(MCSymbol *Sled, const MachineInstr &MI, - SledKind Kind) { - auto Fn = MI.getParent()->getParent()->getFunction(); - auto Attr = Fn->getFnAttribute("function-instrument"); - bool AlwaysInstrument = - Attr.isStringAttribute() && Attr.getValueAsString() == "xray-always"; - Sleds.emplace_back( - XRayFunctionEntry{Sled, CurrentFnSym, Kind, AlwaysInstrument, Fn}); -} - void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI, X86MCInstLower &MCIL) { // We want to emit the following pattern: // + // .p2align 1, ... // .Lxray_sled_N: - // .palign 2, ... // jmp .tmpN // # 9 bytes worth of noops // .tmpN @@ -1051,8 +1034,8 @@ void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI, // call <relative offset, 32-bits> // 5 bytes // auto CurSled = OutContext.createTempSymbol("xray_sled_", true); + OutStreamer->EmitCodeAlignment(2); OutStreamer->EmitLabel(CurSled); - OutStreamer->EmitCodeAlignment(4); auto Target = OutContext.createTempSymbol(); // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as @@ -1074,12 +1057,14 @@ void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI, // // We should emit the RET followed by sleds. // + // .p2align 1, ... // .Lxray_sled_N: // ret # or equivalent instruction // # 10 bytes worth of noops // // This just makes sure that the alignment for the next instruction is 2. auto CurSled = OutContext.createTempSymbol("xray_sled_", true); + OutStreamer->EmitCodeAlignment(2); OutStreamer->EmitLabel(CurSled); unsigned OpCode = MI.getOperand(0).getImm(); MCInst Ret; @@ -1092,29 +1077,37 @@ void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI, recordSled(CurSled, MI, SledKind::FUNCTION_EXIT); } -void X86AsmPrinter::EmitXRayTable() { - if (Sleds.empty()) - return; - if (Subtarget->isTargetELF()) { - auto *Section = OutContext.getELFSection( - "xray_instr_map", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC | ELF::SHF_GROUP | ELF::SHF_MERGE, 0, - CurrentFnSym->getName()); - auto PrevSection = OutStreamer->getCurrentSectionOnly(); - OutStreamer->SwitchSection(Section); - for (const auto &Sled : Sleds) { - OutStreamer->EmitSymbolValue(Sled.Sled, 8); - OutStreamer->EmitSymbolValue(CurrentFnSym, 8); - auto Kind = static_cast<uint8_t>(Sled.Kind); - OutStreamer->EmitBytes( - StringRef(reinterpret_cast<const char *>(&Kind), 1)); - OutStreamer->EmitBytes( - StringRef(reinterpret_cast<const char *>(&Sled.AlwaysInstrument), 1)); - OutStreamer->EmitZeros(14); - } - OutStreamer->SwitchSection(PrevSection); - } - Sleds.clear(); +void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI, X86MCInstLower &MCIL) { + // Like PATCHABLE_RET, we have the actual instruction in the operands to this + // instruction so we lower that particular instruction and its operands. + // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how + // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to + // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual + // tail call much like how we have it in PATCHABLE_RET. + auto CurSled = OutContext.createTempSymbol("xray_sled_", true); + OutStreamer->EmitCodeAlignment(2); + OutStreamer->EmitLabel(CurSled); + auto Target = OutContext.createTempSymbol(); + + // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as + // an operand (computed as an offset from the jmp instruction). + // FIXME: Find another less hacky way do force the relative jump. + OutStreamer->EmitBytes("\xeb\x09"); + EmitNops(*OutStreamer, 9, Subtarget->is64Bit(), getSubtargetInfo()); + OutStreamer->EmitLabel(Target); + recordSled(CurSled, MI, SledKind::TAIL_CALL); + + unsigned OpCode = MI.getOperand(0).getImm(); + MCInst TC; + TC.setOpcode(OpCode); + + // Before emitting the instruction, add a comment to indicate that this is + // indeed a tail call. + OutStreamer->AddComment("TAILCALL"); + for (auto &MO : make_range(MI.operands_begin() + 1, MI.operands_end())) + if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO)) + TC.addOperand(MaybeOperand.getValue()); + OutStreamer->EmitInstruction(TC, getSubtargetInfo()); } // Returns instruction preceding MBBI in MachineFunction. @@ -1152,9 +1145,9 @@ static const Constant *getConstantFromPool(const MachineInstr &MI, return C; } -static std::string getShuffleComment(const MachineOperand &DstOp, - const MachineOperand &SrcOp1, - const MachineOperand &SrcOp2, +static std::string getShuffleComment(const MachineInstr *MI, + unsigned SrcOp1Idx, + unsigned SrcOp2Idx, ArrayRef<int> Mask) { std::string Comment; @@ -1167,7 +1160,10 @@ static std::string getShuffleComment(const MachineOperand &DstOp, return X86ATTInstPrinter::getRegisterName(RegNum); }; - // TODO: Add support for specifying an AVX512 style mask register in the comment. + const MachineOperand &DstOp = MI->getOperand(0); + const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx); + const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx); + StringRef DstName = DstOp.isReg() ? GetRegisterName(DstOp.getReg()) : "mem"; StringRef Src1Name = SrcOp1.isReg() ? GetRegisterName(SrcOp1.getReg()) : "mem"; @@ -1182,7 +1178,26 @@ static std::string getShuffleComment(const MachineOperand &DstOp, ShuffleMask[i] -= e; raw_string_ostream CS(Comment); - CS << DstName << " = "; + CS << DstName; + + // Handle AVX512 MASK/MASXZ write mask comments. + // MASK: zmmX {%kY} + // MASKZ: zmmX {%kY} {z} + if (SrcOp1Idx > 1) { + assert((SrcOp1Idx == 2 || SrcOp1Idx == 3) && "Unexpected writemask"); + + const MachineOperand &WriteMaskOp = MI->getOperand(SrcOp1Idx - 1); + if (WriteMaskOp.isReg()) { + CS << " {%" << GetRegisterName(WriteMaskOp.getReg()) << "}"; + + if (SrcOp1Idx == 2) { + CS << " {z}"; + } + } + } + + CS << " = "; + for (int i = 0, e = ShuffleMask.size(); i != e; ++i) { if (i != 0) CS << ","; @@ -1221,6 +1236,13 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { X86MCInstLower MCInstLowering(*MF, *this); const X86RegisterInfo *RI = MF->getSubtarget<X86Subtarget>().getRegisterInfo(); + // Add a comment about EVEX-2-VEX compression for AVX-512 instrs that + // are compressed from EVEX encoding to VEX encoding. + if (TM.Options.MCOptions.ShowMCEncoding) { + if (MI->getAsmPrinterFlags() & AC_EVEX_2_VEX) + OutStreamer->AddComment("EVEX TO VEX Compression ", false); + } + switch (MI->getOpcode()) { case TargetOpcode::DBG_VALUE: llvm_unreachable("Should be handled target independently"); @@ -1259,7 +1281,6 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { case X86::TAILJMPd64: case X86::TAILJMPr64_REX: case X86::TAILJMPm64_REX: - case X86::TAILJMPd64_REX: // Lower these as normal, but add some comments. OutStreamer->AddComment("TAILCALL"); break; @@ -1364,6 +1385,9 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { case TargetOpcode::PATCHABLE_RET: return LowerPATCHABLE_RET(*MI, MCInstLowering); + case TargetOpcode::PATCHABLE_TAIL_CALL: + return LowerPATCHABLE_TAIL_CALL(*MI, MCInstLowering); + case X86::MORESTACK_RET: EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget))); return; @@ -1377,37 +1401,45 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { return; case X86::SEH_PushReg: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFIPushReg(RI->getSEHRegNum(MI->getOperand(0).getImm())); return; case X86::SEH_SaveReg: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFISaveReg(RI->getSEHRegNum(MI->getOperand(0).getImm()), MI->getOperand(1).getImm()); return; case X86::SEH_SaveXMM: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFISaveXMM(RI->getSEHRegNum(MI->getOperand(0).getImm()), MI->getOperand(1).getImm()); return; case X86::SEH_StackAlloc: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFIAllocStack(MI->getOperand(0).getImm()); return; case X86::SEH_SetFrame: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFISetFrame(RI->getSEHRegNum(MI->getOperand(0).getImm()), MI->getOperand(1).getImm()); return; case X86::SEH_PushFrame: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFIPushFrame(MI->getOperand(0).getImm()); return; case X86::SEH_EndPrologue: + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); OutStreamer->EmitWinCFIEndProlog(); return; case X86::SEH_Epilogue: { + assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); MachineBasicBlock::const_iterator MBBI(MI); // Check if preceded by a call and emit nop if so. for (MBBI = PrevCrossBBInst(MBBI); @@ -1463,59 +1495,84 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { assert(MI->getNumOperands() >= 6 && "We should always have at least 6 operands!"); - const MachineOperand &DstOp = MI->getOperand(0); - const MachineOperand &SrcOp = MI->getOperand(SrcIdx); - const MachineOperand &MaskOp = MI->getOperand(MaskIdx); + const MachineOperand &MaskOp = MI->getOperand(MaskIdx); if (auto *C = getConstantFromPool(*MI, MaskOp)) { - SmallVector<int, 16> Mask; + SmallVector<int, 64> Mask; DecodePSHUFBMask(C, Mask); if (!Mask.empty()) - OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, SrcOp, Mask)); + OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask)); } break; } + case X86::VPERMILPSrm: + case X86::VPERMILPSYrm: + case X86::VPERMILPSZ128rm: + case X86::VPERMILPSZ128rmk: + case X86::VPERMILPSZ128rmkz: + case X86::VPERMILPSZ256rm: + case X86::VPERMILPSZ256rmk: + case X86::VPERMILPSZ256rmkz: + case X86::VPERMILPSZrm: + case X86::VPERMILPSZrmk: + case X86::VPERMILPSZrmkz: case X86::VPERMILPDrm: case X86::VPERMILPDYrm: case X86::VPERMILPDZ128rm: + case X86::VPERMILPDZ128rmk: + case X86::VPERMILPDZ128rmkz: case X86::VPERMILPDZ256rm: - case X86::VPERMILPDZrm: { + case X86::VPERMILPDZ256rmk: + case X86::VPERMILPDZ256rmkz: + case X86::VPERMILPDZrm: + case X86::VPERMILPDZrmk: + case X86::VPERMILPDZrmkz: { if (!OutStreamer->isVerboseAsm()) break; - assert(MI->getNumOperands() > 5 && - "We should always have at least 5 operands!"); - const MachineOperand &DstOp = MI->getOperand(0); - const MachineOperand &SrcOp = MI->getOperand(1); - const MachineOperand &MaskOp = MI->getOperand(5); - - if (auto *C = getConstantFromPool(*MI, MaskOp)) { - SmallVector<int, 8> Mask; - DecodeVPERMILPMask(C, 64, Mask); - if (!Mask.empty()) - OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, SrcOp, Mask)); + unsigned SrcIdx, MaskIdx; + unsigned ElSize; + switch (MI->getOpcode()) { + default: llvm_unreachable("Invalid opcode"); + case X86::VPERMILPSrm: + case X86::VPERMILPSYrm: + case X86::VPERMILPSZ128rm: + case X86::VPERMILPSZ256rm: + case X86::VPERMILPSZrm: + SrcIdx = 1; MaskIdx = 5; ElSize = 32; break; + case X86::VPERMILPSZ128rmkz: + case X86::VPERMILPSZ256rmkz: + case X86::VPERMILPSZrmkz: + SrcIdx = 2; MaskIdx = 6; ElSize = 32; break; + case X86::VPERMILPSZ128rmk: + case X86::VPERMILPSZ256rmk: + case X86::VPERMILPSZrmk: + SrcIdx = 3; MaskIdx = 7; ElSize = 32; break; + case X86::VPERMILPDrm: + case X86::VPERMILPDYrm: + case X86::VPERMILPDZ128rm: + case X86::VPERMILPDZ256rm: + case X86::VPERMILPDZrm: + SrcIdx = 1; MaskIdx = 5; ElSize = 64; break; + case X86::VPERMILPDZ128rmkz: + case X86::VPERMILPDZ256rmkz: + case X86::VPERMILPDZrmkz: + SrcIdx = 2; MaskIdx = 6; ElSize = 64; break; + case X86::VPERMILPDZ128rmk: + case X86::VPERMILPDZ256rmk: + case X86::VPERMILPDZrmk: + SrcIdx = 3; MaskIdx = 7; ElSize = 64; break; } - break; - } - case X86::VPERMILPSrm: - case X86::VPERMILPSYrm: - case X86::VPERMILPSZ128rm: - case X86::VPERMILPSZ256rm: - case X86::VPERMILPSZrm: { - if (!OutStreamer->isVerboseAsm()) - break; - assert(MI->getNumOperands() > 5 && - "We should always have at least 5 operands!"); - const MachineOperand &DstOp = MI->getOperand(0); - const MachineOperand &SrcOp = MI->getOperand(1); - const MachineOperand &MaskOp = MI->getOperand(5); + assert(MI->getNumOperands() >= 6 && + "We should always have at least 6 operands!"); + const MachineOperand &MaskOp = MI->getOperand(MaskIdx); if (auto *C = getConstantFromPool(*MI, MaskOp)) { SmallVector<int, 16> Mask; - DecodeVPERMILPMask(C, 32, Mask); + DecodeVPERMILPMask(C, ElSize, Mask); if (!Mask.empty()) - OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, SrcOp, Mask)); + OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask)); } break; } @@ -1526,14 +1583,10 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { case X86::VPERMIL2PSrmY: { if (!OutStreamer->isVerboseAsm()) break; - assert(MI->getNumOperands() > 7 && - "We should always have at least 7 operands!"); - const MachineOperand &DstOp = MI->getOperand(0); - const MachineOperand &SrcOp1 = MI->getOperand(1); - const MachineOperand &SrcOp2 = MI->getOperand(2); - const MachineOperand &MaskOp = MI->getOperand(6); - const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1); + assert(MI->getNumOperands() >= 8 && + "We should always have at least 8 operands!"); + const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1); if (!CtrlOp.isImm()) break; @@ -1544,11 +1597,12 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { case X86::VPERMIL2PDrm: case X86::VPERMIL2PDrmY: ElSize = 64; break; } + const MachineOperand &MaskOp = MI->getOperand(6); if (auto *C = getConstantFromPool(*MI, MaskOp)) { SmallVector<int, 16> Mask; DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Mask); if (!Mask.empty()) - OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp1, SrcOp2, Mask)); + OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask)); } break; } @@ -1556,18 +1610,15 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { case X86::VPPERMrrm: { if (!OutStreamer->isVerboseAsm()) break; - assert(MI->getNumOperands() > 6 && - "We should always have at least 6 operands!"); - const MachineOperand &DstOp = MI->getOperand(0); - const MachineOperand &SrcOp1 = MI->getOperand(1); - const MachineOperand &SrcOp2 = MI->getOperand(2); - const MachineOperand &MaskOp = MI->getOperand(6); + assert(MI->getNumOperands() >= 7 && + "We should always have at least 7 operands!"); + const MachineOperand &MaskOp = MI->getOperand(6); if (auto *C = getConstantFromPool(*MI, MaskOp)) { SmallVector<int, 16> Mask; DecodeVPPERMMask(C, Mask); if (!Mask.empty()) - OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp1, SrcOp2, Mask)); + OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask)); } break; } @@ -1605,7 +1656,8 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) { CASE_ALL_MOV_RM() if (!OutStreamer->isVerboseAsm()) break; - if (MI->getNumOperands() > 4) + if (MI->getNumOperands() <= 4) + break; if (auto *C = getConstantFromPool(*MI, MI->getOperand(4))) { std::string Comment; raw_string_ostream CS(Comment); |