summaryrefslogtreecommitdiffstats
path: root/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm/lib/Target/X86/X86MCInstLower.cpp')
-rw-r--r--contrib/llvm/lib/Target/X86/X86MCInstLower.cpp264
1 files changed, 158 insertions, 106 deletions
diff --git a/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp b/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp
index 906e342..feeb2fd 100644
--- a/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/contrib/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -16,6 +16,7 @@
#include "X86RegisterInfo.h"
#include "X86ShuffleDecodeConstantPool.h"
#include "InstPrinter/X86ATTInstPrinter.h"
+#include "InstPrinter/X86InstComments.h"
#include "MCTargetDesc/X86BaseInfo.h"
#include "Utils/X86ShuffleDecode.h"
#include "llvm/ADT/Optional.h"
@@ -41,6 +42,7 @@
#include "llvm/MC/MCSymbol.h"
#include "llvm/MC/MCSymbolELF.h"
#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCSectionMachO.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/ELF.h"
#include "llvm/Target/TargetLoweringObjectFile.h"
@@ -68,9 +70,6 @@ public:
private:
MachineModuleInfoMachO &getMachOMMI() const;
- Mangler *getMang() const {
- return AsmPrinter.Mang;
- }
};
} // end anonymous namespace
@@ -499,18 +498,13 @@ ReSimplify:
break;
}
- // TAILJMPd, TAILJMPd64 - Lower to the correct jump instructions.
- case X86::TAILJMPr:
+ // TAILJMPd, TAILJMPd64 - Lower to the correct jump instruction.
+ { unsigned Opcode;
+ case X86::TAILJMPr: Opcode = X86::JMP32r; goto SetTailJmpOpcode;
case X86::TAILJMPd:
- case X86::TAILJMPd64: {
- unsigned Opcode;
- switch (OutMI.getOpcode()) {
- default: llvm_unreachable("Invalid opcode");
- case X86::TAILJMPr: Opcode = X86::JMP32r; break;
- case X86::TAILJMPd:
- case X86::TAILJMPd64: Opcode = X86::JMP_1; break;
- }
+ case X86::TAILJMPd64: Opcode = X86::JMP_1; goto SetTailJmpOpcode;
+ SetTailJmpOpcode:
MCOperand Saved = OutMI.getOperand(0);
OutMI = MCInst();
OutMI.setOpcode(Opcode);
@@ -979,8 +973,7 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
PatchPointOpers opers(&MI);
unsigned ScratchIdx = opers.getNextScratchIdx();
unsigned EncodedBytes = 0;
- const MachineOperand &CalleeMO =
- opers.getMetaOper(PatchPointOpers::TargetPos);
+ const MachineOperand &CalleeMO = opers.getCallTarget();
// Check for null target. If target is non-null (i.e. is non-zero or is
// symbolic) then emit a call.
@@ -1016,7 +1009,7 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
}
// Emit padding.
- unsigned NumBytes = opers.getMetaOper(PatchPointOpers::NBytesPos).getImm();
+ unsigned NumBytes = opers.getNumPatchBytes();
assert(NumBytes >= EncodedBytes &&
"Patchpoint can't request size less than the length of a call.");
@@ -1024,22 +1017,12 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
getSubtargetInfo());
}
-void X86AsmPrinter::recordSled(MCSymbol *Sled, const MachineInstr &MI,
- SledKind Kind) {
- auto Fn = MI.getParent()->getParent()->getFunction();
- auto Attr = Fn->getFnAttribute("function-instrument");
- bool AlwaysInstrument =
- Attr.isStringAttribute() && Attr.getValueAsString() == "xray-always";
- Sleds.emplace_back(
- XRayFunctionEntry{Sled, CurrentFnSym, Kind, AlwaysInstrument, Fn});
-}
-
void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
X86MCInstLower &MCIL) {
// We want to emit the following pattern:
//
+ // .p2align 1, ...
// .Lxray_sled_N:
- // .palign 2, ...
// jmp .tmpN
// # 9 bytes worth of noops
// .tmpN
@@ -1051,8 +1034,8 @@ void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
// call <relative offset, 32-bits> // 5 bytes
//
auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
+ OutStreamer->EmitCodeAlignment(2);
OutStreamer->EmitLabel(CurSled);
- OutStreamer->EmitCodeAlignment(4);
auto Target = OutContext.createTempSymbol();
// Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
@@ -1074,12 +1057,14 @@ void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
//
// We should emit the RET followed by sleds.
//
+ // .p2align 1, ...
// .Lxray_sled_N:
// ret # or equivalent instruction
// # 10 bytes worth of noops
//
// This just makes sure that the alignment for the next instruction is 2.
auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
+ OutStreamer->EmitCodeAlignment(2);
OutStreamer->EmitLabel(CurSled);
unsigned OpCode = MI.getOperand(0).getImm();
MCInst Ret;
@@ -1092,29 +1077,37 @@ void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
recordSled(CurSled, MI, SledKind::FUNCTION_EXIT);
}
-void X86AsmPrinter::EmitXRayTable() {
- if (Sleds.empty())
- return;
- if (Subtarget->isTargetELF()) {
- auto *Section = OutContext.getELFSection(
- "xray_instr_map", ELF::SHT_PROGBITS,
- ELF::SHF_ALLOC | ELF::SHF_GROUP | ELF::SHF_MERGE, 0,
- CurrentFnSym->getName());
- auto PrevSection = OutStreamer->getCurrentSectionOnly();
- OutStreamer->SwitchSection(Section);
- for (const auto &Sled : Sleds) {
- OutStreamer->EmitSymbolValue(Sled.Sled, 8);
- OutStreamer->EmitSymbolValue(CurrentFnSym, 8);
- auto Kind = static_cast<uint8_t>(Sled.Kind);
- OutStreamer->EmitBytes(
- StringRef(reinterpret_cast<const char *>(&Kind), 1));
- OutStreamer->EmitBytes(
- StringRef(reinterpret_cast<const char *>(&Sled.AlwaysInstrument), 1));
- OutStreamer->EmitZeros(14);
- }
- OutStreamer->SwitchSection(PrevSection);
- }
- Sleds.clear();
+void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI, X86MCInstLower &MCIL) {
+ // Like PATCHABLE_RET, we have the actual instruction in the operands to this
+ // instruction so we lower that particular instruction and its operands.
+ // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how
+ // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to
+ // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual
+ // tail call much like how we have it in PATCHABLE_RET.
+ auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
+ OutStreamer->EmitCodeAlignment(2);
+ OutStreamer->EmitLabel(CurSled);
+ auto Target = OutContext.createTempSymbol();
+
+ // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
+ // an operand (computed as an offset from the jmp instruction).
+ // FIXME: Find another less hacky way do force the relative jump.
+ OutStreamer->EmitBytes("\xeb\x09");
+ EmitNops(*OutStreamer, 9, Subtarget->is64Bit(), getSubtargetInfo());
+ OutStreamer->EmitLabel(Target);
+ recordSled(CurSled, MI, SledKind::TAIL_CALL);
+
+ unsigned OpCode = MI.getOperand(0).getImm();
+ MCInst TC;
+ TC.setOpcode(OpCode);
+
+ // Before emitting the instruction, add a comment to indicate that this is
+ // indeed a tail call.
+ OutStreamer->AddComment("TAILCALL");
+ for (auto &MO : make_range(MI.operands_begin() + 1, MI.operands_end()))
+ if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
+ TC.addOperand(MaybeOperand.getValue());
+ OutStreamer->EmitInstruction(TC, getSubtargetInfo());
}
// Returns instruction preceding MBBI in MachineFunction.
@@ -1152,9 +1145,9 @@ static const Constant *getConstantFromPool(const MachineInstr &MI,
return C;
}
-static std::string getShuffleComment(const MachineOperand &DstOp,
- const MachineOperand &SrcOp1,
- const MachineOperand &SrcOp2,
+static std::string getShuffleComment(const MachineInstr *MI,
+ unsigned SrcOp1Idx,
+ unsigned SrcOp2Idx,
ArrayRef<int> Mask) {
std::string Comment;
@@ -1167,7 +1160,10 @@ static std::string getShuffleComment(const MachineOperand &DstOp,
return X86ATTInstPrinter::getRegisterName(RegNum);
};
- // TODO: Add support for specifying an AVX512 style mask register in the comment.
+ const MachineOperand &DstOp = MI->getOperand(0);
+ const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx);
+ const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx);
+
StringRef DstName = DstOp.isReg() ? GetRegisterName(DstOp.getReg()) : "mem";
StringRef Src1Name =
SrcOp1.isReg() ? GetRegisterName(SrcOp1.getReg()) : "mem";
@@ -1182,7 +1178,26 @@ static std::string getShuffleComment(const MachineOperand &DstOp,
ShuffleMask[i] -= e;
raw_string_ostream CS(Comment);
- CS << DstName << " = ";
+ CS << DstName;
+
+ // Handle AVX512 MASK/MASXZ write mask comments.
+ // MASK: zmmX {%kY}
+ // MASKZ: zmmX {%kY} {z}
+ if (SrcOp1Idx > 1) {
+ assert((SrcOp1Idx == 2 || SrcOp1Idx == 3) && "Unexpected writemask");
+
+ const MachineOperand &WriteMaskOp = MI->getOperand(SrcOp1Idx - 1);
+ if (WriteMaskOp.isReg()) {
+ CS << " {%" << GetRegisterName(WriteMaskOp.getReg()) << "}";
+
+ if (SrcOp1Idx == 2) {
+ CS << " {z}";
+ }
+ }
+ }
+
+ CS << " = ";
+
for (int i = 0, e = ShuffleMask.size(); i != e; ++i) {
if (i != 0)
CS << ",";
@@ -1221,6 +1236,13 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
X86MCInstLower MCInstLowering(*MF, *this);
const X86RegisterInfo *RI = MF->getSubtarget<X86Subtarget>().getRegisterInfo();
+ // Add a comment about EVEX-2-VEX compression for AVX-512 instrs that
+ // are compressed from EVEX encoding to VEX encoding.
+ if (TM.Options.MCOptions.ShowMCEncoding) {
+ if (MI->getAsmPrinterFlags() & AC_EVEX_2_VEX)
+ OutStreamer->AddComment("EVEX TO VEX Compression ", false);
+ }
+
switch (MI->getOpcode()) {
case TargetOpcode::DBG_VALUE:
llvm_unreachable("Should be handled target independently");
@@ -1259,7 +1281,6 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
case X86::TAILJMPd64:
case X86::TAILJMPr64_REX:
case X86::TAILJMPm64_REX:
- case X86::TAILJMPd64_REX:
// Lower these as normal, but add some comments.
OutStreamer->AddComment("TAILCALL");
break;
@@ -1364,6 +1385,9 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
case TargetOpcode::PATCHABLE_RET:
return LowerPATCHABLE_RET(*MI, MCInstLowering);
+ case TargetOpcode::PATCHABLE_TAIL_CALL:
+ return LowerPATCHABLE_TAIL_CALL(*MI, MCInstLowering);
+
case X86::MORESTACK_RET:
EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
return;
@@ -1377,37 +1401,45 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
return;
case X86::SEH_PushReg:
+ assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
OutStreamer->EmitWinCFIPushReg(RI->getSEHRegNum(MI->getOperand(0).getImm()));
return;
case X86::SEH_SaveReg:
+ assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
OutStreamer->EmitWinCFISaveReg(RI->getSEHRegNum(MI->getOperand(0).getImm()),
MI->getOperand(1).getImm());
return;
case X86::SEH_SaveXMM:
+ assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
OutStreamer->EmitWinCFISaveXMM(RI->getSEHRegNum(MI->getOperand(0).getImm()),
MI->getOperand(1).getImm());
return;
case X86::SEH_StackAlloc:
+ assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
OutStreamer->EmitWinCFIAllocStack(MI->getOperand(0).getImm());
return;
case X86::SEH_SetFrame:
+ assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
OutStreamer->EmitWinCFISetFrame(RI->getSEHRegNum(MI->getOperand(0).getImm()),
MI->getOperand(1).getImm());
return;
case X86::SEH_PushFrame:
+ assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
OutStreamer->EmitWinCFIPushFrame(MI->getOperand(0).getImm());
return;
case X86::SEH_EndPrologue:
+ assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
OutStreamer->EmitWinCFIEndProlog();
return;
case X86::SEH_Epilogue: {
+ assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
MachineBasicBlock::const_iterator MBBI(MI);
// Check if preceded by a call and emit nop if so.
for (MBBI = PrevCrossBBInst(MBBI);
@@ -1463,59 +1495,84 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
assert(MI->getNumOperands() >= 6 &&
"We should always have at least 6 operands!");
- const MachineOperand &DstOp = MI->getOperand(0);
- const MachineOperand &SrcOp = MI->getOperand(SrcIdx);
- const MachineOperand &MaskOp = MI->getOperand(MaskIdx);
+ const MachineOperand &MaskOp = MI->getOperand(MaskIdx);
if (auto *C = getConstantFromPool(*MI, MaskOp)) {
- SmallVector<int, 16> Mask;
+ SmallVector<int, 64> Mask;
DecodePSHUFBMask(C, Mask);
if (!Mask.empty())
- OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, SrcOp, Mask));
+ OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
}
break;
}
+ case X86::VPERMILPSrm:
+ case X86::VPERMILPSYrm:
+ case X86::VPERMILPSZ128rm:
+ case X86::VPERMILPSZ128rmk:
+ case X86::VPERMILPSZ128rmkz:
+ case X86::VPERMILPSZ256rm:
+ case X86::VPERMILPSZ256rmk:
+ case X86::VPERMILPSZ256rmkz:
+ case X86::VPERMILPSZrm:
+ case X86::VPERMILPSZrmk:
+ case X86::VPERMILPSZrmkz:
case X86::VPERMILPDrm:
case X86::VPERMILPDYrm:
case X86::VPERMILPDZ128rm:
+ case X86::VPERMILPDZ128rmk:
+ case X86::VPERMILPDZ128rmkz:
case X86::VPERMILPDZ256rm:
- case X86::VPERMILPDZrm: {
+ case X86::VPERMILPDZ256rmk:
+ case X86::VPERMILPDZ256rmkz:
+ case X86::VPERMILPDZrm:
+ case X86::VPERMILPDZrmk:
+ case X86::VPERMILPDZrmkz: {
if (!OutStreamer->isVerboseAsm())
break;
- assert(MI->getNumOperands() > 5 &&
- "We should always have at least 5 operands!");
- const MachineOperand &DstOp = MI->getOperand(0);
- const MachineOperand &SrcOp = MI->getOperand(1);
- const MachineOperand &MaskOp = MI->getOperand(5);
-
- if (auto *C = getConstantFromPool(*MI, MaskOp)) {
- SmallVector<int, 8> Mask;
- DecodeVPERMILPMask(C, 64, Mask);
- if (!Mask.empty())
- OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, SrcOp, Mask));
+ unsigned SrcIdx, MaskIdx;
+ unsigned ElSize;
+ switch (MI->getOpcode()) {
+ default: llvm_unreachable("Invalid opcode");
+ case X86::VPERMILPSrm:
+ case X86::VPERMILPSYrm:
+ case X86::VPERMILPSZ128rm:
+ case X86::VPERMILPSZ256rm:
+ case X86::VPERMILPSZrm:
+ SrcIdx = 1; MaskIdx = 5; ElSize = 32; break;
+ case X86::VPERMILPSZ128rmkz:
+ case X86::VPERMILPSZ256rmkz:
+ case X86::VPERMILPSZrmkz:
+ SrcIdx = 2; MaskIdx = 6; ElSize = 32; break;
+ case X86::VPERMILPSZ128rmk:
+ case X86::VPERMILPSZ256rmk:
+ case X86::VPERMILPSZrmk:
+ SrcIdx = 3; MaskIdx = 7; ElSize = 32; break;
+ case X86::VPERMILPDrm:
+ case X86::VPERMILPDYrm:
+ case X86::VPERMILPDZ128rm:
+ case X86::VPERMILPDZ256rm:
+ case X86::VPERMILPDZrm:
+ SrcIdx = 1; MaskIdx = 5; ElSize = 64; break;
+ case X86::VPERMILPDZ128rmkz:
+ case X86::VPERMILPDZ256rmkz:
+ case X86::VPERMILPDZrmkz:
+ SrcIdx = 2; MaskIdx = 6; ElSize = 64; break;
+ case X86::VPERMILPDZ128rmk:
+ case X86::VPERMILPDZ256rmk:
+ case X86::VPERMILPDZrmk:
+ SrcIdx = 3; MaskIdx = 7; ElSize = 64; break;
}
- break;
- }
- case X86::VPERMILPSrm:
- case X86::VPERMILPSYrm:
- case X86::VPERMILPSZ128rm:
- case X86::VPERMILPSZ256rm:
- case X86::VPERMILPSZrm: {
- if (!OutStreamer->isVerboseAsm())
- break;
- assert(MI->getNumOperands() > 5 &&
- "We should always have at least 5 operands!");
- const MachineOperand &DstOp = MI->getOperand(0);
- const MachineOperand &SrcOp = MI->getOperand(1);
- const MachineOperand &MaskOp = MI->getOperand(5);
+ assert(MI->getNumOperands() >= 6 &&
+ "We should always have at least 6 operands!");
+ const MachineOperand &MaskOp = MI->getOperand(MaskIdx);
if (auto *C = getConstantFromPool(*MI, MaskOp)) {
SmallVector<int, 16> Mask;
- DecodeVPERMILPMask(C, 32, Mask);
+ DecodeVPERMILPMask(C, ElSize, Mask);
if (!Mask.empty())
- OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp, SrcOp, Mask));
+ OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
}
break;
}
@@ -1526,14 +1583,10 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
case X86::VPERMIL2PSrmY: {
if (!OutStreamer->isVerboseAsm())
break;
- assert(MI->getNumOperands() > 7 &&
- "We should always have at least 7 operands!");
- const MachineOperand &DstOp = MI->getOperand(0);
- const MachineOperand &SrcOp1 = MI->getOperand(1);
- const MachineOperand &SrcOp2 = MI->getOperand(2);
- const MachineOperand &MaskOp = MI->getOperand(6);
- const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1);
+ assert(MI->getNumOperands() >= 8 &&
+ "We should always have at least 8 operands!");
+ const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1);
if (!CtrlOp.isImm())
break;
@@ -1544,11 +1597,12 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
case X86::VPERMIL2PDrm: case X86::VPERMIL2PDrmY: ElSize = 64; break;
}
+ const MachineOperand &MaskOp = MI->getOperand(6);
if (auto *C = getConstantFromPool(*MI, MaskOp)) {
SmallVector<int, 16> Mask;
DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Mask);
if (!Mask.empty())
- OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp1, SrcOp2, Mask));
+ OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask));
}
break;
}
@@ -1556,18 +1610,15 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
case X86::VPPERMrrm: {
if (!OutStreamer->isVerboseAsm())
break;
- assert(MI->getNumOperands() > 6 &&
- "We should always have at least 6 operands!");
- const MachineOperand &DstOp = MI->getOperand(0);
- const MachineOperand &SrcOp1 = MI->getOperand(1);
- const MachineOperand &SrcOp2 = MI->getOperand(2);
- const MachineOperand &MaskOp = MI->getOperand(6);
+ assert(MI->getNumOperands() >= 7 &&
+ "We should always have at least 7 operands!");
+ const MachineOperand &MaskOp = MI->getOperand(6);
if (auto *C = getConstantFromPool(*MI, MaskOp)) {
SmallVector<int, 16> Mask;
DecodeVPPERMMask(C, Mask);
if (!Mask.empty())
- OutStreamer->AddComment(getShuffleComment(DstOp, SrcOp1, SrcOp2, Mask));
+ OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask));
}
break;
}
@@ -1605,7 +1656,8 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
CASE_ALL_MOV_RM()
if (!OutStreamer->isVerboseAsm())
break;
- if (MI->getNumOperands() > 4)
+ if (MI->getNumOperands() <= 4)
+ break;
if (auto *C = getConstantFromPool(*MI, MI->getOperand(4))) {
std::string Comment;
raw_string_ostream CS(Comment);
OpenPOWER on IntegriCloud