diff options
Diffstat (limited to 'lib/CodeGen')
103 files changed, 7016 insertions, 4402 deletions
diff --git a/lib/CodeGen/Analysis.cpp b/lib/CodeGen/Analysis.cpp index 125e641..fafc010 100644 --- a/lib/CodeGen/Analysis.cpp +++ b/lib/CodeGen/Analysis.cpp @@ -31,7 +31,7 @@ using namespace llvm; /// of insertvalue or extractvalue indices that identify a member, return /// the linearized index of the start of the member. /// -unsigned llvm::ComputeLinearIndex(const Type *Ty, +unsigned llvm::ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex) { @@ -40,7 +40,7 @@ unsigned llvm::ComputeLinearIndex(const Type *Ty, return CurIndex; // Given a struct type, recursively traverse the elements. - if (const StructType *STy = dyn_cast<StructType>(Ty)) { + if (StructType *STy = dyn_cast<StructType>(Ty)) { for (StructType::element_iterator EB = STy->element_begin(), EI = EB, EE = STy->element_end(); @@ -52,8 +52,8 @@ unsigned llvm::ComputeLinearIndex(const Type *Ty, return CurIndex; } // Given an array type, recursively traverse the elements. - else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { - const Type *EltTy = ATy->getElementType(); + else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { + Type *EltTy = ATy->getElementType(); for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) { if (Indices && *Indices == i) return ComputeLinearIndex(EltTy, Indices+1, IndicesEnd, CurIndex); @@ -72,12 +72,12 @@ unsigned llvm::ComputeLinearIndex(const Type *Ty, /// If Offsets is non-null, it points to a vector to be filled in /// with the in-memory offsets of each of the individual values. /// -void llvm::ComputeValueVTs(const TargetLowering &TLI, const Type *Ty, +void llvm::ComputeValueVTs(const TargetLowering &TLI, Type *Ty, SmallVectorImpl<EVT> &ValueVTs, SmallVectorImpl<uint64_t> *Offsets, uint64_t StartingOffset) { // Given a struct type, recursively traverse the elements. - if (const StructType *STy = dyn_cast<StructType>(Ty)) { + if (StructType *STy = dyn_cast<StructType>(Ty)) { const StructLayout *SL = TLI.getTargetData()->getStructLayout(STy); for (StructType::element_iterator EB = STy->element_begin(), EI = EB, @@ -88,8 +88,8 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, const Type *Ty, return; } // Given an array type, recursively traverse the elements. - if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { - const Type *EltTy = ATy->getElementType(); + if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { + Type *EltTy = ATy->getElementType(); uint64_t EltSize = TLI.getTargetData()->getTypeAllocSize(EltTy); for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) ComputeValueVTs(TLI, EltTy, ValueVTs, Offsets, diff --git a/lib/CodeGen/AsmPrinter/ARMException.cpp b/lib/CodeGen/AsmPrinter/ARMException.cpp index 5861fa4..3f23873 100644 --- a/lib/CodeGen/AsmPrinter/ARMException.cpp +++ b/lib/CodeGen/AsmPrinter/ARMException.cpp @@ -17,7 +17,6 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineLocation.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" @@ -27,7 +26,6 @@ #include "llvm/Target/Mangler.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetFrameLowering.h" -#include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetRegisterInfo.h" diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 7f314ee..1999f36 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -33,7 +33,6 @@ #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Target/Mangler.h" -#include "llvm/Target/TargetAsmInfo.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" @@ -45,6 +44,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" +#include "llvm/Support/MathExtras.h" #include "llvm/Support/Timer.h" using namespace llvm; @@ -290,10 +290,10 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { // Handle common and BSS local symbols (.lcomm). if (GVKind.isCommon() || GVKind.isBSSLocal()) { if (Size == 0) Size = 1; // .comm Foo, 0 is undefined, avoid it. + unsigned Align = 1 << AlignLog; // Handle common symbols. if (GVKind.isCommon()) { - unsigned Align = 1 << AlignLog; if (!getObjFileLowering().getCommDirectiveSupportsAlignment()) Align = 0; @@ -307,17 +307,17 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { const MCSection *TheSection = getObjFileLowering().SectionForGlobal(GV, GVKind, Mang, TM); // .zerofill __DATA, __bss, _foo, 400, 5 - OutStreamer.EmitZerofill(TheSection, GVSym, Size, 1 << AlignLog); + OutStreamer.EmitZerofill(TheSection, GVSym, Size, Align); return; } - if (MAI->hasLCOMMDirective()) { + if (MAI->getLCOMMDirectiveType() != LCOMM::None && + (MAI->getLCOMMDirectiveType() != LCOMM::NoAlignment || Align == 1)) { // .lcomm _foo, 42 - OutStreamer.EmitLocalCommonSymbol(GVSym, Size); + OutStreamer.EmitLocalCommonSymbol(GVSym, Size, Align); return; } - unsigned Align = 1 << AlignLog; if (!getObjFileLowering().getCommDirectiveSupportsAlignment()) Align = 0; @@ -474,8 +474,10 @@ void AsmPrinter::EmitFunctionHeader() { void AsmPrinter::EmitFunctionEntryLabel() { // The function label could have already been emitted if two symbols end up // conflicting due to asm renaming. Detect this and emit an error. - if (CurrentFnSym->isUndefined()) + if (CurrentFnSym->isUndefined()) { + OutStreamer.ForceCodeRegion(); return OutStreamer.EmitLabel(CurrentFnSym); + } report_fatal_error("'" + Twine(CurrentFnSym->getName()) + "' label emitted multiple times to assembly file"); @@ -620,6 +622,9 @@ void AsmPrinter::emitPrologLabel(const MachineInstr &MI) { if (needsCFIMoves() == CFI_M_None) return; + if (MMI->getCompactUnwindEncoding() != 0) + OutStreamer.EmitCompactUnwindEncoding(MMI->getCompactUnwindEncoding()); + MachineModuleInfo &MMI = MF->getMMI(); std::vector<MachineMove> &Moves = MMI.getFrameMoves(); bool FoundOne = false; @@ -878,7 +883,7 @@ bool AsmPrinter::doFinalization(Module &M) { I != E; ++I) { MCSymbol *Name = Mang->getSymbol(I); - const GlobalValue *GV = cast<GlobalValue>(I->getAliasedGlobal()); + const GlobalValue *GV = I->getAliasedGlobal(); MCSymbol *Target = Mang->getSymbol(GV); if (I->hasExternalLinkage() || !MAI->getWeakRefDirective()) @@ -1009,7 +1014,7 @@ void AsmPrinter::EmitConstantPool() { unsigned NewOffset = (Offset + AlignMask) & ~AlignMask; OutStreamer.EmitFill(NewOffset - Offset, 0/*fillval*/, 0/*addrspace*/); - const Type *Ty = CPE.getType(); + Type *Ty = CPE.getType(); Offset = NewOffset + TM.getTargetData()->getTypeAllocSize(Ty); OutStreamer.EmitLabel(GetCPISymbol(CPI)); @@ -1055,6 +1060,15 @@ void AsmPrinter::EmitJumpTableInfo() { EmitAlignment(Log2_32(MJTI->getEntryAlignment(*TM.getTargetData()))); + // If we know the form of the jump table, go ahead and tag it as such. + if (!JTInDiffSection) { + if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32) { + OutStreamer.EmitJumpTable32Region(); + } else { + OutStreamer.EmitDataRegion(); + } + } + for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) { const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs; @@ -1226,22 +1240,53 @@ void AsmPrinter::EmitLLVMUsedList(const Constant *List) { } } -/// EmitXXStructorList - Emit the ctor or dtor list. This just prints out the -/// function pointers, ignoring the init priority. +typedef std::pair<int, Constant*> Structor; + +static bool priority_order(const Structor& lhs, const Structor& rhs) { + return lhs.first < rhs.first; +} + +/// EmitXXStructorList - Emit the ctor or dtor list taking into account the init +/// priority. void AsmPrinter::EmitXXStructorList(const Constant *List) { // Should be an array of '{ int, void ()* }' structs. The first value is the - // init priority, which we ignore. + // init priority. if (!isa<ConstantArray>(List)) return; - const ConstantArray *InitList = cast<ConstantArray>(List); - for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i) - if (ConstantStruct *CS = dyn_cast<ConstantStruct>(InitList->getOperand(i))){ - if (CS->getNumOperands() != 2) return; // Not array of 2-element structs. - - if (CS->getOperand(1)->isNullValue()) - return; // Found a null terminator, exit printing. - // Emit the function pointer. - EmitGlobalConstant(CS->getOperand(1)); - } + + // Sanity check the structors list. + const ConstantArray *InitList = dyn_cast<ConstantArray>(List); + if (!InitList) return; // Not an array! + StructType *ETy = dyn_cast<StructType>(InitList->getType()->getElementType()); + if (!ETy || ETy->getNumElements() != 2) return; // Not an array of pairs! + if (!isa<IntegerType>(ETy->getTypeAtIndex(0U)) || + !isa<PointerType>(ETy->getTypeAtIndex(1U))) return; // Not (int, ptr). + + // Gather the structors in a form that's convenient for sorting by priority. + SmallVector<Structor, 8> Structors; + for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i) { + ConstantStruct *CS = dyn_cast<ConstantStruct>(InitList->getOperand(i)); + if (!CS) continue; // Malformed. + if (CS->getOperand(1)->isNullValue()) + break; // Found a null terminator, skip the rest. + ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0)); + if (!Priority) continue; // Malformed. + Structors.push_back(std::make_pair(Priority->getLimitedValue(65535), + CS->getOperand(1))); + } + + // Emit the function pointers in reverse priority order. + switch (MAI->getStructorOutputOrder()) { + case Structors::None: + break; + case Structors::PriorityOrder: + std::sort(Structors.begin(), Structors.end(), priority_order); + break; + case Structors::ReversePriorityOrder: + std::sort(Structors.rbegin(), Structors.rend(), priority_order); + break; + } + for (unsigned i = 0, e = Structors.size(); i != e; ++i) + EmitGlobalConstant(Structors[i].second); } //===--------------------------------------------------------------------===// @@ -1406,8 +1451,7 @@ static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) { // Generate a symbolic expression for the byte address const Constant *PtrVal = CE->getOperand(0); SmallVector<Value*, 8> IdxVec(CE->op_begin()+1, CE->op_end()); - int64_t Offset = TD.getIndexedOffset(PtrVal->getType(), &IdxVec[0], - IdxVec.size()); + int64_t Offset = TD.getIndexedOffset(PtrVal->getType(), IdxVec); const MCExpr *Base = LowerConstant(CE->getOperand(0), AP); if (Offset == 0) @@ -1447,7 +1491,7 @@ static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) { // Support only foldable casts to/from pointers that can be eliminated by // changing the pointer to the appropriately sized integer type. Constant *Op = CE->getOperand(0); - const Type *Ty = CE->getType(); + Type *Ty = CE->getType(); const MCExpr *OpExpr = LowerConstant(Op, AP); @@ -1496,12 +1540,67 @@ static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) { static void EmitGlobalConstantImpl(const Constant *C, unsigned AddrSpace, AsmPrinter &AP); +/// isRepeatedByteSequence - Determine whether the given value is +/// composed of a repeated sequence of identical bytes and return the +/// byte value. If it is not a repeated sequence, return -1. +static int isRepeatedByteSequence(const Value *V, TargetMachine &TM) { + + if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) { + if (CI->getBitWidth() > 64) return -1; + + uint64_t Size = TM.getTargetData()->getTypeAllocSize(V->getType()); + uint64_t Value = CI->getZExtValue(); + + // Make sure the constant is at least 8 bits long and has a power + // of 2 bit width. This guarantees the constant bit width is + // always a multiple of 8 bits, avoiding issues with padding out + // to Size and other such corner cases. + if (CI->getBitWidth() < 8 || !isPowerOf2_64(CI->getBitWidth())) return -1; + + uint8_t Byte = static_cast<uint8_t>(Value); + + for (unsigned i = 1; i < Size; ++i) { + Value >>= 8; + if (static_cast<uint8_t>(Value) != Byte) return -1; + } + return Byte; + } + if (const ConstantArray *CA = dyn_cast<ConstantArray>(V)) { + // Make sure all array elements are sequences of the same repeated + // byte. + if (CA->getNumOperands() == 0) return -1; + + int Byte = isRepeatedByteSequence(CA->getOperand(0), TM); + if (Byte == -1) return -1; + + for (unsigned i = 1, e = CA->getNumOperands(); i != e; ++i) { + int ThisByte = isRepeatedByteSequence(CA->getOperand(i), TM); + if (ThisByte == -1) return -1; + if (Byte != ThisByte) return -1; + } + return Byte; + } + + return -1; +} + static void EmitGlobalConstantArray(const ConstantArray *CA, unsigned AddrSpace, AsmPrinter &AP) { if (AddrSpace != 0 || !CA->isString()) { - // Not a string. Print the values in successive locations - for (unsigned i = 0, e = CA->getNumOperands(); i != e; ++i) - EmitGlobalConstantImpl(CA->getOperand(i), AddrSpace, AP); + // Not a string. Print the values in successive locations. + + // See if we can aggregate some values. Make sure it can be + // represented as a series of bytes of the constant value. + int Value = isRepeatedByteSequence(CA, AP.TM); + + if (Value != -1) { + uint64_t Bytes = AP.TM.getTargetData()->getTypeAllocSize(CA->getType()); + AP.OutStreamer.EmitFill(Bytes, Value, AddrSpace); + } + else { + for (unsigned i = 0, e = CA->getNumOperands(); i != e; ++i) + EmitGlobalConstantImpl(CA->getOperand(i), AddrSpace, AP); + } return; } diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp index dd5b0e2..4d6c281 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp @@ -13,7 +13,7 @@ #define DEBUG_TYPE "asm-printer" #include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/CodeGen/MachineLocation.h" +#include "llvm/MC/MachineLocation.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp index 5ac455e..8eda889 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp @@ -23,15 +23,15 @@ #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/MCSymbol.h" -#include "llvm/Target/TargetAsmParser.h" +#include "llvm/MC/MCTargetAsmParser.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetRegistry.h" #include "llvm/ADT/OwningPtr.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/Twine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SourceMgr.h" +#include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -49,7 +49,7 @@ namespace { static void SrcMgrDiagHandler(const SMDiagnostic &Diag, void *diagInfo) { SrcMgrDiagInfo *DiagInfo = static_cast<SrcMgrDiagInfo *>(diagInfo); assert(DiagInfo && "Diagnostic context not passed down?"); - + // If the inline asm had metadata associated with it, pull out a location // cookie corresponding to which line the error occurred on. unsigned LocCookie = 0; @@ -57,13 +57,13 @@ static void SrcMgrDiagHandler(const SMDiagnostic &Diag, void *diagInfo) { unsigned ErrorLine = Diag.getLineNo()-1; if (ErrorLine >= LocInfo->getNumOperands()) ErrorLine = 0; - + if (LocInfo->getNumOperands() != 0) if (const ConstantInt *CI = dyn_cast<ConstantInt>(LocInfo->getOperand(ErrorLine))) LocCookie = CI->getZExtValue(); } - + DiagInfo->DiagHandler(Diag, DiagInfo->DiagContext, LocCookie); } @@ -109,7 +109,7 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode) const { // Tell SrcMgr about this buffer, it takes ownership of the buffer. SrcMgr.AddNewSourceBuffer(Buffer, SMLoc()); - OwningPtr<MCAsmParser> Parser(createMCAsmParser(TM.getTarget(), SrcMgr, + OwningPtr<MCAsmParser> Parser(createMCAsmParser(SrcMgr, OutContext, OutStreamer, *MAI)); @@ -121,7 +121,8 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode) const { STI(TM.getTarget().createMCSubtargetInfo(TM.getTargetTriple(), TM.getTargetCPU(), TM.getTargetFeatureString())); - OwningPtr<TargetAsmParser> TAP(TM.getTarget().createAsmParser(*STI, *Parser)); + OwningPtr<MCTargetAsmParser> + TAP(TM.getTarget().createMCAsmParser(*STI, *Parser)); if (!TAP) report_fatal_error("Inline asm not supported by this streamer because" " we don't have an asm parser for this target\n"); diff --git a/lib/CodeGen/AsmPrinter/CMakeLists.txt b/lib/CodeGen/AsmPrinter/CMakeLists.txt index 4da7876..67d9273 100644 --- a/lib/CodeGen/AsmPrinter/CMakeLists.txt +++ b/lib/CodeGen/AsmPrinter/CMakeLists.txt @@ -12,3 +12,12 @@ add_llvm_library(LLVMAsmPrinter Win64Exception.cpp ) +add_llvm_library_dependencies(LLVMAsmPrinter + LLVMAnalysis + LLVMCodeGen + LLVMCore + LLVMMC + LLVMMCParser + LLVMSupport + LLVMTarget + ) diff --git a/lib/CodeGen/AsmPrinter/DIE.cpp b/lib/CodeGen/AsmPrinter/DIE.cpp index 21396ca..9c1ce76 100644 --- a/lib/CodeGen/AsmPrinter/DIE.cpp +++ b/lib/CodeGen/AsmPrinter/DIE.cpp @@ -69,7 +69,7 @@ void DIEAbbrev::Emit(AsmPrinter *AP) const { // Emit attribute type. // FIXME: Doing work even in non-asm-verbose runs. AP->EmitULEB128(AttrData.getAttribute(), - dwarf::AttributeString(AttrData.getAttribute())); + dwarf::AttributeString(AttrData.getAttribute())); // Emit form type. // FIXME: Doing work even in non-asm-verbose runs. diff --git a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp index 91b7d08..8ed4f4c 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp @@ -17,7 +17,7 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineLocation.h" +#include "llvm/MC/MachineLocation.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" @@ -77,7 +77,8 @@ void DwarfCFIException::EndModule() { // This is a temporary hack to keep sections in the same order they // were before. This lets us produce bit identical outputs while // transitioning to CFI. - Asm->OutStreamer.SwitchSection(TLOF.getEHFrameSection()); + Asm->OutStreamer.SwitchSection( + const_cast<TargetLoweringObjectFile&>(TLOF).getEHFrameSection()); } } diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp index 1fe035e..88b7524 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp @@ -16,7 +16,10 @@ #include "DwarfCompileUnit.h" #include "DwarfDebug.h" #include "llvm/Constants.h" +#include "llvm/GlobalVariable.h" +#include "llvm/Instructions.h" #include "llvm/Analysis/DIBuilder.h" +#include "llvm/Target/Mangler.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetMachine.h" @@ -132,8 +135,8 @@ void CompileUnit::addSourceLine(DIE *Die, DIGlobalVariable G) { unsigned Line = G.getLineNumber(); if (Line == 0) return; - unsigned FileID = DD->GetOrCreateSourceID(G.getContext().getFilename(), - G.getContext().getDirectory()); + unsigned FileID = DD->GetOrCreateSourceID(G.getFilename(), + G.getDirectory()); assert(FileID && "Invalid file id"); addUInt(Die, dwarf::DW_AT_decl_file, 0, FileID); addUInt(Die, dwarf::DW_AT_decl_line, 0, Line); @@ -439,27 +442,36 @@ void CompileUnit::addBlockByrefAddress(DbgVariable *&DV, DIE *Die, addBlock(Die, Attribute, 0, Block); } +/// isTypeSigned - Return true if the type is signed. +static bool isTypeSigned(DIType Ty, int *SizeInBits) { + if (Ty.isDerivedType()) + return isTypeSigned(DIDerivedType(Ty).getTypeDerivedFrom(), SizeInBits); + if (Ty.isBasicType()) + if (DIBasicType(Ty).getEncoding() == dwarf::DW_ATE_signed + || DIBasicType(Ty).getEncoding() == dwarf::DW_ATE_signed_char) { + *SizeInBits = Ty.getSizeInBits(); + return true; + } + return false; +} + /// addConstantValue - Add constant value entry in variable DIE. bool CompileUnit::addConstantValue(DIE *Die, const MachineOperand &MO, DIType Ty) { assert (MO.isImm() && "Invalid machine operand!"); DIEBlock *Block = new (DIEValueAllocator) DIEBlock(); - unsigned form = dwarf::DW_FORM_udata; - switch (Ty.getSizeInBits()) { - case 8: form = dwarf::DW_FORM_data1; break; - case 16: form = dwarf::DW_FORM_data2; break; - case 32: form = dwarf::DW_FORM_data4; break; - case 64: form = dwarf::DW_FORM_data8; break; + int SizeInBits = -1; + bool SignedConstant = isTypeSigned(Ty, &SizeInBits); + unsigned Form = SignedConstant ? dwarf::DW_FORM_sdata : dwarf::DW_FORM_udata; + switch (SizeInBits) { + case 8: Form = dwarf::DW_FORM_data1; break; + case 16: Form = dwarf::DW_FORM_data2; break; + case 32: Form = dwarf::DW_FORM_data4; break; + case 64: Form = dwarf::DW_FORM_data8; break; default: break; } - - DIBasicType BTy(Ty); - if (BTy.Verify() && - (BTy.getEncoding() == dwarf::DW_ATE_signed - || BTy.getEncoding() == dwarf::DW_ATE_signed_char)) - addSInt(Block, 0, form, MO.getImm()); - else - addUInt(Block, 0, form, MO.getImm()); + SignedConstant ? addSInt(Block, 0, Form, MO.getImm()) + : addUInt(Block, 0, Form, MO.getImm()); addBlock(Die, dwarf::DW_AT_const_value, 0, Block); return true; @@ -555,7 +567,7 @@ void CompileUnit::addToContextOwner(DIE *Die, DIDescriptor Context) { DIE *ContextDIE = getOrCreateNameSpace(DINameSpace(Context)); ContextDIE->addChild(Die); } else if (Context.isSubprogram()) { - DIE *ContextDIE = DD->createSubprogramDIE(DISubprogram(Context)); + DIE *ContextDIE = getOrCreateSubprogramDIE(DISubprogram(Context)); ContextDIE->addChild(Die); } else if (DIE *ContextDIE = getDIE(Context)) ContextDIE->addChild(Die); @@ -565,7 +577,10 @@ void CompileUnit::addToContextOwner(DIE *Die, DIDescriptor Context) { /// getOrCreateTypeDIE - Find existing DIE or create new DIE for the /// given DIType. -DIE *CompileUnit::getOrCreateTypeDIE(DIType Ty) { +DIE *CompileUnit::getOrCreateTypeDIE(const MDNode *TyNode) { + DIType Ty(TyNode); + if (!Ty.Verify()) + return NULL; DIE *TyDIE = getDIE(Ty); if (TyDIE) return TyDIE; @@ -617,7 +632,8 @@ void CompileUnit::addType(DIE *Entity, DIType Ty) { void CompileUnit::addGlobalType(DIType Ty) { DIDescriptor Context = Ty.getContext(); if (Ty.isCompositeType() && !Ty.getName().empty() && !Ty.isForwardDecl() - && (Context.isCompileUnit() || Context.isFile() || Context.isNameSpace())) + && (!Context || Context.isCompileUnit() || Context.isFile() + || Context.isNameSpace())) if (DIEEntry *Entry = getDIEEntry(Ty)) GlobalTypes[Ty.getName()] = Entry->getEntry(); } @@ -642,13 +658,20 @@ void CompileUnit::addPubTypes(DISubprogram SP) { void CompileUnit::constructTypeDIE(DIE &Buffer, DIBasicType BTy) { // Get core information. StringRef Name = BTy.getName(); - Buffer.setTag(dwarf::DW_TAG_base_type); - addUInt(&Buffer, dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, - BTy.getEncoding()); - // Add name if not anonymous or intermediate type. if (!Name.empty()) addString(&Buffer, dwarf::DW_AT_name, dwarf::DW_FORM_string, Name); + + if (BTy.getTag() == dwarf::DW_TAG_unspecified_type) { + Buffer.setTag(dwarf::DW_TAG_unspecified_type); + // Unspecified types has only name, nothing else. + return; + } + + Buffer.setTag(dwarf::DW_TAG_base_type); + addUInt(&Buffer, dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, + BTy.getEncoding()); + uint64_t Size = BTy.getSizeInBits() >> 3; addUInt(&Buffer, dwarf::DW_AT_byte_size, 0, Size); } @@ -752,7 +775,7 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { DIE *ElemDie = NULL; if (Element.isSubprogram()) { DISubprogram SP(Element); - ElemDie = DD->createSubprogramDIE(DISubprogram(Element)); + ElemDie = getOrCreateSubprogramDIE(DISubprogram(Element)); if (SP.isProtected()) addUInt(ElemDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag, dwarf::DW_ACCESS_protected); @@ -880,6 +903,218 @@ DIE *CompileUnit::getOrCreateNameSpace(DINameSpace NS) { return NDie; } +/// getRealLinkageName - If special LLVM prefix that is used to inform the asm +/// printer to not emit usual symbol prefix before the symbol name is used then +/// return linkage name after skipping this special LLVM prefix. +static StringRef getRealLinkageName(StringRef LinkageName) { + char One = '\1'; + if (LinkageName.startswith(StringRef(&One, 1))) + return LinkageName.substr(1); + return LinkageName; +} + +/// getOrCreateSubprogramDIE - Create new DIE using SP. +DIE *CompileUnit::getOrCreateSubprogramDIE(DISubprogram SP) { + DIE *SPDie = getDIE(SP); + if (SPDie) + return SPDie; + + SPDie = new DIE(dwarf::DW_TAG_subprogram); + + // DW_TAG_inlined_subroutine may refer to this DIE. + insertDIE(SP, SPDie); + + // Add to context owner. + addToContextOwner(SPDie, SP.getContext()); + + // Add function template parameters. + addTemplateParams(*SPDie, SP.getTemplateParams()); + + StringRef LinkageName = SP.getLinkageName(); + if (!LinkageName.empty()) + addString(SPDie, dwarf::DW_AT_MIPS_linkage_name, + dwarf::DW_FORM_string, + getRealLinkageName(LinkageName)); + + // If this DIE is going to refer declaration info using AT_specification + // then there is no need to add other attributes. + if (SP.getFunctionDeclaration().isSubprogram()) + return SPDie; + + // Constructors and operators for anonymous aggregates do not have names. + if (!SP.getName().empty()) + addString(SPDie, dwarf::DW_AT_name, dwarf::DW_FORM_string, + SP.getName()); + + addSourceLine(SPDie, SP); + + if (SP.isPrototyped()) + addUInt(SPDie, dwarf::DW_AT_prototyped, dwarf::DW_FORM_flag, 1); + + // Add Return Type. + DICompositeType SPTy = SP.getType(); + DIArray Args = SPTy.getTypeArray(); + unsigned SPTag = SPTy.getTag(); + + if (Args.getNumElements() == 0 || SPTag != dwarf::DW_TAG_subroutine_type) + addType(SPDie, SPTy); + else + addType(SPDie, DIType(Args.getElement(0))); + + unsigned VK = SP.getVirtuality(); + if (VK) { + addUInt(SPDie, dwarf::DW_AT_virtuality, dwarf::DW_FORM_flag, VK); + DIEBlock *Block = getDIEBlock(); + addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_constu); + addUInt(Block, 0, dwarf::DW_FORM_udata, SP.getVirtualIndex()); + addBlock(SPDie, dwarf::DW_AT_vtable_elem_location, 0, Block); + ContainingTypeMap.insert(std::make_pair(SPDie, + SP.getContainingType())); + } + + if (!SP.isDefinition()) { + addUInt(SPDie, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1); + + // Add arguments. Do not add arguments for subprogram definition. They will + // be handled while processing variables. + DICompositeType SPTy = SP.getType(); + DIArray Args = SPTy.getTypeArray(); + unsigned SPTag = SPTy.getTag(); + + if (SPTag == dwarf::DW_TAG_subroutine_type) + for (unsigned i = 1, N = Args.getNumElements(); i < N; ++i) { + DIE *Arg = new DIE(dwarf::DW_TAG_formal_parameter); + DIType ATy = DIType(DIType(Args.getElement(i))); + addType(Arg, ATy); + if (ATy.isArtificial()) + addUInt(Arg, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1); + SPDie->addChild(Arg); + } + } + + if (SP.isArtificial()) + addUInt(SPDie, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1); + + if (!SP.isLocalToUnit()) + addUInt(SPDie, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1); + + if (SP.isOptimized()) + addUInt(SPDie, dwarf::DW_AT_APPLE_optimized, dwarf::DW_FORM_flag, 1); + + if (unsigned isa = Asm->getISAEncoding()) { + addUInt(SPDie, dwarf::DW_AT_APPLE_isa, dwarf::DW_FORM_flag, isa); + } + + return SPDie; +} + +// Return const expression if value is a GEP to access merged global +// constant. e.g. +// i8* getelementptr ({ i8, i8, i8, i8 }* @_MergedGlobals, i32 0, i32 0) +static const ConstantExpr *getMergedGlobalExpr(const Value *V) { + const ConstantExpr *CE = dyn_cast_or_null<ConstantExpr>(V); + if (!CE || CE->getNumOperands() != 3 || + CE->getOpcode() != Instruction::GetElementPtr) + return NULL; + + // First operand points to a global struct. + Value *Ptr = CE->getOperand(0); + if (!isa<GlobalValue>(Ptr) || + !isa<StructType>(cast<PointerType>(Ptr->getType())->getElementType())) + return NULL; + + // Second operand is zero. + const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(CE->getOperand(1)); + if (!CI || !CI->isZero()) + return NULL; + + // Third operand is offset. + if (!isa<ConstantInt>(CE->getOperand(2))) + return NULL; + + return CE; +} + +/// createGlobalVariableDIE - create global variable DIE. +void CompileUnit::createGlobalVariableDIE(const MDNode *N) { + // Check for pre-existence. + if (getDIE(N)) + return; + + DIGlobalVariable GV(N); + if (!GV.Verify()) + return; + + DIE *VariableDIE = new DIE(GV.getTag()); + // Add to map. + insertDIE(N, VariableDIE); + + // Add name. + addString(VariableDIE, dwarf::DW_AT_name, dwarf::DW_FORM_string, + GV.getDisplayName()); + StringRef LinkageName = GV.getLinkageName(); + bool isGlobalVariable = GV.getGlobal() != NULL; + if (!LinkageName.empty() && isGlobalVariable) + addString(VariableDIE, dwarf::DW_AT_MIPS_linkage_name, + dwarf::DW_FORM_string, + getRealLinkageName(LinkageName)); + // Add type. + DIType GTy = GV.getType(); + addType(VariableDIE, GTy); + + // Add scoping info. + if (!GV.isLocalToUnit()) { + addUInt(VariableDIE, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1); + // Expose as global. + addGlobal(GV.getName(), VariableDIE); + } + // Add line number info. + addSourceLine(VariableDIE, GV); + // Add to context owner. + DIDescriptor GVContext = GV.getContext(); + addToContextOwner(VariableDIE, GVContext); + // Add location. + if (isGlobalVariable) { + DIEBlock *Block = new (DIEValueAllocator) DIEBlock(); + addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr); + addLabel(Block, 0, dwarf::DW_FORM_udata, + Asm->Mang->getSymbol(GV.getGlobal())); + // Do not create specification DIE if context is either compile unit + // or a subprogram. + if (GVContext && GV.isDefinition() && !GVContext.isCompileUnit() && + !GVContext.isFile() && !isSubprogramContext(GVContext)) { + // Create specification DIE. + DIE *VariableSpecDIE = new DIE(dwarf::DW_TAG_variable); + addDIEEntry(VariableSpecDIE, dwarf::DW_AT_specification, + dwarf::DW_FORM_ref4, VariableDIE); + addBlock(VariableSpecDIE, dwarf::DW_AT_location, 0, Block); + addUInt(VariableDIE, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, + 1); + addDie(VariableSpecDIE); + } else { + addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block); + } + } else if (const ConstantInt *CI = + dyn_cast_or_null<ConstantInt>(GV.getConstant())) + addConstantValue(VariableDIE, CI, GTy.isUnsignedDIType()); + else if (const ConstantExpr *CE = getMergedGlobalExpr(N->getOperand(11))) { + // GV is a merged global. + DIEBlock *Block = new (DIEValueAllocator) DIEBlock(); + Value *Ptr = CE->getOperand(0); + addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr); + addLabel(Block, 0, dwarf::DW_FORM_udata, + Asm->Mang->getSymbol(cast<GlobalValue>(Ptr))); + addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_constu); + SmallVector<Value*, 3> Idx(CE->op_begin()+1, CE->op_end()); + addUInt(Block, 0, dwarf::DW_FORM_udata, + Asm->getTargetData().getIndexedOffset(Ptr->getType(), Idx)); + addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_plus); + addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block); + } + + return; +} + /// constructSubrangeDIE - Construct subrange DIE from DISubrange. void CompileUnit::constructSubrangeDIE(DIE &Buffer, DISubrange SR, DIE *IndexTy){ DIE *DW_Subrange = new DIE(dwarf::DW_TAG_subrange_type); @@ -944,6 +1179,128 @@ DIE *CompileUnit::constructEnumTypeDIE(DIEnumerator ETy) { return Enumerator; } +/// constructContainingTypeDIEs - Construct DIEs for types that contain +/// vtables. +void CompileUnit::constructContainingTypeDIEs() { + for (DenseMap<DIE *, const MDNode *>::iterator CI = ContainingTypeMap.begin(), + CE = ContainingTypeMap.end(); CI != CE; ++CI) { + DIE *SPDie = CI->first; + const MDNode *N = CI->second; + if (!N) continue; + DIE *NDie = getDIE(N); + if (!NDie) continue; + addDIEEntry(SPDie, dwarf::DW_AT_containing_type, dwarf::DW_FORM_ref4, NDie); + } +} + +/// constructVariableDIE - Construct a DIE for the given DbgVariable. +DIE *CompileUnit::constructVariableDIE(DbgVariable *DV, bool isScopeAbstract) { + StringRef Name = DV->getName(); + if (Name.empty()) + return NULL; + + // Translate tag to proper Dwarf tag. + unsigned Tag = DV->getTag(); + + // Define variable debug information entry. + DIE *VariableDie = new DIE(Tag); + DbgVariable *AbsVar = DV->getAbstractVariable(); + DIE *AbsDIE = AbsVar ? AbsVar->getDIE() : NULL; + if (AbsDIE) + addDIEEntry(VariableDie, dwarf::DW_AT_abstract_origin, + dwarf::DW_FORM_ref4, AbsDIE); + else { + addString(VariableDie, dwarf::DW_AT_name, + dwarf::DW_FORM_string, Name); + addSourceLine(VariableDie, DV->getVariable()); + addType(VariableDie, DV->getType()); + } + + if (DV->isArtificial()) + addUInt(VariableDie, dwarf::DW_AT_artificial, + dwarf::DW_FORM_flag, 1); + + if (isScopeAbstract) { + DV->setDIE(VariableDie); + return VariableDie; + } + + // Add variable address. + + unsigned Offset = DV->getDotDebugLocOffset(); + if (Offset != ~0U) { + addLabel(VariableDie, dwarf::DW_AT_location, + dwarf::DW_FORM_data4, + Asm->GetTempSymbol("debug_loc", Offset)); + DV->setDIE(VariableDie); + return VariableDie; + } + + // Check if variable is described by a DBG_VALUE instruction. + if (const MachineInstr *DVInsn = DV->getMInsn()) { + bool updated = false; + if (DVInsn->getNumOperands() == 3) { + if (DVInsn->getOperand(0).isReg()) { + const MachineOperand RegOp = DVInsn->getOperand(0); + const TargetRegisterInfo *TRI = Asm->TM.getRegisterInfo(); + if (DVInsn->getOperand(1).isImm() && + TRI->getFrameRegister(*Asm->MF) == RegOp.getReg()) { + unsigned FrameReg = 0; + const TargetFrameLowering *TFI = Asm->TM.getFrameLowering(); + int Offset = + TFI->getFrameIndexReference(*Asm->MF, + DVInsn->getOperand(1).getImm(), + FrameReg); + MachineLocation Location(FrameReg, Offset); + addVariableAddress(DV, VariableDie, Location); + + } else if (RegOp.getReg()) + addVariableAddress(DV, VariableDie, + MachineLocation(RegOp.getReg())); + updated = true; + } + else if (DVInsn->getOperand(0).isImm()) + updated = + addConstantValue(VariableDie, DVInsn->getOperand(0), + DV->getType()); + else if (DVInsn->getOperand(0).isFPImm()) + updated = + addConstantFPValue(VariableDie, DVInsn->getOperand(0)); + else if (DVInsn->getOperand(0).isCImm()) + updated = + addConstantValue(VariableDie, + DVInsn->getOperand(0).getCImm(), + DV->getType().isUnsignedDIType()); + } else { + addVariableAddress(DV, VariableDie, + Asm->getDebugValueLocation(DVInsn)); + updated = true; + } + if (!updated) { + // If variableDie is not updated then DBG_VALUE instruction does not + // have valid variable info. + delete VariableDie; + return NULL; + } + DV->setDIE(VariableDie); + return VariableDie; + } else { + // .. else use frame index. + int FI = DV->getFrameIndex(); + if (FI != ~0) { + unsigned FrameReg = 0; + const TargetFrameLowering *TFI = Asm->TM.getFrameLowering(); + int Offset = + TFI->getFrameIndexReference(*Asm->MF, FI, FrameReg); + MachineLocation Location(FrameReg, Offset); + addVariableAddress(DV, VariableDie, Location); + } + } + + DV->setDIE(VariableDie); + return VariableDie; +} + /// createMemberDIE - Create new member DIE. DIE *CompileUnit::createMemberDIE(DIDerivedType DT) { DIE *MemberDie = new DIE(DT.getTag()); @@ -1013,7 +1370,7 @@ DIE *CompileUnit::createMemberDIE(DIDerivedType DT) { addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag, dwarf::DW_ACCESS_private); // Otherwise C++ member and base classes are considered public. - else if (DT.getCompileUnit().getLanguage() == dwarf::DW_LANG_C_plus_plus) + else addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag, dwarf::DW_ACCESS_public); if (DT.isVirtual()) diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h index 213c7fc..7859265 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h +++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h @@ -67,6 +67,11 @@ class CompileUnit { /// DIEBlocks - A list of all the DIEBlocks in use. std::vector<DIEBlock *> DIEBlocks; + /// ContainingTypeMap - This map is used to keep track of subprogram DIEs that + /// need DW_AT_containing_type attribute. This attribute points to a DIE that + /// corresponds to the MDNode mapped with the subprogram DIE. + DenseMap<DIE *, const MDNode *> ContainingTypeMap; + public: CompileUnit(unsigned I, DIE *D, AsmPrinter *A, DwarfDebug *DW); ~CompileUnit(); @@ -226,9 +231,12 @@ public: /// getOrCreateNameSpace - Create a DIE for DINameSpace. DIE *getOrCreateNameSpace(DINameSpace NS); + /// getOrCreateSubprogramDIE - Create new DIE using SP. + DIE *getOrCreateSubprogramDIE(DISubprogram SP); + /// getOrCreateTypeDIE - Find existing DIE or create new DIE for the /// given DIType. - DIE *getOrCreateTypeDIE(DIType Ty); + DIE *getOrCreateTypeDIE(const MDNode *N); /// getOrCreateTemplateTypeParameterDIE - Find existing DIE or create new DIE /// for the given DITemplateTypeParameter. @@ -242,6 +250,9 @@ public: /// information entry. DIEEntry *createDIEEntry(DIE *Entry); + /// createGlobalVariableDIE - create global variable DIE. + void createGlobalVariableDIE(const MDNode *N); + void addPubTypes(DISubprogram SP); /// constructTypeDIE - Construct basic type die from DIBasicType. @@ -266,6 +277,13 @@ public: /// constructEnumTypeDIE - Construct enum type DIE from DIEnumerator. DIE *constructEnumTypeDIE(DIEnumerator ETy); + /// constructContainingTypeDIEs - Construct DIEs for types that contain + /// vtables. + void constructContainingTypeDIEs(); + + /// constructVariableDIE - Construct a DIE for the given DbgVariable. + DIE *constructVariableDIE(DbgVariable *DV, bool isScopeAbstract); + /// createMemberDIE - Create new member DIE. DIE *createMemberDIE(DIDerivedType DT); diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index 125e1e8..1b7e370 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -24,7 +24,6 @@ #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" -#include "llvm/Target/Mangler.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetLoweringObjectFile.h" @@ -45,9 +44,6 @@ #include "llvm/Support/Path.h" using namespace llvm; -static cl::opt<bool> PrintDbgScope("print-dbgscope", cl::Hidden, - cl::desc("Print DbgScope information for each machine instruction")); - static cl::opt<bool> DisableDebugInfoPrinting("disable-debug-info-print", cl::Hidden, cl::desc("Disable debug info printing")); @@ -69,7 +65,7 @@ static const unsigned InitAbbreviationsSetSize = 9; // log2(512) namespace llvm { -DIType DbgVariable::getType() const { +DIType DbgVariable::getType() const { DIType Ty = Var.getType(); // FIXME: isBlockByrefVariable should be reformulated in terms of complex // addresses instead. @@ -120,141 +116,12 @@ DIType DbgVariable::getType() const { return Ty; } -//===----------------------------------------------------------------------===// -/// DbgRange - This is used to track range of instructions with identical -/// debug info scope. -/// -typedef std::pair<const MachineInstr *, const MachineInstr *> DbgRange; - -//===----------------------------------------------------------------------===// -/// DbgScope - This class is used to track scope information. -/// -class DbgScope { - DbgScope *Parent; // Parent to this scope. - DIDescriptor Desc; // Debug info descriptor for scope. - // Location at which this scope is inlined. - AssertingVH<const MDNode> InlinedAtLocation; - bool AbstractScope; // Abstract Scope - const MachineInstr *LastInsn; // Last instruction of this scope. - const MachineInstr *FirstInsn; // First instruction of this scope. - unsigned DFSIn, DFSOut; - // Scopes defined in scope. Contents not owned. - SmallVector<DbgScope *, 4> Scopes; - // Variables declared in scope. Contents owned. - SmallVector<DbgVariable *, 8> Variables; - SmallVector<DbgRange, 4> Ranges; - // Private state for dump() - mutable unsigned IndentLevel; -public: - DbgScope(DbgScope *P, DIDescriptor D, const MDNode *I = 0) - : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(false), - LastInsn(0), FirstInsn(0), - DFSIn(0), DFSOut(0), IndentLevel(0) {} - virtual ~DbgScope(); - - // Accessors. - DbgScope *getParent() const { return Parent; } - void setParent(DbgScope *P) { Parent = P; } - DIDescriptor getDesc() const { return Desc; } - const MDNode *getInlinedAt() const { return InlinedAtLocation; } - const MDNode *getScopeNode() const { return Desc; } - const SmallVector<DbgScope *, 4> &getScopes() { return Scopes; } - const SmallVector<DbgVariable *, 8> &getDbgVariables() { return Variables; } - const SmallVector<DbgRange, 4> &getRanges() { return Ranges; } - - /// openInsnRange - This scope covers instruction range starting from MI. - void openInsnRange(const MachineInstr *MI) { - if (!FirstInsn) - FirstInsn = MI; - - if (Parent) - Parent->openInsnRange(MI); - } - - /// extendInsnRange - Extend the current instruction range covered by - /// this scope. - void extendInsnRange(const MachineInstr *MI) { - assert (FirstInsn && "MI Range is not open!"); - LastInsn = MI; - if (Parent) - Parent->extendInsnRange(MI); - } - - /// closeInsnRange - Create a range based on FirstInsn and LastInsn collected - /// until now. This is used when a new scope is encountered while walking - /// machine instructions. - void closeInsnRange(DbgScope *NewScope = NULL) { - assert (LastInsn && "Last insn missing!"); - Ranges.push_back(DbgRange(FirstInsn, LastInsn)); - FirstInsn = NULL; - LastInsn = NULL; - // If Parent dominates NewScope then do not close Parent's instruction - // range. - if (Parent && (!NewScope || !Parent->dominates(NewScope))) - Parent->closeInsnRange(NewScope); - } - - void setAbstractScope() { AbstractScope = true; } - bool isAbstractScope() const { return AbstractScope; } - - // Depth First Search support to walk and mainpluate DbgScope hierarchy. - unsigned getDFSOut() const { return DFSOut; } - void setDFSOut(unsigned O) { DFSOut = O; } - unsigned getDFSIn() const { return DFSIn; } - void setDFSIn(unsigned I) { DFSIn = I; } - bool dominates(const DbgScope *S) { - if (S == this) - return true; - if (DFSIn < S->getDFSIn() && DFSOut > S->getDFSOut()) - return true; - return false; - } - - /// addScope - Add a scope to the scope. - /// - void addScope(DbgScope *S) { Scopes.push_back(S); } - - /// addVariable - Add a variable to the scope. - /// - void addVariable(DbgVariable *V) { Variables.push_back(V); } - -#ifndef NDEBUG - void dump() const; -#endif -}; - } // end llvm namespace -#ifndef NDEBUG -void DbgScope::dump() const { - raw_ostream &err = dbgs(); - err.indent(IndentLevel); - err << "DFSIn: " << DFSIn << " DFSOut: " << DFSOut << "\n"; - const MDNode *N = Desc; - N->dump(); - if (AbstractScope) - err << "Abstract Scope\n"; - - IndentLevel += 2; - if (!Scopes.empty()) - err << "Children ...\n"; - for (unsigned i = 0, e = Scopes.size(); i != e; ++i) - if (Scopes[i] != this) - Scopes[i]->dump(); - - IndentLevel -= 2; -} -#endif - -DbgScope::~DbgScope() { - for (unsigned j = 0, M = Variables.size(); j < M; ++j) - delete Variables[j]; -} - DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M) : Asm(A), MMI(Asm->MMI), FirstCU(0), AbbreviationsSet(InitAbbreviationsSetSize), - CurrentFnDbgScope(0), PrevLabel(NULL) { + PrevLabel(NULL) { NextStringPoolNumber = 0; DwarfInfoSectionSym = DwarfAbbrevSectionSym = 0; @@ -311,147 +178,12 @@ static StringRef getRealLinkageName(StringRef LinkageName) { return LinkageName; } -/// createSubprogramDIE - Create new DIE using SP. -DIE *DwarfDebug::createSubprogramDIE(DISubprogram SP) { - CompileUnit *SPCU = getCompileUnit(SP); - DIE *SPDie = SPCU->getDIE(SP); - if (SPDie) - return SPDie; - - SPDie = new DIE(dwarf::DW_TAG_subprogram); - - // DW_TAG_inlined_subroutine may refer to this DIE. - SPCU->insertDIE(SP, SPDie); - - // Add to context owner. - SPCU->addToContextOwner(SPDie, SP.getContext()); - - // Add function template parameters. - SPCU->addTemplateParams(*SPDie, SP.getTemplateParams()); - - StringRef LinkageName = SP.getLinkageName(); - if (!LinkageName.empty()) - SPCU->addString(SPDie, dwarf::DW_AT_MIPS_linkage_name, dwarf::DW_FORM_string, - getRealLinkageName(LinkageName)); - - // If this DIE is going to refer declaration info using AT_specification - // then there is no need to add other attributes. - if (SP.getFunctionDeclaration().isSubprogram()) - return SPDie; - - // Constructors and operators for anonymous aggregates do not have names. - if (!SP.getName().empty()) - SPCU->addString(SPDie, dwarf::DW_AT_name, dwarf::DW_FORM_string, - SP.getName()); - - SPCU->addSourceLine(SPDie, SP); - - if (SP.isPrototyped()) - SPCU->addUInt(SPDie, dwarf::DW_AT_prototyped, dwarf::DW_FORM_flag, 1); - - // Add Return Type. - DICompositeType SPTy = SP.getType(); - DIArray Args = SPTy.getTypeArray(); - unsigned SPTag = SPTy.getTag(); - - if (Args.getNumElements() == 0 || SPTag != dwarf::DW_TAG_subroutine_type) - SPCU->addType(SPDie, SPTy); - else - SPCU->addType(SPDie, DIType(Args.getElement(0))); - - unsigned VK = SP.getVirtuality(); - if (VK) { - SPCU->addUInt(SPDie, dwarf::DW_AT_virtuality, dwarf::DW_FORM_flag, VK); - DIEBlock *Block = SPCU->getDIEBlock(); - SPCU->addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_constu); - SPCU->addUInt(Block, 0, dwarf::DW_FORM_udata, SP.getVirtualIndex()); - SPCU->addBlock(SPDie, dwarf::DW_AT_vtable_elem_location, 0, Block); - ContainingTypeMap.insert(std::make_pair(SPDie, - SP.getContainingType())); - } - - if (!SP.isDefinition()) { - SPCU->addUInt(SPDie, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1); - - // Add arguments. Do not add arguments for subprogram definition. They will - // be handled while processing variables. - DICompositeType SPTy = SP.getType(); - DIArray Args = SPTy.getTypeArray(); - unsigned SPTag = SPTy.getTag(); - - if (SPTag == dwarf::DW_TAG_subroutine_type) - for (unsigned i = 1, N = Args.getNumElements(); i < N; ++i) { - DIE *Arg = new DIE(dwarf::DW_TAG_formal_parameter); - DIType ATy = DIType(DIType(Args.getElement(i))); - SPCU->addType(Arg, ATy); - if (ATy.isArtificial()) - SPCU->addUInt(Arg, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1); - SPDie->addChild(Arg); - } - } - - if (SP.isArtificial()) - SPCU->addUInt(SPDie, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1); - - if (!SP.isLocalToUnit()) - SPCU->addUInt(SPDie, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1); - - if (SP.isOptimized()) - SPCU->addUInt(SPDie, dwarf::DW_AT_APPLE_optimized, dwarf::DW_FORM_flag, 1); - - if (unsigned isa = Asm->getISAEncoding()) { - SPCU->addUInt(SPDie, dwarf::DW_AT_APPLE_isa, dwarf::DW_FORM_flag, isa); - } - - return SPDie; -} - -DbgScope *DwarfDebug::getOrCreateAbstractScope(const MDNode *N) { - assert(N && "Invalid Scope encoding!"); - - DbgScope *AScope = AbstractScopes.lookup(N); - if (AScope) - return AScope; - - DbgScope *Parent = NULL; - - DIDescriptor Scope(N); - if (Scope.isLexicalBlock()) { - DILexicalBlock DB(N); - DIDescriptor ParentDesc = DB.getContext(); - Parent = getOrCreateAbstractScope(ParentDesc); - } - - AScope = new DbgScope(Parent, DIDescriptor(N), NULL); - - if (Parent) - Parent->addScope(AScope); - AScope->setAbstractScope(); - AbstractScopes[N] = AScope; - if (DIDescriptor(N).isSubprogram()) - AbstractScopesList.push_back(AScope); - return AScope; -} - -/// isSubprogramContext - Return true if Context is either a subprogram -/// or another context nested inside a subprogram. -static bool isSubprogramContext(const MDNode *Context) { - if (!Context) - return false; - DIDescriptor D(Context); - if (D.isSubprogram()) - return true; - if (D.isType()) - return isSubprogramContext(DIType(Context).getContext()); - return false; -} - /// updateSubprogramScopeDIE - Find DIE for the given subprogram and /// attach appropriate DW_AT_low_pc and DW_AT_high_pc attributes. /// If there are global variables in this scope then create and insert /// DIEs for these variables. -DIE *DwarfDebug::updateSubprogramScopeDIE(const MDNode *SPNode) { - CompileUnit *SPCU = getCompileUnit(SPNode); +DIE *DwarfDebug::updateSubprogramScopeDIE(CompileUnit *SPCU, + const MDNode *SPNode) { DIE *SPDie = SPCU->getDIE(SPNode); assert(SPDie && "Unable to find subprogram DIE!"); @@ -461,7 +193,7 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(const MDNode *SPNode) { if (SPDecl.isSubprogram()) // Refer function declaration directly. SPCU->addDIEEntry(SPDie, dwarf::DW_AT_specification, dwarf::DW_FORM_ref4, - createSubprogramDIE(SPDecl)); + SPCU->getOrCreateSubprogramDIE(SPDecl)); else { // There is not any need to generate specification DIE for a function // defined at compile unit level. If a function is defined inside another @@ -514,25 +246,26 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(const MDNode *SPNode) { /// constructLexicalScope - Construct new DW_TAG_lexical_block /// for this scope and attach DW_AT_low_pc/DW_AT_high_pc labels. -DIE *DwarfDebug::constructLexicalScopeDIE(DbgScope *Scope) { +DIE *DwarfDebug::constructLexicalScopeDIE(CompileUnit *TheCU, + LexicalScope *Scope) { DIE *ScopeDIE = new DIE(dwarf::DW_TAG_lexical_block); if (Scope->isAbstractScope()) return ScopeDIE; - const SmallVector<DbgRange, 4> &Ranges = Scope->getRanges(); + const SmallVector<InsnRange, 4> &Ranges = Scope->getRanges(); if (Ranges.empty()) return 0; - CompileUnit *TheCU = getCompileUnit(Scope->getScopeNode()); - SmallVector<DbgRange, 4>::const_iterator RI = Ranges.begin(); + SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(); if (Ranges.size() > 1) { // .debug_range section has not been laid out yet. Emit offset in // .debug_range as a uint, size 4, for now. emitDIE will handle // DW_AT_ranges appropriately. TheCU->addUInt(ScopeDIE, dwarf::DW_AT_ranges, dwarf::DW_FORM_data4, - DebugRangeSymbols.size() * Asm->getTargetData().getPointerSize()); - for (SmallVector<DbgRange, 4>::const_iterator RI = Ranges.begin(), + DebugRangeSymbols.size() + * Asm->getTargetData().getPointerSize()); + for (SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(), RE = Ranges.end(); RI != RE; ++RI) { DebugRangeSymbols.push_back(getLabelBeforeInsn(RI->first)); DebugRangeSymbols.push_back(getLabelAfterInsn(RI->second)); @@ -559,22 +292,29 @@ DIE *DwarfDebug::constructLexicalScopeDIE(DbgScope *Scope) { /// constructInlinedScopeDIE - This scope represents inlined body of /// a function. Construct DIE to represent this concrete inlined copy /// of the function. -DIE *DwarfDebug::constructInlinedScopeDIE(DbgScope *Scope) { +DIE *DwarfDebug::constructInlinedScopeDIE(CompileUnit *TheCU, + LexicalScope *Scope) { - const SmallVector<DbgRange, 4> &Ranges = Scope->getRanges(); + const SmallVector<InsnRange, 4> &Ranges = Scope->getRanges(); assert (Ranges.empty() == false - && "DbgScope does not have instruction markers!"); + && "LexicalScope does not have instruction markers!"); - // FIXME : .debug_inlined section specification does not clearly state how - // to emit inlined scope that is split into multiple instruction ranges. - // For now, use first instruction range and emit low_pc/high_pc pair and - // corresponding .debug_inlined section entry for this pair. - SmallVector<DbgRange, 4>::const_iterator RI = Ranges.begin(); + if (!Scope->getScopeNode()) + return NULL; + DIScope DS(Scope->getScopeNode()); + DISubprogram InlinedSP = getDISubprogram(DS); + DIE *OriginDIE = TheCU->getDIE(InlinedSP); + if (!OriginDIE) { + DEBUG(dbgs() << "Unable to find original DIE for inlined subprogram."); + return NULL; + } + + SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(); const MCSymbol *StartLabel = getLabelBeforeInsn(RI->first); const MCSymbol *EndLabel = getLabelAfterInsn(RI->second); if (StartLabel == 0 || EndLabel == 0) { - assert (0 && "Unexpected Start and End labels for a inlined scope!"); + assert (0 && "Unexpected Start and End labels for a inlined scope!"); return 0; } assert(StartLabel->isDefined() && @@ -582,26 +322,38 @@ DIE *DwarfDebug::constructInlinedScopeDIE(DbgScope *Scope) { assert(EndLabel->isDefined() && "Invalid end label for an inlined scope!"); - if (!Scope->getScopeNode()) - return NULL; - DIScope DS(Scope->getScopeNode()); - DISubprogram InlinedSP = getDISubprogram(DS); - CompileUnit *TheCU = getCompileUnit(InlinedSP); - DIE *OriginDIE = TheCU->getDIE(InlinedSP); - if (!OriginDIE) { - DEBUG(dbgs() << "Unable to find original DIE for inlined subprogram."); - return NULL; - } DIE *ScopeDIE = new DIE(dwarf::DW_TAG_inlined_subroutine); TheCU->addDIEEntry(ScopeDIE, dwarf::DW_AT_abstract_origin, dwarf::DW_FORM_ref4, OriginDIE); - TheCU->addLabel(ScopeDIE, dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, StartLabel); - TheCU->addLabel(ScopeDIE, dwarf::DW_AT_high_pc, dwarf::DW_FORM_addr, EndLabel); + if (Ranges.size() > 1) { + // .debug_range section has not been laid out yet. Emit offset in + // .debug_range as a uint, size 4, for now. emitDIE will handle + // DW_AT_ranges appropriately. + TheCU->addUInt(ScopeDIE, dwarf::DW_AT_ranges, dwarf::DW_FORM_data4, + DebugRangeSymbols.size() + * Asm->getTargetData().getPointerSize()); + for (SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(), + RE = Ranges.end(); RI != RE; ++RI) { + DebugRangeSymbols.push_back(getLabelBeforeInsn(RI->first)); + DebugRangeSymbols.push_back(getLabelAfterInsn(RI->second)); + } + DebugRangeSymbols.push_back(NULL); + DebugRangeSymbols.push_back(NULL); + } else { + TheCU->addLabel(ScopeDIE, dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, + StartLabel); + TheCU->addLabel(ScopeDIE, dwarf::DW_AT_high_pc, dwarf::DW_FORM_addr, + EndLabel); + } InlinedSubprogramDIEs.insert(OriginDIE); // Track the start label for this inlined function. + //.debug_inlined section specification does not clearly state how + // to emit inlined scope that is split into multiple instruction ranges. + // For now, use first instruction range and emit low_pc/high_pc pair and + // corresponding .debug_inlined section entry for this pair. DenseMap<const MDNode *, SmallVector<InlineInfoLabels, 4> >::iterator I = InlineInfo.find(InlinedSP); @@ -619,200 +371,51 @@ DIE *DwarfDebug::constructInlinedScopeDIE(DbgScope *Scope) { return ScopeDIE; } -/// isUnsignedDIType - Return true if type encoding is unsigned. -static bool isUnsignedDIType(DIType Ty) { - DIDerivedType DTy(Ty); - if (DTy.Verify()) - return isUnsignedDIType(DTy.getTypeDerivedFrom()); - - DIBasicType BTy(Ty); - if (BTy.Verify()) { - unsigned Encoding = BTy.getEncoding(); - if (Encoding == dwarf::DW_ATE_unsigned || - Encoding == dwarf::DW_ATE_unsigned_char) - return true; - } - return false; -} - -/// constructVariableDIE - Construct a DIE for the given DbgVariable. -DIE *DwarfDebug::constructVariableDIE(DbgVariable *DV, DbgScope *Scope) { - StringRef Name = DV->getName(); - if (Name.empty()) - return NULL; - - // Translate tag to proper Dwarf tag. The result variable is dropped for - // now. - unsigned Tag; - switch (DV->getTag()) { - case dwarf::DW_TAG_return_variable: - return NULL; - case dwarf::DW_TAG_arg_variable: - Tag = dwarf::DW_TAG_formal_parameter; - break; - case dwarf::DW_TAG_auto_variable: // fall thru - default: - Tag = dwarf::DW_TAG_variable; - break; - } - - // Define variable debug information entry. - DIE *VariableDie = new DIE(Tag); - CompileUnit *VariableCU = getCompileUnit(DV->getVariable()); - DIE *AbsDIE = NULL; - DenseMap<const DbgVariable *, const DbgVariable *>::iterator - V2AVI = VarToAbstractVarMap.find(DV); - if (V2AVI != VarToAbstractVarMap.end()) - AbsDIE = V2AVI->second->getDIE(); - - if (AbsDIE) - VariableCU->addDIEEntry(VariableDie, dwarf::DW_AT_abstract_origin, - dwarf::DW_FORM_ref4, AbsDIE); - else { - VariableCU->addString(VariableDie, dwarf::DW_AT_name, dwarf::DW_FORM_string, - Name); - VariableCU->addSourceLine(VariableDie, DV->getVariable()); - - // Add variable type. - VariableCU->addType(VariableDie, DV->getType()); - } - - if (Tag == dwarf::DW_TAG_formal_parameter && DV->getType().isArtificial()) - VariableCU->addUInt(VariableDie, dwarf::DW_AT_artificial, - dwarf::DW_FORM_flag, 1); - else if (DIVariable(DV->getVariable()).isArtificial()) - VariableCU->addUInt(VariableDie, dwarf::DW_AT_artificial, - dwarf::DW_FORM_flag, 1); - - if (Scope->isAbstractScope()) { - DV->setDIE(VariableDie); - return VariableDie; - } - - // Add variable address. - - unsigned Offset = DV->getDotDebugLocOffset(); - if (Offset != ~0U) { - VariableCU->addLabel(VariableDie, dwarf::DW_AT_location, dwarf::DW_FORM_data4, - Asm->GetTempSymbol("debug_loc", Offset)); - DV->setDIE(VariableDie); - UseDotDebugLocEntry.insert(VariableDie); - return VariableDie; - } - - // Check if variable is described by a DBG_VALUE instruction. - DenseMap<const DbgVariable *, const MachineInstr *>::iterator DVI = - DbgVariableToDbgInstMap.find(DV); - if (DVI != DbgVariableToDbgInstMap.end()) { - const MachineInstr *DVInsn = DVI->second; - bool updated = false; - // FIXME : Handle getNumOperands != 3 - if (DVInsn->getNumOperands() == 3) { - if (DVInsn->getOperand(0).isReg()) { - const MachineOperand RegOp = DVInsn->getOperand(0); - const TargetRegisterInfo *TRI = Asm->TM.getRegisterInfo(); - if (DVInsn->getOperand(1).isImm() && - TRI->getFrameRegister(*Asm->MF) == RegOp.getReg()) { - unsigned FrameReg = 0; - const TargetFrameLowering *TFI = Asm->TM.getFrameLowering(); - int Offset = - TFI->getFrameIndexReference(*Asm->MF, - DVInsn->getOperand(1).getImm(), - FrameReg); - MachineLocation Location(FrameReg, Offset); - VariableCU->addVariableAddress(DV, VariableDie, Location); - - } else if (RegOp.getReg()) - VariableCU->addVariableAddress(DV, VariableDie, - MachineLocation(RegOp.getReg())); - updated = true; - } - else if (DVInsn->getOperand(0).isImm()) - updated = - VariableCU->addConstantValue(VariableDie, DVInsn->getOperand(0), - DV->getType()); - else if (DVInsn->getOperand(0).isFPImm()) - updated = - VariableCU->addConstantFPValue(VariableDie, DVInsn->getOperand(0)); - else if (DVInsn->getOperand(0).isCImm()) - updated = - VariableCU->addConstantValue(VariableDie, - DVInsn->getOperand(0).getCImm(), - isUnsignedDIType(DV->getType())); - } else { - VariableCU->addVariableAddress(DV, VariableDie, - Asm->getDebugValueLocation(DVInsn)); - updated = true; - } - if (!updated) { - // If variableDie is not updated then DBG_VALUE instruction does not - // have valid variable info. - delete VariableDie; - return NULL; - } - DV->setDIE(VariableDie); - return VariableDie; - } - - // .. else use frame index, if available. - int FI = 0; - if (findVariableFrameIndex(DV, &FI)) { - unsigned FrameReg = 0; - const TargetFrameLowering *TFI = Asm->TM.getFrameLowering(); - int Offset = - TFI->getFrameIndexReference(*Asm->MF, FI, FrameReg); - MachineLocation Location(FrameReg, Offset); - VariableCU->addVariableAddress(DV, VariableDie, Location); - } - - DV->setDIE(VariableDie); - return VariableDie; - -} - /// constructScopeDIE - Construct a DIE for this scope. -DIE *DwarfDebug::constructScopeDIE(DbgScope *Scope) { +DIE *DwarfDebug::constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) { if (!Scope || !Scope->getScopeNode()) return NULL; SmallVector <DIE *, 8> Children; // Collect arguments for current function. - if (Scope == CurrentFnDbgScope) + if (LScopes.isCurrentFunctionScope(Scope)) for (unsigned i = 0, N = CurrentFnArguments.size(); i < N; ++i) if (DbgVariable *ArgDV = CurrentFnArguments[i]) - if (DIE *Arg = constructVariableDIE(ArgDV, Scope)) + if (DIE *Arg = + TheCU->constructVariableDIE(ArgDV, Scope->isAbstractScope())) Children.push_back(Arg); - // Collect lexical scope childrens first. - const SmallVector<DbgVariable *, 8> &Variables = Scope->getDbgVariables(); + // Collect lexical scope children first. + const SmallVector<DbgVariable *, 8> &Variables = ScopeVariables.lookup(Scope); for (unsigned i = 0, N = Variables.size(); i < N; ++i) - if (DIE *Variable = constructVariableDIE(Variables[i], Scope)) + if (DIE *Variable = + TheCU->constructVariableDIE(Variables[i], Scope->isAbstractScope())) Children.push_back(Variable); - const SmallVector<DbgScope *, 4> &Scopes = Scope->getScopes(); + const SmallVector<LexicalScope *, 4> &Scopes = Scope->getChildren(); for (unsigned j = 0, M = Scopes.size(); j < M; ++j) - if (DIE *Nested = constructScopeDIE(Scopes[j])) + if (DIE *Nested = constructScopeDIE(TheCU, Scopes[j])) Children.push_back(Nested); DIScope DS(Scope->getScopeNode()); DIE *ScopeDIE = NULL; if (Scope->getInlinedAt()) - ScopeDIE = constructInlinedScopeDIE(Scope); + ScopeDIE = constructInlinedScopeDIE(TheCU, Scope); else if (DS.isSubprogram()) { ProcessedSPNodes.insert(DS); if (Scope->isAbstractScope()) { - ScopeDIE = getCompileUnit(DS)->getDIE(DS); + ScopeDIE = TheCU->getDIE(DS); // Note down abstract DIE. if (ScopeDIE) AbstractSPDies.insert(std::make_pair(DS, ScopeDIE)); } else - ScopeDIE = updateSubprogramScopeDIE(DS); + ScopeDIE = updateSubprogramScopeDIE(TheCU, DS); } else { // There is no need to emit empty lexical block DIE. if (Children.empty()) return NULL; - ScopeDIE = constructLexicalScopeDIE(Scope); + ScopeDIE = constructLexicalScopeDIE(TheCU, Scope); } if (!ScopeDIE) return NULL; @@ -823,7 +426,7 @@ DIE *DwarfDebug::constructScopeDIE(DbgScope *Scope) { ScopeDIE->addChild(*I); if (DS.isSubprogram()) - getCompileUnit(DS)->addPubTypes(DISubprogram(DS)); + TheCU->addPubTypes(DISubprogram(DS)); return ScopeDIE; } @@ -862,7 +465,7 @@ unsigned DwarfDebug::GetOrCreateSourceID(StringRef FileName, /// constructCompileUnit - Create new CompileUnit for the given /// metadata node with tag DW_TAG_compile_unit. -void DwarfDebug::constructCompileUnit(const MDNode *N) { +CompileUnit *DwarfDebug::constructCompileUnit(const MDNode *N) { DICompileUnit DIUnit(N); StringRef FN = DIUnit.getFilename(); StringRef Dir = DIUnit.getDirectory(); @@ -893,7 +496,8 @@ void DwarfDebug::constructCompileUnit(const MDNode *N) { StringRef Flags = DIUnit.getFlags(); if (!Flags.empty()) - NewCU->addString(Die, dwarf::DW_AT_APPLE_flags, dwarf::DW_FORM_string, Flags); + NewCU->addString(Die, dwarf::DW_AT_APPLE_flags, dwarf::DW_FORM_string, + Flags); unsigned RVer = DIUnit.getRunTimeVersion(); if (RVer) @@ -903,159 +507,19 @@ void DwarfDebug::constructCompileUnit(const MDNode *N) { if (!FirstCU) FirstCU = NewCU; CUMap.insert(std::make_pair(N, NewCU)); -} - -/// getCompielUnit - Get CompileUnit DIE. -CompileUnit *DwarfDebug::getCompileUnit(const MDNode *N) const { - assert (N && "Invalid DwarfDebug::getCompileUnit argument!"); - DIDescriptor D(N); - const MDNode *CUNode = NULL; - if (D.isCompileUnit()) - CUNode = N; - else if (D.isSubprogram()) - CUNode = DISubprogram(N).getCompileUnit(); - else if (D.isType()) - CUNode = DIType(N).getCompileUnit(); - else if (D.isGlobalVariable()) - CUNode = DIGlobalVariable(N).getCompileUnit(); - else if (D.isVariable()) - CUNode = DIVariable(N).getCompileUnit(); - else if (D.isNameSpace()) - CUNode = DINameSpace(N).getCompileUnit(); - else if (D.isFile()) - CUNode = DIFile(N).getCompileUnit(); - else - return FirstCU; - - DenseMap<const MDNode *, CompileUnit *>::const_iterator I - = CUMap.find(CUNode); - if (I == CUMap.end()) - return FirstCU; - return I->second; -} - -// Return const exprssion if value is a GEP to access merged global -// constant. e.g. -// i8* getelementptr ({ i8, i8, i8, i8 }* @_MergedGlobals, i32 0, i32 0) -static const ConstantExpr *getMergedGlobalExpr(const Value *V) { - const ConstantExpr *CE = dyn_cast_or_null<ConstantExpr>(V); - if (!CE || CE->getNumOperands() != 3 || - CE->getOpcode() != Instruction::GetElementPtr) - return NULL; - - // First operand points to a global value. - if (!isa<GlobalValue>(CE->getOperand(0))) - return NULL; - - // Second operand is zero. - const ConstantInt *CI = - dyn_cast_or_null<ConstantInt>(CE->getOperand(1)); - if (!CI || !CI->isZero()) - return NULL; - - // Third operand is offset. - if (!isa<ConstantInt>(CE->getOperand(2))) - return NULL; - - return CE; -} - -/// constructGlobalVariableDIE - Construct global variable DIE. -void DwarfDebug::constructGlobalVariableDIE(const MDNode *N) { - DIGlobalVariable GV(N); - - // If debug information is malformed then ignore it. - if (GV.Verify() == false) - return; - - // Check for pre-existence. - CompileUnit *TheCU = getCompileUnit(N); - if (TheCU->getDIE(GV)) - return; - - DIType GTy = GV.getType(); - DIE *VariableDIE = new DIE(GV.getTag()); - - bool isGlobalVariable = GV.getGlobal() != NULL; - - // Add name. - TheCU->addString(VariableDIE, dwarf::DW_AT_name, dwarf::DW_FORM_string, - GV.getDisplayName()); - StringRef LinkageName = GV.getLinkageName(); - if (!LinkageName.empty() && isGlobalVariable) - TheCU->addString(VariableDIE, dwarf::DW_AT_MIPS_linkage_name, - dwarf::DW_FORM_string, - getRealLinkageName(LinkageName)); - // Add type. - TheCU->addType(VariableDIE, GTy); - - // Add scoping info. - if (!GV.isLocalToUnit()) { - TheCU->addUInt(VariableDIE, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1); - // Expose as global. - TheCU->addGlobal(GV.getName(), VariableDIE); - } - // Add line number info. - TheCU->addSourceLine(VariableDIE, GV); - // Add to map. - TheCU->insertDIE(N, VariableDIE); - // Add to context owner. - DIDescriptor GVContext = GV.getContext(); - TheCU->addToContextOwner(VariableDIE, GVContext); - // Add location. - if (isGlobalVariable) { - DIEBlock *Block = new (DIEValueAllocator) DIEBlock(); - TheCU->addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr); - TheCU->addLabel(Block, 0, dwarf::DW_FORM_udata, - Asm->Mang->getSymbol(GV.getGlobal())); - // Do not create specification DIE if context is either compile unit - // or a subprogram. - if (GV.isDefinition() && !GVContext.isCompileUnit() && - !GVContext.isFile() && !isSubprogramContext(GVContext)) { - // Create specification DIE. - DIE *VariableSpecDIE = new DIE(dwarf::DW_TAG_variable); - TheCU->addDIEEntry(VariableSpecDIE, dwarf::DW_AT_specification, - dwarf::DW_FORM_ref4, VariableDIE); - TheCU->addBlock(VariableSpecDIE, dwarf::DW_AT_location, 0, Block); - TheCU->addUInt(VariableDIE, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1); - TheCU->addDie(VariableSpecDIE); - } else { - TheCU->addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block); - } - } else if (const ConstantInt *CI = - dyn_cast_or_null<ConstantInt>(GV.getConstant())) - TheCU->addConstantValue(VariableDIE, CI, isUnsignedDIType(GTy)); - else if (const ConstantExpr *CE = getMergedGlobalExpr(N->getOperand(11))) { - // GV is a merged global. - DIEBlock *Block = new (DIEValueAllocator) DIEBlock(); - TheCU->addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr); - TheCU->addLabel(Block, 0, dwarf::DW_FORM_udata, - Asm->Mang->getSymbol(cast<GlobalValue>(CE->getOperand(0)))); - ConstantInt *CII = cast<ConstantInt>(CE->getOperand(2)); - TheCU->addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_constu); - TheCU->addUInt(Block, 0, dwarf::DW_FORM_udata, CII->getZExtValue()); - TheCU->addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_plus); - TheCU->addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block); - } - - return; + return NewCU; } /// construct SubprogramDIE - Construct subprogram DIE. -void DwarfDebug::constructSubprogramDIE(const MDNode *N) { +void DwarfDebug::constructSubprogramDIE(CompileUnit *TheCU, + const MDNode *N) { DISubprogram SP(N); - - // Check for pre-existence. - CompileUnit *TheCU = getCompileUnit(N); - if (TheCU->getDIE(N)) - return; - if (!SP.isDefinition()) // This is a method declaration which will be handled while constructing // class type. return; - DIE *SubprogramDie = createSubprogramDIE(SP); + DIE *SubprogramDie = TheCU->getOrCreateSubprogramDIE(SP); // Add to map. TheCU->insertDIE(N, SubprogramDie); @@ -1066,71 +530,115 @@ void DwarfDebug::constructSubprogramDIE(const MDNode *N) { // Expose as global. TheCU->addGlobal(SP.getName(), SubprogramDie); + SPMap[N] = TheCU; return; } +/// collectInfoFromNamedMDNodes - Collect debug info from named mdnodes such +/// as llvm.dbg.enum and llvm.dbg.ty +void DwarfDebug::collectInfoFromNamedMDNodes(Module *M) { + if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.sp")) + for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { + const MDNode *N = NMD->getOperand(i); + if (CompileUnit *CU = CUMap.lookup(DISubprogram(N).getCompileUnit())) + constructSubprogramDIE(CU, N); + } + + if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.gv")) + for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { + const MDNode *N = NMD->getOperand(i); + if (CompileUnit *CU = CUMap.lookup(DIGlobalVariable(N).getCompileUnit())) + CU->createGlobalVariableDIE(N); + } + + if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.enum")) + for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { + DIType Ty(NMD->getOperand(i)); + if (CompileUnit *CU = CUMap.lookup(Ty.getCompileUnit())) + CU->getOrCreateTypeDIE(Ty); + } + + if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.ty")) + for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { + DIType Ty(NMD->getOperand(i)); + if (CompileUnit *CU = CUMap.lookup(Ty.getCompileUnit())) + CU->getOrCreateTypeDIE(Ty); + } +} + +/// collectLegacyDebugInfo - Collect debug info using DebugInfoFinder. +/// FIXME - Remove this when dragon-egg and llvm-gcc switch to DIBuilder. +bool DwarfDebug::collectLegacyDebugInfo(Module *M) { + DebugInfoFinder DbgFinder; + DbgFinder.processModule(*M); + + bool HasDebugInfo = false; + // Scan all the compile-units to see if there are any marked as the main + // unit. If not, we do not generate debug info. + for (DebugInfoFinder::iterator I = DbgFinder.compile_unit_begin(), + E = DbgFinder.compile_unit_end(); I != E; ++I) { + if (DICompileUnit(*I).isMain()) { + HasDebugInfo = true; + break; + } + } + if (!HasDebugInfo) return false; + + // Create all the compile unit DIEs. + for (DebugInfoFinder::iterator I = DbgFinder.compile_unit_begin(), + E = DbgFinder.compile_unit_end(); I != E; ++I) + constructCompileUnit(*I); + + // Create DIEs for each global variable. + for (DebugInfoFinder::iterator I = DbgFinder.global_variable_begin(), + E = DbgFinder.global_variable_end(); I != E; ++I) { + const MDNode *N = *I; + if (CompileUnit *CU = CUMap.lookup(DIGlobalVariable(N).getCompileUnit())) + CU->createGlobalVariableDIE(N); + } + + // Create DIEs for each subprogram. + for (DebugInfoFinder::iterator I = DbgFinder.subprogram_begin(), + E = DbgFinder.subprogram_end(); I != E; ++I) { + const MDNode *N = *I; + if (CompileUnit *CU = CUMap.lookup(DISubprogram(N).getCompileUnit())) + constructSubprogramDIE(CU, N); + } + + return HasDebugInfo; +} + /// beginModule - Emit all Dwarf sections that should come prior to the /// content. Create global DIEs and emit initial debug info sections. -/// This is inovked by the target AsmPrinter. +/// This is invoked by the target AsmPrinter. void DwarfDebug::beginModule(Module *M) { if (DisableDebugInfoPrinting) return; - // If module has named metadata anchors then use them, otherwise scan the module - // using debug info finder to collect debug info. + // If module has named metadata anchors then use them, otherwise scan the + // module using debug info finder to collect debug info. NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu"); if (CU_Nodes) { - - NamedMDNode *GV_Nodes = M->getNamedMetadata("llvm.dbg.gv"); - NamedMDNode *SP_Nodes = M->getNamedMetadata("llvm.dbg.sp"); - if (!GV_Nodes && !SP_Nodes) - // If there are not any global variables or any functions then - // there is not any debug info in this module. - return; - - for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) - constructCompileUnit(CU_Nodes->getOperand(i)); - - if (GV_Nodes) - for (unsigned i = 0, e = GV_Nodes->getNumOperands(); i != e; ++i) - constructGlobalVariableDIE(GV_Nodes->getOperand(i)); - - if (SP_Nodes) - for (unsigned i = 0, e = SP_Nodes->getNumOperands(); i != e; ++i) - constructSubprogramDIE(SP_Nodes->getOperand(i)); - - } else { - - DebugInfoFinder DbgFinder; - DbgFinder.processModule(*M); - - bool HasDebugInfo = false; - // Scan all the compile-units to see if there are any marked as the main unit. - // if not, we do not generate debug info. - for (DebugInfoFinder::iterator I = DbgFinder.compile_unit_begin(), - E = DbgFinder.compile_unit_end(); I != E; ++I) { - if (DICompileUnit(*I).isMain()) { - HasDebugInfo = true; - break; - } + for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) { + DICompileUnit CUNode(CU_Nodes->getOperand(i)); + CompileUnit *CU = constructCompileUnit(CUNode); + DIArray GVs = CUNode.getGlobalVariables(); + for (unsigned i = 0, e = GVs.getNumElements(); i != e; ++i) + CU->createGlobalVariableDIE(GVs.getElement(i)); + DIArray SPs = CUNode.getSubprograms(); + for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i) + constructSubprogramDIE(CU, SPs.getElement(i)); + DIArray EnumTypes = CUNode.getEnumTypes(); + for (unsigned i = 0, e = EnumTypes.getNumElements(); i != e; ++i) + CU->getOrCreateTypeDIE(EnumTypes.getElement(i)); + DIArray RetainedTypes = CUNode.getRetainedTypes(); + for (unsigned i = 0, e = RetainedTypes.getNumElements(); i != e; ++i) + CU->getOrCreateTypeDIE(RetainedTypes.getElement(i)); } - if (!HasDebugInfo) return; - - // Create all the compile unit DIEs. - for (DebugInfoFinder::iterator I = DbgFinder.compile_unit_begin(), - E = DbgFinder.compile_unit_end(); I != E; ++I) - constructCompileUnit(*I); - - // Create DIEs for each global variable. - for (DebugInfoFinder::iterator I = DbgFinder.global_variable_begin(), - E = DbgFinder.global_variable_end(); I != E; ++I) - constructGlobalVariableDIE(*I); - - // Create DIEs for each subprogram. - for (DebugInfoFinder::iterator I = DbgFinder.subprogram_begin(), - E = DbgFinder.subprogram_end(); I != E; ++I) - constructSubprogramDIE(*I); - } + } else if (!collectLegacyDebugInfo(M)) + return; + + collectInfoFromNamedMDNodes(M); // Tell MMI that we have debug info. MMI->setDebugInfoAvailability(true); @@ -1138,19 +646,6 @@ void DwarfDebug::beginModule(Module *M) { // Emit initial sections. EmitSectionLabels(); - //getOrCreateTypeDIE - if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.enum")) - for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { - DIType Ty(NMD->getOperand(i)); - getCompileUnit(Ty)->getOrCreateTypeDIE(Ty); - } - - if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.ty")) - for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { - DIType Ty(NMD->getOperand(i)); - getCompileUnit(Ty)->getOrCreateTypeDIE(Ty); - } - // Prime section data. SectionMap.insert(Asm->getObjFileLowering().getTextSection()); } @@ -1160,38 +655,38 @@ void DwarfDebug::beginModule(Module *M) { void DwarfDebug::endModule() { if (!FirstCU) return; const Module *M = MMI->getModule(); - DenseMap<const MDNode *, DbgScope *> DeadFnScopeMap; - if (NamedMDNode *AllSPs = M->getNamedMetadata("llvm.dbg.sp")) { - for (unsigned SI = 0, SE = AllSPs->getNumOperands(); SI != SE; ++SI) { - if (ProcessedSPNodes.count(AllSPs->getOperand(SI)) != 0) continue; - DISubprogram SP(AllSPs->getOperand(SI)); - if (!SP.Verify()) continue; - - // Collect info for variables that were optimized out. - if (!SP.isDefinition()) continue; - StringRef FName = SP.getLinkageName(); - if (FName.empty()) - FName = SP.getName(); - NamedMDNode *NMD = getFnSpecificMDNode(*(MMI->getModule()), FName); - if (!NMD) continue; - unsigned E = NMD->getNumOperands(); - if (!E) continue; - DbgScope *Scope = new DbgScope(NULL, DIDescriptor(SP), NULL); - DeadFnScopeMap[SP] = Scope; - for (unsigned I = 0; I != E; ++I) { - DIVariable DV(NMD->getOperand(I)); - if (!DV.Verify()) continue; - Scope->addVariable(new DbgVariable(DV)); - } + DenseMap<const MDNode *, LexicalScope *> DeadFnScopeMap; - // Construct subprogram DIE and add variables DIEs. - constructSubprogramDIE(SP); - DIE *ScopeDIE = getCompileUnit(SP)->getDIE(SP); - const SmallVector<DbgVariable *, 8> &Variables = Scope->getDbgVariables(); - for (unsigned i = 0, N = Variables.size(); i < N; ++i) { - DIE *VariableDIE = constructVariableDIE(Variables[i], Scope); - if (VariableDIE) - ScopeDIE->addChild(VariableDIE); + // Collect info for variables that were optimized out. + if (NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu")) { + for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) { + DICompileUnit TheCU(CU_Nodes->getOperand(i)); + DIArray Subprograms = TheCU.getSubprograms(); + for (unsigned i = 0, e = Subprograms.getNumElements(); i != e; ++i) { + DISubprogram SP(Subprograms.getElement(i)); + if (ProcessedSPNodes.count(SP) != 0) continue; + if (!SP.Verify()) continue; + if (!SP.isDefinition()) continue; + DIArray Variables = SP.getVariables(); + if (Variables.getNumElements() == 0) continue; + + LexicalScope *Scope = + new LexicalScope(NULL, DIDescriptor(SP), NULL, false); + DeadFnScopeMap[SP] = Scope; + + // Construct subprogram DIE and add variables DIEs. + CompileUnit *SPCU = CUMap.lookup(TheCU); + assert (SPCU && "Unable to find Compile Unit!"); + constructSubprogramDIE(SPCU, SP); + DIE *ScopeDIE = SPCU->getDIE(SP); + for (unsigned vi = 0, ve = Variables.getNumElements(); vi != ve; ++vi) { + DIVariable DV(Variables.getElement(vi)); + if (!DV.Verify()) continue; + DbgVariable *NewVar = new DbgVariable(DV, NULL); + if (DIE *VariableDIE = + SPCU->constructVariableDIE(NewVar, Scope->isAbstractScope())) + ScopeDIE->addChild(VariableDIE); + } } } } @@ -1203,15 +698,12 @@ void DwarfDebug::endModule() { FirstCU->addUInt(ISP, dwarf::DW_AT_inline, 0, dwarf::DW_INL_inlined); } - for (DenseMap<DIE *, const MDNode *>::iterator CI = ContainingTypeMap.begin(), - CE = ContainingTypeMap.end(); CI != CE; ++CI) { - DIE *SPDie = CI->first; - const MDNode *N = dyn_cast_or_null<MDNode>(CI->second); - if (!N) continue; - DIE *NDie = getCompileUnit(N)->getDIE(N); - if (!NDie) continue; - getCompileUnit(N)->addDIEEntry(SPDie, dwarf::DW_AT_containing_type, - dwarf::DW_FORM_ref4, NDie); + // Emit DW_AT_containing_type attribute to connect types with their + // vtable holding type. + for (DenseMap<const MDNode *, CompileUnit *>::iterator CUI = CUMap.begin(), + CUE = CUMap.end(); CUI != CUE; ++CUI) { + CompileUnit *TheCU = CUI->second; + TheCU->constructContainingTypeDIEs(); } // Standard sections final addresses. @@ -1261,6 +753,7 @@ void DwarfDebug::endModule() { // clean up. DeleteContainerSeconds(DeadFnScopeMap); + SPMap.clear(); for (DenseMap<const MDNode *, CompileUnit *>::iterator I = CUMap.begin(), E = CUMap.end(); I != E; ++I) delete I->second; @@ -1268,29 +761,30 @@ void DwarfDebug::endModule() { } /// findAbstractVariable - Find abstract variable, if any, associated with Var. -DbgVariable *DwarfDebug::findAbstractVariable(DIVariable &Var, +DbgVariable *DwarfDebug::findAbstractVariable(DIVariable &DV, DebugLoc ScopeLoc) { - + LLVMContext &Ctx = DV->getContext(); + // More then one inlined variable corresponds to one abstract variable. + DIVariable Var = cleanseInlinedVariable(DV, Ctx); DbgVariable *AbsDbgVariable = AbstractVariables.lookup(Var); if (AbsDbgVariable) return AbsDbgVariable; - LLVMContext &Ctx = Var->getContext(); - DbgScope *Scope = AbstractScopes.lookup(ScopeLoc.getScope(Ctx)); + LexicalScope *Scope = LScopes.findAbstractScope(ScopeLoc.getScope(Ctx)); if (!Scope) return NULL; - AbsDbgVariable = new DbgVariable(Var); - Scope->addVariable(AbsDbgVariable); + AbsDbgVariable = new DbgVariable(Var, NULL); + addScopeVariable(Scope, AbsDbgVariable); AbstractVariables[Var] = AbsDbgVariable; return AbsDbgVariable; } -/// addCurrentFnArgument - If Var is an current function argument that add -/// it in CurrentFnArguments list. +/// addCurrentFnArgument - If Var is a current function argument then add +/// it to CurrentFnArguments list. bool DwarfDebug::addCurrentFnArgument(const MachineFunction *MF, - DbgVariable *Var, DbgScope *Scope) { - if (Scope != CurrentFnDbgScope) + DbgVariable *Var, LexicalScope *Scope) { + if (!LScopes.isCurrentFunctionScope(Scope)) return false; DIVariable DV = Var->getVariable(); if (DV.getTag() != dwarf::DW_TAG_arg_variable) @@ -1313,7 +807,7 @@ bool DwarfDebug::addCurrentFnArgument(const MachineFunction *MF, /// collectVariableInfoFromMMITable - Collect variable information from /// side table maintained by MMI. void -DwarfDebug::collectVariableInfoFromMMITable(const MachineFunction * MF, +DwarfDebug::collectVariableInfoFromMMITable(const MachineFunction *MF, SmallPtrSet<const MDNode *, 16> &Processed) { MachineModuleInfo::VariableDbgInfoMapTy &VMap = MMI->getVariableDbgInfo(); for (MachineModuleInfo::VariableDbgInfoMapTy::iterator VI = VMap.begin(), @@ -1324,21 +818,19 @@ DwarfDebug::collectVariableInfoFromMMITable(const MachineFunction * MF, DIVariable DV(Var); const std::pair<unsigned, DebugLoc> &VP = VI->second; - DbgScope *Scope = findDbgScope(VP.second); + LexicalScope *Scope = LScopes.findLexicalScope(VP.second); // If variable scope is not found then skip this variable. if (Scope == 0) continue; DbgVariable *AbsDbgVariable = findAbstractVariable(DV, VP.second); - DbgVariable *RegVar = new DbgVariable(DV); - recordVariableFrameIndex(RegVar, VP.first); + DbgVariable *RegVar = new DbgVariable(DV, AbsDbgVariable); + RegVar->setFrameIndex(VP.first); if (!addCurrentFnArgument(MF, RegVar, Scope)) - Scope->addVariable(RegVar); - if (AbsDbgVariable) { - recordVariableFrameIndex(AbsDbgVariable, VP.first); - VarToAbstractVarMap[RegVar] = AbsDbgVariable; - } + addScopeVariable(Scope, RegVar); + if (AbsDbgVariable) + AbsDbgVariable->setFrameIndex(VP.first); } } @@ -1351,7 +843,7 @@ static bool isDbgValueInDefinedReg(const MachineInstr *MI) { MI->getOperand(1).isImm() && MI->getOperand(1).getImm() == 0; } -/// getDebugLocEntry - Get .debug_loc entry for the instraction range starting +/// getDebugLocEntry - Get .debug_loc entry for the instruction range starting /// at MI. static DotDebugLocEntry getDebugLocEntry(AsmPrinter *Asm, const MCSymbol *FLabel, @@ -1379,7 +871,7 @@ static DotDebugLocEntry getDebugLocEntry(AsmPrinter *Asm, return DotDebugLocEntry(); } -/// collectVariableInfo - Populate DbgScope entries with variables' info. +/// collectVariableInfo - Find variables for each lexical scope. void DwarfDebug::collectVariableInfo(const MachineFunction *MF, SmallPtrSet<const MDNode *, 16> &Processed) { @@ -1402,30 +894,37 @@ DwarfDebug::collectVariableInfo(const MachineFunction *MF, const MachineInstr *MInsn = History.front(); DIVariable DV(Var); - DbgScope *Scope = NULL; + LexicalScope *Scope = NULL; if (DV.getTag() == dwarf::DW_TAG_arg_variable && DISubprogram(DV.getContext()).describes(MF->getFunction())) - Scope = CurrentFnDbgScope; - else - Scope = findDbgScope(MInsn->getDebugLoc()); + Scope = LScopes.getCurrentFunctionScope(); + else { + if (DV.getVersion() <= LLVMDebugVersion9) + Scope = LScopes.findLexicalScope(MInsn->getDebugLoc()); + else { + if (MDNode *IA = DV.getInlinedAt()) + Scope = LScopes.findInlinedScope(DebugLoc::getFromDILocation(IA)); + else + Scope = LScopes.findLexicalScope(cast<MDNode>(DV->getOperand(1))); + } + } // If variable scope is not found then skip this variable. if (!Scope) continue; Processed.insert(DV); assert(MInsn->isDebugValue() && "History must begin with debug value"); - DbgVariable *RegVar = new DbgVariable(DV); + DbgVariable *AbsVar = findAbstractVariable(DV, MInsn->getDebugLoc()); + DbgVariable *RegVar = new DbgVariable(DV, AbsVar); if (!addCurrentFnArgument(MF, RegVar, Scope)) - Scope->addVariable(RegVar); - if (DbgVariable *AbsVar = findAbstractVariable(DV, MInsn->getDebugLoc())) { - DbgVariableToDbgInstMap[AbsVar] = MInsn; - VarToAbstractVarMap[RegVar] = AbsVar; - } + addScopeVariable(Scope, RegVar); + if (AbsVar) + AbsVar->setMInsn(MInsn); // Simple ranges that are fully coalesced. if (History.size() <= 1 || (History.size() == 2 && MInsn->isIdenticalTo(History.back()))) { - DbgVariableToDbgInstMap[RegVar] = MInsn; + RegVar->setMInsn(MInsn); continue; } @@ -1471,16 +970,14 @@ DwarfDebug::collectVariableInfo(const MachineFunction *MF, } // Collect info for variables that were optimized out. - const Function *F = MF->getFunction(); - if (NamedMDNode *NMD = getFnSpecificMDNode(*(F->getParent()), F->getName())) { - for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { - DIVariable DV(cast<MDNode>(NMD->getOperand(i))); - if (!DV || !Processed.insert(DV)) - continue; - DbgScope *Scope = DbgScopeMap.lookup(DV.getContext()); - if (Scope) - Scope->addVariable(new DbgVariable(DV)); - } + LexicalScope *FnScope = LScopes.getCurrentFunctionScope(); + DIArray Variables = DISubprogram(FnScope->getScopeNode()).getVariables(); + for (unsigned i = 0, e = Variables.getNumElements(); i != e; ++i) { + DIVariable DV(Variables.getElement(i)); + if (!DV || !DV.Verify() || !Processed.insert(DV)) + continue; + if (LexicalScope *Scope = LScopes.findLexicalScope(DV.getContext())) + addScopeVariable(Scope, new DbgVariable(DV, NULL)); } } @@ -1561,237 +1058,33 @@ void DwarfDebug::endInstruction(const MachineInstr *MI) { I->second = PrevLabel; } -/// getOrCreateDbgScope - Create DbgScope for the scope. -DbgScope *DwarfDebug::getOrCreateDbgScope(DebugLoc DL) { - LLVMContext &Ctx = Asm->MF->getFunction()->getContext(); - MDNode *Scope = NULL; - MDNode *InlinedAt = NULL; - DL.getScopeAndInlinedAt(Scope, InlinedAt, Ctx); - - if (!InlinedAt) { - DbgScope *WScope = DbgScopeMap.lookup(Scope); - if (WScope) - return WScope; - WScope = new DbgScope(NULL, DIDescriptor(Scope), NULL); - DbgScopeMap.insert(std::make_pair(Scope, WScope)); - if (DIDescriptor(Scope).isLexicalBlock()) { - DbgScope *Parent = - getOrCreateDbgScope(DebugLoc::getFromDILexicalBlock(Scope)); - WScope->setParent(Parent); - Parent->addScope(WScope); - } else if (DIDescriptor(Scope).isSubprogram() - && DISubprogram(Scope).describes(Asm->MF->getFunction())) - CurrentFnDbgScope = WScope; - - return WScope; - } - - getOrCreateAbstractScope(Scope); - DbgScope *WScope = DbgScopeMap.lookup(InlinedAt); - if (WScope) - return WScope; - - WScope = new DbgScope(NULL, DIDescriptor(Scope), InlinedAt); - DbgScopeMap.insert(std::make_pair(InlinedAt, WScope)); - InlinedDbgScopeMap[DebugLoc::getFromDILocation(InlinedAt)] = WScope; - DbgScope *Parent = - getOrCreateDbgScope(DebugLoc::getFromDILocation(InlinedAt)); - WScope->setParent(Parent); - Parent->addScope(WScope); - return WScope; -} - -/// calculateDominanceGraph - Calculate dominance graph for DbgScope -/// hierarchy. -static void calculateDominanceGraph(DbgScope *Scope) { - assert (Scope && "Unable to calculate scop edominance graph!"); - SmallVector<DbgScope *, 4> WorkStack; - WorkStack.push_back(Scope); - unsigned Counter = 0; - while (!WorkStack.empty()) { - DbgScope *WS = WorkStack.back(); - const SmallVector<DbgScope *, 4> &Children = WS->getScopes(); - bool visitedChildren = false; - for (SmallVector<DbgScope *, 4>::const_iterator SI = Children.begin(), - SE = Children.end(); SI != SE; ++SI) { - DbgScope *ChildScope = *SI; - if (!ChildScope->getDFSOut()) { - WorkStack.push_back(ChildScope); - visitedChildren = true; - ChildScope->setDFSIn(++Counter); - break; - } - } - if (!visitedChildren) { - WorkStack.pop_back(); - WS->setDFSOut(++Counter); - } - } -} - -/// printDbgScopeInfo - Print DbgScope info for each machine instruction. -static -void printDbgScopeInfo(const MachineFunction *MF, - DenseMap<const MachineInstr *, DbgScope *> &MI2ScopeMap) -{ -#ifndef NDEBUG - LLVMContext &Ctx = MF->getFunction()->getContext(); - unsigned PrevDFSIn = 0; - for (MachineFunction::const_iterator I = MF->begin(), E = MF->end(); - I != E; ++I) { - for (MachineBasicBlock::const_iterator II = I->begin(), IE = I->end(); - II != IE; ++II) { - const MachineInstr *MInsn = II; - MDNode *Scope = NULL; - MDNode *InlinedAt = NULL; - - // Check if instruction has valid location information. - DebugLoc MIDL = MInsn->getDebugLoc(); - if (!MIDL.isUnknown()) { - MIDL.getScopeAndInlinedAt(Scope, InlinedAt, Ctx); - dbgs() << " [ "; - if (InlinedAt) - dbgs() << "*"; - DenseMap<const MachineInstr *, DbgScope *>::iterator DI = - MI2ScopeMap.find(MInsn); - if (DI != MI2ScopeMap.end()) { - DbgScope *S = DI->second; - dbgs() << S->getDFSIn(); - PrevDFSIn = S->getDFSIn(); - } else - dbgs() << PrevDFSIn; - } else - dbgs() << " [ x" << PrevDFSIn; - dbgs() << " ]"; - MInsn->dump(); - } - dbgs() << "\n"; - } -#endif -} -/// extractScopeInformation - Scan machine instructions in this function -/// and collect DbgScopes. Return true, if at least one scope was found. -bool DwarfDebug::extractScopeInformation() { - // If scope information was extracted using .dbg intrinsics then there is not - // any need to extract these information by scanning each instruction. - if (!DbgScopeMap.empty()) - return false; - - // Scan each instruction and create scopes. First build working set of scopes. - SmallVector<DbgRange, 4> MIRanges; - DenseMap<const MachineInstr *, DbgScope *> MI2ScopeMap; - DebugLoc PrevDL; - const MachineInstr *RangeBeginMI = NULL; - const MachineInstr *PrevMI = NULL; - for (MachineFunction::const_iterator I = Asm->MF->begin(), E = Asm->MF->end(); - I != E; ++I) { - for (MachineBasicBlock::const_iterator II = I->begin(), IE = I->end(); - II != IE; ++II) { - const MachineInstr *MInsn = II; - - // Check if instruction has valid location information. - const DebugLoc MIDL = MInsn->getDebugLoc(); - if (MIDL.isUnknown()) { - PrevMI = MInsn; - continue; - } - - // If scope has not changed then skip this instruction. - if (MIDL == PrevDL) { - PrevMI = MInsn; - continue; - } - - // Ignore DBG_VALUE. It does not contribute any instruction in output. - if (MInsn->isDebugValue()) - continue; - - if (RangeBeginMI) { - // If we have alread seen a beginning of a instruction range and - // current instruction scope does not match scope of first instruction - // in this range then create a new instruction range. - DEBUG(dbgs() << "Creating new instruction range :\n"); - DEBUG(dbgs() << "Begin Range at " << *RangeBeginMI); - DEBUG(dbgs() << "End Range at " << *PrevMI); - DEBUG(dbgs() << "Next Range starting at " << *MInsn); - DEBUG(dbgs() << "------------------------\n"); - DbgRange R(RangeBeginMI, PrevMI); - MI2ScopeMap[RangeBeginMI] = getOrCreateDbgScope(PrevDL); - MIRanges.push_back(R); - } - - // This is a beginning of a new instruction range. - RangeBeginMI = MInsn; - - // Reset previous markers. - PrevMI = MInsn; - PrevDL = MIDL; - } - } - - // Create last instruction range. - if (RangeBeginMI && PrevMI && !PrevDL.isUnknown()) { - DbgRange R(RangeBeginMI, PrevMI); - MIRanges.push_back(R); - MI2ScopeMap[RangeBeginMI] = getOrCreateDbgScope(PrevDL); - } - - if (!CurrentFnDbgScope) - return false; - - calculateDominanceGraph(CurrentFnDbgScope); - if (PrintDbgScope) - printDbgScopeInfo(Asm->MF, MI2ScopeMap); - - // Find ranges of instructions covered by each DbgScope; - DbgScope *PrevDbgScope = NULL; - for (SmallVector<DbgRange, 4>::const_iterator RI = MIRanges.begin(), - RE = MIRanges.end(); RI != RE; ++RI) { - const DbgRange &R = *RI; - DbgScope *S = MI2ScopeMap.lookup(R.first); - assert (S && "Lost DbgScope for a machine instruction!"); - if (PrevDbgScope && !PrevDbgScope->dominates(S)) - PrevDbgScope->closeInsnRange(S); - S->openInsnRange(R.first); - S->extendInsnRange(R.second); - PrevDbgScope = S; - } - - if (PrevDbgScope) - PrevDbgScope->closeInsnRange(); - - identifyScopeMarkers(); - - return !DbgScopeMap.empty(); -} - /// identifyScopeMarkers() - -/// Each DbgScope has first instruction and last instruction to mark beginning -/// and end of a scope respectively. Create an inverse map that list scopes -/// starts (and ends) with an instruction. One instruction may start (or end) -/// multiple scopes. Ignore scopes that are not reachable. +/// Each LexicalScope has first instruction and last instruction to mark +/// beginning and end of a scope respectively. Create an inverse map that list +/// scopes starts (and ends) with an instruction. One instruction may start (or +/// end) multiple scopes. Ignore scopes that are not reachable. void DwarfDebug::identifyScopeMarkers() { - SmallVector<DbgScope *, 4> WorkList; - WorkList.push_back(CurrentFnDbgScope); + SmallVector<LexicalScope *, 4> WorkList; + WorkList.push_back(LScopes.getCurrentFunctionScope()); while (!WorkList.empty()) { - DbgScope *S = WorkList.pop_back_val(); + LexicalScope *S = WorkList.pop_back_val(); - const SmallVector<DbgScope *, 4> &Children = S->getScopes(); + const SmallVector<LexicalScope *, 4> &Children = S->getChildren(); if (!Children.empty()) - for (SmallVector<DbgScope *, 4>::const_iterator SI = Children.begin(), + for (SmallVector<LexicalScope *, 4>::const_iterator SI = Children.begin(), SE = Children.end(); SI != SE; ++SI) WorkList.push_back(*SI); if (S->isAbstractScope()) continue; - const SmallVector<DbgRange, 4> &Ranges = S->getRanges(); + const SmallVector<InsnRange, 4> &Ranges = S->getRanges(); if (Ranges.empty()) continue; - for (SmallVector<DbgRange, 4>::const_iterator RI = Ranges.begin(), + for (SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(), RE = Ranges.end(); RI != RE; ++RI) { - assert(RI->first && "DbgRange does not have first instruction!"); - assert(RI->second && "DbgRange does not have second instruction!"); + assert(RI->first && "InsnRange does not have first instruction!"); + assert(RI->second && "InsnRange does not have second instruction!"); requestLabelBeforeInsn(RI->first); requestLabelAfterInsn(RI->second); } @@ -1819,7 +1112,9 @@ static DebugLoc getFnDebugLoc(DebugLoc DL, const LLVMContext &Ctx) { /// emitted immediately after the function entry point. void DwarfDebug::beginFunction(const MachineFunction *MF) { if (!MMI->hasDebugInfo()) return; - if (!extractScopeInformation()) return; + LScopes.initialize(*MF); + if (LScopes.empty()) return; + identifyScopeMarkers(); FunctionBeginSym = Asm->GetTempSymbol("func_begin", Asm->getFunctionNumber()); @@ -1953,7 +1248,8 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { const MachineInstr *Prev = History.back(); if (Prev->isDebugValue() && isDbgValueInDefinedReg(Prev)) { const MachineBasicBlock *PrevMBB = Prev->getParent(); - MachineBasicBlock::const_iterator LastMI = PrevMBB->getLastNonDebugInstr(); + MachineBasicBlock::const_iterator LastMI = + PrevMBB->getLastNonDebugInstr(); if (LastMI == PrevMBB->end()) // Drop DBG_VALUE for empty range. History.pop_back(); @@ -1985,110 +1281,73 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { } } +void DwarfDebug::addScopeVariable(LexicalScope *LS, DbgVariable *Var) { +// SmallVector<DbgVariable *, 8> &Vars = ScopeVariables.lookup(LS); + ScopeVariables[LS].push_back(Var); +// Vars.push_back(Var); +} + /// endFunction - Gather and emit post-function debug information. /// void DwarfDebug::endFunction(const MachineFunction *MF) { - if (!MMI->hasDebugInfo() || DbgScopeMap.empty()) return; - - if (CurrentFnDbgScope) { + if (!MMI->hasDebugInfo() || LScopes.empty()) return; - // Define end label for subprogram. - FunctionEndSym = Asm->GetTempSymbol("func_end", - Asm->getFunctionNumber()); - // Assumes in correct section after the entry point. - Asm->OutStreamer.EmitLabel(FunctionEndSym); - - SmallPtrSet<const MDNode *, 16> ProcessedVars; - collectVariableInfo(MF, ProcessedVars); - - // Construct abstract scopes. - for (SmallVector<DbgScope *, 4>::iterator AI = AbstractScopesList.begin(), - AE = AbstractScopesList.end(); AI != AE; ++AI) { - DISubprogram SP((*AI)->getScopeNode()); - if (SP.Verify()) { - // Collect info for variables that were optimized out. - StringRef FName = SP.getLinkageName(); - if (FName.empty()) - FName = SP.getName(); - if (NamedMDNode *NMD = - getFnSpecificMDNode(*(MF->getFunction()->getParent()), FName)) { - for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { - DIVariable DV(cast<MDNode>(NMD->getOperand(i))); - if (!DV || !ProcessedVars.insert(DV)) - continue; - DbgScope *Scope = AbstractScopes.lookup(DV.getContext()); - if (Scope) - Scope->addVariable(new DbgVariable(DV)); - } - } + // Define end label for subprogram. + FunctionEndSym = Asm->GetTempSymbol("func_end", + Asm->getFunctionNumber()); + // Assumes in correct section after the entry point. + Asm->OutStreamer.EmitLabel(FunctionEndSym); + + SmallPtrSet<const MDNode *, 16> ProcessedVars; + collectVariableInfo(MF, ProcessedVars); + + LexicalScope *FnScope = LScopes.getCurrentFunctionScope(); + CompileUnit *TheCU = SPMap.lookup(FnScope->getScopeNode()); + assert (TheCU && "Unable to find compile unit!"); + + // Construct abstract scopes. + ArrayRef<LexicalScope *> AList = LScopes.getAbstractScopesList(); + for (unsigned i = 0, e = AList.size(); i != e; ++i) { + LexicalScope *AScope = AList[i]; + DISubprogram SP(AScope->getScopeNode()); + if (SP.Verify()) { + // Collect info for variables that were optimized out. + DIArray Variables = SP.getVariables(); + for (unsigned i = 0, e = Variables.getNumElements(); i != e; ++i) { + DIVariable DV(Variables.getElement(i)); + if (!DV || !DV.Verify() || !ProcessedVars.insert(DV)) + continue; + if (LexicalScope *Scope = LScopes.findAbstractScope(DV.getContext())) + addScopeVariable(Scope, new DbgVariable(DV, NULL)); } - if (ProcessedSPNodes.count((*AI)->getScopeNode()) == 0) - constructScopeDIE(*AI); } - - DIE *CurFnDIE = constructScopeDIE(CurrentFnDbgScope); - - if (!DisableFramePointerElim(*MF)) - getCompileUnit(CurrentFnDbgScope->getScopeNode())->addUInt(CurFnDIE, - dwarf::DW_AT_APPLE_omit_frame_ptr, - dwarf::DW_FORM_flag, 1); - - - DebugFrames.push_back(FunctionDebugFrameInfo(Asm->getFunctionNumber(), - MMI->getFrameMoves())); + if (ProcessedSPNodes.count(AScope->getScopeNode()) == 0) + constructScopeDIE(TheCU, AScope); } + + DIE *CurFnDIE = constructScopeDIE(TheCU, FnScope); + + if (!DisableFramePointerElim(*MF)) + TheCU->addUInt(CurFnDIE, dwarf::DW_AT_APPLE_omit_frame_ptr, + dwarf::DW_FORM_flag, 1); + + DebugFrames.push_back(FunctionDebugFrameInfo(Asm->getFunctionNumber(), + MMI->getFrameMoves())); // Clear debug info - CurrentFnDbgScope = NULL; + for (DenseMap<LexicalScope *, SmallVector<DbgVariable *, 8> >::iterator + I = ScopeVariables.begin(), E = ScopeVariables.end(); I != E; ++I) + DeleteContainerPointers(I->second); + ScopeVariables.clear(); DeleteContainerPointers(CurrentFnArguments); - DbgVariableToFrameIndexMap.clear(); - VarToAbstractVarMap.clear(); - DbgVariableToDbgInstMap.clear(); - InlinedDbgScopeMap.clear(); - DeleteContainerSeconds(DbgScopeMap); UserVariables.clear(); DbgValues.clear(); - DeleteContainerSeconds(AbstractScopes); - AbstractScopesList.clear(); AbstractVariables.clear(); LabelsBeforeInsn.clear(); LabelsAfterInsn.clear(); PrevLabel = NULL; } -/// recordVariableFrameIndex - Record a variable's index. -void DwarfDebug::recordVariableFrameIndex(const DbgVariable *V, int Index) { - assert (V && "Invalid DbgVariable!"); - DbgVariableToFrameIndexMap[V] = Index; -} - -/// findVariableFrameIndex - Return true if frame index for the variable -/// is found. Update FI to hold value of the index. -bool DwarfDebug::findVariableFrameIndex(const DbgVariable *V, int *FI) { - assert (V && "Invalid DbgVariable!"); - DenseMap<const DbgVariable *, int>::iterator I = - DbgVariableToFrameIndexMap.find(V); - if (I == DbgVariableToFrameIndexMap.end()) - return false; - *FI = I->second; - return true; -} - -/// findDbgScope - Find DbgScope for the debug loc. -DbgScope *DwarfDebug::findDbgScope(DebugLoc DL) { - if (DL.isUnknown()) - return NULL; - - DbgScope *Scope = NULL; - LLVMContext &Ctx = Asm->MF->getFunction()->getContext(); - if (MDNode *IA = DL.getInlinedAt(Ctx)) - Scope = InlinedDbgScopeMap.lookup(DebugLoc::getFromDILocation(IA)); - else - Scope = DbgScopeMap.lookup(DL.getScope(Ctx)); - return Scope; -} - - /// recordSourceLine - Register a source line with debug info. Returns the /// unique label that was emitted and which provides correspondence to /// the source line list. @@ -2112,6 +1371,10 @@ void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S, DISubprogram SP(S); Fn = SP.getFilename(); Dir = SP.getDirectory(); + } else if (Scope.isLexicalBlockFile()) { + DILexicalBlockFile DBF(S); + Fn = DBF.getFilename(); + Dir = DBF.getDirectory(); } else if (Scope.isLexicalBlock()) { DILexicalBlock DB(S); Fn = DB.getFilename(); @@ -2121,8 +1384,7 @@ void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S, Src = GetOrCreateSourceID(Fn, Dir); } - Asm->OutStreamer.EmitDwarfLocDirective(Src, Line, Col, Flags, - 0, 0, Fn); + Asm->OutStreamer.EmitDwarfLocDirective(Src, Line, Col, Flags, 0, 0, Fn); } //===----------------------------------------------------------------------===// @@ -2235,7 +1497,7 @@ void DwarfDebug::EmitSectionLabels() { EmitSectionSym(Asm, TLOF.getDataSection()); } -/// emitDIE - Recusively Emits a debug information entry. +/// emitDIE - Recursively emits a debug information entry. /// void DwarfDebug::emitDIE(DIE *Die) { // Get the abbreviation for this DIE. @@ -2290,10 +1552,9 @@ void DwarfDebug::emitDIE(DIE *Die) { break; } case dwarf::DW_AT_location: { - if (UseDotDebugLocEntry.count(Die) != 0) { - DIELabel *L = cast<DIELabel>(Values[i]); + if (DIELabel *L = dyn_cast<DIELabel>(Values[i])) Asm->EmitLabelDifference(L->getValue(), DwarfDebugLocSectionSym, 4); - } else + else Values[i]->EmitValue(Asm, Form); break; } @@ -2464,7 +1725,7 @@ void DwarfDebug::emitDebugPubNames() { Asm->OutStreamer.AddComment("End Mark"); Asm->EmitInt32(0); Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubnames_end", - TheCU->getID())); + TheCU->getID())); } } @@ -2499,7 +1760,7 @@ void DwarfDebug::emitDebugPubTypes() { for (StringMap<DIE*>::const_iterator GI = Globals.begin(), GE = Globals.end(); GI != GE; ++GI) { const char *Name = GI->getKeyData(); - DIE * Entity = GI->second; + DIE *Entity = GI->second; if (Asm->isVerbose()) Asm->OutStreamer.AddComment("DIE offset"); Asm->EmitInt32(Entity->getOffset()); diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.h b/lib/CodeGen/AsmPrinter/DwarfDebug.h index b245006..35653be 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.h +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.h @@ -15,7 +15,8 @@ #define CODEGEN_ASMPRINTER_DWARFDEBUG_H__ #include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/CodeGen/MachineLocation.h" +#include "llvm/CodeGen/LexicalScopes.h" +#include "llvm/MC/MachineLocation.h" #include "llvm/Analysis/DebugInfo.h" #include "DIE.h" #include "llvm/ADT/DenseMap.h" @@ -30,7 +31,6 @@ namespace llvm { class CompileUnit; class DbgConcreteScope; -class DbgScope; class DbgVariable; class MachineFrameInfo; class MachineModuleInfo; @@ -125,9 +125,14 @@ class DbgVariable { DIVariable Var; // Variable Descriptor. DIE *TheDIE; // Variable DIE. unsigned DotDebugLocOffset; // Offset in DotDebugLocEntries. + DbgVariable *AbsVar; // Corresponding Abstract variable, if any. + const MachineInstr *MInsn; // DBG_VALUE instruction of the variable. + int FrameIndex; public: // AbsVar may be NULL. - DbgVariable(DIVariable V) : Var(V), TheDIE(0), DotDebugLocOffset(~0U) {} + DbgVariable(DIVariable V, DbgVariable *AV) + : Var(V), TheDIE(0), DotDebugLocOffset(~0U), AbsVar(AV), MInsn(0), + FrameIndex(~0) {} // Accessors. DIVariable getVariable() const { return Var; } @@ -136,7 +141,27 @@ public: void setDotDebugLocOffset(unsigned O) { DotDebugLocOffset = O; } unsigned getDotDebugLocOffset() const { return DotDebugLocOffset; } StringRef getName() const { return Var.getName(); } - unsigned getTag() const { return Var.getTag(); } + DbgVariable *getAbstractVariable() const { return AbsVar; } + const MachineInstr *getMInsn() const { return MInsn; } + void setMInsn(const MachineInstr *M) { MInsn = M; } + int getFrameIndex() const { return FrameIndex; } + void setFrameIndex(int FI) { FrameIndex = FI; } + // Translate tag to proper Dwarf tag. + unsigned getTag() const { + if (Var.getTag() == dwarf::DW_TAG_arg_variable) + return dwarf::DW_TAG_formal_parameter; + + return dwarf::DW_TAG_variable; + } + /// isArtificial - Return true if DbgVariable is artificial. + bool isArtificial() const { + if (Var.isArtificial()) + return true; + if (Var.getTag() == dwarf::DW_TAG_arg_variable + && getType().isArtificial()) + return true; + return false; + } bool variableHasComplexAddress() const { assert(Var.Verify() && "Invalid complex DbgVariable!"); return Var.hasComplexAddress(); @@ -167,8 +192,13 @@ class DwarfDebug { // CompileUnit *FirstCU; + + /// Maps MDNode with its corresponding CompileUnit. DenseMap <const MDNode *, CompileUnit *> CUMap; + /// Maps subprogram MDNode with its corresponding CompileUnit. + DenseMap <const MDNode *, CompileUnit *> SPMap; + /// AbbreviationsSet - Used to uniquely define abbreviations. /// FoldingSet<DIEAbbrev> AbbreviationsSet; @@ -192,63 +222,27 @@ class DwarfDebug { /// UniqueVector<const MCSection*> SectionMap; - /// CurrentFnDbgScope - Top level scope for the current function. - /// - DbgScope *CurrentFnDbgScope; - /// CurrentFnArguments - List of Arguments (DbgValues) for current function. SmallVector<DbgVariable *, 8> CurrentFnArguments; - /// DbgScopeMap - Tracks the scopes in the current function. Owns the - /// contained DbgScope*s. - DenseMap<const MDNode *, DbgScope *> DbgScopeMap; - - /// InlinedDbgScopeMap - Tracks inlined function scopes in current function. - DenseMap<DebugLoc, DbgScope *> InlinedDbgScopeMap; - - /// AbstractScopes - Tracks the abstract scopes a module. These scopes are - /// not included DbgScopeMap. AbstractScopes owns its DbgScope*s. - DenseMap<const MDNode *, DbgScope *> AbstractScopes; + LexicalScopes LScopes; /// AbstractSPDies - Collection of abstract subprogram DIEs. DenseMap<const MDNode *, DIE *> AbstractSPDies; - /// AbstractScopesList - Tracks abstract scopes constructed while processing - /// a function. This list is cleared during endFunction(). - SmallVector<DbgScope *, 4>AbstractScopesList; + /// ScopeVariables - Collection of dbg variables of a scope. + DenseMap<LexicalScope *, SmallVector<DbgVariable *, 8> > ScopeVariables; - /// AbstractVariables - Collection on abstract variables. Owned by the - /// DbgScopes in AbstractScopes. + /// AbstractVariables - Collection on abstract variables. DenseMap<const MDNode *, DbgVariable *> AbstractVariables; - /// DbgVariableToFrameIndexMap - Tracks frame index used to find - /// variable's value. - DenseMap<const DbgVariable *, int> DbgVariableToFrameIndexMap; - - /// DbgVariableToDbgInstMap - Maps DbgVariable to corresponding DBG_VALUE - /// machine instruction. - DenseMap<const DbgVariable *, const MachineInstr *> DbgVariableToDbgInstMap; - /// DotDebugLocEntries - Collection of DotDebugLocEntry. SmallVector<DotDebugLocEntry, 4> DotDebugLocEntries; - /// UseDotDebugLocEntry - DW_AT_location attributes for the DIEs in this set - /// idetifies corresponding .debug_loc entry offset. - SmallPtrSet<const DIE *, 4> UseDotDebugLocEntry; - - /// VarToAbstractVarMap - Maps DbgVariable with corresponding Abstract - /// DbgVariable, if any. - DenseMap<const DbgVariable *, const DbgVariable *> VarToAbstractVarMap; - /// InliendSubprogramDIEs - Collection of subprgram DIEs that are marked /// (at the end of the module) as DW_AT_inline. SmallPtrSet<DIE *, 4> InlinedSubprogramDIEs; - /// ContainingTypeMap - This map is used to keep track of subprogram DIEs that - /// need DW_AT_containing_type attribute. This attribute points to a DIE that - /// corresponds to the MDNode mapped with the subprogram DIE. - DenseMap<DIE *, const MDNode *> ContainingTypeMap; - /// InlineInfo - Keep track of inlined functions and their location. This /// information is used to populate debug_inlined section. typedef std::pair<const MCSymbol *, DIE *> InlineInfoLabels; @@ -316,10 +310,7 @@ private: /// void assignAbbrevNumber(DIEAbbrev &Abbrev); - /// getOrCreateDbgScope - Create DbgScope for the scope. - DbgScope *getOrCreateDbgScope(DebugLoc DL); - - DbgScope *getOrCreateAbstractScope(const MDNode *N); + void addScopeVariable(LexicalScope *LS, DbgVariable *Var); /// findAbstractVariable - Find abstract variable associated with Var. DbgVariable *findAbstractVariable(DIVariable &Var, DebugLoc Loc); @@ -328,22 +319,22 @@ private: /// attach appropriate DW_AT_low_pc and DW_AT_high_pc attributes. /// If there are global variables in this scope then create and insert /// DIEs for these variables. - DIE *updateSubprogramScopeDIE(const MDNode *SPNode); + DIE *updateSubprogramScopeDIE(CompileUnit *SPCU, const MDNode *SPNode); /// constructLexicalScope - Construct new DW_TAG_lexical_block /// for this scope and attach DW_AT_low_pc/DW_AT_high_pc labels. - DIE *constructLexicalScopeDIE(DbgScope *Scope); + DIE *constructLexicalScopeDIE(CompileUnit *TheCU, LexicalScope *Scope); /// constructInlinedScopeDIE - This scope represents inlined body of /// a function. Construct DIE to represent this concrete inlined copy /// of the function. - DIE *constructInlinedScopeDIE(DbgScope *Scope); + DIE *constructInlinedScopeDIE(CompileUnit *TheCU, LexicalScope *Scope); /// constructVariableDIE - Construct a DIE for the given DbgVariable. - DIE *constructVariableDIE(DbgVariable *DV, DbgScope *S); + DIE *constructVariableDIE(DbgVariable *DV, LexicalScope *S); /// constructScopeDIE - Construct a DIE for this scope. - DIE *constructScopeDIE(DbgScope *Scope); + DIE *constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope); /// EmitSectionLabels - Emit initial Dwarf sections with a label at /// the start of each one. @@ -424,16 +415,10 @@ private: /// constructCompileUnit - Create new CompileUnit for the given /// metadata node with tag DW_TAG_compile_unit. - void constructCompileUnit(const MDNode *N); - - /// getCompielUnit - Get CompileUnit DIE. - CompileUnit *getCompileUnit(const MDNode *N) const; - - /// constructGlobalVariableDIE - Construct global variable DIE. - void constructGlobalVariableDIE(const MDNode *N); + CompileUnit *constructCompileUnit(const MDNode *N); /// construct SubprogramDIE - Construct subprogram DIE. - void constructSubprogramDIE(const MDNode *N); + void constructSubprogramDIE(CompileUnit *TheCU, const MDNode *N); /// recordSourceLine - Register a source line with debug info. Returns the /// unique label that was emitted and which provides correspondence to @@ -441,30 +426,16 @@ private: void recordSourceLine(unsigned Line, unsigned Col, const MDNode *Scope, unsigned Flags); - /// recordVariableFrameIndex - Record a variable's index. - void recordVariableFrameIndex(const DbgVariable *V, int Index); - - /// findVariableFrameIndex - Return true if frame index for the variable - /// is found. Update FI to hold value of the index. - bool findVariableFrameIndex(const DbgVariable *V, int *FI); - - /// findDbgScope - Find DbgScope for the debug loc. - DbgScope *findDbgScope(DebugLoc DL); - /// identifyScopeMarkers() - Indentify instructions that are marking /// beginning of or end of a scope. void identifyScopeMarkers(); - /// extractScopeInformation - Scan machine instructions in this function - /// and collect DbgScopes. Return true, if atleast one scope was found. - bool extractScopeInformation(); - /// addCurrentFnArgument - If Var is an current function argument that add /// it in CurrentFnArguments list. bool addCurrentFnArgument(const MachineFunction *MF, - DbgVariable *Var, DbgScope *Scope); + DbgVariable *Var, LexicalScope *Scope); - /// collectVariableInfo - Populate DbgScope entries with variables' info. + /// collectVariableInfo - Populate LexicalScope entries with variables' info. void collectVariableInfo(const MachineFunction *, SmallPtrSet<const MDNode *, 16> &ProcessedVars); @@ -496,6 +467,14 @@ public: DwarfDebug(AsmPrinter *A, Module *M); ~DwarfDebug(); + /// collectInfoFromNamedMDNodes - Collect debug info from named mdnodes such + /// as llvm.dbg.enum and llvm.dbg.ty + void collectInfoFromNamedMDNodes(Module *M); + + /// collectLegacyDebugInfo - Collect debug info using DebugInfoFinder. + /// FIXME - Remove this when dragon-egg and llvm-gcc switch to DIBuilder. + bool collectLegacyDebugInfo(Module *M); + /// beginModule - Emit all Dwarf sections that should come prior to the /// content. void beginModule(Module *M); diff --git a/lib/CodeGen/AsmPrinter/DwarfException.cpp b/lib/CodeGen/AsmPrinter/DwarfException.cpp index 1f992fa..18b726b 100644 --- a/lib/CodeGen/AsmPrinter/DwarfException.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfException.cpp @@ -17,7 +17,6 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineLocation.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" @@ -527,29 +526,26 @@ void DwarfException::EmitExceptionTable() { I = CallSites.begin(), E = CallSites.end(); I != E; ++I, ++idx) { const CallSiteEntry &S = *I; + // Offset of the landing pad, counted in 16-byte bundles relative to the + // @LPStart address. if (VerboseAsm) { - // Emit comments that decode the call site. Asm->OutStreamer.AddComment(Twine(">> Call Site ") + llvm::utostr(idx) + " <<"); Asm->OutStreamer.AddComment(Twine(" On exception at call site ") + llvm::utostr(idx)); + } + Asm->EmitULEB128(idx); + // Offset of the first associated action record, relative to the start of + // the action table. This value is biased by 1 (1 indicates the start of + // the action table), and 0 indicates that there are no actions. + if (VerboseAsm) { if (S.Action == 0) Asm->OutStreamer.AddComment(" Action: cleanup"); else Asm->OutStreamer.AddComment(Twine(" Action: ") + llvm::utostr((S.Action - 1) / 2 + 1)); - - Asm->OutStreamer.AddBlankLine(); } - - // Offset of the landing pad, counted in 16-byte bundles relative to the - // @LPStart address. - Asm->EmitULEB128(idx); - - // Offset of the first associated action record, relative to the start of - // the action table. This value is biased by 1 (1 indicates the start of - // the action table), and 0 indicates that there are no actions. Asm->EmitULEB128(S.Action); } } else { @@ -595,46 +591,43 @@ void DwarfException::EmitExceptionTable() { if (EndLabel == 0) EndLabel = Asm->GetTempSymbol("eh_func_end", Asm->getFunctionNumber()); - if (VerboseAsm) { - // Emit comments that decode the call site. - Asm->OutStreamer.AddComment(Twine(">> Call Site ") + - llvm::utostr(++Entry) + " <<"); - Asm->OutStreamer.AddComment(Twine(" Call between ") + - BeginLabel->getName() + " and " + - EndLabel->getName()); - - if (!S.PadLabel) { - Asm->OutStreamer.AddComment(" has no landing pad"); - } else { - Asm->OutStreamer.AddComment(Twine(" jumps to ") + - S.PadLabel->getName()); - - if (S.Action == 0) - Asm->OutStreamer.AddComment(" On action: cleanup"); - else - Asm->OutStreamer.AddComment(Twine(" On action: ") + - llvm::utostr((S.Action - 1) / 2 + 1)); - } - - Asm->OutStreamer.AddBlankLine(); - } // Offset of the call site relative to the previous call site, counted in // number of 16-byte bundles. The first call site is counted relative to // the start of the procedure fragment. + if (VerboseAsm) + Asm->OutStreamer.AddComment(Twine(">> Call Site ") + + llvm::utostr(++Entry) + " <<"); Asm->EmitLabelDifference(BeginLabel, EHFuncBeginSym, 4); + if (VerboseAsm) + Asm->OutStreamer.AddComment(Twine(" Call between ") + + BeginLabel->getName() + " and " + + EndLabel->getName()); Asm->EmitLabelDifference(EndLabel, BeginLabel, 4); // Offset of the landing pad, counted in 16-byte bundles relative to the // @LPStart address. - if (!S.PadLabel) + if (!S.PadLabel) { + if (VerboseAsm) + Asm->OutStreamer.AddComment(" has no landing pad"); Asm->OutStreamer.EmitIntValue(0, 4/*size*/, 0/*addrspace*/); - else + } else { + if (VerboseAsm) + Asm->OutStreamer.AddComment(Twine(" jumps to ") + + S.PadLabel->getName()); Asm->EmitLabelDifference(S.PadLabel, EHFuncBeginSym, 4); + } // Offset of the first associated action record, relative to the start of // the action table. This value is biased by 1 (1 indicates the start of // the action table), and 0 indicates that there are no actions. + if (VerboseAsm) { + if (S.Action == 0) + Asm->OutStreamer.AddComment(" On action: cleanup"); + else + Asm->OutStreamer.AddComment(Twine(" On action: ") + + llvm::utostr((S.Action - 1) / 2 + 1)); + } Asm->EmitULEB128(S.Action); } } @@ -649,13 +642,29 @@ void DwarfException::EmitExceptionTable() { // Emit comments that decode the action table. Asm->OutStreamer.AddComment(Twine(">> Action Record ") + llvm::utostr(++Entry) + " <<"); - if (Action.ValueForTypeID >= 0) + } + + // Type Filter + // + // Used by the runtime to match the type of the thrown exception to the + // type of the catch clauses or the types in the exception specification. + if (VerboseAsm) { + if (Action.ValueForTypeID > 0) Asm->OutStreamer.AddComment(Twine(" Catch TypeInfo ") + llvm::itostr(Action.ValueForTypeID)); - else + else if (Action.ValueForTypeID < 0) Asm->OutStreamer.AddComment(Twine(" Filter TypeInfo ") + llvm::itostr(Action.ValueForTypeID)); + else + Asm->OutStreamer.AddComment(" Cleanup"); + } + Asm->EmitSLEB128(Action.ValueForTypeID); + // Action Record + // + // Self-relative signed displacement in bytes of the next action record, + // or 0 if there is no next action record. + if (VerboseAsm) { if (Action.NextAction == 0) { Asm->OutStreamer.AddComment(" No further actions"); } else { @@ -663,20 +672,7 @@ void DwarfException::EmitExceptionTable() { Asm->OutStreamer.AddComment(Twine(" Continue to action ") + llvm::utostr(NextAction)); } - - Asm->OutStreamer.AddBlankLine(); } - - // Type Filter - // - // Used by the runtime to match the type of the thrown exception to the - // type of the catch clauses or the types in the exception specification. - Asm->EmitSLEB128(Action.ValueForTypeID); - - // Action Record - // - // Self-relative signed displacement in bytes of the next action record, - // or 0 if there is no next action record. Asm->EmitSLEB128(Action.NextAction); } diff --git a/lib/CodeGen/AsmPrinter/Win64Exception.cpp b/lib/CodeGen/AsmPrinter/Win64Exception.cpp index c2ad5eb..b83aa5a 100644 --- a/lib/CodeGen/AsmPrinter/Win64Exception.cpp +++ b/lib/CodeGen/AsmPrinter/Win64Exception.cpp @@ -17,7 +17,6 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineLocation.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp index 99090a8..75288b0 100644 --- a/lib/CodeGen/BranchFolding.cpp +++ b/lib/CodeGen/BranchFolding.cpp @@ -1624,26 +1624,29 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) { if (!TIB->isSafeToMove(TII, 0, DontMoveAcrossStore)) break; + // Remove kills from LocalDefsSet, these registers had short live ranges. + for (unsigned i = 0, e = TIB->getNumOperands(); i != e; ++i) { + MachineOperand &MO = TIB->getOperand(i); + if (!MO.isReg() || !MO.isUse() || !MO.isKill()) + continue; + unsigned Reg = MO.getReg(); + if (!Reg || !LocalDefsSet.count(Reg)) + continue; + for (const unsigned *OR = TRI->getOverlaps(Reg); *OR; ++OR) + LocalDefsSet.erase(*OR); + } + // Track local defs so we can update liveins. for (unsigned i = 0, e = TIB->getNumOperands(); i != e; ++i) { MachineOperand &MO = TIB->getOperand(i); - if (!MO.isReg()) + if (!MO.isReg() || !MO.isDef() || MO.isDead()) continue; unsigned Reg = MO.getReg(); if (!Reg) continue; - if (MO.isDef()) { - if (!MO.isDead()) { - LocalDefs.push_back(Reg); - LocalDefsSet.insert(Reg); - for (const unsigned *SR = TRI->getSubRegisters(Reg); *SR; ++SR) - LocalDefsSet.insert(*SR); - } - } else if (MO.isKill() && LocalDefsSet.count(Reg)) { - LocalDefsSet.erase(Reg); - for (const unsigned *SR = TRI->getSubRegisters(Reg); *SR; ++SR) - LocalDefsSet.erase(*SR); - } + LocalDefs.push_back(Reg); + for (const unsigned *OR = TRI->getOverlaps(Reg); *OR; ++OR) + LocalDefsSet.insert(*OR); } HasDups = true;; diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt index 06d2a95..9a5e551 100644 --- a/lib/CodeGen/CMakeLists.txt +++ b/lib/CodeGen/CMakeLists.txt @@ -13,7 +13,9 @@ add_llvm_library(LLVMCodeGen EdgeBundles.cpp ELFCodeEmitter.cpp ELFWriter.cpp + ExecutionDepsFix.cpp ExpandISelPseudos.cpp + ExpandPostRAPseudos.cpp GCMetadata.cpp GCMetadataPrinter.cpp GCStrategy.cpp @@ -23,17 +25,18 @@ add_llvm_library(LLVMCodeGen IntrinsicLowering.cpp LLVMTargetMachine.cpp LatencyPriorityQueue.cpp + LexicalScopes.cpp LiveDebugVariables.cpp LiveInterval.cpp LiveIntervalAnalysis.cpp LiveIntervalUnion.cpp LiveStackAnalysis.cpp LiveVariables.cpp + LiveRangeCalc.cpp LiveRangeEdit.cpp LocalStackSlotAllocation.cpp - LowerSubregs.cpp MachineBasicBlock.cpp - MachineBlockFrequency.cpp + MachineBlockFrequencyInfo.cpp MachineBranchProbabilityInfo.cpp MachineCSE.cpp MachineDominators.cpp @@ -97,5 +100,15 @@ add_llvm_library(LLVMCodeGen VirtRegRewriter.cpp ) +add_llvm_library_dependencies(LLVMCodeGen + LLVMAnalysis + LLVMCore + LLVMMC + LLVMScalarOpts + LLVMSupport + LLVMTarget + LLVMTransformUtils + ) + add_subdirectory(SelectionDAG) add_subdirectory(AsmPrinter) diff --git a/lib/CodeGen/CalcSpillWeights.cpp b/lib/CodeGen/CalcSpillWeights.cpp index e6b3bbc..ea16a25 100644 --- a/lib/CodeGen/CalcSpillWeights.cpp +++ b/lib/CodeGen/CalcSpillWeights.cpp @@ -185,35 +185,3 @@ void VirtRegAuxInfo::CalculateWeightAndHint(LiveInterval &li) { li.weight = normalizeSpillWeight(totalWeight, li.getSize()); } - -void VirtRegAuxInfo::CalculateRegClass(unsigned reg) { - MachineRegisterInfo &MRI = MF.getRegInfo(); - const TargetInstrInfo *TII = MF.getTarget().getInstrInfo(); - const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); - const TargetRegisterClass *OldRC = MRI.getRegClass(reg); - const TargetRegisterClass *NewRC = TRI->getLargestLegalSuperClass(OldRC); - - // Stop early if there is no room to grow. - if (NewRC == OldRC) - return; - - // Accumulate constraints from all uses. - for (MachineRegisterInfo::reg_nodbg_iterator I = MRI.reg_nodbg_begin(reg), - E = MRI.reg_nodbg_end(); I != E; ++I) { - // TRI doesn't have accurate enough information to model this yet. - if (I.getOperand().getSubReg()) - return; - // Inline asm instuctions don't remember their constraints. - if (I->isInlineAsm()) - return; - const TargetRegisterClass *OpRC = - TII->getRegClass(I->getDesc(), I.getOperandNo(), TRI); - if (OpRC) - NewRC = getCommonSubClass(NewRC, OpRC); - if (!NewRC || NewRC == OldRC) - return; - } - DEBUG(dbgs() << "Inflating " << OldRC->getName() << ':' << PrintReg(reg) - << " to " << NewRC->getName() <<".\n"); - MRI.setRegClass(reg, NewRC); -} diff --git a/lib/CodeGen/CodeGen.cpp b/lib/CodeGen/CodeGen.cpp index 489746c..424535b 100644 --- a/lib/CodeGen/CodeGen.cpp +++ b/lib/CodeGen/CodeGen.cpp @@ -27,6 +27,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeLiveIntervalsPass(Registry); initializeLiveStacksPass(Registry); initializeLiveVariablesPass(Registry); + initializeMachineBlockFrequencyInfoPass(Registry); initializeMachineCSEPass(Registry); initializeMachineDominatorTreePass(Registry); initializeMachineLICMPass(Registry); diff --git a/lib/CodeGen/DwarfEHPrepare.cpp b/lib/CodeGen/DwarfEHPrepare.cpp index 03604b0..ed9e409 100644 --- a/lib/CodeGen/DwarfEHPrepare.cpp +++ b/lib/CodeGen/DwarfEHPrepare.cpp @@ -63,6 +63,8 @@ namespace { typedef SmallPtrSet<BasicBlock*, 8> BBSet; BBSet LandingPads; + bool InsertUnwindResumeCalls(); + bool NormalizeLandingPads(); bool LowerUnwindsAndResumes(); bool MoveExceptionValueCalls(); @@ -658,13 +660,76 @@ Instruction *DwarfEHPrepare::CreateExceptionValueCall(BasicBlock *BB) { return CallInst::Create(ExceptionValueIntrinsic, "eh.value.call", Start); } +/// InsertUnwindResumeCalls - Convert the ResumeInsts that are still present +/// into calls to the appropriate _Unwind_Resume function. +bool DwarfEHPrepare::InsertUnwindResumeCalls() { + bool UsesNewEH = false; + SmallVector<ResumeInst*, 16> Resumes; + for (Function::iterator I = F->begin(), E = F->end(); I != E; ++I) { + TerminatorInst *TI = I->getTerminator(); + if (ResumeInst *RI = dyn_cast<ResumeInst>(TI)) + Resumes.push_back(RI); + else if (InvokeInst *II = dyn_cast<InvokeInst>(TI)) + UsesNewEH = II->getUnwindDest()->isLandingPad(); + } + + if (Resumes.empty()) + return UsesNewEH; + + // Find the rewind function if we didn't already. + if (!RewindFunction) { + LLVMContext &Ctx = Resumes[0]->getContext(); + FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), + Type::getInt8PtrTy(Ctx), false); + const char *RewindName = TLI->getLibcallName(RTLIB::UNWIND_RESUME); + RewindFunction = F->getParent()->getOrInsertFunction(RewindName, FTy); + } + + // Create the basic block where the _Unwind_Resume call will live. + LLVMContext &Ctx = F->getContext(); + BasicBlock *UnwindBB = BasicBlock::Create(Ctx, "unwind_resume", F); + PHINode *PN = PHINode::Create(Type::getInt8PtrTy(Ctx), Resumes.size(), + "exn.obj", UnwindBB); + + // Extract the exception object from the ResumeInst and add it to the PHI node + // that feeds the _Unwind_Resume call. + BasicBlock *UnwindBBDom = Resumes[0]->getParent(); + for (SmallVectorImpl<ResumeInst*>::iterator + I = Resumes.begin(), E = Resumes.end(); I != E; ++I) { + ResumeInst *RI = *I; + BranchInst::Create(UnwindBB, RI->getParent()); + ExtractValueInst *ExnObj = ExtractValueInst::Create(RI->getOperand(0), + 0, "exn.obj", RI); + PN->addIncoming(ExnObj, RI->getParent()); + UnwindBBDom = DT->findNearestCommonDominator(RI->getParent(), UnwindBBDom); + RI->eraseFromParent(); + } + + // Call the function. + CallInst *CI = CallInst::Create(RewindFunction, PN, "", UnwindBB); + CI->setCallingConv(TLI->getLibcallCallingConv(RTLIB::UNWIND_RESUME)); + + // We never expect _Unwind_Resume to return. + new UnreachableInst(Ctx, UnwindBB); + + // Now update DominatorTree analysis information. + DT->addNewBlock(UnwindBB, UnwindBBDom); + return true; +} + bool DwarfEHPrepare::runOnFunction(Function &Fn) { bool Changed = false; // Initialize internal state. - DT = &getAnalysis<DominatorTree>(); + DT = &getAnalysis<DominatorTree>(); // FIXME: We won't need this with the new EH. F = &Fn; + if (InsertUnwindResumeCalls()) { + // FIXME: The reset of this function can go once the new EH is done. + LandingPads.clear(); + return true; + } + // Ensure that only unwind edges end at landing pads (a landing pad is a // basic block where an invoke unwind edge ends). Changed |= NormalizeLandingPads(); diff --git a/lib/CodeGen/ELFCodeEmitter.cpp b/lib/CodeGen/ELFCodeEmitter.cpp index 3fb087c..660424c3 100644 --- a/lib/CodeGen/ELFCodeEmitter.cpp +++ b/lib/CodeGen/ELFCodeEmitter.cpp @@ -155,7 +155,7 @@ void ELFCodeEmitter::emitConstantPool(MachineConstantPool *MCP) { CPSections.push_back(CstPool.SectionIdx); if (CPE.isMachineConstantPoolEntry()) - assert("CPE.isMachineConstantPoolEntry not supported yet"); + assert(0 && "CPE.isMachineConstantPoolEntry not supported yet"); // Emit the constant to constant pool section EW.EmitGlobalConstant(CPE.Val.ConstVal, CstPool); diff --git a/lib/CodeGen/ELFCodeEmitter.h b/lib/CodeGen/ELFCodeEmitter.h index 2ec1f6e..8671c67 100644 --- a/lib/CodeGen/ELFCodeEmitter.h +++ b/lib/CodeGen/ELFCodeEmitter.h @@ -58,13 +58,13 @@ namespace llvm { /// emitLabel - Emits a label virtual void emitLabel(MCSymbol *Label) { - assert("emitLabel not implemented"); + assert(0 && "emitLabel not implemented"); } /// getLabelAddress - Return the address of the specified LabelID, /// only usable after the LabelID has been emitted. virtual uintptr_t getLabelAddress(MCSymbol *Label) const { - assert("getLabelAddress not implemented"); + assert(0 && "getLabelAddress not implemented"); return 0; } diff --git a/lib/CodeGen/ELFWriter.cpp b/lib/CodeGen/ELFWriter.cpp index d977651..f2c2185 100644 --- a/lib/CodeGen/ELFWriter.cpp +++ b/lib/CodeGen/ELFWriter.cpp @@ -45,12 +45,12 @@ #include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/Target/Mangler.h" -#include "llvm/Target/TargetAsmInfo.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetELFWriterInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" @@ -65,7 +65,8 @@ char ELFWriter::ID = 0; ELFWriter::ELFWriter(raw_ostream &o, TargetMachine &tm) : MachineFunctionPass(ID), O(o), TM(tm), - OutContext(*new MCContext(*TM.getMCAsmInfo(), new TargetAsmInfo(tm))), + OutContext(*new MCContext(*TM.getMCAsmInfo(), *TM.getRegisterInfo(), + &TM.getTargetLowering()->getObjFileLowering())), TLOF(TM.getTargetLowering()->getObjFileLowering()), is64Bit(TM.getTargetData()->getPointerSizeInBits() == 64), isLittleEndian(TM.getTargetData()->isLittleEndian()), @@ -482,7 +483,7 @@ void ELFWriter::EmitGlobalConstant(const Constant *CV, ELFSection &GblS) { EmitGlobalConstantLargeInt(CI, GblS); return; } else if (const ConstantVector *CP = dyn_cast<ConstantVector>(CV)) { - const VectorType *PTy = CP->getType(); + VectorType *PTy = CP->getType(); for (unsigned I = 0, E = PTy->getNumElements(); I < E; ++I) EmitGlobalConstant(CP->getOperand(I), GblS); return; @@ -540,8 +541,7 @@ CstExprResTy ELFWriter::ResolveConstantExpr(const Constant *CV) { case Instruction::GetElementPtr: { const Constant *ptrVal = CE->getOperand(0); SmallVector<Value*, 8> idxVec(CE->op_begin()+1, CE->op_end()); - int64_t Offset = TD->getIndexedOffset(ptrVal->getType(), &idxVec[0], - idxVec.size()); + int64_t Offset = TD->getIndexedOffset(ptrVal->getType(), idxVec); return std::make_pair(ptrVal, Offset); } case Instruction::IntToPtr: { @@ -552,7 +552,7 @@ CstExprResTy ELFWriter::ResolveConstantExpr(const Constant *CV) { } case Instruction::PtrToInt: { Constant *Op = CE->getOperand(0); - const Type *Ty = CE->getType(); + Type *Ty = CE->getType(); // We can emit the pointer value into this slot if the slot is an // integer slot greater or equal to the size of the pointer. diff --git a/lib/CodeGen/ExecutionDepsFix.cpp b/lib/CodeGen/ExecutionDepsFix.cpp new file mode 100644 index 0000000..01dccdb --- /dev/null +++ b/lib/CodeGen/ExecutionDepsFix.cpp @@ -0,0 +1,523 @@ +//===- ExecutionDepsFix.cpp - Fix execution dependecy issues ----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains the execution dependency fix pass. +// +// Some X86 SSE instructions like mov, and, or, xor are available in different +// variants for different operand types. These variant instructions are +// equivalent, but on Nehalem and newer cpus there is extra latency +// transferring data between integer and floating point domains. ARM cores +// have similar issues when they are configured with both VFP and NEON +// pipelines. +// +// This pass changes the variant instructions to minimize domain crossings. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "execution-fix" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +using namespace llvm; + +/// A DomainValue is a bit like LiveIntervals' ValNo, but it also keeps track +/// of execution domains. +/// +/// An open DomainValue represents a set of instructions that can still switch +/// execution domain. Multiple registers may refer to the same open +/// DomainValue - they will eventually be collapsed to the same execution +/// domain. +/// +/// A collapsed DomainValue represents a single register that has been forced +/// into one of more execution domains. There is a separate collapsed +/// DomainValue for each register, but it may contain multiple execution +/// domains. A register value is initially created in a single execution +/// domain, but if we were forced to pay the penalty of a domain crossing, we +/// keep track of the fact the the register is now available in multiple +/// domains. +namespace { +struct DomainValue { + // Basic reference counting. + unsigned Refs; + + // Bitmask of available domains. For an open DomainValue, it is the still + // possible domains for collapsing. For a collapsed DomainValue it is the + // domains where the register is available for free. + unsigned AvailableDomains; + + // Position of the last defining instruction. + unsigned Dist; + + // Twiddleable instructions using or defining these registers. + SmallVector<MachineInstr*, 8> Instrs; + + // A collapsed DomainValue has no instructions to twiddle - it simply keeps + // track of the domains where the registers are already available. + bool isCollapsed() const { return Instrs.empty(); } + + // Is domain available? + bool hasDomain(unsigned domain) const { + return AvailableDomains & (1u << domain); + } + + // Mark domain as available. + void addDomain(unsigned domain) { + AvailableDomains |= 1u << domain; + } + + // Restrict to a single domain available. + void setSingleDomain(unsigned domain) { + AvailableDomains = 1u << domain; + } + + // Return bitmask of domains that are available and in mask. + unsigned getCommonDomains(unsigned mask) const { + return AvailableDomains & mask; + } + + // First domain available. + unsigned getFirstDomain() const { + return CountTrailingZeros_32(AvailableDomains); + } + + DomainValue() { clear(); } + + void clear() { + Refs = AvailableDomains = Dist = 0; + Instrs.clear(); + } +}; +} + +namespace { +class ExeDepsFix : public MachineFunctionPass { + static char ID; + SpecificBumpPtrAllocator<DomainValue> Allocator; + SmallVector<DomainValue*,16> Avail; + + const TargetRegisterClass *const RC; + MachineFunction *MF; + const TargetInstrInfo *TII; + const TargetRegisterInfo *TRI; + MachineBasicBlock *MBB; + std::vector<int> AliasMap; + const unsigned NumRegs; + DomainValue **LiveRegs; + typedef DenseMap<MachineBasicBlock*,DomainValue**> LiveOutMap; + LiveOutMap LiveOuts; + unsigned Distance; + +public: + ExeDepsFix(const TargetRegisterClass *rc) + : MachineFunctionPass(ID), RC(rc), NumRegs(RC->getNumRegs()) {} + + virtual void getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesAll(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + virtual bool runOnMachineFunction(MachineFunction &MF); + + virtual const char *getPassName() const { + return "SSE execution domain fixup"; + } + +private: + // Register mapping. + int RegIndex(unsigned Reg); + + // DomainValue allocation. + DomainValue *Alloc(int domain = -1); + void Recycle(DomainValue*); + + // LiveRegs manipulations. + void SetLiveReg(int rx, DomainValue *DV); + void Kill(int rx); + void Force(int rx, unsigned domain); + void Collapse(DomainValue *dv, unsigned domain); + bool Merge(DomainValue *A, DomainValue *B); + + void enterBasicBlock(); + void visitGenericInstr(MachineInstr*); + void visitSoftInstr(MachineInstr*, unsigned mask); + void visitHardInstr(MachineInstr*, unsigned domain); +}; +} + +char ExeDepsFix::ID = 0; + +/// Translate TRI register number to an index into our smaller tables of +/// interesting registers. Return -1 for boring registers. +int ExeDepsFix::RegIndex(unsigned Reg) { + assert(Reg < AliasMap.size() && "Invalid register"); + return AliasMap[Reg]; +} + +DomainValue *ExeDepsFix::Alloc(int domain) { + DomainValue *dv = Avail.empty() ? + new(Allocator.Allocate()) DomainValue : + Avail.pop_back_val(); + dv->Dist = Distance; + if (domain >= 0) + dv->addDomain(domain); + return dv; +} + +void ExeDepsFix::Recycle(DomainValue *dv) { + assert(dv && "Cannot recycle NULL"); + dv->clear(); + Avail.push_back(dv); +} + +/// Set LiveRegs[rx] = dv, updating reference counts. +void ExeDepsFix::SetLiveReg(int rx, DomainValue *dv) { + assert(unsigned(rx) < NumRegs && "Invalid index"); + if (!LiveRegs) { + LiveRegs = new DomainValue*[NumRegs]; + std::fill(LiveRegs, LiveRegs+NumRegs, (DomainValue*)0); + } + + if (LiveRegs[rx] == dv) + return; + if (LiveRegs[rx]) { + assert(LiveRegs[rx]->Refs && "Bad refcount"); + if (--LiveRegs[rx]->Refs == 0) Recycle(LiveRegs[rx]); + } + LiveRegs[rx] = dv; + if (dv) ++dv->Refs; +} + +// Kill register rx, recycle or collapse any DomainValue. +void ExeDepsFix::Kill(int rx) { + assert(unsigned(rx) < NumRegs && "Invalid index"); + if (!LiveRegs || !LiveRegs[rx]) return; + + // Before killing the last reference to an open DomainValue, collapse it to + // the first available domain. + if (LiveRegs[rx]->Refs == 1 && !LiveRegs[rx]->isCollapsed()) + Collapse(LiveRegs[rx], LiveRegs[rx]->getFirstDomain()); + else + SetLiveReg(rx, 0); +} + +/// Force register rx into domain. +void ExeDepsFix::Force(int rx, unsigned domain) { + assert(unsigned(rx) < NumRegs && "Invalid index"); + DomainValue *dv; + if (LiveRegs && (dv = LiveRegs[rx])) { + if (dv->isCollapsed()) + dv->addDomain(domain); + else if (dv->hasDomain(domain)) + Collapse(dv, domain); + else { + // This is an incompatible open DomainValue. Collapse it to whatever and + // force the new value into domain. This costs a domain crossing. + Collapse(dv, dv->getFirstDomain()); + assert(LiveRegs[rx] && "Not live after collapse?"); + LiveRegs[rx]->addDomain(domain); + } + } else { + // Set up basic collapsed DomainValue. + SetLiveReg(rx, Alloc(domain)); + } +} + +/// Collapse open DomainValue into given domain. If there are multiple +/// registers using dv, they each get a unique collapsed DomainValue. +void ExeDepsFix::Collapse(DomainValue *dv, unsigned domain) { + assert(dv->hasDomain(domain) && "Cannot collapse"); + + // Collapse all the instructions. + while (!dv->Instrs.empty()) + TII->setExecutionDomain(dv->Instrs.pop_back_val(), domain); + dv->setSingleDomain(domain); + + // If there are multiple users, give them new, unique DomainValues. + if (LiveRegs && dv->Refs > 1) + for (unsigned rx = 0; rx != NumRegs; ++rx) + if (LiveRegs[rx] == dv) + SetLiveReg(rx, Alloc(domain)); +} + +/// Merge - All instructions and registers in B are moved to A, and B is +/// released. +bool ExeDepsFix::Merge(DomainValue *A, DomainValue *B) { + assert(!A->isCollapsed() && "Cannot merge into collapsed"); + assert(!B->isCollapsed() && "Cannot merge from collapsed"); + if (A == B) + return true; + // Restrict to the domains that A and B have in common. + unsigned common = A->getCommonDomains(B->AvailableDomains); + if (!common) + return false; + A->AvailableDomains = common; + A->Dist = std::max(A->Dist, B->Dist); + A->Instrs.append(B->Instrs.begin(), B->Instrs.end()); + for (unsigned rx = 0; rx != NumRegs; ++rx) + if (LiveRegs[rx] == B) + SetLiveReg(rx, A); + return true; +} + +void ExeDepsFix::enterBasicBlock() { + // Try to coalesce live-out registers from predecessors. + for (MachineBasicBlock::livein_iterator i = MBB->livein_begin(), + e = MBB->livein_end(); i != e; ++i) { + int rx = RegIndex(*i); + if (rx < 0) continue; + for (MachineBasicBlock::const_pred_iterator pi = MBB->pred_begin(), + pe = MBB->pred_end(); pi != pe; ++pi) { + LiveOutMap::const_iterator fi = LiveOuts.find(*pi); + if (fi == LiveOuts.end()) continue; + DomainValue *pdv = fi->second[rx]; + if (!pdv) continue; + if (!LiveRegs || !LiveRegs[rx]) { + SetLiveReg(rx, pdv); + continue; + } + + // We have a live DomainValue from more than one predecessor. + if (LiveRegs[rx]->isCollapsed()) { + // We are already collapsed, but predecessor is not. Force him. + unsigned domain = LiveRegs[rx]->getFirstDomain(); + if (!pdv->isCollapsed() && pdv->hasDomain(domain)) + Collapse(pdv, domain); + continue; + } + + // Currently open, merge in predecessor. + if (!pdv->isCollapsed()) + Merge(LiveRegs[rx], pdv); + else + Force(rx, pdv->getFirstDomain()); + } + } +} + +// A hard instruction only works in one domain. All input registers will be +// forced into that domain. +void ExeDepsFix::visitHardInstr(MachineInstr *mi, unsigned domain) { + // Collapse all uses. + for (unsigned i = mi->getDesc().getNumDefs(), + e = mi->getDesc().getNumOperands(); i != e; ++i) { + MachineOperand &mo = mi->getOperand(i); + if (!mo.isReg()) continue; + int rx = RegIndex(mo.getReg()); + if (rx < 0) continue; + Force(rx, domain); + } + + // Kill all defs and force them. + for (unsigned i = 0, e = mi->getDesc().getNumDefs(); i != e; ++i) { + MachineOperand &mo = mi->getOperand(i); + if (!mo.isReg()) continue; + int rx = RegIndex(mo.getReg()); + if (rx < 0) continue; + Kill(rx); + Force(rx, domain); + } +} + +// A soft instruction can be changed to work in other domains given by mask. +void ExeDepsFix::visitSoftInstr(MachineInstr *mi, unsigned mask) { + // Bitmask of available domains for this instruction after taking collapsed + // operands into account. + unsigned available = mask; + + // Scan the explicit use operands for incoming domains. + SmallVector<int, 4> used; + if (LiveRegs) + for (unsigned i = mi->getDesc().getNumDefs(), + e = mi->getDesc().getNumOperands(); i != e; ++i) { + MachineOperand &mo = mi->getOperand(i); + if (!mo.isReg()) continue; + int rx = RegIndex(mo.getReg()); + if (rx < 0) continue; + if (DomainValue *dv = LiveRegs[rx]) { + // Bitmask of domains that dv and available have in common. + unsigned common = dv->getCommonDomains(available); + // Is it possible to use this collapsed register for free? + if (dv->isCollapsed()) { + // Restrict available domains to the ones in common with the operand. + // If there are no common domains, we must pay the cross-domain + // penalty for this operand. + if (common) available = common; + } else if (common) + // Open DomainValue is compatible, save it for merging. + used.push_back(rx); + else + // Open DomainValue is not compatible with instruction. It is useless + // now. + Kill(rx); + } + } + + // If the collapsed operands force a single domain, propagate the collapse. + if (isPowerOf2_32(available)) { + unsigned domain = CountTrailingZeros_32(available); + TII->setExecutionDomain(mi, domain); + visitHardInstr(mi, domain); + return; + } + + // Kill off any remaining uses that don't match available, and build a list of + // incoming DomainValues that we want to merge. + SmallVector<DomainValue*,4> doms; + for (SmallVector<int, 4>::iterator i=used.begin(), e=used.end(); i!=e; ++i) { + int rx = *i; + DomainValue *dv = LiveRegs[rx]; + // This useless DomainValue could have been missed above. + if (!dv->getCommonDomains(available)) { + Kill(*i); + continue; + } + // sorted, uniqued insert. + bool inserted = false; + for (SmallVector<DomainValue*,4>::iterator i = doms.begin(), e = doms.end(); + i != e && !inserted; ++i) { + if (dv == *i) + inserted = true; + else if (dv->Dist < (*i)->Dist) { + inserted = true; + doms.insert(i, dv); + } + } + if (!inserted) + doms.push_back(dv); + } + + // doms are now sorted in order of appearance. Try to merge them all, giving + // priority to the latest ones. + DomainValue *dv = 0; + while (!doms.empty()) { + if (!dv) { + dv = doms.pop_back_val(); + continue; + } + + DomainValue *latest = doms.pop_back_val(); + if (Merge(dv, latest)) continue; + + // If latest didn't merge, it is useless now. Kill all registers using it. + for (SmallVector<int,4>::iterator i=used.begin(), e=used.end(); i != e; ++i) + if (LiveRegs[*i] == latest) + Kill(*i); + } + + // dv is the DomainValue we are going to use for this instruction. + if (!dv) + dv = Alloc(); + dv->Dist = Distance; + dv->AvailableDomains = available; + dv->Instrs.push_back(mi); + + // Finally set all defs and non-collapsed uses to dv. + for (unsigned i = 0, e = mi->getDesc().getNumOperands(); i != e; ++i) { + MachineOperand &mo = mi->getOperand(i); + if (!mo.isReg()) continue; + int rx = RegIndex(mo.getReg()); + if (rx < 0) continue; + if (!LiveRegs || !LiveRegs[rx] || (mo.isDef() && LiveRegs[rx]!=dv)) { + Kill(rx); + SetLiveReg(rx, dv); + } + } +} + +void ExeDepsFix::visitGenericInstr(MachineInstr *mi) { + // Process explicit defs, kill any relevant registers redefined. + for (unsigned i = 0, e = mi->getDesc().getNumDefs(); i != e; ++i) { + MachineOperand &mo = mi->getOperand(i); + if (!mo.isReg()) continue; + int rx = RegIndex(mo.getReg()); + if (rx < 0) continue; + Kill(rx); + } +} + +bool ExeDepsFix::runOnMachineFunction(MachineFunction &mf) { + MF = &mf; + TII = MF->getTarget().getInstrInfo(); + TRI = MF->getTarget().getRegisterInfo(); + MBB = 0; + LiveRegs = 0; + Distance = 0; + assert(NumRegs == RC->getNumRegs() && "Bad regclass"); + + // If no relevant registers are used in the function, we can skip it + // completely. + bool anyregs = false; + for (TargetRegisterClass::const_iterator I = RC->begin(), E = RC->end(); + I != E; ++I) + if (MF->getRegInfo().isPhysRegUsed(*I)) { + anyregs = true; + break; + } + if (!anyregs) return false; + + // Initialize the AliasMap on the first use. + if (AliasMap.empty()) { + // Given a PhysReg, AliasMap[PhysReg] is either the relevant index into RC, + // or -1. + AliasMap.resize(TRI->getNumRegs(), -1); + for (unsigned i = 0, e = RC->getNumRegs(); i != e; ++i) + for (const unsigned *AI = TRI->getOverlaps(RC->getRegister(i)); *AI; ++AI) + AliasMap[*AI] = i; + } + + MachineBasicBlock *Entry = MF->begin(); + SmallPtrSet<MachineBasicBlock*, 16> Visited; + for (df_ext_iterator<MachineBasicBlock*, SmallPtrSet<MachineBasicBlock*, 16> > + DFI = df_ext_begin(Entry, Visited), DFE = df_ext_end(Entry, Visited); + DFI != DFE; ++DFI) { + MBB = *DFI; + enterBasicBlock(); + for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E; + ++I) { + MachineInstr *mi = I; + if (mi->isDebugValue()) continue; + ++Distance; + std::pair<uint16_t, uint16_t> domp = TII->getExecutionDomain(mi); + if (domp.first) + if (domp.second) + visitSoftInstr(mi, domp.second); + else + visitHardInstr(mi, domp.first); + else if (LiveRegs) + visitGenericInstr(mi); + } + + // Save live registers at end of MBB - used by enterBasicBlock(). + if (LiveRegs) + LiveOuts.insert(std::make_pair(MBB, LiveRegs)); + LiveRegs = 0; + } + + // Clear the LiveOuts vectors. Should we also collapse any remaining + // DomainValues? + for (LiveOutMap::const_iterator i = LiveOuts.begin(), e = LiveOuts.end(); + i != e; ++i) + delete[] i->second; + LiveOuts.clear(); + Avail.clear(); + Allocator.DestroyAll(); + + return false; +} + +FunctionPass * +llvm::createExecutionDependencyFixPass(const TargetRegisterClass *RC) { + return new ExeDepsFix(RC); +} diff --git a/lib/CodeGen/LowerSubregs.cpp b/lib/CodeGen/ExpandPostRAPseudos.cpp index 7871ba9..e2a14a8 100644 --- a/lib/CodeGen/LowerSubregs.cpp +++ b/lib/CodeGen/ExpandPostRAPseudos.cpp @@ -1,4 +1,4 @@ -//===-- LowerSubregs.cpp - Subregister Lowering instruction pass ----------===// +//===-- ExpandPostRAPseudos.cpp - Pseudo instruction expansion pass -------===// // // The LLVM Compiler Infrastructure // @@ -7,14 +7,12 @@ // //===----------------------------------------------------------------------===// // -// This file defines a MachineFunction pass which runs after register -// allocation that turns subreg insert/extract instructions into register -// copies, as needed. This ensures correct codegen even if the coalescer -// isn't able to remove all subreg instructions. +// This file defines a pass that expands COPY and SUBREG_TO_REG pseudo +// instructions after register allocation. // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "lowersubregs" +#define DEBUG_TYPE "postrapseudos" #include "llvm/CodeGen/Passes.h" #include "llvm/Function.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -29,52 +27,51 @@ using namespace llvm; namespace { - struct LowerSubregsInstructionPass : public MachineFunctionPass { - private: - const TargetRegisterInfo *TRI; - const TargetInstrInfo *TII; +struct ExpandPostRA : public MachineFunctionPass { +private: + const TargetRegisterInfo *TRI; + const TargetInstrInfo *TII; - public: - static char ID; // Pass identification, replacement for typeid - LowerSubregsInstructionPass() : MachineFunctionPass(ID) {} +public: + static char ID; // Pass identification, replacement for typeid + ExpandPostRA() : MachineFunctionPass(ID) {} - const char *getPassName() const { - return "Subregister lowering instruction pass"; - } + const char *getPassName() const { + return "Post-RA pseudo instruction expansion pass"; + } - virtual void getAnalysisUsage(AnalysisUsage &AU) const { - AU.setPreservesCFG(); - AU.addPreservedID(MachineLoopInfoID); - AU.addPreservedID(MachineDominatorsID); - MachineFunctionPass::getAnalysisUsage(AU); - } + virtual void getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesCFG(); + AU.addPreservedID(MachineLoopInfoID); + AU.addPreservedID(MachineDominatorsID); + MachineFunctionPass::getAnalysisUsage(AU); + } - /// runOnMachineFunction - pass entry point - bool runOnMachineFunction(MachineFunction&); + /// runOnMachineFunction - pass entry point + bool runOnMachineFunction(MachineFunction&); - private: - bool LowerSubregToReg(MachineInstr *MI); - bool LowerCopy(MachineInstr *MI); +private: + bool LowerSubregToReg(MachineInstr *MI); + bool LowerCopy(MachineInstr *MI); - void TransferDeadFlag(MachineInstr *MI, unsigned DstReg, - const TargetRegisterInfo *TRI); - void TransferImplicitDefs(MachineInstr *MI); - }; + void TransferDeadFlag(MachineInstr *MI, unsigned DstReg, + const TargetRegisterInfo *TRI); + void TransferImplicitDefs(MachineInstr *MI); +}; +} // end anonymous namespace - char LowerSubregsInstructionPass::ID = 0; -} +char ExpandPostRA::ID = 0; -FunctionPass *llvm::createLowerSubregsPass() { - return new LowerSubregsInstructionPass(); +FunctionPass *llvm::createExpandPostRAPseudosPass() { + return new ExpandPostRA(); } /// TransferDeadFlag - MI is a pseudo-instruction with DstReg dead, /// and the lowered replacement instructions immediately precede it. /// Mark the replacement instructions with the dead flag. void -LowerSubregsInstructionPass::TransferDeadFlag(MachineInstr *MI, - unsigned DstReg, - const TargetRegisterInfo *TRI) { +ExpandPostRA::TransferDeadFlag(MachineInstr *MI, unsigned DstReg, + const TargetRegisterInfo *TRI) { for (MachineBasicBlock::iterator MII = prior(MachineBasicBlock::iterator(MI)); ; --MII) { if (MII->addRegisterDead(DstReg, TRI)) @@ -88,7 +85,7 @@ LowerSubregsInstructionPass::TransferDeadFlag(MachineInstr *MI, /// replacement instructions immediately precede it. Copy any implicit-def /// operands from MI to the replacement instruction. void -LowerSubregsInstructionPass::TransferImplicitDefs(MachineInstr *MI) { +ExpandPostRA::TransferImplicitDefs(MachineInstr *MI) { MachineBasicBlock::iterator CopyMI = MI; --CopyMI; @@ -100,7 +97,7 @@ LowerSubregsInstructionPass::TransferImplicitDefs(MachineInstr *MI) { } } -bool LowerSubregsInstructionPass::LowerSubregToReg(MachineInstr *MI) { +bool ExpandPostRA::LowerSubregToReg(MachineInstr *MI) { MachineBasicBlock *MBB = MI->getParent(); assert((MI->getOperand(0).isReg() && MI->getOperand(0).isDef()) && MI->getOperand(1).isImm() && @@ -152,7 +149,7 @@ bool LowerSubregsInstructionPass::LowerSubregToReg(MachineInstr *MI) { return true; } -bool LowerSubregsInstructionPass::LowerCopy(MachineInstr *MI) { +bool ExpandPostRA::LowerCopy(MachineInstr *MI) { MachineOperand &DstMO = MI->getOperand(0); MachineOperand &SrcMO = MI->getOperand(1); @@ -191,9 +188,9 @@ bool LowerSubregsInstructionPass::LowerCopy(MachineInstr *MI) { /// runOnMachineFunction - Reduce subregister inserts and extracts to register /// copies. /// -bool LowerSubregsInstructionPass::runOnMachineFunction(MachineFunction &MF) { +bool ExpandPostRA::runOnMachineFunction(MachineFunction &MF) { DEBUG(dbgs() << "Machine Function\n" - << "********** LOWERING SUBREG INSTRS **********\n" + << "********** EXPANDING POST-RA PSEUDO INSTRS **********\n" << "********** Function: " << MF.getFunction()->getName() << '\n'); TRI = MF.getTarget().getRegisterInfo(); @@ -205,17 +202,34 @@ bool LowerSubregsInstructionPass::runOnMachineFunction(MachineFunction &MF) { mbbi != mbbe; ++mbbi) { for (MachineBasicBlock::iterator mi = mbbi->begin(), me = mbbi->end(); mi != me;) { - MachineBasicBlock::iterator nmi = llvm::next(mi); MachineInstr *MI = mi; - assert(!MI->isInsertSubreg() && "INSERT_SUBREG should no longer appear"); - assert(MI->getOpcode() != TargetOpcode::EXTRACT_SUBREG && - "EXTRACT_SUBREG should no longer appear"); - if (MI->isSubregToReg()) { + // Advance iterator here because MI may be erased. + ++mi; + + // Only expand pseudos. + if (!MI->getDesc().isPseudo()) + continue; + + // Give targets a chance to expand even standard pseudos. + if (TII->expandPostRAPseudo(MI)) { + MadeChange = true; + continue; + } + + // Expand standard pseudos. + switch (MI->getOpcode()) { + case TargetOpcode::SUBREG_TO_REG: MadeChange |= LowerSubregToReg(MI); - } else if (MI->isCopy()) { + break; + case TargetOpcode::COPY: MadeChange |= LowerCopy(MI); + break; + case TargetOpcode::DBG_VALUE: + continue; + case TargetOpcode::INSERT_SUBREG: + case TargetOpcode::EXTRACT_SUBREG: + llvm_unreachable("Sub-register pseudos should have been eliminated."); } - mi = nmi; } } diff --git a/lib/CodeGen/IfConversion.cpp b/lib/CodeGen/IfConversion.cpp index 6cb2277..ce7ed29 100644 --- a/lib/CodeGen/IfConversion.cpp +++ b/lib/CodeGen/IfConversion.cpp @@ -16,14 +16,13 @@ #include "llvm/Function.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/MC/MCInstrItineraries.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Support/BranchProbability.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -154,7 +153,8 @@ namespace { const TargetInstrInfo *TII; const TargetRegisterInfo *TRI; const InstrItineraryData *InstrItins; - const MachineLoopInfo *MLI; + const MachineBranchProbabilityInfo *MBPI; + bool MadeChange; int FnNum; public: @@ -162,9 +162,9 @@ namespace { IfConverter() : MachineFunctionPass(ID), FnNum(-1) { initializeIfConverterPass(*PassRegistry::getPassRegistry()); } - + virtual void getAnalysisUsage(AnalysisUsage &AU) const { - AU.addRequired<MachineLoopInfo>(); + AU.addRequired<MachineBranchProbabilityInfo>(); MachineFunctionPass::getAnalysisUsage(AU); } @@ -252,7 +252,7 @@ namespace { } INITIALIZE_PASS_BEGIN(IfConverter, "if-converter", "If Converter", false, false) -INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) +INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo) INITIALIZE_PASS_END(IfConverter, "if-converter", "If Converter", false, false) FunctionPass *llvm::createIfConverterPass() { return new IfConverter(); } @@ -261,7 +261,7 @@ bool IfConverter::runOnMachineFunction(MachineFunction &MF) { TLI = MF.getTarget().getTargetLowering(); TII = MF.getTarget().getInstrInfo(); TRI = MF.getTarget().getRegisterInfo(); - MLI = &getAnalysis<MachineLoopInfo>(); + MBPI = &getAnalysis<MachineBranchProbabilityInfo>(); InstrItins = MF.getTarget().getInstrItineraryData(); if (!TII) return false; @@ -790,28 +790,9 @@ IfConverter::BBInfo &IfConverter::AnalyzeBlock(MachineBasicBlock *BB, bool TNeedSub = TrueBBI.Predicate.size() > 0; bool FNeedSub = FalseBBI.Predicate.size() > 0; bool Enqueued = false; - - // Try to predict the branch, using loop info to guide us. - // General heuristics are: - // - backedge -> 90% taken - // - early exit -> 20% taken - // - branch predictor confidence -> 90% - BranchProbability Prediction(5, 10); - MachineLoop *Loop = MLI->getLoopFor(BB); - if (Loop) { - if (TrueBBI.BB == Loop->getHeader()) - Prediction = BranchProbability(9, 10); - else if (FalseBBI.BB == Loop->getHeader()) - Prediction = BranchProbability(1, 10); - - MachineLoop *TrueLoop = MLI->getLoopFor(TrueBBI.BB); - MachineLoop *FalseLoop = MLI->getLoopFor(FalseBBI.BB); - if (!TrueLoop || TrueLoop->getParentLoop() == Loop) - Prediction = BranchProbability(2, 10); - else if (!FalseLoop || FalseLoop->getParentLoop() == Loop) - Prediction = BranchProbability(8, 10); - } - + + BranchProbability Prediction = MBPI->getEdgeProbability(BB, TrueBBI.BB); + if (CanRevCond && ValidDiamond(TrueBBI, FalseBBI, Dups, Dups2) && MeetIfcvtSizeLimit(*TrueBBI.BB, (TrueBBI.NonPredSize - (Dups + Dups2) + TrueBBI.ExtraCost), TrueBBI.ExtraCost2, diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp index 5547f73..726af46 100644 --- a/lib/CodeGen/InlineSpiller.cpp +++ b/lib/CodeGen/InlineSpiller.cpp @@ -17,6 +17,7 @@ #include "LiveRangeEdit.h" #include "VirtRegMap.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/TinyPtrVector.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveStackAnalysis.h" @@ -27,22 +28,26 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; STATISTIC(NumSpilledRanges, "Number of spilled live ranges"); -STATISTIC(NumSnippets, "Number of snippets included in spills"); +STATISTIC(NumSnippets, "Number of spilled snippets"); STATISTIC(NumSpills, "Number of spills inserted"); +STATISTIC(NumSpillsRemoved, "Number of spills removed"); STATISTIC(NumReloads, "Number of reloads inserted"); +STATISTIC(NumReloadsRemoved, "Number of reloads removed"); STATISTIC(NumFolded, "Number of folded stack accesses"); STATISTIC(NumFoldedLoads, "Number of folded loads"); STATISTIC(NumRemats, "Number of rematerialized defs for spilling"); -STATISTIC(NumOmitReloadSpill, "Number of omitted spills after reloads"); -STATISTIC(NumHoistLocal, "Number of locally hoisted spills"); -STATISTIC(NumHoistGlobal, "Number of globally hoisted spills"); -STATISTIC(NumRedundantSpills, "Number of redundant spills identified"); +STATISTIC(NumOmitReloadSpill, "Number of omitted spills of reloads"); +STATISTIC(NumHoists, "Number of hoisted spills"); + +static cl::opt<bool> DisableHoisting("disable-spill-hoist", cl::Hidden, + cl::desc("Disable inline spill hoisting")); namespace { class InlineSpiller : public Spiller { @@ -75,26 +80,49 @@ class InlineSpiller : public Spiller { // Values that failed to remat at some point. SmallPtrSet<VNInfo*, 8> UsedValues; +public: // Information about a value that was defined by a copy from a sibling // register. struct SibValueInfo { // True when all reaching defs were reloads: No spill is necessary. bool AllDefsAreReloads; + // True when value is defined by an original PHI not from splitting. + bool DefByOrigPHI; + + // True when the COPY defining this value killed its source. + bool KillsSource; + // The preferred register to spill. unsigned SpillReg; // The value of SpillReg that should be spilled. VNInfo *SpillVNI; + // The block where SpillVNI should be spilled. Currently, this must be the + // block containing SpillVNI->def. + MachineBasicBlock *SpillMBB; + // A defining instruction that is not a sibling copy or a reload, or NULL. // This can be used as a template for rematerialization. MachineInstr *DefMI; + // List of values that depend on this one. These values are actually the + // same, but live range splitting has placed them in different registers, + // or SSA update needed to insert PHI-defs to preserve SSA form. This is + // copies of the current value and phi-kills. Usually only phi-kills cause + // more than one dependent value. + TinyPtrVector<VNInfo*> Deps; + SibValueInfo(unsigned Reg, VNInfo *VNI) - : AllDefsAreReloads(false), SpillReg(Reg), SpillVNI(VNI), DefMI(0) {} + : AllDefsAreReloads(true), DefByOrigPHI(false), KillsSource(false), + SpillReg(Reg), SpillVNI(VNI), SpillMBB(0), DefMI(0) {} + + // Returns true when a def has been found. + bool hasDef() const { return DefByOrigPHI || DefMI; } }; +private: // Values in RegsToSpill defined by sibling copies. typedef DenseMap<VNInfo*, SibValueInfo> SibValueMap; SibValueMap SibValues; @@ -134,6 +162,7 @@ private: bool isSibling(unsigned Reg); MachineInstr *traceSiblingValue(unsigned, VNInfo*, VNInfo*); + void propagateSiblingValue(SibValueMap::iterator, VNInfo *VNI = 0); void analyzeSiblingValues(); bool hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI); @@ -282,6 +311,156 @@ bool InlineSpiller::isSibling(unsigned Reg) { VRM.getOriginal(Reg) == Original; } +#ifndef NDEBUG +static raw_ostream &operator<<(raw_ostream &OS, + const InlineSpiller::SibValueInfo &SVI) { + OS << "spill " << PrintReg(SVI.SpillReg) << ':' + << SVI.SpillVNI->id << '@' << SVI.SpillVNI->def; + if (SVI.SpillMBB) + OS << " in BB#" << SVI.SpillMBB->getNumber(); + if (SVI.AllDefsAreReloads) + OS << " all-reloads"; + if (SVI.DefByOrigPHI) + OS << " orig-phi"; + if (SVI.KillsSource) + OS << " kill"; + OS << " deps["; + for (unsigned i = 0, e = SVI.Deps.size(); i != e; ++i) + OS << ' ' << SVI.Deps[i]->id << '@' << SVI.Deps[i]->def; + OS << " ]"; + if (SVI.DefMI) + OS << " def: " << *SVI.DefMI; + else + OS << '\n'; + return OS; +} +#endif + +/// propagateSiblingValue - Propagate the value in SVI to dependents if it is +/// known. Otherwise remember the dependency for later. +/// +/// @param SVI SibValues entry to propagate. +/// @param VNI Dependent value, or NULL to propagate to all saved dependents. +void InlineSpiller::propagateSiblingValue(SibValueMap::iterator SVI, + VNInfo *VNI) { + // When VNI is non-NULL, add it to SVI's deps, and only propagate to that. + TinyPtrVector<VNInfo*> FirstDeps; + if (VNI) { + FirstDeps.push_back(VNI); + SVI->second.Deps.push_back(VNI); + } + + // Has the value been completely determined yet? If not, defer propagation. + if (!SVI->second.hasDef()) + return; + + // Work list of values to propagate. It would be nice to use a SetVector + // here, but then we would be forced to use a SmallSet. + SmallVector<SibValueMap::iterator, 8> WorkList(1, SVI); + SmallPtrSet<VNInfo*, 8> WorkSet; + + do { + SVI = WorkList.pop_back_val(); + WorkSet.erase(SVI->first); + TinyPtrVector<VNInfo*> *Deps = VNI ? &FirstDeps : &SVI->second.Deps; + VNI = 0; + + SibValueInfo &SV = SVI->second; + if (!SV.SpillMBB) + SV.SpillMBB = LIS.getMBBFromIndex(SV.SpillVNI->def); + + DEBUG(dbgs() << " prop to " << Deps->size() << ": " + << SVI->first->id << '@' << SVI->first->def << ":\t" << SV); + + assert(SV.hasDef() && "Propagating undefined value"); + + // Should this value be propagated as a preferred spill candidate? We don't + // propagate values of registers that are about to spill. + bool PropSpill = !DisableHoisting && !isRegToSpill(SV.SpillReg); + unsigned SpillDepth = ~0u; + + for (TinyPtrVector<VNInfo*>::iterator DepI = Deps->begin(), + DepE = Deps->end(); DepI != DepE; ++DepI) { + SibValueMap::iterator DepSVI = SibValues.find(*DepI); + assert(DepSVI != SibValues.end() && "Dependent value not in SibValues"); + SibValueInfo &DepSV = DepSVI->second; + if (!DepSV.SpillMBB) + DepSV.SpillMBB = LIS.getMBBFromIndex(DepSV.SpillVNI->def); + + bool Changed = false; + + // Propagate defining instruction. + if (!DepSV.hasDef()) { + Changed = true; + DepSV.DefMI = SV.DefMI; + DepSV.DefByOrigPHI = SV.DefByOrigPHI; + } + + // Propagate AllDefsAreReloads. For PHI values, this computes an AND of + // all predecessors. + if (!SV.AllDefsAreReloads && DepSV.AllDefsAreReloads) { + Changed = true; + DepSV.AllDefsAreReloads = false; + } + + // Propagate best spill value. + if (PropSpill && SV.SpillVNI != DepSV.SpillVNI) { + if (SV.SpillMBB == DepSV.SpillMBB) { + // DepSV is in the same block. Hoist when dominated. + if (DepSV.KillsSource && SV.SpillVNI->def < DepSV.SpillVNI->def) { + // This is an alternative def earlier in the same MBB. + // Hoist the spill as far as possible in SpillMBB. This can ease + // register pressure: + // + // x = def + // y = use x + // s = copy x + // + // Hoisting the spill of s to immediately after the def removes the + // interference between x and y: + // + // x = def + // spill x + // y = use x<kill> + // + // This hoist only helps when the DepSV copy kills its source. + Changed = true; + DepSV.SpillReg = SV.SpillReg; + DepSV.SpillVNI = SV.SpillVNI; + DepSV.SpillMBB = SV.SpillMBB; + } + } else { + // DepSV is in a different block. + if (SpillDepth == ~0u) + SpillDepth = Loops.getLoopDepth(SV.SpillMBB); + + // Also hoist spills to blocks with smaller loop depth, but make sure + // that the new value dominates. Non-phi dependents are always + // dominated, phis need checking. + if ((Loops.getLoopDepth(DepSV.SpillMBB) > SpillDepth) && + (!DepSVI->first->isPHIDef() || + MDT.dominates(SV.SpillMBB, DepSV.SpillMBB))) { + Changed = true; + DepSV.SpillReg = SV.SpillReg; + DepSV.SpillVNI = SV.SpillVNI; + DepSV.SpillMBB = SV.SpillMBB; + } + } + } + + if (!Changed) + continue; + + // Something changed in DepSVI. Propagate to dependents. + if (WorkSet.insert(DepSVI->first)) + WorkList.push_back(DepSVI); + + DEBUG(dbgs() << " update " << DepSVI->first->id << '@' + << DepSVI->first->def << " to:\t" << DepSV); + } + } while (!WorkList.empty()); +} + /// traceSiblingValue - Trace a value that is about to be spilled back to the /// real defining instructions by looking through sibling copies. Always stay /// within the range of OrigVNI so the registers are known to carry the same @@ -294,84 +473,101 @@ bool InlineSpiller::isSibling(unsigned Reg) { /// MachineInstr *InlineSpiller::traceSiblingValue(unsigned UseReg, VNInfo *UseVNI, VNInfo *OrigVNI) { + // Check if a cached value already exists. + SibValueMap::iterator SVI; + bool Inserted; + tie(SVI, Inserted) = + SibValues.insert(std::make_pair(UseVNI, SibValueInfo(UseReg, UseVNI))); + if (!Inserted) { + DEBUG(dbgs() << "Cached value " << PrintReg(UseReg) << ':' + << UseVNI->id << '@' << UseVNI->def << ' ' << SVI->second); + return SVI->second.DefMI; + } + DEBUG(dbgs() << "Tracing value " << PrintReg(UseReg) << ':' << UseVNI->id << '@' << UseVNI->def << '\n'); - SmallPtrSet<VNInfo*, 8> Visited; + + // List of (Reg, VNI) that have been inserted into SibValues, but need to be + // processed. SmallVector<std::pair<unsigned, VNInfo*>, 8> WorkList; WorkList.push_back(std::make_pair(UseReg, UseVNI)); - // Best spill candidate seen so far. This must dominate UseVNI. - SibValueInfo SVI(UseReg, UseVNI); - MachineBasicBlock *UseMBB = LIS.getMBBFromIndex(UseVNI->def); - MachineBasicBlock *SpillMBB = UseMBB; - unsigned SpillDepth = Loops.getLoopDepth(SpillMBB); - bool SeenOrigPHI = false; // Original PHI met. - do { unsigned Reg; VNInfo *VNI; tie(Reg, VNI) = WorkList.pop_back_val(); - if (!Visited.insert(VNI)) - continue; + DEBUG(dbgs() << " " << PrintReg(Reg) << ':' << VNI->id << '@' << VNI->def + << ":\t"); - // Is this value a better spill candidate? - if (!isRegToSpill(Reg)) { - MachineBasicBlock *MBB = LIS.getMBBFromIndex(VNI->def); - if (MBB == SpillMBB) { - // This is an alternative def earlier in the same MBB. - // Hoist the spill as far as possible in SpillMBB. This can ease - // register pressure: - // - // x = def - // y = use x - // s = copy x - // - // Hoisting the spill of s to immediately after the def removes the - // interference between x and y: - // - // x = def - // spill x - // y = use x<kill> - // - if (VNI->def < SVI.SpillVNI->def) { - DEBUG(dbgs() << " hoist in BB#" << MBB->getNumber() << ": " - << PrintReg(Reg) << ':' << VNI->id << '@' << VNI->def - << '\n'); - SVI.SpillReg = Reg; - SVI.SpillVNI = VNI; - } - } else if (MBB != UseMBB && MDT.dominates(MBB, UseMBB)) { - // This is a valid spill location dominating UseVNI. - // Prefer to spill at a smaller loop depth. - unsigned Depth = Loops.getLoopDepth(MBB); - if (Depth < SpillDepth) { - DEBUG(dbgs() << " spill depth " << Depth << ": " << PrintReg(Reg) - << ':' << VNI->id << '@' << VNI->def << '\n'); - SVI.SpillReg = Reg; - SVI.SpillVNI = VNI; - SpillMBB = MBB; - SpillDepth = Depth; - } - } - } + // First check if this value has already been computed. + SVI = SibValues.find(VNI); + assert(SVI != SibValues.end() && "Missing SibValues entry"); // Trace through PHI-defs created by live range splitting. if (VNI->isPHIDef()) { + // Stop at original PHIs. We don't know the value at the predecessors. if (VNI->def == OrigVNI->def) { - DEBUG(dbgs() << " orig phi value " << PrintReg(Reg) << ':' - << VNI->id << '@' << VNI->def << '\n'); - SeenOrigPHI = true; + DEBUG(dbgs() << "orig phi value\n"); + SVI->second.DefByOrigPHI = true; + SVI->second.AllDefsAreReloads = false; + propagateSiblingValue(SVI); continue; } - // Get values live-out of predecessors. + + // This is a PHI inserted by live range splitting. We could trace the + // live-out value from predecessor blocks, but that search can be very + // expensive if there are many predecessors and many more PHIs as + // generated by tail-dup when it sees an indirectbr. Instead, look at + // all the non-PHI defs that have the same value as OrigVNI. They must + // jointly dominate VNI->def. This is not optimal since VNI may actually + // be jointly dominated by a smaller subset of defs, so there is a change + // we will miss a AllDefsAreReloads optimization. + + // Separate all values dominated by OrigVNI into PHIs and non-PHIs. + SmallVector<VNInfo*, 8> PHIs, NonPHIs; LiveInterval &LI = LIS.getInterval(Reg); - MachineBasicBlock *MBB = LIS.getMBBFromIndex(VNI->def); - for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), - PE = MBB->pred_end(); PI != PE; ++PI) { - VNInfo *PVNI = LI.getVNInfoAt(LIS.getMBBEndIdx(*PI).getPrevSlot()); - if (PVNI) - WorkList.push_back(std::make_pair(Reg, PVNI)); + LiveInterval &OrigLI = LIS.getInterval(Original); + + for (LiveInterval::vni_iterator VI = LI.vni_begin(), VE = LI.vni_end(); + VI != VE; ++VI) { + VNInfo *VNI2 = *VI; + if (VNI2->isUnused()) + continue; + if (!OrigLI.containsOneValue() && + OrigLI.getVNInfoAt(VNI2->def) != OrigVNI) + continue; + if (VNI2->isPHIDef() && VNI2->def != OrigVNI->def) + PHIs.push_back(VNI2); + else + NonPHIs.push_back(VNI2); + } + DEBUG(dbgs() << "split phi value, checking " << PHIs.size() + << " phi-defs, and " << NonPHIs.size() + << " non-phi/orig defs\n"); + + // Create entries for all the PHIs. Don't add them to the worklist, we + // are processing all of them in one go here. + for (unsigned i = 0, e = PHIs.size(); i != e; ++i) + SibValues.insert(std::make_pair(PHIs[i], SibValueInfo(Reg, PHIs[i]))); + + // Add every PHI as a dependent of all the non-PHIs. + for (unsigned i = 0, e = NonPHIs.size(); i != e; ++i) { + VNInfo *NonPHI = NonPHIs[i]; + // Known value? Try an insertion. + tie(SVI, Inserted) = + SibValues.insert(std::make_pair(NonPHI, SibValueInfo(Reg, NonPHI))); + // Add all the PHIs as dependents of NonPHI. + for (unsigned pi = 0, pe = PHIs.size(); pi != pe; ++pi) + SVI->second.Deps.push_back(PHIs[pi]); + // This is the first time we see NonPHI, add it to the worklist. + if (Inserted) + WorkList.push_back(std::make_pair(Reg, NonPHI)); + else + // Propagate to all inserted PHIs, not just VNI. + propagateSiblingValue(SVI); } + + // Next work list item. continue; } @@ -382,48 +578,49 @@ MachineInstr *InlineSpiller::traceSiblingValue(unsigned UseReg, VNInfo *UseVNI, if (unsigned SrcReg = isFullCopyOf(MI, Reg)) { if (isSibling(SrcReg)) { LiveInterval &SrcLI = LIS.getInterval(SrcReg); - VNInfo *SrcVNI = SrcLI.getVNInfoAt(VNI->def.getUseIndex()); - assert(SrcVNI && "Copy from non-existing value"); - DEBUG(dbgs() << " copy of " << PrintReg(SrcReg) << ':' - << SrcVNI->id << '@' << SrcVNI->def << '\n'); - WorkList.push_back(std::make_pair(SrcReg, SrcVNI)); + LiveRange *SrcLR = SrcLI.getLiveRangeContaining(VNI->def.getUseIndex()); + assert(SrcLR && "Copy from non-existing value"); + // Check if this COPY kills its source. + SVI->second.KillsSource = (SrcLR->end == VNI->def); + VNInfo *SrcVNI = SrcLR->valno; + DEBUG(dbgs() << "copy of " << PrintReg(SrcReg) << ':' + << SrcVNI->id << '@' << SrcVNI->def + << " kill=" << unsigned(SVI->second.KillsSource) << '\n'); + // Known sibling source value? Try an insertion. + tie(SVI, Inserted) = SibValues.insert(std::make_pair(SrcVNI, + SibValueInfo(SrcReg, SrcVNI))); + // This is the first time we see Src, add it to the worklist. + if (Inserted) + WorkList.push_back(std::make_pair(SrcReg, SrcVNI)); + propagateSiblingValue(SVI, VNI); + // Next work list item. continue; } } // Track reachable reloads. + SVI->second.DefMI = MI; + SVI->second.SpillMBB = MI->getParent(); int FI; if (Reg == TII.isLoadFromStackSlot(MI, FI) && FI == StackSlot) { - DEBUG(dbgs() << " reload " << PrintReg(Reg) << ':' - << VNI->id << "@" << VNI->def << '\n'); - SVI.AllDefsAreReloads = true; + DEBUG(dbgs() << "reload\n"); + propagateSiblingValue(SVI); + // Next work list item. continue; } - // We have an 'original' def. Don't record trivial cases. - if (VNI == UseVNI) { - DEBUG(dbgs() << "Not a sibling copy.\n"); - return MI; - } - // Potential remat candidate. - DEBUG(dbgs() << " def " << PrintReg(Reg) << ':' - << VNI->id << '@' << VNI->def << '\t' << *MI); - SVI.DefMI = MI; + DEBUG(dbgs() << "def " << *MI); + SVI->second.AllDefsAreReloads = false; + propagateSiblingValue(SVI); } while (!WorkList.empty()); - if (SeenOrigPHI || SVI.DefMI) - SVI.AllDefsAreReloads = false; - - DEBUG({ - if (SVI.AllDefsAreReloads) - dbgs() << "All defs are reloads.\n"; - else - dbgs() << "Prefer to spill " << PrintReg(SVI.SpillReg) << ':' - << SVI.SpillVNI->id << '@' << SVI.SpillVNI->def << '\n'; - }); - SibValues.insert(std::make_pair(UseVNI, SVI)); - return SVI.DefMI; + // Look up the value we were looking for. We already did this lokup at the + // top of the function, but SibValues may have been invalidated. + SVI = SibValues.find(UseVNI); + assert(SVI != SibValues.end() && "Didn't compute requested info"); + DEBUG(dbgs() << " traced to:\t" << SVI->second); + return SVI->second.DefMI; } /// analyzeSiblingValues - Trace values defined by sibling copies back to @@ -506,6 +703,7 @@ bool InlineSpiller::hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI) { // Already spilled everywhere. if (SVI.AllDefsAreReloads) { + DEBUG(dbgs() << "\tno spill needed: " << SVI); ++NumOmitReloadSpill; return true; } @@ -531,10 +729,8 @@ bool InlineSpiller::hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI) { VRM.addSpillSlotUse(StackSlot, MII); DEBUG(dbgs() << "\thoisted: " << SVI.SpillVNI->def << '\t' << *MII); - if (MBB == CopyMI->getParent()) - ++NumHoistLocal; - else - ++NumHoistGlobal; + ++NumSpills; + ++NumHoists; return true; } @@ -589,7 +785,8 @@ void InlineSpiller::eliminateRedundantSpills(LiveInterval &SLI, VNInfo *VNI) { // eliminateDeadDefs won't normally remove stores, so switch opcode. MI->setDesc(TII.get(TargetOpcode::KILL)); DeadDefs.push_back(MI); - ++NumRedundantSpills; + ++NumSpillsRemoved; + --NumSpills; } } } while (!WorkList.empty()); @@ -637,7 +834,7 @@ void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) { bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineBasicBlock::iterator MI) { SlotIndex UseIdx = LIS.getInstructionIndex(MI).getUseIndex(); - VNInfo *ParentVNI = VirtReg.getVNInfoAt(UseIdx); + VNInfo *ParentVNI = VirtReg.getVNInfoAt(UseIdx.getBaseIndex()); if (!ParentVNI) { DEBUG(dbgs() << "\tadding <undef> flags: "); @@ -787,10 +984,10 @@ void InlineSpiller::reMaterializeAll() { /// If MI is a load or store of StackSlot, it can be removed. bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, unsigned Reg) { int FI = 0; - unsigned InstrReg; - if (!(InstrReg = TII.isLoadFromStackSlot(MI, FI)) && - !(InstrReg = TII.isStoreToStackSlot(MI, FI))) - return false; + unsigned InstrReg = TII.isLoadFromStackSlot(MI, FI); + bool IsLoad = InstrReg; + if (!IsLoad) + InstrReg = TII.isStoreToStackSlot(MI, FI); // We have a stack access. Is it the right register and slot? if (InstrReg != Reg || FI != StackSlot) @@ -799,6 +996,15 @@ bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, unsigned Reg) { DEBUG(dbgs() << "Coalescing stack access: " << *MI); LIS.RemoveMachineInstrFromMaps(MI); MI->eraseFromParent(); + + if (IsLoad) { + ++NumReloadsRemoved; + --NumReloads; + } else { + ++NumSpillsRemoved; + --NumSpills; + } + return true; } @@ -810,6 +1016,7 @@ bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, unsigned Reg) { bool InlineSpiller::foldMemoryOperand(MachineBasicBlock::iterator MI, const SmallVectorImpl<unsigned> &Ops, MachineInstr *LoadMI) { + bool WasCopy = MI->isCopy(); // TargetInstrInfo::foldMemoryOperand only expects explicit, non-tied // operands. SmallVector<unsigned, 8> FoldOps; @@ -839,7 +1046,12 @@ bool InlineSpiller::foldMemoryOperand(MachineBasicBlock::iterator MI, VRM.addSpillSlotUse(StackSlot, FoldMI); MI->eraseFromParent(); DEBUG(dbgs() << "\tfolded: " << *FoldMI); - ++NumFolded; + if (!WasCopy) + ++NumFolded; + else if (Ops.front() == 0) + ++NumSpills; + else + ++NumReloads; return true; } @@ -975,8 +1187,16 @@ void InlineSpiller::spillAroundUses(unsigned Reg) { DEBUG(dbgs() << "\trewrite: " << Idx << '\t' << *MI); // FIXME: Use a second vreg if instruction has no tied ops. - if (Writes && hasLiveDef) + if (Writes) { + if (hasLiveDef) insertSpill(NewLI, OldLI, Idx, MI); + else { + // This instruction defines a dead value. We don't need to spill it, + // but do create a live range for the dead value. + VNInfo *VNI = NewLI.getNextValue(Idx, 0, LIS.getVNInfoAllocator()); + NewLI.addRange(LiveRange(Idx, Idx.getNextSlot(), VNI)); + } + } DEBUG(dbgs() << "\tinterval: " << NewLI << '\n'); } diff --git a/lib/CodeGen/InterferenceCache.cpp b/lib/CodeGen/InterferenceCache.cpp index a09bb39..29b47bd 100644 --- a/lib/CodeGen/InterferenceCache.cpp +++ b/lib/CodeGen/InterferenceCache.cpp @@ -18,10 +18,13 @@ using namespace llvm; +// Static member used for null interference cursors. +InterferenceCache::BlockInterference InterferenceCache::Cursor::NoInterference; + void InterferenceCache::init(MachineFunction *mf, LiveIntervalUnion *liuarray, SlotIndexes *indexes, - const TargetRegisterInfo *tri) { + const TargetRegisterInfo *tri) { MF = mf; LIUArray = liuarray; TRI = tri; diff --git a/lib/CodeGen/InterferenceCache.h b/lib/CodeGen/InterferenceCache.h index 7f0a27a..4df0a9e 100644 --- a/lib/CodeGen/InterferenceCache.h +++ b/lib/CodeGen/InterferenceCache.h @@ -138,6 +138,7 @@ public: class Cursor { Entry *CacheEntry; BlockInterference *Current; + static BlockInterference NoInterference; void setEntry(Entry *E) { Current = 0; @@ -175,7 +176,7 @@ public: /// moveTo - Move cursor to basic block MBBNum. void moveToBlock(unsigned MBBNum) { - Current = CacheEntry->get(MBBNum); + Current = CacheEntry ? CacheEntry->get(MBBNum) : &NoInterference; } /// hasInterference - Return true if the current block has any interference. diff --git a/lib/CodeGen/IntrinsicLowering.cpp b/lib/CodeGen/IntrinsicLowering.cpp index 611886f..0f92c2d 100644 --- a/lib/CodeGen/IntrinsicLowering.cpp +++ b/lib/CodeGen/IntrinsicLowering.cpp @@ -27,7 +27,7 @@ using namespace llvm; template <class ArgIt> static void EnsureFunctionExists(Module &M, const char *Name, ArgIt ArgBegin, ArgIt ArgEnd, - const Type *RetTy) { + Type *RetTy) { // Insert a correctly-typed definition now. std::vector<Type *> ParamTys; for (ArgIt I = ArgBegin; I != ArgEnd; ++I) @@ -64,7 +64,7 @@ static void EnsureFPIntrinsicsExist(Module &M, Function *Fn, template <class ArgIt> static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI, ArgIt ArgBegin, ArgIt ArgEnd, - const Type *RetTy) { + Type *RetTy) { // If we haven't already looked up this function, check to see if the // program already contains a function with this name. Module *M = CI->getParent()->getParent()->getParent(); @@ -462,7 +462,7 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) { break; // Strip out annotate intrinsic case Intrinsic::memcpy: { - const IntegerType *IntPtr = TD.getIntPtrType(Context); + IntegerType *IntPtr = TD.getIntPtrType(Context); Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr, /* isSigned */ false); Value *Ops[3]; @@ -473,7 +473,7 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) { break; } case Intrinsic::memmove: { - const IntegerType *IntPtr = TD.getIntPtrType(Context); + IntegerType *IntPtr = TD.getIntPtrType(Context); Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr, /* isSigned */ false); Value *Ops[3]; @@ -484,7 +484,7 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) { break; } case Intrinsic::memset: { - const IntegerType *IntPtr = TD.getIntPtrType(Context); + IntegerType *IntPtr = TD.getIntPtrType(Context); Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr, /* isSigned */ false); Value *Ops[3]; diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp index f985af8..80ecc22 100644 --- a/lib/CodeGen/LLVMTargetMachine.cpp +++ b/lib/CodeGen/LLVMTargetMachine.cpp @@ -27,16 +27,18 @@ #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/Target/TargetAsmInfo.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegistry.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetLoweringObjectFile.h" +#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" #include "llvm/Transforms/Scalar.h" #include "llvm/ADT/OwningPtr.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/FormattedStream.h" +#include "llvm/Support/TargetRegistry.h" using namespace llvm; namespace llvm { @@ -55,8 +57,12 @@ static cl::opt<bool> DisableCodePlace("disable-code-place", cl::Hidden, cl::desc("Disable code placement")); static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden, cl::desc("Disable Stack Slot Coloring")); +static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden, + cl::desc("Disable Machine Dead Code Elimination")); static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden, cl::desc("Disable Machine LICM")); +static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden, + cl::desc("Disable Machine Common Subexpression Elimination")); static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm", cl::Hidden, cl::desc("Disable Machine LICM")); @@ -103,20 +109,17 @@ EnableFastISelOption("fast-isel", cl::Hidden, cl::desc("Enable the \"fast\" instruction selector")); LLVMTargetMachine::LLVMTargetMachine(const Target &T, StringRef Triple, - StringRef CPU, StringRef FS) + StringRef CPU, StringRef FS, + Reloc::Model RM, CodeModel::Model CM) : TargetMachine(T, Triple, CPU, FS) { + CodeGenInfo = T.createMCCodeGenInfo(Triple, RM, CM); AsmInfo = T.createMCAsmInfo(Triple); -} - -// Set the default code model for the JIT for a generic target. -// FIXME: Is small right here? or .is64Bit() ? Large : Small? -void LLVMTargetMachine::setCodeModelForJIT() { - setCodeModel(CodeModel::Small); -} - -// Set the default code model for static compilation for a generic target. -void LLVMTargetMachine::setCodeModelForStatic() { - setCodeModel(CodeModel::Small); + // TargetSelect.h moved to a different directory between LLVM 2.9 and 3.0, + // and if the old one gets included then MCAsmInfo will be NULL and + // we'll crash later. + // Provide the user with a useful error message about what's wrong. + assert(AsmInfo && "MCAsmInfo not initialized." + "Make sure you include the correct TargetSelect.h!"); } bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, @@ -134,21 +137,22 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, Context->setAllowTemporaryLabels(false); const MCAsmInfo &MAI = *getMCAsmInfo(); + const MCSubtargetInfo &STI = getSubtarget<MCSubtargetInfo>(); OwningPtr<MCStreamer> AsmStreamer; switch (FileType) { default: return true; case CGFT_AssemblyFile: { MCInstPrinter *InstPrinter = - getTarget().createMCInstPrinter(MAI.getAssemblerDialect(), MAI); + getTarget().createMCInstPrinter(MAI.getAssemblerDialect(), MAI, STI); // Create a code emitter if asked to show the encoding. MCCodeEmitter *MCE = 0; - TargetAsmBackend *TAB = 0; + MCAsmBackend *MAB = 0; if (ShowMCEncoding) { const MCSubtargetInfo &STI = getSubtarget<MCSubtargetInfo>(); - MCE = getTarget().createCodeEmitter(*getInstrInfo(), STI, *Context); - TAB = getTarget().createAsmBackend(getTargetTriple()); + MCE = getTarget().createMCCodeEmitter(*getInstrInfo(), STI, *Context); + MAB = getTarget().createMCAsmBackend(getTargetTriple()); } MCStreamer *S = getTarget().createAsmStreamer(*Context, Out, @@ -156,7 +160,7 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, hasMCUseLoc(), hasMCUseCFI(), InstPrinter, - MCE, TAB, + MCE, MAB, ShowMCInst); AsmStreamer.reset(S); break; @@ -164,17 +168,16 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, case CGFT_ObjectFile: { // Create the code emitter for the target if it exists. If not, .o file // emission fails. - const MCSubtargetInfo &STI = getSubtarget<MCSubtargetInfo>(); - MCCodeEmitter *MCE = getTarget().createCodeEmitter(*getInstrInfo(), STI, - *Context); - TargetAsmBackend *TAB = getTarget().createAsmBackend(getTargetTriple()); - if (MCE == 0 || TAB == 0) + MCCodeEmitter *MCE = getTarget().createMCCodeEmitter(*getInstrInfo(), STI, + *Context); + MCAsmBackend *MAB = getTarget().createMCAsmBackend(getTargetTriple()); + if (MCE == 0 || MAB == 0) return true; - AsmStreamer.reset(getTarget().createObjectStreamer(getTargetTriple(), - *Context, *TAB, Out, MCE, - hasMCRelaxAll(), - hasMCNoExecStack())); + AsmStreamer.reset(getTarget().createMCObjectStreamer(getTargetTriple(), + *Context, *MAB, Out, + MCE, hasMCRelaxAll(), + hasMCNoExecStack())); AsmStreamer.get()->InitSections(); break; } @@ -198,8 +201,6 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, PM.add(Printer); - // Make sure the code model is set. - setCodeModelForStatic(); PM.add(createGCInfoDeleter()); return false; } @@ -214,9 +215,6 @@ bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM, JITCodeEmitter &JCE, CodeGenOpt::Level OptLevel, bool DisableVerify) { - // Make sure the code model is set. - setCodeModelForJIT(); - // Add common CodeGen passes. MCContext *Ctx = 0; if (addCommonCodeGenPasses(PM, OptLevel, DisableVerify, Ctx)) @@ -248,16 +246,16 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM, // Create the code emitter for the target if it exists. If not, .o file // emission fails. const MCSubtargetInfo &STI = getSubtarget<MCSubtargetInfo>(); - MCCodeEmitter *MCE = getTarget().createCodeEmitter(*getInstrInfo(),STI, *Ctx); - TargetAsmBackend *TAB = getTarget().createAsmBackend(getTargetTriple()); - if (MCE == 0 || TAB == 0) + MCCodeEmitter *MCE = getTarget().createMCCodeEmitter(*getInstrInfo(),STI, *Ctx); + MCAsmBackend *MAB = getTarget().createMCAsmBackend(getTargetTriple()); + if (MCE == 0 || MAB == 0) return true; OwningPtr<MCStreamer> AsmStreamer; - AsmStreamer.reset(getTarget().createObjectStreamer(getTargetTriple(), *Ctx, - *TAB, Out, MCE, - hasMCRelaxAll(), - hasMCNoExecStack())); + AsmStreamer.reset(getTarget().createMCObjectStreamer(getTargetTriple(), *Ctx, + *MAB, Out, MCE, + hasMCRelaxAll(), + hasMCNoExecStack())); AsmStreamer.get()->InitSections(); // Create the AsmPrinter, which takes ownership of AsmStreamer if successful. @@ -270,9 +268,6 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM, PM.add(Printer); - // Make sure the code model is set. - setCodeModelForJIT(); - return false; // success! } @@ -369,8 +364,9 @@ bool LLVMTargetMachine::addCommonCodeGenPasses(PassManagerBase &PM, // Install a MachineModuleInfo class, which is an immutable pass that holds // all the per-module stuff we're generating, including MCContext. - TargetAsmInfo *TAI = new TargetAsmInfo(*this); - MachineModuleInfo *MMI = new MachineModuleInfo(*getMCAsmInfo(), TAI); + MachineModuleInfo *MMI = new MachineModuleInfo(*getMCAsmInfo(), + *getRegisterInfo(), + &getTargetLowering()->getObjFileLowering()); PM.add(MMI); OutContext = &MMI->getContext(); // Return the MCContext specifically by-ref. @@ -412,12 +408,14 @@ bool LLVMTargetMachine::addCommonCodeGenPasses(PassManagerBase &PM, // there is one known exception: lowered code for arguments that are only // used by tail calls, where the tail calls reuse the incoming stack // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll). - PM.add(createDeadMachineInstructionElimPass()); + if (!DisableMachineDCE) + PM.add(createDeadMachineInstructionElimPass()); printAndVerify(PM, "After codegen DCE pass"); if (!DisableMachineLICM) PM.add(createMachineLICMPass()); - PM.add(createMachineCSEPass()); + if (!DisableMachineCSE) + PM.add(createMachineCSEPass()); if (!DisableMachineSink) PM.add(createMachineSinkingPass()); printAndVerify(PM, "After Machine LICM, CSE and Sinking passes"); @@ -452,8 +450,8 @@ bool LLVMTargetMachine::addCommonCodeGenPasses(PassManagerBase &PM, if (addPostRegAlloc(PM, OptLevel)) printAndVerify(PM, "After PostRegAlloc passes"); - PM.add(createLowerSubregsPass()); - printAndVerify(PM, "After LowerSubregs"); + PM.add(createExpandPostRAPseudosPass()); + printAndVerify(PM, "After ExpandPostRAPseudos"); // Insert prolog/epilog code. Eliminate abstract frame index references... PM.add(createPrologEpilogCodeInserter()); diff --git a/lib/CodeGen/LexicalScopes.cpp b/lib/CodeGen/LexicalScopes.cpp new file mode 100644 index 0000000..a12e1a3 --- /dev/null +++ b/lib/CodeGen/LexicalScopes.cpp @@ -0,0 +1,335 @@ +//===- LexicalScopes.cpp - Collecting lexical scope info ------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements LexicalScopes analysis. +// +// This pass collects lexical scope information and maps machine instructions +// to respective lexical scopes. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "lexicalscopes" +#include "llvm/CodeGen/LexicalScopes.h" +#include "llvm/Function.h" +#include "llvm/Analysis/DebugInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormattedStream.h" +using namespace llvm; + +LexicalScopes::~LexicalScopes() { + releaseMemory(); +} + +/// releaseMemory - release memory. +void LexicalScopes::releaseMemory() { + MF = NULL; + CurrentFnLexicalScope = NULL; + DeleteContainerSeconds(LexicalScopeMap); + DeleteContainerSeconds(AbstractScopeMap); + InlinedLexicalScopeMap.clear(); + AbstractScopesList.clear(); +} + +/// initialize - Scan machine function and constuct lexical scope nest. +void LexicalScopes::initialize(const MachineFunction &Fn) { + releaseMemory(); + MF = &Fn; + SmallVector<InsnRange, 4> MIRanges; + DenseMap<const MachineInstr *, LexicalScope *> MI2ScopeMap; + extractLexicalScopes(MIRanges, MI2ScopeMap); + if (CurrentFnLexicalScope) { + constructScopeNest(CurrentFnLexicalScope); + assignInstructionRanges(MIRanges, MI2ScopeMap); + } +} + +/// extractLexicalScopes - Extract instruction ranges for each lexical scopes +/// for the given machine function. +void LexicalScopes:: +extractLexicalScopes(SmallVectorImpl<InsnRange> &MIRanges, + DenseMap<const MachineInstr *, LexicalScope *> &MI2ScopeMap) { + + // Scan each instruction and create scopes. First build working set of scopes. + for (MachineFunction::const_iterator I = MF->begin(), E = MF->end(); + I != E; ++I) { + const MachineInstr *RangeBeginMI = NULL; + const MachineInstr *PrevMI = NULL; + DebugLoc PrevDL; + for (MachineBasicBlock::const_iterator II = I->begin(), IE = I->end(); + II != IE; ++II) { + const MachineInstr *MInsn = II; + + // Check if instruction has valid location information. + const DebugLoc MIDL = MInsn->getDebugLoc(); + if (MIDL.isUnknown()) { + PrevMI = MInsn; + continue; + } + + // If scope has not changed then skip this instruction. + if (MIDL == PrevDL) { + PrevMI = MInsn; + continue; + } + + // Ignore DBG_VALUE. It does not contribute to any instruction in output. + if (MInsn->isDebugValue()) + continue; + + if (RangeBeginMI) { + // If we have already seen a beginning of an instruction range and + // current instruction scope does not match scope of first instruction + // in this range then create a new instruction range. + InsnRange R(RangeBeginMI, PrevMI); + MI2ScopeMap[RangeBeginMI] = getOrCreateLexicalScope(PrevDL); + MIRanges.push_back(R); + } + + // This is a beginning of a new instruction range. + RangeBeginMI = MInsn; + + // Reset previous markers. + PrevMI = MInsn; + PrevDL = MIDL; + } + + // Create last instruction range. + if (RangeBeginMI && PrevMI && !PrevDL.isUnknown()) { + InsnRange R(RangeBeginMI, PrevMI); + MIRanges.push_back(R); + MI2ScopeMap[RangeBeginMI] = getOrCreateLexicalScope(PrevDL); + } + } +} + +/// findLexicalScope - Find lexical scope, either regular or inlined, for the +/// given DebugLoc. Return NULL if not found. +LexicalScope *LexicalScopes::findLexicalScope(DebugLoc DL) { + MDNode *Scope = NULL; + MDNode *IA = NULL; + DL.getScopeAndInlinedAt(Scope, IA, MF->getFunction()->getContext()); + if (!Scope) return NULL; + + // The scope that we were created with could have an extra file - which + // isn't what we care about in this case. + DIDescriptor D = DIDescriptor(Scope); + if (D.isLexicalBlockFile()) + Scope = DILexicalBlockFile(Scope).getScope(); + + if (IA) + return InlinedLexicalScopeMap.lookup(DebugLoc::getFromDILocation(IA)); + return LexicalScopeMap.lookup(Scope); +} + +/// getOrCreateLexicalScope - Find lexical scope for the given DebugLoc. If +/// not available then create new lexical scope. +LexicalScope *LexicalScopes::getOrCreateLexicalScope(DebugLoc DL) { + MDNode *Scope = NULL; + MDNode *InlinedAt = NULL; + DL.getScopeAndInlinedAt(Scope, InlinedAt, MF->getFunction()->getContext()); + + if (InlinedAt) { + // Create an abstract scope for inlined function. + getOrCreateAbstractScope(Scope); + // Create an inlined scope for inlined function. + return getOrCreateInlinedScope(Scope, InlinedAt); + } + + return getOrCreateRegularScope(Scope); +} + +/// getOrCreateRegularScope - Find or create a regular lexical scope. +LexicalScope *LexicalScopes::getOrCreateRegularScope(MDNode *Scope) { + DIDescriptor D = DIDescriptor(Scope); + if (D.isLexicalBlockFile()) { + Scope = DILexicalBlockFile(Scope).getScope(); + D = DIDescriptor(Scope); + } + + LexicalScope *WScope = LexicalScopeMap.lookup(Scope); + if (WScope) + return WScope; + + LexicalScope *Parent = NULL; + if (D.isLexicalBlock()) + Parent = getOrCreateLexicalScope(DebugLoc::getFromDILexicalBlock(Scope)); + WScope = new LexicalScope(Parent, DIDescriptor(Scope), NULL, false); + LexicalScopeMap.insert(std::make_pair(Scope, WScope)); + if (!Parent && DIDescriptor(Scope).isSubprogram() + && DISubprogram(Scope).describes(MF->getFunction())) + CurrentFnLexicalScope = WScope; + + return WScope; +} + +/// getOrCreateInlinedScope - Find or create an inlined lexical scope. +LexicalScope *LexicalScopes::getOrCreateInlinedScope(MDNode *Scope, + MDNode *InlinedAt) { + LexicalScope *InlinedScope = LexicalScopeMap.lookup(InlinedAt); + if (InlinedScope) + return InlinedScope; + + DebugLoc InlinedLoc = DebugLoc::getFromDILocation(InlinedAt); + InlinedScope = new LexicalScope(getOrCreateLexicalScope(InlinedLoc), + DIDescriptor(Scope), InlinedAt, false); + InlinedLexicalScopeMap[InlinedLoc] = InlinedScope; + LexicalScopeMap[InlinedAt] = InlinedScope; + return InlinedScope; +} + +/// getOrCreateAbstractScope - Find or create an abstract lexical scope. +LexicalScope *LexicalScopes::getOrCreateAbstractScope(const MDNode *N) { + assert(N && "Invalid Scope encoding!"); + + DIDescriptor Scope(N); + if (Scope.isLexicalBlockFile()) + Scope = DILexicalBlockFile(Scope).getScope(); + LexicalScope *AScope = AbstractScopeMap.lookup(N); + if (AScope) + return AScope; + + LexicalScope *Parent = NULL; + if (Scope.isLexicalBlock()) { + DILexicalBlock DB(N); + DIDescriptor ParentDesc = DB.getContext(); + Parent = getOrCreateAbstractScope(ParentDesc); + } + AScope = new LexicalScope(Parent, DIDescriptor(N), NULL, true); + AbstractScopeMap[N] = AScope; + if (DIDescriptor(N).isSubprogram()) + AbstractScopesList.push_back(AScope); + return AScope; +} + +/// constructScopeNest +void LexicalScopes::constructScopeNest(LexicalScope *Scope) { + assert (Scope && "Unable to calculate scop edominance graph!"); + SmallVector<LexicalScope *, 4> WorkStack; + WorkStack.push_back(Scope); + unsigned Counter = 0; + while (!WorkStack.empty()) { + LexicalScope *WS = WorkStack.back(); + const SmallVector<LexicalScope *, 4> &Children = WS->getChildren(); + bool visitedChildren = false; + for (SmallVector<LexicalScope *, 4>::const_iterator SI = Children.begin(), + SE = Children.end(); SI != SE; ++SI) { + LexicalScope *ChildScope = *SI; + if (!ChildScope->getDFSOut()) { + WorkStack.push_back(ChildScope); + visitedChildren = true; + ChildScope->setDFSIn(++Counter); + break; + } + } + if (!visitedChildren) { + WorkStack.pop_back(); + WS->setDFSOut(++Counter); + } + } +} + +/// assignInstructionRanges - Find ranges of instructions covered by each +/// lexical scope. +void LexicalScopes:: +assignInstructionRanges(SmallVectorImpl<InsnRange> &MIRanges, + DenseMap<const MachineInstr *, LexicalScope *> &MI2ScopeMap) +{ + + LexicalScope *PrevLexicalScope = NULL; + for (SmallVectorImpl<InsnRange>::const_iterator RI = MIRanges.begin(), + RE = MIRanges.end(); RI != RE; ++RI) { + const InsnRange &R = *RI; + LexicalScope *S = MI2ScopeMap.lookup(R.first); + assert (S && "Lost LexicalScope for a machine instruction!"); + if (PrevLexicalScope && !PrevLexicalScope->dominates(S)) + PrevLexicalScope->closeInsnRange(S); + S->openInsnRange(R.first); + S->extendInsnRange(R.second); + PrevLexicalScope = S; + } + + if (PrevLexicalScope) + PrevLexicalScope->closeInsnRange(); +} + +/// getMachineBasicBlocks - Populate given set using machine basic blocks which +/// have machine instructions that belong to lexical scope identified by +/// DebugLoc. +void LexicalScopes:: +getMachineBasicBlocks(DebugLoc DL, + SmallPtrSet<const MachineBasicBlock*, 4> &MBBs) { + MBBs.clear(); + LexicalScope *Scope = getOrCreateLexicalScope(DL); + if (!Scope) + return; + + if (Scope == CurrentFnLexicalScope) { + for (MachineFunction::const_iterator I = MF->begin(), E = MF->end(); + I != E; ++I) + MBBs.insert(I); + return; + } + + SmallVector<InsnRange, 4> &InsnRanges = Scope->getRanges(); + for (SmallVector<InsnRange, 4>::iterator I = InsnRanges.begin(), + E = InsnRanges.end(); I != E; ++I) { + InsnRange &R = *I; + MBBs.insert(R.first->getParent()); + } +} + +/// dominates - Return true if DebugLoc's lexical scope dominates at least one +/// machine instruction's lexical scope in a given machine basic block. +bool LexicalScopes::dominates(DebugLoc DL, MachineBasicBlock *MBB) { + LexicalScope *Scope = getOrCreateLexicalScope(DL); + if (!Scope) + return false; + + // Current function scope covers all basic blocks in the function. + if (Scope == CurrentFnLexicalScope && MBB->getParent() == MF) + return true; + + bool Result = false; + for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); + I != E; ++I) { + DebugLoc IDL = I->getDebugLoc(); + if (IDL.isUnknown()) + continue; + if (LexicalScope *IScope = getOrCreateLexicalScope(IDL)) + if (Scope->dominates(IScope)) + return true; + } + return Result; +} + +/// dump - Print data structures. +void LexicalScope::dump() const { +#ifndef NDEBUG + raw_ostream &err = dbgs(); + err.indent(IndentLevel); + err << "DFSIn: " << DFSIn << " DFSOut: " << DFSOut << "\n"; + const MDNode *N = Desc; + N->dump(); + if (AbstractScope) + err << "Abstract Scope\n"; + + IndentLevel += 2; + if (!Children.empty()) + err << "Children ...\n"; + for (unsigned i = 0, e = Children.size(); i != e; ++i) + if (Children[i] != this) + Children[i]->dump(); + + IndentLevel -= 2; +#endif +} + diff --git a/lib/CodeGen/LiveDebugVariables.cpp b/lib/CodeGen/LiveDebugVariables.cpp index 5d38c83..3dfe4c0 100644 --- a/lib/CodeGen/LiveDebugVariables.cpp +++ b/lib/CodeGen/LiveDebugVariables.cpp @@ -25,7 +25,10 @@ #include "llvm/Constants.h" #include "llvm/Metadata.h" #include "llvm/Value.h" +#include "llvm/Analysis/DebugInfo.h" #include "llvm/ADT/IntervalMap.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/LexicalScopes.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" @@ -44,6 +47,7 @@ static cl::opt<bool> EnableLDV("live-debug-variables", cl::init(true), cl::desc("Enable the live debug variables pass"), cl::Hidden); +STATISTIC(NumInsertedDebugValues, "Number of DBG_VALUEs inserted"); char LiveDebugVariables::ID = 0; INITIALIZE_PASS_BEGIN(LiveDebugVariables, "livedebugvars", @@ -67,6 +71,29 @@ LiveDebugVariables::LiveDebugVariables() : MachineFunctionPass(ID), pImpl(0) { /// LocMap - Map of where a user value is live, and its location. typedef IntervalMap<SlotIndex, unsigned, 4> LocMap; +namespace { +/// UserValueScopes - Keeps track of lexical scopes associated with an +/// user value's source location. +class UserValueScopes { + DebugLoc DL; + LexicalScopes &LS; + SmallPtrSet<const MachineBasicBlock *, 4> LBlocks; + +public: + UserValueScopes(DebugLoc D, LexicalScopes &L) : DL(D), LS(L) {} + + /// dominates - Return true if current scope dominates at least one machine + /// instruction in a given machine basic block. + bool dominates(MachineBasicBlock *MBB) { + if (LBlocks.empty()) + LS.getMachineBasicBlocks(DL, LBlocks); + if (LBlocks.count(MBB) != 0 || LS.dominates(DL, MBB)) + return true; + return false; + } +}; +} // end anonymous namespace + /// UserValue - A user value is a part of a debug info user variable. /// /// A DBG_VALUE instruction notes that (a sub-register of) a virtual register @@ -179,6 +206,9 @@ public: LocMap::iterator I = locInts.find(Idx); if (!I.valid() || I.start() != Idx) I.insert(Idx, Idx.getNextSlot(), getLocationNo(LocMO)); + else + // A later DBG_VALUE at the same SlotIndex overrides the old location. + I.setValue(getLocationNo(LocMO)); } /// extendDef - Extend the current definition as far as possible down the @@ -195,7 +225,8 @@ public: void extendDef(SlotIndex Idx, unsigned LocNo, LiveInterval *LI, const VNInfo *VNI, SmallVectorImpl<SlotIndex> *Kills, - LiveIntervals &LIS, MachineDominatorTree &MDT); + LiveIntervals &LIS, MachineDominatorTree &MDT, + UserValueScopes &UVS); /// addDefsFromCopies - The value in LI/LocNo may be copies to other /// registers. Determine if any of the copies are available at the kill @@ -213,7 +244,8 @@ public: /// computeIntervals - Compute the live intervals of all locations after /// collecting all their def points. void computeIntervals(MachineRegisterInfo &MRI, - LiveIntervals &LIS, MachineDominatorTree &MDT); + LiveIntervals &LIS, MachineDominatorTree &MDT, + UserValueScopes &UVS); /// renameRegister - Update locations to rewrite OldReg as NewReg:SubIdx. void renameRegister(unsigned OldReg, unsigned NewReg, unsigned SubIdx, @@ -236,6 +268,9 @@ public: /// Only first one needs DebugLoc to identify variable's lexical scope /// in source file. DebugLoc findDebugLoc(); + + /// getDebugLoc - Return DebugLoc of this UserValue. + DebugLoc getDebugLoc() { return dl;} void print(raw_ostream&, const TargetMachine*); }; } // namespace @@ -247,6 +282,7 @@ class LDVImpl { LocMap::Allocator allocator; MachineFunction *MF; LiveIntervals *LIS; + LexicalScopes LS; MachineDominatorTree *MDT; const TargetRegisterInfo *TRI; @@ -312,8 +348,10 @@ public: } // namespace void UserValue::print(raw_ostream &OS, const TargetMachine *TM) { - if (const MDString *MDS = dyn_cast<MDString>(variable->getOperand(2))) - OS << "!\"" << MDS->getString() << "\"\t"; + DIVariable DV(variable); + OS << "!\""; + DV.printExtendedName(OS); + OS << "\"\t"; if (offset) OS << '+' << offset; for (LocMap::const_iterator I = locInts.begin(); I.valid(); ++I) { @@ -447,10 +485,10 @@ bool LDVImpl::collectDebugValues(MachineFunction &mf) { void UserValue::extendDef(SlotIndex Idx, unsigned LocNo, LiveInterval *LI, const VNInfo *VNI, SmallVectorImpl<SlotIndex> *Kills, - LiveIntervals &LIS, MachineDominatorTree &MDT) { + LiveIntervals &LIS, MachineDominatorTree &MDT, + UserValueScopes &UVS) { SmallVector<SlotIndex, 16> Todo; Todo.push_back(Idx); - do { SlotIndex Start = Todo.pop_back_val(); MachineBasicBlock *MBB = LIS.getMBBFromIndex(Start); @@ -497,8 +535,11 @@ void UserValue::extendDef(SlotIndex Idx, unsigned LocNo, continue; const std::vector<MachineDomTreeNode*> &Children = MDT.getNode(MBB)->getChildren(); - for (unsigned i = 0, e = Children.size(); i != e; ++i) - Todo.push_back(LIS.getMBBStartIdx(Children[i]->getBlock())); + for (unsigned i = 0, e = Children.size(); i != e; ++i) { + MachineBasicBlock *MBB = Children[i]->getBlock(); + if (UVS.dominates(MBB)) + Todo.push_back(LIS.getMBBStartIdx(MBB)); + } } while (!Todo.empty()); } @@ -578,7 +619,8 @@ UserValue::addDefsFromCopies(LiveInterval *LI, unsigned LocNo, void UserValue::computeIntervals(MachineRegisterInfo &MRI, LiveIntervals &LIS, - MachineDominatorTree &MDT) { + MachineDominatorTree &MDT, + UserValueScopes &UVS) { SmallVector<std::pair<SlotIndex, unsigned>, 16> Defs; // Collect all defs to be extended (Skipping undefs). @@ -597,10 +639,10 @@ UserValue::computeIntervals(MachineRegisterInfo &MRI, LiveInterval *LI = &LIS.getInterval(Loc.getReg()); const VNInfo *VNI = LI->getVNInfoAt(Idx); SmallVector<SlotIndex, 16> Kills; - extendDef(Idx, LocNo, LI, VNI, &Kills, LIS, MDT); + extendDef(Idx, LocNo, LI, VNI, &Kills, LIS, MDT, UVS); addDefsFromCopies(LI, LocNo, Kills, Defs, MRI, LIS); } else - extendDef(Idx, LocNo, 0, 0, 0, LIS, MDT); + extendDef(Idx, LocNo, 0, 0, 0, LIS, MDT, UVS); } // Finally, erase all the undefs. @@ -613,7 +655,8 @@ UserValue::computeIntervals(MachineRegisterInfo &MRI, void LDVImpl::computeIntervals() { for (unsigned i = 0, e = userValues.size(); i != e; ++i) { - userValues[i]->computeIntervals(MF->getRegInfo(), *LIS, *MDT); + UserValueScopes UVS(userValues[i]->getDebugLoc(), LS); + userValues[i]->computeIntervals(MF->getRegInfo(), *LIS, *MDT, UVS); userValues[i]->mapVirtRegs(this); } } @@ -624,6 +667,7 @@ bool LDVImpl::runOnMachineFunction(MachineFunction &mf) { MDT = &pass.getAnalysis<MachineDominatorTree>(); TRI = mf.getTarget().getRegisterInfo(); clear(); + LS.initialize(mf); DEBUG(dbgs() << "********** COMPUTING LIVE DEBUG VARIABLES: " << ((Value*)mf.getFunction())->getName() << " **********\n"); @@ -631,6 +675,7 @@ bool LDVImpl::runOnMachineFunction(MachineFunction &mf) { bool Changed = collectDebugValues(mf); computeIntervals(); DEBUG(print(dbgs())); + LS.releaseMemory(); return Changed; } @@ -891,6 +936,7 @@ void UserValue::insertDebugValue(MachineBasicBlock *MBB, SlotIndex Idx, const TargetInstrInfo &TII) { MachineBasicBlock::iterator I = findInsertLocation(MBB, Idx, LIS); MachineOperand &Loc = locations[LocNo]; + ++NumInsertedDebugValues; // Frame index locations may require a target callback. if (Loc.isFI()) { @@ -921,7 +967,6 @@ void UserValue::emitDebugValues(VirtRegMap *VRM, LiveIntervals &LIS, DEBUG(dbgs() << " BB#" << MBB->getNumber() << '-' << MBBEnd); insertDebugValue(MBB, Start, LocNo, LIS, TII); - // This interval may span multiple basic blocks. // Insert a DBG_VALUE into each one. while(Stop > MBBEnd) { diff --git a/lib/CodeGen/LiveInterval.cpp b/lib/CodeGen/LiveInterval.cpp index cfade24..b69945a 100644 --- a/lib/CodeGen/LiveInterval.cpp +++ b/lib/CodeGen/LiveInterval.cpp @@ -148,7 +148,6 @@ void LiveInterval::markValNoForDeletion(VNInfo *ValNo) { /// remaining unused values. void LiveInterval::RenumberValues(LiveIntervals &lis) { SmallPtrSet<VNInfo*, 8> Seen; - bool seenPHIDef = false; valnos.clear(); for (const_iterator I = begin(), E = end(); I != E; ++I) { VNInfo *VNI = I->valno; @@ -157,26 +156,6 @@ void LiveInterval::RenumberValues(LiveIntervals &lis) { assert(!VNI->isUnused() && "Unused valno used by live range"); VNI->id = (unsigned)valnos.size(); valnos.push_back(VNI); - VNI->setHasPHIKill(false); - if (VNI->isPHIDef()) - seenPHIDef = true; - } - - // Recompute phi kill flags. - if (!seenPHIDef) - return; - for (const_vni_iterator I = vni_begin(), E = vni_end(); I != E; ++I) { - VNInfo *VNI = *I; - if (!VNI->isPHIDef()) - continue; - const MachineBasicBlock *PHIBB = lis.getMBBFromIndex(VNI->def); - assert(PHIBB && "No basic block for phi-def"); - for (MachineBasicBlock::const_pred_iterator PI = PHIBB->pred_begin(), - PE = PHIBB->pred_end(); PI != PE; ++PI) { - VNInfo *KVNI = getVNInfoAt(lis.getMBBEndIdx(*PI).getPrevSlot()); - if (KVNI) - KVNI->setHasPHIKill(true); - } } } @@ -294,20 +273,20 @@ LiveInterval::addRangeFrom(LiveRange LR, iterator From) { return ranges.insert(it, LR); } -/// extendInBlock - If this interval is live before UseIdx in the basic -/// block that starts at StartIdx, extend it to be live at UseIdx and return -/// the value. If there is no live range before UseIdx, return NULL. -VNInfo *LiveInterval::extendInBlock(SlotIndex StartIdx, SlotIndex UseIdx) { +/// extendInBlock - If this interval is live before Kill in the basic +/// block that starts at StartIdx, extend it to be live up to Kill and return +/// the value. If there is no live range before Kill, return NULL. +VNInfo *LiveInterval::extendInBlock(SlotIndex StartIdx, SlotIndex Kill) { if (empty()) return 0; - iterator I = std::upper_bound(begin(), end(), UseIdx); + iterator I = std::upper_bound(begin(), end(), Kill.getPrevSlot()); if (I == begin()) return 0; --I; if (I->end <= StartIdx) return 0; - if (I->end <= UseIdx) - extendIntervalEndTo(I, UseIdx.getNextSlot()); + if (I->end < Kill) + extendIntervalEndTo(I, Kill); return I->valno; } diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp index 9257191..b1e202a 100644 --- a/lib/CodeGen/LiveIntervalAnalysis.cpp +++ b/lib/CodeGen/LiveIntervalAnalysis.cpp @@ -304,8 +304,19 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, // Make sure the first definition is not a partial redefinition. Add an // <imp-def> of the full register. - if (MO.getSubReg()) + // FIXME: LiveIntervals shouldn't modify the code like this. Whoever + // created the machine instruction should annotate it with <undef> flags + // as needed. Then we can simply assert here. The REG_SEQUENCE lowering + // is the main suspect. + if (MO.getSubReg()) { mi->addRegisterDefined(interval.reg); + // Mark all defs of interval.reg on this instruction as reading <undef>. + for (unsigned i = MOIdx, e = mi->getNumOperands(); i != e; ++i) { + MachineOperand &MO2 = mi->getOperand(i); + if (MO2.isReg() && MO2.getReg() == interval.reg && MO2.getSubReg()) + MO2.setIsUndef(); + } + } MachineInstr *CopyMI = NULL; if (mi->isCopyLike()) { @@ -747,6 +758,9 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li, // Find all the values used, including PHI kills. SmallVector<std::pair<SlotIndex, VNInfo*>, 16> WorkList; + // Blocks that have already been added to WorkList as live-out. + SmallPtrSet<MachineBasicBlock*, 16> LiveOut; + // Visit all instructions reading li->reg. for (MachineRegisterInfo::reg_iterator I = mri_->reg_begin(li->reg); MachineInstr *UseMI = I.skipInstruction();) { @@ -780,8 +794,6 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li, VNInfo *VNI = *I; if (VNI->isUnused()) continue; - // We may eliminate PHI values, so recompute PHIKill flags. - VNI->setHasPHIKill(false); NewLI.addRange(LiveRange(VNI->def, VNI->def.getNextSlot(), VNI)); // A use tied to an early-clobber def ends at the load slot and isn't caught @@ -804,7 +816,7 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li, SlotIndex BlockStart = getMBBStartIdx(MBB); // Extend the live range for VNI to be live at Idx. - if (VNInfo *ExtVNI = NewLI.extendInBlock(BlockStart, Idx)) { + if (VNInfo *ExtVNI = NewLI.extendInBlock(BlockStart, Idx.getNextSlot())) { (void)ExtVNI; assert(ExtVNI == VNI && "Unexpected existing value number"); // Is this a PHIDef we haven't seen before? @@ -813,13 +825,12 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li, // The PHI is live, make sure the predecessors are live-out. for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { + if (!LiveOut.insert(*PI)) + continue; SlotIndex Stop = getMBBEndIdx(*PI).getPrevSlot(); - VNInfo *PVNI = li->getVNInfoAt(Stop); // A predecessor is not required to have a live-out value for a PHI. - if (PVNI) { - PVNI->setHasPHIKill(true); + if (VNInfo *PVNI = li->getVNInfoAt(Stop)) WorkList.push_back(std::make_pair(Stop, PVNI)); - } } continue; } @@ -831,6 +842,8 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li, // Make sure VNI is live-out from the predecessors. for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { + if (!LiveOut.insert(*PI)) + continue; SlotIndex Stop = getMBBEndIdx(*PI).getPrevSlot(); assert(li->getVNInfoAt(Stop) == VNI && "Wrong value out of predecessor"); WorkList.push_back(std::make_pair(Stop, VNI)); diff --git a/lib/CodeGen/LiveIntervalUnion.cpp b/lib/CodeGen/LiveIntervalUnion.cpp index 70003e7..110fe1e 100644 --- a/lib/CodeGen/LiveIntervalUnion.cpp +++ b/lib/CodeGen/LiveIntervalUnion.cpp @@ -91,25 +91,6 @@ LiveIntervalUnion::print(raw_ostream &OS, const TargetRegisterInfo *TRI) const { OS << '\n'; } -void LiveIntervalUnion::InterferenceResult::print(raw_ostream &OS, - const TargetRegisterInfo *TRI) const { - OS << '[' << start() << ';' << stop() << "):" - << PrintReg(interference()->reg, TRI); -} - -void LiveIntervalUnion::Query::print(raw_ostream &OS, - const TargetRegisterInfo *TRI) { - OS << "Interferences with "; - LiveUnion->print(OS, TRI); - InterferenceResult IR = firstInterference(); - while (isInterference(IR)) { - OS << " "; - IR.print(OS, TRI); - OS << '\n'; - nextInterference(IR); - } -} - #ifndef NDEBUG // Verify the live intervals in this union and add them to the visited set. void LiveIntervalUnion::verify(LiveVirtRegBitSet& VisitedVRegs) { @@ -118,114 +99,6 @@ void LiveIntervalUnion::verify(LiveVirtRegBitSet& VisitedVRegs) { } #endif //!NDEBUG -// Private interface accessed by Query. -// -// Find a pair of segments that intersect, one in the live virtual register -// (LiveInterval), and the other in this LiveIntervalUnion. The caller (Query) -// is responsible for advancing the LiveIntervalUnion segments to find a -// "notable" intersection, which requires query-specific logic. -// -// This design assumes only a fast mechanism for intersecting a single live -// virtual register segment with a set of LiveIntervalUnion segments. This may -// be ok since most virtual registers have very few segments. If we had a data -// structure that optimizd MxN intersection of segments, then we would bypass -// the loop that advances within the LiveInterval. -// -// If no intersection exists, set VirtRegI = VirtRegEnd, and set SI to the first -// segment whose start point is greater than LiveInterval's end point. -// -// Assumes that segments are sorted by start position in both -// LiveInterval and LiveSegments. -void LiveIntervalUnion::Query::findIntersection(InterferenceResult &IR) const { - // Search until reaching the end of the LiveUnion segments. - LiveInterval::iterator VirtRegEnd = VirtReg->end(); - if (IR.VirtRegI == VirtRegEnd) - return; - while (IR.LiveUnionI.valid()) { - // Slowly advance the live virtual reg iterator until we surpass the next - // segment in LiveUnion. - // - // Note: If this is ever used for coalescing of fixed registers and we have - // a live vreg with thousands of segments, then change this code to use - // upperBound instead. - IR.VirtRegI = VirtReg->advanceTo(IR.VirtRegI, IR.LiveUnionI.start()); - if (IR.VirtRegI == VirtRegEnd) - break; // Retain current (nonoverlapping) LiveUnionI - - // VirtRegI may have advanced far beyond LiveUnionI, catch up. - IR.LiveUnionI.advanceTo(IR.VirtRegI->start); - - // Check if no LiveUnionI exists with VirtRegI->Start < LiveUnionI.end - if (!IR.LiveUnionI.valid()) - break; - if (IR.LiveUnionI.start() < IR.VirtRegI->end) { - assert(overlap(*IR.VirtRegI, IR.LiveUnionI) && - "upperBound postcondition"); - break; - } - } - if (!IR.LiveUnionI.valid()) - IR.VirtRegI = VirtRegEnd; -} - -// Find the first intersection, and cache interference info -// (retain segment iterators into both VirtReg and LiveUnion). -const LiveIntervalUnion::InterferenceResult & -LiveIntervalUnion::Query::firstInterference() { - if (CheckedFirstInterference) - return FirstInterference; - CheckedFirstInterference = true; - InterferenceResult &IR = FirstInterference; - IR.LiveUnionI.setMap(LiveUnion->getMap()); - - // Quickly skip interference check for empty sets. - if (VirtReg->empty() || LiveUnion->empty()) { - IR.VirtRegI = VirtReg->end(); - } else if (VirtReg->beginIndex() < LiveUnion->startIndex()) { - // VirtReg starts first, perform double binary search. - IR.VirtRegI = VirtReg->find(LiveUnion->startIndex()); - if (IR.VirtRegI != VirtReg->end()) - IR.LiveUnionI.find(IR.VirtRegI->start); - } else { - // LiveUnion starts first, perform double binary search. - IR.LiveUnionI.find(VirtReg->beginIndex()); - if (IR.LiveUnionI.valid()) - IR.VirtRegI = VirtReg->find(IR.LiveUnionI.start()); - else - IR.VirtRegI = VirtReg->end(); - } - findIntersection(FirstInterference); - assert((IR.VirtRegI == VirtReg->end() || IR.LiveUnionI.valid()) - && "Uninitialized iterator"); - return FirstInterference; -} - -// Treat the result as an iterator and advance to the next interfering pair -// of segments. This is a plain iterator with no filter. -bool LiveIntervalUnion::Query::nextInterference(InterferenceResult &IR) const { - assert(isInterference(IR) && "iteration past end of interferences"); - - // Advance either the VirtReg or LiveUnion segment to ensure that we visit all - // unique overlapping pairs. - if (IR.VirtRegI->end < IR.LiveUnionI.stop()) { - if (++IR.VirtRegI == VirtReg->end()) - return false; - } - else { - if (!(++IR.LiveUnionI).valid()) { - IR.VirtRegI = VirtReg->end(); - return false; - } - } - // Short-circuit findIntersection() if possible. - if (overlap(*IR.VirtRegI, IR.LiveUnionI)) - return true; - - // Find the next intersection. - findIntersection(IR); - return isInterference(IR); -} - // Scan the vector of interfering virtual registers in this union. Assume it's // quite small. bool LiveIntervalUnion::Query::isSeenInterference(LiveInterval *VirtReg) const { @@ -234,64 +107,75 @@ bool LiveIntervalUnion::Query::isSeenInterference(LiveInterval *VirtReg) const { return I != InterferingVRegs.end(); } -// Count the number of virtual registers in this union that interfere with this +// Collect virtual registers in this union that interfere with this // query's live virtual register. // -// The number of times that we either advance IR.VirtRegI or call -// LiveUnion.upperBound() will be no more than the number of holes in -// VirtReg. So each invocation of collectInterferingVRegs() takes -// time proportional to |VirtReg Holes| * time(LiveUnion.upperBound()). +// The query state is one of: +// +// 1. CheckedFirstInterference == false: Iterators are uninitialized. +// 2. SeenAllInterferences == true: InterferingVRegs complete, iterators unused. +// 3. Iterators left at the last seen intersection. // -// For comments on how to speed it up, see Query::findIntersection(). unsigned LiveIntervalUnion::Query:: collectInterferingVRegs(unsigned MaxInterferingRegs) { - InterferenceResult IR = firstInterference(); - LiveInterval::iterator VirtRegEnd = VirtReg->end(); - LiveInterval *RecentInterferingVReg = NULL; - if (IR.VirtRegI != VirtRegEnd) while (IR.LiveUnionI.valid()) { - // Advance the union's iterator to reach an unseen interfering vreg. - do { - if (IR.LiveUnionI.value() == RecentInterferingVReg) - continue; + // Fast path return if we already have the desired information. + if (SeenAllInterferences || InterferingVRegs.size() >= MaxInterferingRegs) + return InterferingVRegs.size(); + + // Set up iterators on the first call. + if (!CheckedFirstInterference) { + CheckedFirstInterference = true; + + // Quickly skip interference check for empty sets. + if (VirtReg->empty() || LiveUnion->empty()) { + SeenAllInterferences = true; + return 0; + } - if (!isSeenInterference(IR.LiveUnionI.value())) - break; + // In most cases, the union will start before VirtReg. + VirtRegI = VirtReg->begin(); + LiveUnionI.setMap(LiveUnion->getMap()); + LiveUnionI.find(VirtRegI->start); + } - // Cache the most recent interfering vreg to bypass isSeenInterference. - RecentInterferingVReg = IR.LiveUnionI.value(); + LiveInterval::iterator VirtRegEnd = VirtReg->end(); + LiveInterval *RecentReg = 0; + while (LiveUnionI.valid()) { + assert(VirtRegI != VirtRegEnd && "Reached end of VirtReg"); + + // Check for overlapping interference. + while (VirtRegI->start < LiveUnionI.stop() && + VirtRegI->end > LiveUnionI.start()) { + // This is an overlap, record the interfering register. + LiveInterval *VReg = LiveUnionI.value(); + if (VReg != RecentReg && !isSeenInterference(VReg)) { + RecentReg = VReg; + InterferingVRegs.push_back(VReg); + if (InterferingVRegs.size() >= MaxInterferingRegs) + return InterferingVRegs.size(); + } + // This LiveUnion segment is no longer interesting. + if (!(++LiveUnionI).valid()) { + SeenAllInterferences = true; + return InterferingVRegs.size(); + } + } - } while ((++IR.LiveUnionI).valid()); - if (!IR.LiveUnionI.valid()) - break; + // The iterators are now not overlapping, LiveUnionI has been advanced + // beyond VirtRegI. + assert(VirtRegI->end <= LiveUnionI.start() && "Expected non-overlap"); - // Advance the VirtReg iterator until surpassing the next segment in - // LiveUnion. - IR.VirtRegI = VirtReg->advanceTo(IR.VirtRegI, IR.LiveUnionI.start()); - if (IR.VirtRegI == VirtRegEnd) + // Advance the iterator that ends first. + VirtRegI = VirtReg->advanceTo(VirtRegI, LiveUnionI.start()); + if (VirtRegI == VirtRegEnd) break; - // Check for intersection with the union's segment. - if (overlap(*IR.VirtRegI, IR.LiveUnionI)) { - - if (!IR.LiveUnionI.value()->isSpillable()) - SeenUnspillableVReg = true; - - if (InterferingVRegs.size() == MaxInterferingRegs) - // Leave SeenAllInterferences set to false to indicate that at least one - // interference exists beyond those we collected. - return MaxInterferingRegs; - - InterferingVRegs.push_back(IR.LiveUnionI.value()); - - // Cache the most recent interfering vreg to bypass isSeenInterference. - RecentInterferingVReg = IR.LiveUnionI.value(); - ++IR.LiveUnionI; - + // Detect overlap, handle above. + if (VirtRegI->start < LiveUnionI.stop()) continue; - } - // VirtRegI may have advanced far beyond LiveUnionI, - // do a fast intersection test to "catch up" - IR.LiveUnionI.advanceTo(IR.VirtRegI->start); + + // Still not overlapping. Catch up LiveUnionI. + LiveUnionI.advanceTo(VirtRegI->start); } SeenAllInterferences = true; return InterferingVRegs.size(); diff --git a/lib/CodeGen/LiveIntervalUnion.h b/lib/CodeGen/LiveIntervalUnion.h index 5e78d5e..5d64d28 100644 --- a/lib/CodeGen/LiveIntervalUnion.h +++ b/lib/CodeGen/LiveIntervalUnion.h @@ -59,7 +59,6 @@ public: // LiveIntervalUnions share an external allocator. typedef LiveSegments::Allocator Allocator; - class InterferenceResult; class Query; private: @@ -106,62 +105,13 @@ public: void verify(LiveVirtRegBitSet& VisitedVRegs); #endif - /// Cache a single interference test result in the form of two intersecting - /// segments. This allows efficiently iterating over the interferences. The - /// iteration logic is handled by LiveIntervalUnion::Query which may - /// filter interferences depending on the type of query. - class InterferenceResult { - friend class Query; - - LiveInterval::iterator VirtRegI; // current position in VirtReg - SegmentIter LiveUnionI; // current position in LiveUnion - - // Internal ctor. - InterferenceResult(LiveInterval::iterator VRegI, SegmentIter UnionI) - : VirtRegI(VRegI), LiveUnionI(UnionI) {} - - public: - // Public default ctor. - InterferenceResult(): VirtRegI(), LiveUnionI() {} - - /// start - Return the start of the current overlap. - SlotIndex start() const { - return std::max(VirtRegI->start, LiveUnionI.start()); - } - - /// stop - Return the end of the current overlap. - SlotIndex stop() const { - return std::min(VirtRegI->end, LiveUnionI.stop()); - } - - /// interference - Return the register that is interfering here. - LiveInterval *interference() const { return LiveUnionI.value(); } - - // Note: this interface provides raw access to the iterators because the - // result has no way to tell if it's valid to dereference them. - - // Access the VirtReg segment. - LiveInterval::iterator virtRegPos() const { return VirtRegI; } - - // Access the LiveUnion segment. - const SegmentIter &liveUnionPos() const { return LiveUnionI; } - - bool operator==(const InterferenceResult &IR) const { - return VirtRegI == IR.VirtRegI && LiveUnionI == IR.LiveUnionI; - } - bool operator!=(const InterferenceResult &IR) const { - return !operator==(IR); - } - - void print(raw_ostream &OS, const TargetRegisterInfo *TRI) const; - }; - /// Query interferences between a single live virtual register and a live /// interval union. class Query { LiveIntervalUnion *LiveUnion; LiveInterval *VirtReg; - InterferenceResult FirstInterference; + LiveInterval::iterator VirtRegI; // current position in VirtReg + SegmentIter LiveUnionI; // current position in LiveUnion SmallVector<LiveInterval*,4> InterferingVRegs; bool CheckedFirstInterference; bool SeenAllInterferences; @@ -206,26 +156,8 @@ public: return *VirtReg; } - bool isInterference(const InterferenceResult &IR) const { - if (IR.VirtRegI != VirtReg->end()) { - assert(overlap(*IR.VirtRegI, IR.LiveUnionI) && - "invalid segment iterators"); - return true; - } - return false; - } - // Does this live virtual register interfere with the union? - bool checkInterference() { return isInterference(firstInterference()); } - - // Get the first pair of interfering segments, or a noninterfering result. - // This initializes the firstInterference_ cache. - const InterferenceResult &firstInterference(); - - // Treat the result as an iterator and advance to the next interfering pair - // of segments. Visiting each unique interfering pairs means that the same - // VirtReg or LiveUnion segment may be visited multiple times. - bool nextInterference(InterferenceResult &IR) const; + bool checkInterference() { return collectInterferingVRegs(1); } // Count the virtual registers in this union that interfere with this // query's live virtual register, up to maxInterferingRegs. @@ -249,13 +181,9 @@ public: /// Loop. bool checkLoopInterference(MachineLoopRange*); - void print(raw_ostream &OS, const TargetRegisterInfo *TRI); private: Query(const Query&); // DO NOT IMPLEMENT void operator=(const Query&); // DO NOT IMPLEMENT - - // Private interface for queries - void findIntersection(InterferenceResult &IR) const; }; }; diff --git a/lib/CodeGen/LiveRangeCalc.cpp b/lib/CodeGen/LiveRangeCalc.cpp new file mode 100644 index 0000000..a7d5af5 --- /dev/null +++ b/lib/CodeGen/LiveRangeCalc.cpp @@ -0,0 +1,270 @@ +//===---- LiveRangeCalc.cpp - Calculate live ranges -----------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implementation of the LiveRangeCalc class. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "regalloc" +#include "LiveRangeCalc.h" +#include "llvm/CodeGen/MachineDominators.h" + +using namespace llvm; + +void LiveRangeCalc::reset(const MachineFunction *MF) { + unsigned N = MF->getNumBlockIDs(); + Seen.clear(); + Seen.resize(N); + LiveOut.resize(N); + LiveIn.clear(); +} + + +// Transfer information from the LiveIn vector to the live ranges. +void LiveRangeCalc::updateLiveIns(VNInfo *OverrideVNI, SlotIndexes *Indexes) { + for (SmallVectorImpl<LiveInBlock>::iterator I = LiveIn.begin(), + E = LiveIn.end(); I != E; ++I) { + if (!I->DomNode) + continue; + MachineBasicBlock *MBB = I->DomNode->getBlock(); + + VNInfo *VNI = OverrideVNI ? OverrideVNI : I->Value; + assert(VNI && "No live-in value found"); + + SlotIndex Start, End; + tie(Start, End) = Indexes->getMBBRange(MBB); + + if (I->Kill.isValid()) + I->LI->addRange(LiveRange(Start, I->Kill, VNI)); + else { + I->LI->addRange(LiveRange(Start, End, VNI)); + // The value is live-through, update LiveOut as well. Defer the Domtree + // lookup until it is needed. + assert(Seen.test(MBB->getNumber())); + LiveOut[MBB] = LiveOutPair(VNI, (MachineDomTreeNode *)0); + } + } + LiveIn.clear(); +} + + +void LiveRangeCalc::extend(LiveInterval *LI, + SlotIndex Kill, + SlotIndexes *Indexes, + MachineDominatorTree *DomTree, + VNInfo::Allocator *Alloc) { + assert(LI && "Missing live range"); + assert(Kill.isValid() && "Invalid SlotIndex"); + assert(Indexes && "Missing SlotIndexes"); + assert(DomTree && "Missing dominator tree"); + + MachineBasicBlock *KillMBB = Indexes->getMBBFromIndex(Kill.getPrevSlot()); + assert(Kill && "No MBB at Kill"); + + // Is there a def in the same MBB we can extend? + if (LI->extendInBlock(Indexes->getMBBStartIdx(KillMBB), Kill)) + return; + + // Find the single reaching def, or determine if Kill is jointly dominated by + // multiple values, and we may need to create even more phi-defs to preserve + // VNInfo SSA form. Perform a search for all predecessor blocks where we + // know the dominating VNInfo. + VNInfo *VNI = findReachingDefs(LI, KillMBB, Kill, Indexes, DomTree); + + // When there were multiple different values, we may need new PHIs. + if (!VNI) + updateSSA(Indexes, DomTree, Alloc); + + updateLiveIns(VNI, Indexes); +} + + +// This function is called by a client after using the low-level API to add +// live-out and live-in blocks. The unique value optimization is not +// available, SplitEditor::transferValues handles that case directly anyway. +void LiveRangeCalc::calculateValues(SlotIndexes *Indexes, + MachineDominatorTree *DomTree, + VNInfo::Allocator *Alloc) { + assert(Indexes && "Missing SlotIndexes"); + assert(DomTree && "Missing dominator tree"); + updateSSA(Indexes, DomTree, Alloc); + updateLiveIns(0, Indexes); +} + + +VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI, + MachineBasicBlock *KillMBB, + SlotIndex Kill, + SlotIndexes *Indexes, + MachineDominatorTree *DomTree) { + // Blocks where LI should be live-in. + SmallVector<MachineBasicBlock*, 16> WorkList(1, KillMBB); + + // Remember if we have seen more than one value. + bool UniqueVNI = true; + VNInfo *TheVNI = 0; + + // Using Seen as a visited set, perform a BFS for all reaching defs. + for (unsigned i = 0; i != WorkList.size(); ++i) { + MachineBasicBlock *MBB = WorkList[i]; + assert(!MBB->pred_empty() && "Value live-in to entry block?"); + for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), + PE = MBB->pred_end(); PI != PE; ++PI) { + MachineBasicBlock *Pred = *PI; + + // Is this a known live-out block? + if (Seen.test(Pred->getNumber())) { + if (VNInfo *VNI = LiveOut[Pred].first) { + if (TheVNI && TheVNI != VNI) + UniqueVNI = false; + TheVNI = VNI; + } + continue; + } + + SlotIndex Start, End; + tie(Start, End) = Indexes->getMBBRange(Pred); + + // First time we see Pred. Try to determine the live-out value, but set + // it as null if Pred is live-through with an unknown value. + VNInfo *VNI = LI->extendInBlock(Start, End); + setLiveOutValue(Pred, VNI); + if (VNI) { + if (TheVNI && TheVNI != VNI) + UniqueVNI = false; + TheVNI = VNI; + continue; + } + + // No, we need a live-in value for Pred as well + if (Pred != KillMBB) + WorkList.push_back(Pred); + else + // Loopback to KillMBB, so value is really live through. + Kill = SlotIndex(); + } + } + + // Transfer WorkList to LiveInBlocks in reverse order. + // This ordering works best with updateSSA(). + LiveIn.clear(); + LiveIn.reserve(WorkList.size()); + while(!WorkList.empty()) + addLiveInBlock(LI, DomTree->getNode(WorkList.pop_back_val())); + + // The kill block may not be live-through. + assert(LiveIn.back().DomNode->getBlock() == KillMBB); + LiveIn.back().Kill = Kill; + + return UniqueVNI ? TheVNI : 0; +} + + +// This is essentially the same iterative algorithm that SSAUpdater uses, +// except we already have a dominator tree, so we don't have to recompute it. +void LiveRangeCalc::updateSSA(SlotIndexes *Indexes, + MachineDominatorTree *DomTree, + VNInfo::Allocator *Alloc) { + assert(Indexes && "Missing SlotIndexes"); + assert(DomTree && "Missing dominator tree"); + + // Interate until convergence. + unsigned Changes; + do { + Changes = 0; + // Propagate live-out values down the dominator tree, inserting phi-defs + // when necessary. + for (SmallVectorImpl<LiveInBlock>::iterator I = LiveIn.begin(), + E = LiveIn.end(); I != E; ++I) { + MachineDomTreeNode *Node = I->DomNode; + // Skip block if the live-in value has already been determined. + if (!Node) + continue; + MachineBasicBlock *MBB = Node->getBlock(); + MachineDomTreeNode *IDom = Node->getIDom(); + LiveOutPair IDomValue; + + // We need a live-in value to a block with no immediate dominator? + // This is probably an unreachable block that has survived somehow. + bool needPHI = !IDom || !Seen.test(IDom->getBlock()->getNumber()); + + // IDom dominates all of our predecessors, but it may not be their + // immediate dominator. Check if any of them have live-out values that are + // properly dominated by IDom. If so, we need a phi-def here. + if (!needPHI) { + IDomValue = LiveOut[IDom->getBlock()]; + + // Cache the DomTree node that defined the value. + if (IDomValue.first && !IDomValue.second) + LiveOut[IDom->getBlock()].second = IDomValue.second = + DomTree->getNode(Indexes->getMBBFromIndex(IDomValue.first->def)); + + for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), + PE = MBB->pred_end(); PI != PE; ++PI) { + LiveOutPair &Value = LiveOut[*PI]; + if (!Value.first || Value.first == IDomValue.first) + continue; + + // Cache the DomTree node that defined the value. + if (!Value.second) + Value.second = + DomTree->getNode(Indexes->getMBBFromIndex(Value.first->def)); + + // This predecessor is carrying something other than IDomValue. + // It could be because IDomValue hasn't propagated yet, or it could be + // because MBB is in the dominance frontier of that value. + if (DomTree->dominates(IDom, Value.second)) { + needPHI = true; + break; + } + } + } + + // The value may be live-through even if Kill is set, as can happen when + // we are called from extendRange. In that case LiveOutSeen is true, and + // LiveOut indicates a foreign or missing value. + LiveOutPair &LOP = LiveOut[MBB]; + + // Create a phi-def if required. + if (needPHI) { + ++Changes; + assert(Alloc && "Need VNInfo allocator to create PHI-defs"); + SlotIndex Start, End; + tie(Start, End) = Indexes->getMBBRange(MBB); + VNInfo *VNI = I->LI->getNextValue(Start, 0, *Alloc); + VNI->setIsPHIDef(true); + I->Value = VNI; + // This block is done, we know the final value. + I->DomNode = 0; + + // Add liveness since updateLiveIns now skips this node. + if (I->Kill.isValid()) + I->LI->addRange(LiveRange(Start, I->Kill, VNI)); + else { + I->LI->addRange(LiveRange(Start, End, VNI)); + LOP = LiveOutPair(VNI, Node); + } + } else if (IDomValue.first) { + // No phi-def here. Remember incoming value. + I->Value = IDomValue.first; + + // If the IDomValue is killed in the block, don't propagate through. + if (I->Kill.isValid()) + continue; + + // Propagate IDomValue if it isn't killed: + // MBB is live-out and doesn't define its own value. + if (LOP.first == IDomValue.first) + continue; + ++Changes; + LOP = IDomValue; + } + } + } while (Changes); +} diff --git a/lib/CodeGen/LiveRangeCalc.h b/lib/CodeGen/LiveRangeCalc.h new file mode 100644 index 0000000..b8c8585 --- /dev/null +++ b/lib/CodeGen/LiveRangeCalc.h @@ -0,0 +1,226 @@ +//===---- LiveRangeCalc.h - Calculate live ranges ---------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// The LiveRangeCalc class can be used to compute live ranges from scratch. It +// caches information about values in the CFG to speed up repeated operations +// on the same live range. The cache can be shared by non-overlapping live +// ranges. SplitKit uses that when computing the live range of split products. +// +// A low-level interface is available to clients that know where a variable is +// live, but don't know which value it has as every point. LiveRangeCalc will +// propagate values down the dominator tree, and even insert PHI-defs where +// needed. SplitKit uses this faster interface when possible. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_LIVERANGECALC_H +#define LLVM_CODEGEN_LIVERANGECALC_H + +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/IndexedMap.h" +#include "llvm/CodeGen/LiveInterval.h" + +namespace llvm { + +/// Forward declarations for MachineDominators.h: +class MachineDominatorTree; +template <class NodeT> class DomTreeNodeBase; +typedef DomTreeNodeBase<MachineBasicBlock> MachineDomTreeNode; + +class LiveRangeCalc { + /// Seen - Bit vector of active entries in LiveOut, also used as a visited + /// set by findReachingDefs. One entry per basic block, indexed by block + /// number. This is kept as a separate bit vector because it can be cleared + /// quickly when switching live ranges. + BitVector Seen; + + /// LiveOutPair - A value and the block that defined it. The domtree node is + /// redundant, it can be computed as: MDT[Indexes.getMBBFromIndex(VNI->def)]. + typedef std::pair<VNInfo*, MachineDomTreeNode*> LiveOutPair; + + /// LiveOutMap - Map basic blocks to the value leaving the block. + typedef IndexedMap<LiveOutPair, MBB2NumberFunctor> LiveOutMap; + + /// LiveOut - Map each basic block where a live range is live out to the + /// live-out value and its defining block. + /// + /// For every basic block, MBB, one of these conditions shall be true: + /// + /// 1. !Seen.count(MBB->getNumber()) + /// Blocks without a Seen bit are ignored. + /// 2. LiveOut[MBB].second.getNode() == MBB + /// The live-out value is defined in MBB. + /// 3. forall P in preds(MBB): LiveOut[P] == LiveOut[MBB] + /// The live-out value passses through MBB. All predecessors must carry + /// the same value. + /// + /// The domtree node may be null, it can be computed. + /// + /// The map can be shared by multiple live ranges as long as no two are + /// live-out of the same block. + LiveOutMap LiveOut; + + /// LiveInBlock - Information about a basic block where a live range is known + /// to be live-in, but the value has not yet been determined. + struct LiveInBlock { + // LI - The live range that is live-in to this block. The algorithms can + // handle multiple non-overlapping live ranges simultaneously. + LiveInterval *LI; + + // DomNode - Dominator tree node for the block. + // Cleared when the final value has been determined and LI has been updated. + MachineDomTreeNode *DomNode; + + // Position in block where the live-in range ends, or SlotIndex() if the + // range passes through the block. When the final value has been + // determined, the range from the block start to Kill will be added to LI. + SlotIndex Kill; + + // Live-in value filled in by updateSSA once it is known. + VNInfo *Value; + + LiveInBlock(LiveInterval *li, MachineDomTreeNode *node, SlotIndex kill) + : LI(li), DomNode(node), Kill(kill), Value(0) {} + }; + + /// LiveIn - Work list of blocks where the live-in value has yet to be + /// determined. This list is typically computed by findReachingDefs() and + /// used as a work list by updateSSA(). The low-level interface may also be + /// used to add entries directly. + SmallVector<LiveInBlock, 16> LiveIn; + + /// findReachingDefs - Assuming that LI is live-in to KillMBB and killed at + /// Kill, search for values that can reach KillMBB. All blocks that need LI + /// to be live-in are added to LiveIn. If a unique reaching def is found, + /// its value is returned, if Kill is jointly dominated by multiple values, + /// NULL is returned. + VNInfo *findReachingDefs(LiveInterval *LI, + MachineBasicBlock *KillMBB, + SlotIndex Kill, + SlotIndexes *Indexes, + MachineDominatorTree *DomTree); + + /// updateSSA - Compute the values that will be live in to all requested + /// blocks in LiveIn. Create PHI-def values as required to preserve SSA form. + /// + /// Every live-in block must be jointly dominated by the added live-out + /// blocks. No values are read from the live ranges. + void updateSSA(SlotIndexes *Indexes, + MachineDominatorTree *DomTree, + VNInfo::Allocator *Alloc); + + /// updateLiveIns - Add liveness as specified in the LiveIn vector, using VNI + /// as a wildcard value for LiveIn entries without a value. + void updateLiveIns(VNInfo *VNI, SlotIndexes*); + +public: + //===--------------------------------------------------------------------===// + // High-level interface. + //===--------------------------------------------------------------------===// + // + // Calculate live ranges from scratch. + // + + /// reset - Prepare caches for a new set of non-overlapping live ranges. The + /// caches must be reset before attempting calculations with a live range + /// that may overlap a previously computed live range, and before the first + /// live range in a function. If live ranges are not known to be + /// non-overlapping, call reset before each. + void reset(const MachineFunction *MF); + + /// calculate - Calculate the live range of a virtual register from its defs + /// and uses. LI must be empty with no values. + void calculate(LiveInterval *LI, + MachineRegisterInfo *MRI, + SlotIndexes *Indexes, + VNInfo::Allocator *Alloc); + + //===--------------------------------------------------------------------===// + // Mid-level interface. + //===--------------------------------------------------------------------===// + // + // Modify existing live ranges. + // + + /// extend - Extend the live range of LI to reach Kill. + /// + /// The existing values in LI must be live so they jointly dominate Kill. If + /// Kill is not dominated by a single existing value, PHI-defs are inserted + /// as required to preserve SSA form. If Kill is known to be dominated by a + /// single existing value, Alloc may be null. + void extend(LiveInterval *LI, + SlotIndex Kill, + SlotIndexes *Indexes, + MachineDominatorTree *DomTree, + VNInfo::Allocator *Alloc); + + /// extendToUses - Extend the live range of LI to reach all uses. + /// + /// All uses must be jointly dominated by existing liveness. PHI-defs are + /// inserted as needed to preserve SSA form. + void extendToUses(LiveInterval *LI, + MachineRegisterInfo *MRI, + SlotIndexes *Indexes, + MachineDominatorTree *DomTree, + VNInfo::Allocator *Alloc); + + //===--------------------------------------------------------------------===// + // Low-level interface. + //===--------------------------------------------------------------------===// + // + // These functions can be used to compute live ranges where the live-in and + // live-out blocks are already known, but the SSA value in each block is + // unknown. + // + // After calling reset(), add known live-out values and known live-in blocks. + // Then call calculateValues() to compute the actual value that is + // live-in to each block, and add liveness to the live ranges. + // + + /// setLiveOutValue - Indicate that VNI is live out from MBB. The + /// calculateValues() function will not add liveness for MBB, the caller + /// should take care of that. + /// + /// VNI may be null only if MBB is a live-through block also passed to + /// addLiveInBlock(). + void setLiveOutValue(MachineBasicBlock *MBB, VNInfo *VNI) { + Seen.set(MBB->getNumber()); + LiveOut[MBB] = LiveOutPair(VNI, (MachineDomTreeNode *)0); + } + + /// addLiveInBlock - Add a block with an unknown live-in value. This + /// function can only be called once per basic block. Once the live-in value + /// has been determined, calculateValues() will add liveness to LI. + /// + /// @param LI The live range that is live-in to the block. + /// @param DomNode The domtree node for the block. + /// @param Kill Index in block where LI is killed. If the value is + /// live-through, set Kill = SLotIndex() and also call + /// setLiveOutValue(MBB, 0). + void addLiveInBlock(LiveInterval *LI, + MachineDomTreeNode *DomNode, + SlotIndex Kill = SlotIndex()) { + LiveIn.push_back(LiveInBlock(LI, DomNode, Kill)); + } + + /// calculateValues - Calculate the value that will be live-in to each block + /// added with addLiveInBlock. Add PHI-def values as needed to preserve SSA + /// form. Add liveness to all live-in blocks up to the Kill point, or the + /// whole block for live-through blocks. + /// + /// Every predecessor of a live-in block must have been given a value with + /// setLiveOutValue, the value may be null for live-trough blocks. + void calculateValues(SlotIndexes *Indexes, + MachineDominatorTree *DomTree, + VNInfo::Allocator *Alloc); +}; + +} // end namespace llvm + +#endif diff --git a/lib/CodeGen/LiveRangeEdit.cpp b/lib/CodeGen/LiveRangeEdit.cpp index b385fb3..b23f851 100644 --- a/lib/CodeGen/LiveRangeEdit.cpp +++ b/lib/CodeGen/LiveRangeEdit.cpp @@ -319,9 +319,12 @@ void LiveRangeEdit::calculateRegClassAndHint(MachineFunction &MF, LiveIntervals &LIS, const MachineLoopInfo &Loops) { VirtRegAuxInfo VRAI(MF, LIS, Loops); + MachineRegisterInfo &MRI = MF.getRegInfo(); for (iterator I = begin(), E = end(); I != E; ++I) { LiveInterval &LI = **I; - VRAI.CalculateRegClass(LI.reg); + if (MRI.recomputeRegClass(LI.reg, MF.getTarget())) + DEBUG(dbgs() << "Inflated " << PrintReg(LI.reg) << " to " + << MRI.getRegClass(LI.reg)->getName() << '\n'); VRAI.CalculateWeightAndHint(LI); } } diff --git a/lib/CodeGen/LiveRangeEdit.h b/lib/CodeGen/LiveRangeEdit.h index db6740c..9b0a671 100644 --- a/lib/CodeGen/LiveRangeEdit.h +++ b/lib/CodeGen/LiveRangeEdit.h @@ -115,7 +115,7 @@ public: LiveInterval *get(unsigned idx) const { return newRegs_[idx+firstNew_]; } ArrayRef<LiveInterval*> regs() const { - return ArrayRef<LiveInterval*>(newRegs_).slice(firstNew_); + return makeArrayRef(newRegs_).slice(firstNew_); } /// FIXME: Temporary accessors until we can get rid of diff --git a/lib/CodeGen/LiveStackAnalysis.cpp b/lib/CodeGen/LiveStackAnalysis.cpp index c75196a..939e795 100644 --- a/lib/CodeGen/LiveStackAnalysis.cpp +++ b/lib/CodeGen/LiveStackAnalysis.cpp @@ -44,7 +44,8 @@ void LiveStacks::releaseMemory() { S2RCMap.clear(); } -bool LiveStacks::runOnMachineFunction(MachineFunction &) { +bool LiveStacks::runOnMachineFunction(MachineFunction &MF) { + TRI = MF.getTarget().getRegisterInfo(); // FIXME: No analysis is being done right now. We are relying on the // register allocators to provide the information. return false; @@ -61,7 +62,7 @@ LiveStacks::getOrCreateInterval(int Slot, const TargetRegisterClass *RC) { } else { // Use the largest common subclass register class. const TargetRegisterClass *OldRC = S2RCMap[Slot]; - S2RCMap[Slot] = getCommonSubClass(OldRC, RC); + S2RCMap[Slot] = TRI->getCommonSubClass(OldRC, RC); } return I->second; } diff --git a/lib/CodeGen/LiveVariables.cpp b/lib/CodeGen/LiveVariables.cpp index 20bad60..2ca90f9 100644 --- a/lib/CodeGen/LiveVariables.cpp +++ b/lib/CodeGen/LiveVariables.cpp @@ -662,7 +662,7 @@ void LiveVariables::removeVirtualRegistersKilled(MachineInstr *MI) { if (TargetRegisterInfo::isVirtualRegister(Reg)) { bool removed = getVarInfo(Reg).removeKill(MI); assert(removed && "kill not in register's VarInfo?"); - removed = true; + (void)removed; } } } diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp index 8f0fb46..4c5fe4c 100644 --- a/lib/CodeGen/MachineBasicBlock.cpp +++ b/lib/CodeGen/MachineBasicBlock.cpp @@ -571,6 +571,11 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { if (i->getOperand(ni+1).getMBB() == this) i->getOperand(ni+1).setMBB(NMBB); + // Inherit live-ins from the successor + for (MachineBasicBlock::livein_iterator I = Succ->livein_begin(), + E = Succ->livein_end(); I != E; ++I) + NMBB->addLiveIn(*I); + // Update LiveVariables. if (LV) { // Restore kills of virtual registers that were killed by the terminators. diff --git a/lib/CodeGen/MachineBlockFrequency.cpp b/lib/CodeGen/MachineBlockFrequencyInfo.cpp index 893a320..b92cda9 100644 --- a/lib/CodeGen/MachineBlockFrequency.cpp +++ b/lib/CodeGen/MachineBlockFrequencyInfo.cpp @@ -1,4 +1,4 @@ -//====----- MachineBlockFrequency.cpp - Machine Block Frequency Analysis ----====// +//====----- MachineBlockFrequencyInfo.cpp - Machine Block Frequency Analysis ----====// // // The LLVM Compiler Infrastructure // @@ -13,47 +13,49 @@ #include "llvm/InitializePasses.h" #include "llvm/Analysis/BlockFrequencyImpl.h" -#include "llvm/CodeGen/MachineBlockFrequency.h" +#include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" using namespace llvm; -INITIALIZE_PASS_BEGIN(MachineBlockFrequency, "machine-block-freq", +INITIALIZE_PASS_BEGIN(MachineBlockFrequencyInfo, "machine-block-freq", "Machine Block Frequency Analysis", true, true) INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo) -INITIALIZE_PASS_END(MachineBlockFrequency, "machine-block-freq", +INITIALIZE_PASS_END(MachineBlockFrequencyInfo, "machine-block-freq", "Machine Block Frequency Analysis", true, true) -char MachineBlockFrequency::ID = 0; +char MachineBlockFrequencyInfo::ID = 0; -MachineBlockFrequency::MachineBlockFrequency() : MachineFunctionPass(ID) { - initializeMachineBlockFrequencyPass(*PassRegistry::getPassRegistry()); +MachineBlockFrequencyInfo::MachineBlockFrequencyInfo() : MachineFunctionPass(ID) { + initializeMachineBlockFrequencyInfoPass(*PassRegistry::getPassRegistry()); MBFI = new BlockFrequencyImpl<MachineBasicBlock, MachineFunction, MachineBranchProbabilityInfo>(); } -MachineBlockFrequency::~MachineBlockFrequency() { +MachineBlockFrequencyInfo::~MachineBlockFrequencyInfo() { delete MBFI; } -void MachineBlockFrequency::getAnalysisUsage(AnalysisUsage &AU) const { +void MachineBlockFrequencyInfo::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired<MachineBranchProbabilityInfo>(); AU.setPreservesAll(); + MachineFunctionPass::getAnalysisUsage(AU); } -bool MachineBlockFrequency::runOnMachineFunction(MachineFunction &F) { +bool MachineBlockFrequencyInfo::runOnMachineFunction(MachineFunction &F) { MachineBranchProbabilityInfo &MBPI = getAnalysis<MachineBranchProbabilityInfo>(); MBFI->doFunction(&F, &MBPI); return false; } -/// getblockFreq - Return block frequency. Never return 0, value must be -/// positive. Please note that initial frequency is equal to 1024. It means that -/// we should not rely on the value itself, but only on the comparison to the -/// other block frequencies. We do this to avoid using of floating points. +/// getblockFreq - Return block frequency. Return 0 if we don't have the +/// information. Please note that initial frequency is equal to 1024. It means +/// that we should not rely on the value itself, but only on the comparison to +/// the other block frequencies. We do this to avoid using of floating points. /// -uint32_t MachineBlockFrequency::getBlockFreq(MachineBasicBlock *MBB) { +BlockFrequency MachineBlockFrequencyInfo:: +getBlockFreq(MachineBasicBlock *MBB) const { return MBFI->getBlockFreq(MBB); } diff --git a/lib/CodeGen/MachineCSE.cpp b/lib/CodeGen/MachineCSE.cpp index 3a60a37..7eda8c1 100644 --- a/lib/CodeGen/MachineCSE.cpp +++ b/lib/CodeGen/MachineCSE.cpp @@ -430,13 +430,24 @@ bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) { unsigned NewReg = CSMI->getOperand(i).getReg(); if (OldReg == NewReg) continue; + assert(TargetRegisterInfo::isVirtualRegister(OldReg) && TargetRegisterInfo::isVirtualRegister(NewReg) && "Do not CSE physical register defs!"); + if (!isProfitableToCSE(NewReg, OldReg, CSMI, MI)) { DoCSE = false; break; } + + // Don't perform CSE if the result of the old instruction cannot exist + // within the register class of the new instruction. + const TargetRegisterClass *OldRC = MRI->getRegClass(OldReg); + if (!MRI->constrainRegClass(NewReg, OldRC)) { + DoCSE = false; + break; + } + CSEPairs.push_back(std::make_pair(OldReg, NewReg)); --NumDefs; } diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp index cd25156..20066a0 100644 --- a/lib/CodeGen/MachineFunction.cpp +++ b/lib/CodeGen/MachineFunction.cpp @@ -619,7 +619,7 @@ void MachineJumpTableInfo::dump() const { print(dbgs()); } // MachineConstantPool implementation //===----------------------------------------------------------------------===// -const Type *MachineConstantPoolEntry::getType() const { +Type *MachineConstantPoolEntry::getType() const { if (isMachineConstantPoolEntry()) return Val.MachineCPVal->getType(); return Val.ConstVal->getType(); diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp index 143a29b..a240667 100644 --- a/lib/CodeGen/MachineInstr.cpp +++ b/lib/CodeGen/MachineInstr.cpp @@ -51,7 +51,7 @@ using namespace llvm; /// explicitly nulled out. void MachineOperand::AddRegOperandToRegInfo(MachineRegisterInfo *RegInfo) { assert(isReg() && "Can only add reg operand to use lists"); - + // If the reginfo pointer is null, just explicitly null out or next/prev // pointers, to ensure they are not garbage. if (RegInfo == 0) { @@ -59,23 +59,23 @@ void MachineOperand::AddRegOperandToRegInfo(MachineRegisterInfo *RegInfo) { Contents.Reg.Next = 0; return; } - + // Otherwise, add this operand to the head of the registers use/def list. MachineOperand **Head = &RegInfo->getRegUseDefListHead(getReg()); - + // For SSA values, we prefer to keep the definition at the start of the list. // we do this by skipping over the definition if it is at the head of the // list. if (*Head && (*Head)->isDef()) Head = &(*Head)->Contents.Reg.Next; - + Contents.Reg.Next = *Head; if (Contents.Reg.Next) { assert(getReg() == Contents.Reg.Next->getReg() && "Different regs on the same list!"); Contents.Reg.Next->Contents.Reg.Prev = &Contents.Reg.Next; } - + Contents.Reg.Prev = Head; *Head = this; } @@ -86,7 +86,7 @@ void MachineOperand::RemoveRegOperandFromRegInfo() { assert(isOnRegUseList() && "Reg operand is not on a use list"); // Unlink this from the doubly linked list of operands. MachineOperand *NextOp = Contents.Reg.Next; - *Contents.Reg.Prev = NextOp; + *Contents.Reg.Prev = NextOp; if (NextOp) { assert(NextOp->getReg() == getReg() && "Corrupt reg use/def chain!"); NextOp->Contents.Reg.Prev = Contents.Reg.Prev; @@ -97,7 +97,7 @@ void MachineOperand::RemoveRegOperandFromRegInfo() { void MachineOperand::setReg(unsigned Reg) { if (getReg() == Reg) return; // No change. - + // Otherwise, we have to change the register. If this operand is embedded // into a machine function, we need to update the old and new register's // use/def lists. @@ -109,7 +109,7 @@ void MachineOperand::setReg(unsigned Reg) { AddRegOperandToRegInfo(&MF->getRegInfo()); return; } - + // Otherwise, just change the register, no problem. :) SmallContents.RegNo = Reg; } @@ -144,7 +144,7 @@ void MachineOperand::ChangeToImmediate(int64_t ImmVal) { if (isReg() && getParent() && getParent()->getParent() && getParent()->getParent()->getParent()) RemoveRegOperandFromRegInfo(); - + OpKind = MO_Immediate; Contents.ImmVal = ImmVal; } @@ -155,7 +155,7 @@ void MachineOperand::ChangeToImmediate(int64_t ImmVal) { void MachineOperand::ChangeToRegister(unsigned Reg, bool isDef, bool isImp, bool isKill, bool isDead, bool isUndef, bool isDebug) { - // If this operand is already a register operand, use setReg to update the + // If this operand is already a register operand, use setReg to update the // register's use/def lists. if (isReg()) { assert(!isEarlyClobber()); @@ -189,7 +189,7 @@ bool MachineOperand::isIdenticalTo(const MachineOperand &Other) const { if (getType() != Other.getType() || getTargetFlags() != Other.getTargetFlags()) return false; - + switch (getType()) { default: llvm_unreachable("Unrecognized operand type"); case MachineOperand::MO_Register: @@ -322,7 +322,7 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const { default: llvm_unreachable("Unrecognized operand type"); } - + if (unsigned TF = getTargetFlags()) OS << "[TF=" << TF << ']'; } @@ -408,7 +408,7 @@ uint64_t MachineMemOperand::getAlignment() const { raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) { assert((MMO.isLoad() || MMO.isStore()) && "SV has to be a load, store or both."); - + if (MMO.isVolatile()) OS << "Volatile "; @@ -417,7 +417,7 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) { if (MMO.isStore()) OS << "ST"; OS << MMO.getSize(); - + // Print the address information. OS << "["; if (!MMO.getValue()) @@ -464,7 +464,7 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) { /// MachineInstr ctor - This constructor creates a dummy MachineInstr with /// MCID NULL and no operands. MachineInstr::MachineInstr() - : MCID(0), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), + : MCID(0), Flags(0), AsmPrinterFlags(0), MemRefs(0), MemRefsEnd(0), Parent(0) { // Make sure that we get added to a machine basicblock @@ -484,8 +484,9 @@ void MachineInstr::addImplicitDefUseOperands() { /// implicit operands. It reserves space for the number of operands specified by /// the MCInstrDesc. MachineInstr::MachineInstr(const MCInstrDesc &tid, bool NoImp) - : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), + : MCID(&tid), Flags(0), AsmPrinterFlags(0), MemRefs(0), MemRefsEnd(0), Parent(0) { + unsigned NumImplicitOps = 0; if (!NoImp) NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); Operands.reserve(NumImplicitOps + MCID->getNumOperands()); @@ -498,8 +499,9 @@ MachineInstr::MachineInstr(const MCInstrDesc &tid, bool NoImp) /// MachineInstr ctor - As above, but with a DebugLoc. MachineInstr::MachineInstr(const MCInstrDesc &tid, const DebugLoc dl, bool NoImp) - : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), + : MCID(&tid), Flags(0), AsmPrinterFlags(0), MemRefs(0), MemRefsEnd(0), Parent(0), debugLoc(dl) { + unsigned NumImplicitOps = 0; if (!NoImp) NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); Operands.reserve(NumImplicitOps + MCID->getNumOperands()); @@ -510,13 +512,14 @@ MachineInstr::MachineInstr(const MCInstrDesc &tid, const DebugLoc dl, } /// MachineInstr ctor - Work exactly the same as the ctor two above, except -/// that the MachineInstr is created and added to the end of the specified +/// that the MachineInstr is created and added to the end of the specified /// basic block. MachineInstr::MachineInstr(MachineBasicBlock *MBB, const MCInstrDesc &tid) - : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), + : MCID(&tid), Flags(0), AsmPrinterFlags(0), MemRefs(0), MemRefsEnd(0), Parent(0) { assert(MBB && "Cannot use inserting ctor with null basic block!"); - NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); + unsigned NumImplicitOps = + MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); Operands.reserve(NumImplicitOps + MCID->getNumOperands()); addImplicitDefUseOperands(); // Make sure that we get added to a machine basicblock @@ -528,10 +531,11 @@ MachineInstr::MachineInstr(MachineBasicBlock *MBB, const MCInstrDesc &tid) /// MachineInstr::MachineInstr(MachineBasicBlock *MBB, const DebugLoc dl, const MCInstrDesc &tid) - : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), + : MCID(&tid), Flags(0), AsmPrinterFlags(0), MemRefs(0), MemRefsEnd(0), Parent(0), debugLoc(dl) { assert(MBB && "Cannot use inserting ctor with null basic block!"); - NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); + unsigned NumImplicitOps = + MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); Operands.reserve(NumImplicitOps + MCID->getNumOperands()); addImplicitDefUseOperands(); // Make sure that we get added to a machine basicblock @@ -542,7 +546,7 @@ MachineInstr::MachineInstr(MachineBasicBlock *MBB, const DebugLoc dl, /// MachineInstr ctor - Copies MachineInstr arg exactly /// MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI) - : MCID(&MI.getDesc()), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), + : MCID(&MI.getDesc()), Flags(0), AsmPrinterFlags(0), MemRefs(MI.MemRefs), MemRefsEnd(MI.MemRefsEnd), Parent(0), debugLoc(MI.getDebugLoc()) { Operands.reserve(MI.getNumOperands()); @@ -550,7 +554,6 @@ MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI) // Add operands for (unsigned i = 0; i != MI.getNumOperands(); ++i) addOperand(MI.getOperand(i)); - NumImplicitOps = MI.NumImplicitOps; // Copy all the flags. Flags = MI.Flags; @@ -605,102 +608,74 @@ void MachineInstr::AddRegOperandsToUseLists(MachineRegisterInfo &RegInfo) { /// addOperand - Add the specified operand to the instruction. If it is an /// implicit operand, it is added to the end of the operand list. If it is /// an explicit operand it is added at the end of the explicit operand list -/// (before the first implicit operand). +/// (before the first implicit operand). void MachineInstr::addOperand(const MachineOperand &Op) { + assert(MCID && "Cannot add operands before providing an instr descriptor"); bool isImpReg = Op.isReg() && Op.isImplicit(); - assert((isImpReg || !OperandsComplete()) && - "Trying to add an operand to a machine instr that is already done!"); - MachineRegisterInfo *RegInfo = getRegInfo(); - // If we are adding the operand to the end of the list, our job is simpler. - // This is true most of the time, so this is a reasonable optimization. - if (isImpReg || NumImplicitOps == 0) { - // We can only do this optimization if we know that the operand list won't - // reallocate. - if (Operands.empty() || Operands.size()+1 <= Operands.capacity()) { - Operands.push_back(Op); - - // Set the parent of the operand. - Operands.back().ParentMI = this; - - // If the operand is a register, update the operand's use list. - if (Op.isReg()) { - Operands.back().AddRegOperandToRegInfo(RegInfo); - // If the register operand is flagged as early, mark the operand as such - unsigned OpNo = Operands.size() - 1; - if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1) - Operands[OpNo].setIsEarlyClobber(true); - } - return; + // If the Operands backing store is reallocated, all register operands must + // be removed and re-added to RegInfo. It is storing pointers to operands. + bool Reallocate = RegInfo && + !Operands.empty() && Operands.size() == Operands.capacity(); + + // Find the insert location for the new operand. Implicit registers go at + // the end, everything goes before the implicit regs. + unsigned OpNo = Operands.size(); + + // Remove all the implicit operands from RegInfo if they need to be shifted. + // FIXME: Allow mixed explicit and implicit operands on inline asm. + // InstrEmitter::EmitSpecialNode() is marking inline asm clobbers as + // implicit-defs, but they must not be moved around. See the FIXME in + // InstrEmitter.cpp. + if (!isImpReg && !isInlineAsm()) { + while (OpNo && Operands[OpNo-1].isReg() && Operands[OpNo-1].isImplicit()) { + --OpNo; + if (RegInfo) + Operands[OpNo].RemoveRegOperandFromRegInfo(); } } - - // Otherwise, we have to insert a real operand before any implicit ones. - unsigned OpNo = Operands.size()-NumImplicitOps; - // If this instruction isn't embedded into a function, then we don't need to - // update any operand lists. - if (RegInfo == 0) { - // Simple insertion, no reginfo update needed for other register operands. - Operands.insert(Operands.begin()+OpNo, Op); - Operands[OpNo].ParentMI = this; - - // Do explicitly set the reginfo for this operand though, to ensure the - // next/prev fields are properly nulled out. - if (Operands[OpNo].isReg()) { - Operands[OpNo].AddRegOperandToRegInfo(0); - // If the register operand is flagged as early, mark the operand as such - if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1) - Operands[OpNo].setIsEarlyClobber(true); - } + // OpNo now points as the desired insertion point. Unless this is a variadic + // instruction, only implicit regs are allowed beyond MCID->getNumOperands(). + assert((isImpReg || MCID->isVariadic() || OpNo < MCID->getNumOperands()) && + "Trying to add an operand to a machine instr that is already done!"); - } else if (Operands.size()+1 <= Operands.capacity()) { - // Otherwise, we have to remove register operands from their register use - // list, add the operand, then add the register operands back to their use - // list. This also must handle the case when the operand list reallocates - // to somewhere else. - - // If insertion of this operand won't cause reallocation of the operand - // list, just remove the implicit operands, add the operand, then re-add all - // the rest of the operands. - for (unsigned i = OpNo, e = Operands.size(); i != e; ++i) { - assert(Operands[i].isReg() && "Should only be an implicit reg!"); - Operands[i].RemoveRegOperandFromRegInfo(); - } - - // Add the operand. If it is a register, add it to the reg list. - Operands.insert(Operands.begin()+OpNo, Op); - Operands[OpNo].ParentMI = this; - - if (Operands[OpNo].isReg()) { - Operands[OpNo].AddRegOperandToRegInfo(RegInfo); - // If the register operand is flagged as early, mark the operand as such - if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1) - Operands[OpNo].setIsEarlyClobber(true); - } - - // Re-add all the implicit ops. - for (unsigned i = OpNo+1, e = Operands.size(); i != e; ++i) { + // All operands from OpNo have been removed from RegInfo. If the Operands + // backing store needs to be reallocated, we also need to remove any other + // register operands. + if (Reallocate) + for (unsigned i = 0; i != OpNo; ++i) + if (Operands[i].isReg()) + Operands[i].RemoveRegOperandFromRegInfo(); + + // Insert the new operand at OpNo. + Operands.insert(Operands.begin() + OpNo, Op); + Operands[OpNo].ParentMI = this; + + // The Operands backing store has now been reallocated, so we can re-add the + // operands before OpNo. + if (Reallocate) + for (unsigned i = 0; i != OpNo; ++i) + if (Operands[i].isReg()) + Operands[i].AddRegOperandToRegInfo(RegInfo); + + // When adding a register operand, tell RegInfo about it. + if (Operands[OpNo].isReg()) { + // Add the new operand to RegInfo, even when RegInfo is NULL. + // This will initialize the linked list pointers. + Operands[OpNo].AddRegOperandToRegInfo(RegInfo); + // If the register operand is flagged as early, mark the operand as such. + if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1) + Operands[OpNo].setIsEarlyClobber(true); + } + + // Re-add all the implicit ops. + if (RegInfo) { + for (unsigned i = OpNo + 1, e = Operands.size(); i != e; ++i) { assert(Operands[i].isReg() && "Should only be an implicit reg!"); Operands[i].AddRegOperandToRegInfo(RegInfo); } - } else { - // Otherwise, we will be reallocating the operand list. Remove all reg - // operands from their list, then readd them after the operand list is - // reallocated. - RemoveRegOperandsFromUseLists(); - - Operands.insert(Operands.begin()+OpNo, Op); - Operands[OpNo].ParentMI = this; - - // Re-add all the operands. - AddRegOperandsToUseLists(*RegInfo); - - // If the register operand is flagged as early, mark the operand as such - if (Operands[OpNo].isReg() - && MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1) - Operands[OpNo].setIsEarlyClobber(true); } } @@ -709,13 +684,13 @@ void MachineInstr::addOperand(const MachineOperand &Op) { /// void MachineInstr::RemoveOperand(unsigned OpNo) { assert(OpNo < Operands.size() && "Invalid operand number"); - + // Special case removing the last one. if (OpNo == Operands.size()-1) { // If needed, remove from the reg def/use list. if (Operands.back().isReg() && Operands.back().isOnRegUseList()) Operands.back().RemoveRegOperandFromRegInfo(); - + Operands.pop_back(); return; } @@ -730,7 +705,7 @@ void MachineInstr::RemoveOperand(unsigned OpNo) { Operands[i].RemoveRegOperandFromRegInfo(); } } - + Operands.erase(Operands.begin()+OpNo); if (RegInfo) { @@ -827,15 +802,6 @@ void MachineInstr::eraseFromParent() { } -/// OperandComplete - Return true if it's illegal to add a new operand -/// -bool MachineInstr::OperandsComplete() const { - unsigned short NumOperands = MCID->getNumOperands(); - if (!MCID->isVariadic() && getNumOperands()-NumImplicitOps >= NumOperands) - return true; // Broken: we have all the operands of this instruction! - return false; -} - /// getNumExplicitOperands - Returns the number of non-implicit operands. /// unsigned MachineInstr::getNumExplicitOperands() const { @@ -860,6 +826,67 @@ bool MachineInstr::isStackAligningInlineAsm() const { return false; } +int MachineInstr::findInlineAsmFlagIdx(unsigned OpIdx, + unsigned *GroupNo) const { + assert(isInlineAsm() && "Expected an inline asm instruction"); + assert(OpIdx < getNumOperands() && "OpIdx out of range"); + + // Ignore queries about the initial operands. + if (OpIdx < InlineAsm::MIOp_FirstOperand) + return -1; + + unsigned Group = 0; + unsigned NumOps; + for (unsigned i = InlineAsm::MIOp_FirstOperand, e = getNumOperands(); i < e; + i += NumOps) { + const MachineOperand &FlagMO = getOperand(i); + // If we reach the implicit register operands, stop looking. + if (!FlagMO.isImm()) + return -1; + NumOps = 1 + InlineAsm::getNumOperandRegisters(FlagMO.getImm()); + if (i + NumOps > OpIdx) { + if (GroupNo) + *GroupNo = Group; + return i; + } + ++Group; + } + return -1; +} + +const TargetRegisterClass* +MachineInstr::getRegClassConstraint(unsigned OpIdx, + const TargetInstrInfo *TII, + const TargetRegisterInfo *TRI) const { + // Most opcodes have fixed constraints in their MCInstrDesc. + if (!isInlineAsm()) + return TII->getRegClass(getDesc(), OpIdx, TRI); + + if (!getOperand(OpIdx).isReg()) + return NULL; + + // For tied uses on inline asm, get the constraint from the def. + unsigned DefIdx; + if (getOperand(OpIdx).isUse() && isRegTiedToDefOperand(OpIdx, &DefIdx)) + OpIdx = DefIdx; + + // Inline asm stores register class constraints in the flag word. + int FlagIdx = findInlineAsmFlagIdx(OpIdx); + if (FlagIdx < 0) + return NULL; + + unsigned Flag = getOperand(FlagIdx).getImm(); + unsigned RCID; + if (InlineAsm::hasRegClassConstraint(Flag, RCID)) + return TRI->getRegClass(RCID); + + // Assume that all registers in a memory operand are pointers. + if (InlineAsm::getKind(Flag) == InlineAsm::Kind_Mem) + return TRI->getPointerRegClass(); + + return NULL; +} + /// findRegisterUseOperandIdx() - Returns the MachineOperand that is a use of /// the specific register or -1 if it is not found. It further tightens /// the search criteria to a use that kills the register if isKill is true. @@ -901,7 +928,8 @@ MachineInstr::readsWritesVirtualRegister(unsigned Reg, Ops->push_back(i); if (MO.isUse()) Use |= !MO.isUndef(); - else if (MO.getSubReg()) + else if (MO.getSubReg() && !MO.isUndef()) + // A partial <def,undef> doesn't count as reading the register. PartDef = true; else FullDef = true; @@ -941,6 +969,10 @@ MachineInstr::findRegisterDefOperandIdx(unsigned Reg, bool isDead, bool Overlap, /// operand list that is used to represent the predicate. It returns -1 if /// none is found. int MachineInstr::findFirstPredOperandIdx() const { + // Don't call MCID.findFirstPredOperandIdx() because this variant + // is sometimes called on an instruction that's not yet complete, and + // so the number of operands is less than the MCID indicates. In + // particular, the PTX target does this. const MCInstrDesc &MCID = getDesc(); if (MCID.isPredicable()) { for (unsigned i = 0, e = getNumOperands(); i != e; ++i) @@ -950,7 +982,7 @@ int MachineInstr::findFirstPredOperandIdx() const { return -1; } - + /// isRegTiedToUseOperand - Given the index of a register def operand, /// check if the register def is tied to a source operand, due to either /// two-address elimination or inline assembly constraints. Returns the @@ -964,23 +996,13 @@ isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx) const { return false; // Determine the actual operand index that corresponds to this index. unsigned DefNo = 0; - unsigned DefPart = 0; - for (unsigned i = InlineAsm::MIOp_FirstOperand, e = getNumOperands(); - i < e; ) { - const MachineOperand &FMO = getOperand(i); - // After the normal asm operands there may be additional imp-def regs. - if (!FMO.isImm()) - return false; - // Skip over this def. - unsigned NumOps = InlineAsm::getNumOperandRegisters(FMO.getImm()); - unsigned PrevDef = i + 1; - i = PrevDef + NumOps; - if (i > DefOpIdx) { - DefPart = DefOpIdx - PrevDef; - break; - } - ++DefNo; - } + int FlagIdx = findInlineAsmFlagIdx(DefOpIdx, &DefNo); + if (FlagIdx < 0) + return false; + + // Which part of the group is DefOpIdx? + unsigned DefPart = DefOpIdx - (FlagIdx + 1); + for (unsigned i = InlineAsm::MIOp_FirstOperand, e = getNumOperands(); i != e; ++i) { const MachineOperand &FMO = getOperand(i); @@ -1024,20 +1046,10 @@ isRegTiedToDefOperand(unsigned UseOpIdx, unsigned *DefOpIdx) const { return false; // Find the flag operand corresponding to UseOpIdx - unsigned FlagIdx, NumOps=0; - for (FlagIdx = InlineAsm::MIOp_FirstOperand; - FlagIdx < UseOpIdx; FlagIdx += NumOps+1) { - const MachineOperand &UFMO = getOperand(FlagIdx); - // After the normal asm operands there may be additional imp-def regs. - if (!UFMO.isImm()) - return false; - NumOps = InlineAsm::getNumOperandRegisters(UFMO.getImm()); - assert(NumOps < getNumOperands() && "Invalid inline asm flag"); - if (UseOpIdx < FlagIdx+NumOps+1) - break; - } - if (FlagIdx >= UseOpIdx) + int FlagIdx = findInlineAsmFlagIdx(UseOpIdx); + if (FlagIdx < 0) return false; + const MachineOperand &UFMO = getOperand(FlagIdx); unsigned DefNo; if (InlineAsm::isUseOperandTiedToDef(UFMO.getImm(), DefNo)) { @@ -1211,7 +1223,7 @@ bool MachineInstr::hasVolatileMemoryRef() const { // conservatively assume it wasn't preserved. if (memoperands_empty()) return true; - + // Check the memory reference information for volatile references. for (mmo_iterator I = memoperands_begin(), E = memoperands_end(); I != E; ++I) if ((*I)->isVolatile()) @@ -1318,7 +1330,7 @@ void MachineInstr::dump() const { dbgs() << " " << *this; } -static void printDebugLoc(DebugLoc DL, const MachineFunction *MF, +static void printDebugLoc(DebugLoc DL, const MachineFunction *MF, raw_ostream &CommentOS) { const LLVMContext &Ctx = MF->getFunction()->getContext(); if (!DL.isUnknown()) { // Print source line info. @@ -1380,7 +1392,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { unsigned AsmDescOp = ~0u; unsigned AsmOpCount = 0; - if (isInlineAsm()) { + if (isInlineAsm() && e >= InlineAsm::MIOp_FirstOperand) { // Print asm string. OS << " "; getOperand(InlineAsm::MIOp_AsmString).print(OS, TM); @@ -1451,18 +1463,28 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { OS << '$' << AsmOpCount++; unsigned Flag = MO.getImm(); switch (InlineAsm::getKind(Flag)) { - case InlineAsm::Kind_RegUse: OS << ":[reguse]"; break; - case InlineAsm::Kind_RegDef: OS << ":[regdef]"; break; - case InlineAsm::Kind_RegDefEarlyClobber: OS << ":[regdef-ec]"; break; - case InlineAsm::Kind_Clobber: OS << ":[clobber]"; break; - case InlineAsm::Kind_Imm: OS << ":[imm]"; break; - case InlineAsm::Kind_Mem: OS << ":[mem]"; break; - default: OS << ":[??" << InlineAsm::getKind(Flag) << ']'; break; + case InlineAsm::Kind_RegUse: OS << ":[reguse"; break; + case InlineAsm::Kind_RegDef: OS << ":[regdef"; break; + case InlineAsm::Kind_RegDefEarlyClobber: OS << ":[regdef-ec"; break; + case InlineAsm::Kind_Clobber: OS << ":[clobber"; break; + case InlineAsm::Kind_Imm: OS << ":[imm"; break; + case InlineAsm::Kind_Mem: OS << ":[mem"; break; + default: OS << ":[??" << InlineAsm::getKind(Flag); break; + } + + unsigned RCID = 0; + if (InlineAsm::hasRegClassConstraint(Flag, RCID)) { + if (TM) + OS << ':' << TM->getRegisterInfo()->getRegClass(RCID)->getName(); + else + OS << ":RC" << RCID; } unsigned TiedTo = 0; if (InlineAsm::isUseOperandTiedToDef(Flag, TiedTo)) - OS << " [tiedto:$" << TiedTo << ']'; + OS << " tiedto:$" << TiedTo; + + OS << ']'; // Compute the index of the next operand descriptor. AsmDescOp += 1 + InlineAsm::getNumOperandRegisters(Flag); @@ -1516,7 +1538,19 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { } // Print debug location information. - if (!debugLoc.isUnknown() && MF) { + if (isDebugValue() && getOperand(e - 1).isMetadata()) { + if (!HaveSemi) OS << ";"; HaveSemi = true; + DIVariable DV(getOperand(e - 1).getMetadata()); + OS << " line no:" << DV.getLineNumber(); + if (MDNode *InlinedAt = DV.getInlinedAt()) { + DebugLoc InlinedAtDL = DebugLoc::getFromDILocation(InlinedAt); + if (!InlinedAtDL.isUnknown()) { + OS << " inlined @[ "; + printDebugLoc(InlinedAtDL, MF, OS); + OS << " ]"; + } + } + } else if (!debugLoc.isUnknown() && MF) { if (!HaveSemi) OS << ";"; HaveSemi = true; OS << " dbg:"; printDebugLoc(debugLoc, MF, OS); @@ -1627,7 +1661,7 @@ bool MachineInstr::addRegisterDead(unsigned IncomingReg, // new implicit operand if required. if (Found || !AddIfNotFound) return Found; - + addOperand(MachineOperand::CreateReg(IncomingReg, true /*IsDef*/, true /*IsImp*/, diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp index 722ceb2..a1f80d5 100644 --- a/lib/CodeGen/MachineLICM.cpp +++ b/lib/CodeGen/MachineLICM.cpp @@ -37,10 +37,16 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; +static cl::opt<bool> +AvoidSpeculation("avoid-speculation", + cl::desc("MachineLICM should avoid speculation"), + cl::init(false), cl::Hidden); + STATISTIC(NumHoisted, "Number of machine instructions hoisted out of loops"); STATISTIC(NumLowRP, @@ -91,6 +97,17 @@ namespace { // For each opcode, keep a list of potential CSE instructions. DenseMap<unsigned, std::vector<const MachineInstr*> > CSEMap; + enum { + SpeculateFalse = 0, + SpeculateTrue = 1, + SpeculateUnknown = 2 + }; + + // If a MBB does not dominate loop exiting blocks then it may not safe + // to hoist loads from this block. + // Tri-state: 0 - false, 1 - true, 2 - unknown + unsigned SpeculationState; + public: static char ID; // Pass identification, replacement for typeid MachineLICM() : @@ -194,6 +211,10 @@ namespace { /// hoist the given loop invariant. bool IsProfitableToHoist(MachineInstr &MI); + /// IsGuaranteedToExecute - Check if this mbb is guaranteed to execute. + /// If not then a load from this mbb may not be safe to hoist. + bool IsGuaranteedToExecute(MachineBasicBlock *BB); + /// HoistRegion - Walk the specified region of the CFG (defined by all /// blocks dominated by the specified block, and that are in the current /// loop) in depth first order w.r.t the DominatorTree. This allows us to @@ -202,6 +223,13 @@ namespace { /// void HoistRegion(MachineDomTreeNode *N, bool IsHeader = false); + /// getRegisterClassIDAndCost - For a given MI, register, and the operand + /// index, return the ID and cost of its representative register class by + /// reference. + void getRegisterClassIDAndCost(const MachineInstr *MI, + unsigned Reg, unsigned OpIdx, + unsigned &RCId, unsigned &RCCost) const; + /// InitRegPressure - Find all virtual register references that are liveout /// of the preheader to initialize the starting "register pressure". Note /// this does not count live through (livein but not used) registers. @@ -229,6 +257,10 @@ namespace { bool EliminateCSE(MachineInstr *MI, DenseMap<unsigned, std::vector<const MachineInstr*> >::iterator &CI); + /// MayCSE - Return true if the given instruction will be CSE'd if it's + /// hoisted out of the loop. + bool MayCSE(MachineInstr *MI); + /// Hoist - When an instruction is found to only use loop invariant operands /// that is safe to hoist, this instruction is called to do the dirty work. /// It returns true if the instruction is hoisted. @@ -441,6 +473,12 @@ void MachineLICM::HoistRegionPostRA() { const std::vector<MachineBasicBlock*> Blocks = CurLoop->getBlocks(); for (unsigned i = 0, e = Blocks.size(); i != e; ++i) { MachineBasicBlock *BB = Blocks[i]; + + // If the header of the loop containing this basic block is a landing pad, + // then don't try to hoist instructions out of this loop. + const MachineLoop *ML = MLI->getLoopFor(BB); + if (ML && ML->getHeader()->isLandingPad()) continue; + // Conservatively treat live-in's as an external def. // FIXME: That means a reload that're reused in successor block(s) will not // be LICM'ed. @@ -452,6 +490,7 @@ void MachineLICM::HoistRegionPostRA() { ++PhysRegDefs[*AS]; } + SpeculationState = SpeculateUnknown; for (MachineBasicBlock::iterator MII = BB->begin(), E = BB->end(); MII != E; ++MII) { MachineInstr *MI = &*MII; @@ -545,6 +584,27 @@ void MachineLICM::HoistPostRA(MachineInstr *MI, unsigned Def) { Changed = true; } +// IsGuaranteedToExecute - Check if this mbb is guaranteed to execute. +// If not then a load from this mbb may not be safe to hoist. +bool MachineLICM::IsGuaranteedToExecute(MachineBasicBlock *BB) { + if (SpeculationState != SpeculateUnknown) + return SpeculationState == SpeculateFalse; + + if (BB != CurLoop->getHeader()) { + // Check loop exiting blocks. + SmallVector<MachineBasicBlock*, 8> CurrentLoopExitingBlocks; + CurLoop->getExitingBlocks(CurrentLoopExitingBlocks); + for (unsigned i = 0, e = CurrentLoopExitingBlocks.size(); i != e; ++i) + if (!DT->dominates(BB, CurrentLoopExitingBlocks[i])) { + SpeculationState = SpeculateTrue; + return false; + } + } + + SpeculationState = SpeculateFalse; + return true; +} + /// HoistRegion - Walk the specified region of the CFG (defined by all blocks /// dominated by the specified block, and that are in the current loop) in depth /// first order w.r.t the DominatorTree. This allows us to visit definitions @@ -554,6 +614,11 @@ void MachineLICM::HoistRegion(MachineDomTreeNode *N, bool IsHeader) { assert(N != 0 && "Null dominator tree node?"); MachineBasicBlock *BB = N->getBlock(); + // If the header of the loop containing this basic block is a landing pad, + // then don't try to hoist instructions out of this loop. + const MachineLoop *ML = MLI->getLoopFor(BB); + if (ML && ML->getHeader()->isLandingPad()) return; + // If this subregion is not in the top level loop at all, exit. if (!CurLoop->contains(BB)) return; @@ -571,6 +636,7 @@ void MachineLICM::HoistRegion(MachineDomTreeNode *N, bool IsHeader) { // Remember livein register pressure. BackTrace.push_back(RegPressure); + SpeculationState = SpeculateUnknown; for (MachineBasicBlock::iterator MII = BB->begin(), E = BB->end(); MII != E; ) { MachineBasicBlock::iterator NextMII = MII; ++NextMII; @@ -596,6 +662,23 @@ static bool isOperandKill(const MachineOperand &MO, MachineRegisterInfo *MRI) { return MO.isKill() || MRI->hasOneNonDBGUse(MO.getReg()); } +/// getRegisterClassIDAndCost - For a given MI, register, and the operand +/// index, return the ID and cost of its representative register class. +void +MachineLICM::getRegisterClassIDAndCost(const MachineInstr *MI, + unsigned Reg, unsigned OpIdx, + unsigned &RCId, unsigned &RCCost) const { + const TargetRegisterClass *RC = MRI->getRegClass(Reg); + EVT VT = *RC->vt_begin(); + if (VT == MVT::untyped) { + RCId = RC->getID(); + RCCost = 1; + } else { + RCId = TLI->getRepRegClassFor(VT)->getID(); + RCCost = TLI->getRepRegClassCostFor(VT); + } +} + /// InitRegPressure - Find all virtual register references that are liveout of /// the preheader to initialize the starting "register pressure". Note this /// does not count live through (livein but not used) registers. @@ -625,18 +708,17 @@ void MachineLICM::InitRegPressure(MachineBasicBlock *BB) { continue; bool isNew = RegSeen.insert(Reg); - const TargetRegisterClass *RC = MRI->getRegClass(Reg); - EVT VT = *RC->vt_begin(); - unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); + unsigned RCId, RCCost; + getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); if (MO.isDef()) - RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); + RegPressure[RCId] += RCCost; else { bool isKill = isOperandKill(MO, MRI); if (isNew && !isKill) // Haven't seen this, it must be a livein. - RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); + RegPressure[RCId] += RCCost; else if (!isNew && isKill) - RegPressure[RCId] -= TLI->getRepRegClassCostFor(VT); + RegPressure[RCId] -= RCCost; } } } @@ -661,11 +743,8 @@ void MachineLICM::UpdateRegPressure(const MachineInstr *MI) { if (MO.isDef()) Defs.push_back(Reg); else if (!isNew && isOperandKill(MO, MRI)) { - const TargetRegisterClass *RC = MRI->getRegClass(Reg); - EVT VT = *RC->vt_begin(); - unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); - unsigned RCCost = TLI->getRepRegClassCostFor(VT); - + unsigned RCId, RCCost; + getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); if (RCCost > RegPressure[RCId]) RegPressure[RCId] = 0; else @@ -673,13 +752,13 @@ void MachineLICM::UpdateRegPressure(const MachineInstr *MI) { } } + unsigned Idx = 0; while (!Defs.empty()) { unsigned Reg = Defs.pop_back_val(); - const TargetRegisterClass *RC = MRI->getRegClass(Reg); - EVT VT = *RC->vt_begin(); - unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); - unsigned RCCost = TLI->getRepRegClassCostFor(VT); + unsigned RCId, RCCost; + getRegisterClassIDAndCost(MI, Reg, Idx, RCId, RCCost); RegPressure[RCId] += RCCost; + ++Idx; } } @@ -691,7 +770,14 @@ bool MachineLICM::IsLICMCandidate(MachineInstr &I) { bool DontMoveAcrossStore = true; if (!I.isSafeToMove(TII, AA, DontMoveAcrossStore)) return false; - + + // If it is load then check if it is guaranteed to execute by making sure that + // it dominates all exiting blocks. If it doesn't, then there is a path out of + // the loop which does not execute this load, so we can't hoist it. + // Stores and side effects are already checked by isSafeToMove. + if (I.getDesc().mayLoad() && !IsGuaranteedToExecute(I.getParent())) + return false; + return true; } @@ -879,10 +965,8 @@ void MachineLICM::UpdateBackTraceRegPressure(const MachineInstr *MI) { if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; - const TargetRegisterClass *RC = MRI->getRegClass(Reg); - EVT VT = *RC->vt_begin(); - unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); - unsigned RCCost = TLI->getRepRegClassCostFor(VT); + unsigned RCId, RCCost; + getRegisterClassIDAndCost(MI, Reg, i, RCId, RCCost); if (MO.isDef()) { DenseMap<unsigned, int>::iterator CI = Cost.find(RCId); if (CI != Cost.end()) @@ -941,16 +1025,15 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) { unsigned Reg = MO.getReg(); if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; + + unsigned RCId, RCCost; + getRegisterClassIDAndCost(&MI, Reg, i, RCId, RCCost); if (MO.isDef()) { if (HasHighOperandLatency(MI, i, Reg)) { ++NumHighLatency; return true; } - const TargetRegisterClass *RC = MRI->getRegClass(Reg); - EVT VT = *RC->vt_begin(); - unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); - unsigned RCCost = TLI->getRepRegClassCostFor(VT); DenseMap<unsigned, int>::iterator CI = Cost.find(RCId); if (CI != Cost.end()) CI->second += RCCost; @@ -960,10 +1043,6 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) { // Is a virtual register use is a kill, hoisting it out of the loop // may actually reduce register pressure or be register pressure // neutral. - const TargetRegisterClass *RC = MRI->getRegClass(Reg); - EVT VT = *RC->vt_begin(); - unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); - unsigned RCCost = TLI->getRepRegClassCostFor(VT); DenseMap<unsigned, int>::iterator CI = Cost.find(RCId); if (CI != Cost.end()) CI->second -= RCCost; @@ -979,6 +1058,13 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) { return true; } + // Do not "speculate" in high register pressure situation. If an + // instruction is not guaranteed to be executed in the loop, it's best to be + // conservative. + if (AvoidSpeculation && + (!IsGuaranteedToExecute(MI.getParent()) && !MayCSE(&MI))) + return false; + // High register pressure situation, only hoist if the instruction is going to // be remat'ed. if (!TII->isTriviallyReMaterializable(&MI, AA) && @@ -1116,6 +1202,20 @@ bool MachineLICM::EliminateCSE(MachineInstr *MI, return false; } +/// MayCSE - Return true if the given instruction will be CSE'd if it's +/// hoisted out of the loop. +bool MachineLICM::MayCSE(MachineInstr *MI) { + unsigned Opcode = MI->getOpcode(); + DenseMap<unsigned, std::vector<const MachineInstr*> >::iterator + CI = CSEMap.find(Opcode); + // Do not CSE implicit_def so ProcessImplicitDefs can properly propagate + // the undef property onto uses. + if (CI == CSEMap.end() || MI->isImplicitDef()) + return false; + + return LookForDuplicate(MI, CI->second) != 0; +} + /// Hoist - When an instruction is found to use only loop invariant operands /// that are safe to hoist, this instruction is called to do the dirty work. /// diff --git a/lib/CodeGen/MachineModuleInfo.cpp b/lib/CodeGen/MachineModuleInfo.cpp index fadc594..80c4854 100644 --- a/lib/CodeGen/MachineModuleInfo.cpp +++ b/lib/CodeGen/MachineModuleInfo.cpp @@ -17,9 +17,7 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" +#include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCSymbol.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/Support/Dwarf.h" @@ -254,11 +252,12 @@ void MMIAddrLabelMapCallbackPtr::allUsesReplacedWith(Value *V2) { //===----------------------------------------------------------------------===// MachineModuleInfo::MachineModuleInfo(const MCAsmInfo &MAI, - const TargetAsmInfo *TAI) -: ImmutablePass(ID), Context(MAI, TAI), - ObjFileMMI(0), - CurCallSite(0), CallsEHReturn(0), CallsUnwindInit(0), DbgInfoAvailable(false), - CallsExternalVAFunctionWithFloatingPointArguments(false) { + const MCRegisterInfo &MRI, + const MCObjectFileInfo *MOFI) + : ImmutablePass(ID), Context(MAI, MRI, MOFI), + ObjFileMMI(0), CompactUnwindEncoding(0), CurCallSite(0), CallsEHReturn(0), + CallsUnwindInit(0), DbgInfoAvailable(false), + CallsExternalVAFunctionWithFloatingPointArguments(false) { initializeMachineModuleInfoPass(*PassRegistry::getPassRegistry()); // Always emit some info, by default "no personality" info. Personalities.push_back(NULL); @@ -267,7 +266,8 @@ MachineModuleInfo::MachineModuleInfo(const MCAsmInfo &MAI, } MachineModuleInfo::MachineModuleInfo() -: ImmutablePass(ID), Context(*(MCAsmInfo*)0, NULL) { + : ImmutablePass(ID), + Context(*(MCAsmInfo*)0, *(MCRegisterInfo*)0, (MCObjectFileInfo*)0) { assert(0 && "This MachineModuleInfo constructor should never be called, MMI " "should always be explicitly constructed by LLVMTargetMachine"); abort(); @@ -311,6 +311,7 @@ void MachineModuleInfo::EndFunction() { FilterEnds.clear(); CallsEHReturn = 0; CallsUnwindInit = 0; + CompactUnwindEncoding = 0; VariableDbgInfo.clear(); } @@ -426,8 +427,9 @@ void MachineModuleInfo::addPersonality(MachineBasicBlock *LandingPad, /// addCatchTypeInfo - Provide the catch typeinfo for a landing pad. /// -void MachineModuleInfo::addCatchTypeInfo(MachineBasicBlock *LandingPad, - std::vector<const GlobalVariable *> &TyInfo) { +void MachineModuleInfo:: +addCatchTypeInfo(MachineBasicBlock *LandingPad, + ArrayRef<const GlobalVariable *> TyInfo) { LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad); for (unsigned N = TyInfo.size(); N; --N) LP.TypeIds.push_back(getTypeIDFor(TyInfo[N - 1])); @@ -435,8 +437,9 @@ void MachineModuleInfo::addCatchTypeInfo(MachineBasicBlock *LandingPad, /// addFilterTypeInfo - Provide the filter typeinfo for a landing pad. /// -void MachineModuleInfo::addFilterTypeInfo(MachineBasicBlock *LandingPad, - std::vector<const GlobalVariable *> &TyInfo) { +void MachineModuleInfo:: +addFilterTypeInfo(MachineBasicBlock *LandingPad, + ArrayRef<const GlobalVariable *> TyInfo) { LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad); std::vector<unsigned> IdsInFilter(TyInfo.size()); for (unsigned I = 0, E = TyInfo.size(); I != E; ++I) @@ -496,6 +499,14 @@ void MachineModuleInfo::TidyLandingPads(DenseMap<MCSymbol*, uintptr_t> *LPMap) { } } +/// setCallSiteLandingPad - Map the landing pad's EH symbol to the call site +/// indexes. +void MachineModuleInfo::setCallSiteLandingPad(MCSymbol *Sym, + ArrayRef<unsigned> Sites) { + for (unsigned I = 0, E = Sites.size(); I != E; ++I) + LPadToCallSiteMap[Sym].push_back(Sites[I]); +} + /// getTypeIDFor - Return the type id for the specified typeinfo. This is /// function wide. unsigned MachineModuleInfo::getTypeIDFor(const GlobalVariable *TI) { diff --git a/lib/CodeGen/MachineRegisterInfo.cpp b/lib/CodeGen/MachineRegisterInfo.cpp index 4b3e64c..266ebf6 100644 --- a/lib/CodeGen/MachineRegisterInfo.cpp +++ b/lib/CodeGen/MachineRegisterInfo.cpp @@ -14,10 +14,11 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Support/CommandLine.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; -MachineRegisterInfo::MachineRegisterInfo(const TargetRegisterInfo &TRI) { +MachineRegisterInfo::MachineRegisterInfo(const TargetRegisterInfo &TRI) + : TRI(&TRI), IsSSA(true) { VRegInfo.reserve(256); RegAllocHints.reserve(256); UsedPhysRegs.resize(TRI.getNumRegs()); @@ -48,18 +49,47 @@ MachineRegisterInfo::setRegClass(unsigned Reg, const TargetRegisterClass *RC) { const TargetRegisterClass * MachineRegisterInfo::constrainRegClass(unsigned Reg, - const TargetRegisterClass *RC) { + const TargetRegisterClass *RC, + unsigned MinNumRegs) { const TargetRegisterClass *OldRC = getRegClass(Reg); if (OldRC == RC) return RC; - const TargetRegisterClass *NewRC = getCommonSubClass(OldRC, RC); - if (!NewRC) + const TargetRegisterClass *NewRC = TRI->getCommonSubClass(OldRC, RC); + if (!NewRC || NewRC == OldRC) + return NewRC; + if (NewRC->getNumRegs() < MinNumRegs) return 0; - if (NewRC != OldRC) - setRegClass(Reg, NewRC); + setRegClass(Reg, NewRC); return NewRC; } +bool +MachineRegisterInfo::recomputeRegClass(unsigned Reg, const TargetMachine &TM) { + const TargetInstrInfo *TII = TM.getInstrInfo(); + const TargetRegisterClass *OldRC = getRegClass(Reg); + const TargetRegisterClass *NewRC = TRI->getLargestLegalSuperClass(OldRC); + + // Stop early if there is no room to grow. + if (NewRC == OldRC) + return false; + + // Accumulate constraints from all uses. + for (reg_nodbg_iterator I = reg_nodbg_begin(Reg), E = reg_nodbg_end(); I != E; + ++I) { + // TRI doesn't have accurate enough information to model this yet. + if (I.getOperand().getSubReg()) + return false; + const TargetRegisterClass *OpRC = + I->getRegClassConstraint(I.getOperandNo(), TII, TRI); + if (OpRC) + NewRC = TRI->getCommonSubClass(NewRC, OpRC); + if (!NewRC || NewRC == OldRC) + return false; + } + setRegClass(Reg, NewRC); + return true; +} + /// createVirtualRegister - Create and return a new virtual register in the /// function with the specified register class. /// diff --git a/lib/CodeGen/MachineSink.cpp b/lib/CodeGen/MachineSink.cpp index 916dff7..29cfb49 100644 --- a/lib/CodeGen/MachineSink.cpp +++ b/lib/CodeGen/MachineSink.cpp @@ -382,6 +382,25 @@ static bool AvoidsSinking(MachineInstr *MI, MachineRegisterInfo *MRI) { return MI->isInsertSubreg() || MI->isSubregToReg() || MI->isRegSequence(); } +/// collectDebgValues - Scan instructions following MI and collect any +/// matching DBG_VALUEs. +static void collectDebugValues(MachineInstr *MI, + SmallVector<MachineInstr *, 2> & DbgValues) { + DbgValues.clear(); + if (!MI->getOperand(0).isReg()) + return; + + MachineBasicBlock::iterator DI = MI; ++DI; + for (MachineBasicBlock::iterator DE = MI->getParent()->end(); + DI != DE; ++DI) { + if (!DI->isDebugValue()) + return; + if (DI->getOperand(0).isReg() && + DI->getOperand(0).getReg() == MI->getOperand(0).getReg()) + DbgValues.push_back(DI); + } +} + /// SinkInstruction - Determine whether it is safe to sink the specified machine /// instruction out of its current block into a successor. bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) { @@ -598,10 +617,22 @@ bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) { while (InsertPos != SuccToSinkTo->end() && InsertPos->isPHI()) ++InsertPos; + // collect matching debug values. + SmallVector<MachineInstr *, 2> DbgValuesToSink; + collectDebugValues(MI, DbgValuesToSink); + // Move the instruction. SuccToSinkTo->splice(InsertPos, ParentBlock, MI, ++MachineBasicBlock::iterator(MI)); + // Move debug values. + for (SmallVector<MachineInstr *, 2>::iterator DBI = DbgValuesToSink.begin(), + DBE = DbgValuesToSink.end(); DBI != DBE; ++DBI) { + MachineInstr *DbgMI = *DBI; + SuccToSinkTo->splice(InsertPos, ParentBlock, DbgMI, + ++MachineBasicBlock::iterator(DbgMI)); + } + // Conservatively, clear any kill flags, since it's possible that they are no // longer correct. MI->clearKillInfo(); diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp index 7a55852..26847d3 100644 --- a/lib/CodeGen/MachineVerifier.cpp +++ b/lib/CodeGen/MachineVerifier.cpp @@ -72,6 +72,8 @@ namespace { typedef DenseSet<unsigned> RegSet; typedef DenseMap<unsigned, const MachineInstr*> RegMap; + const MachineInstr *FirstTerminator; + BitVector regsReserved; RegSet regsLive; RegVector regsDefined, regsDead, regsKilled; @@ -389,6 +391,8 @@ static bool matchPair(MachineBasicBlock::const_succ_iterator i, void MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) { + FirstTerminator = 0; + // Count the number of landing pad successors. SmallPtrSet<MachineBasicBlock*, 4> LandingPadSuccs; for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(), @@ -570,6 +574,18 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { } } + // Ensure non-terminators don't follow terminators. + if (MCID.isTerminator()) { + if (!FirstTerminator) + FirstTerminator = MI; + } else if (FirstTerminator) { + report("Non-terminator instruction after the first terminator", MI); + *OS << "First terminator was:\t" << *FirstTerminator; + } + + StringRef ErrorInfo; + if (!TII->verifyInstruction(MI, ErrorInfo)) + report(ErrorInfo.data(), MI); } void @@ -686,6 +702,11 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { else addRegWithSubRegs(regsDefined, Reg); + // Verify SSA form. + if (MRI->isSSA() && TargetRegisterInfo::isVirtualRegister(Reg) && + llvm::next(MRI->def_begin(Reg)) != MRI->def_end()) + report("Multiple virtual register defs in SSA form", MO, MONum); + // Check LiveInts for a live range, but only for virtual registers. if (LiveInts && TargetRegisterInfo::isVirtualRegister(Reg) && !LiveInts->isNotInMIMap(MI)) { @@ -714,20 +735,14 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { unsigned SubIdx = MO->getSubReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) { - unsigned sr = Reg; if (SubIdx) { - unsigned s = TRI->getSubReg(Reg, SubIdx); - if (!s) { - report("Invalid subregister index for physical register", - MO, MONum); - return; - } - sr = s; + report("Illegal subregister index for physical register", MO, MONum); + return; } if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) { - if (!DRC->contains(sr)) { + if (!DRC->contains(Reg)) { report("Illegal physical register for instruction", MO, MONum); - *OS << TRI->getName(sr) << " is not a " + *OS << TRI->getName(Reg) << " is not a " << DRC->getName() << " register.\n"; } } @@ -735,16 +750,35 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { // Virtual register. const TargetRegisterClass *RC = MRI->getRegClass(Reg); if (SubIdx) { - const TargetRegisterClass *SRC = RC->getSubRegisterRegClass(SubIdx); + const TargetRegisterClass *SRC = + TRI->getSubClassWithSubReg(RC, SubIdx); if (!SRC) { report("Invalid subregister index for virtual register", MO, MONum); *OS << "Register class " << RC->getName() << " does not support subreg index " << SubIdx << "\n"; return; } - RC = SRC; + if (RC != SRC) { + report("Invalid register class for subregister index", MO, MONum); + *OS << "Register class " << RC->getName() + << " does not fully support subreg index " << SubIdx << "\n"; + return; + } } if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) { + if (SubIdx) { + const TargetRegisterClass *SuperRC = + TRI->getLargestLegalSuperClass(RC); + if (!SuperRC) { + report("No largest legal super class exists.", MO, MONum); + return; + } + DRC = TRI->getMatchingSuperRegClass(SuperRC, DRC, SubIdx); + if (!DRC) { + report("No matching super-reg register class.", MO, MONum); + return; + } + } if (!RC->hasSuperClassEq(DRC)) { report("Illegal virtual register for instruction", MO, MONum); *OS << "Expected a " << DRC->getName() << " register, but got a " @@ -1161,18 +1195,8 @@ void MachineVerifier::verifyLiveIntervals() { SlotIndex PEnd = LiveInts->getMBBEndIdx(*PI).getPrevSlot(); const VNInfo *PVNI = LI.getVNInfoAt(PEnd); - if (VNI->isPHIDef() && VNI->def == LiveInts->getMBBStartIdx(MFI)) { - if (PVNI && !PVNI->hasPHIKill()) { - report("Value live out of predecessor doesn't have PHIKill", MF); - *OS << "Valno #" << PVNI->id << " live out of BB#" - << (*PI)->getNumber() << '@' << PEnd - << " doesn't have PHIKill, but Valno #" << VNI->id - << " is PHIDef and defined at the beginning of BB#" - << MFI->getNumber() << '@' << LiveInts->getMBBStartIdx(MFI) - << " in " << LI << '\n'; - } + if (VNI->isPHIDef() && VNI->def == LiveInts->getMBBStartIdx(MFI)) continue; - } if (!PVNI) { report("Register not marked live out of predecessor", *PI); diff --git a/lib/CodeGen/PHIElimination.cpp b/lib/CodeGen/PHIElimination.cpp index af65f13..6994aa5 100644 --- a/lib/CodeGen/PHIElimination.cpp +++ b/lib/CodeGen/PHIElimination.cpp @@ -109,6 +109,9 @@ bool PHIElimination::runOnMachineFunction(MachineFunction &MF) { bool Changed = false; + // This pass takes the function out of SSA form. + MRI->leaveSSA(); + // Split critical edges to help the coalescer if (!DisableEdgeSplitting) { if (LiveVariables *LV = getAnalysisIfAvailable<LiveVariables>()) { diff --git a/lib/CodeGen/PeepholeOptimizer.cpp b/lib/CodeGen/PeepholeOptimizer.cpp index c523e39..bbc7ce2 100644 --- a/lib/CodeGen/PeepholeOptimizer.cpp +++ b/lib/CodeGen/PeepholeOptimizer.cpp @@ -295,7 +295,6 @@ bool PeepholeOptimizer::OptimizeBitcastInstr(MachineInstr *MI, if (!DefMI || !DefMI->getDesc().isBitcast()) return false; - unsigned SrcDef = 0; unsigned SrcSrc = 0; NumDefs = DefMI->getDesc().getNumDefs(); NumSrcs = DefMI->getDesc().getNumOperands() - NumDefs; @@ -308,13 +307,13 @@ bool PeepholeOptimizer::OptimizeBitcastInstr(MachineInstr *MI, unsigned Reg = MO.getReg(); if (!Reg) continue; - if (MO.isDef()) - SrcDef = Reg; - else if (SrcSrc) - // Multiple sources? - return false; - else - SrcSrc = Reg; + if (!MO.isDef()) { + if (SrcSrc) + // Multiple sources? + return false; + else + SrcSrc = Reg; + } } if (MRI->getRegClass(SrcSrc) != MRI->getRegClass(Def)) @@ -434,6 +433,7 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) { if (MCID.isBitcast()) { if (OptimizeBitcastInstr(MI, MBB)) { // MI is deleted. + LocalMIs.erase(MI); Changed = true; MII = First ? I->begin() : llvm::next(PMII); continue; @@ -441,6 +441,7 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) { } else if (MCID.isCompare()) { if (OptimizeCmpInstr(MI, MBB)) { // MI is deleted. + LocalMIs.erase(MI); Changed = true; MII = First ? I->begin() : llvm::next(PMII); continue; diff --git a/lib/CodeGen/ProcessImplicitDefs.cpp b/lib/CodeGen/ProcessImplicitDefs.cpp index c04d656..b1d8c97 100644 --- a/lib/CodeGen/ProcessImplicitDefs.cpp +++ b/lib/CodeGen/ProcessImplicitDefs.cpp @@ -125,8 +125,14 @@ bool ProcessImplicitDefs::runOnMachineFunction(MachineFunction &fn) { LiveVariables::VarInfo& vi = LV->getVarInfo(MO.getReg()); vi.removeKill(MI); } + unsigned Reg = MI->getOperand(0).getReg(); MI->eraseFromParent(); Changed = true; + + // A REG_SEQUENCE may have been expanded into partial definitions. + // If this was the last one, mark Reg as implicitly defined. + if (TargetRegisterInfo::isVirtualRegister(Reg) && MRI->def_empty(Reg)) + ImpDefRegs.insert(Reg); continue; } } diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp index a901c5f..32c9325 100644 --- a/lib/CodeGen/PrologEpilogInserter.cpp +++ b/lib/CodeGen/PrologEpilogInserter.cpp @@ -29,6 +29,7 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetInstrInfo.h" @@ -54,6 +55,8 @@ INITIALIZE_PASS_END(PEI, "prologepilog", STATISTIC(NumVirtualFrameRegs, "Number of virtual frame regs encountered"); STATISTIC(NumScavengedRegs, "Number of frame index regs scavenged"); +STATISTIC(NumBytesStackSpace, + "Number of bytes used for stack in all functions"); /// createPrologEpilogCodeInserter - This function returns a pass that inserts /// prolog and epilog code, and eliminates abstract frame references. @@ -677,7 +680,9 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { } // Update frame info to pretend that this is part of the stack... - MFI->setStackSize(Offset - LocalAreaOffset); + int64_t StackSize = Offset - LocalAreaOffset; + MFI->setStackSize(StackSize); + NumBytesStackSpace += StackSize; } /// insertPrologEpilogCode - Scan the function for modified callee saved @@ -696,6 +701,13 @@ void PEI::insertPrologEpilogCode(MachineFunction &Fn) { if (!I->empty() && I->back().getDesc().isReturn()) TFI.emitEpilogue(Fn, *I); } + + // Emit additional code that is required to support segmented stacks, if + // we've been asked for it. This, when linked with a runtime with support + // for segmented stacks (libgcc is one), will result in allocating stack + // space in small chunks instead of one large contiguous block. + if (EnableSegmentedStacks) + TFI.adjustForSegmentedStacks(Fn); } /// replaceFrameIndices - Replace all MO_FrameIndex operands with physical diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp index 5ea26ad..5496d69 100644 --- a/lib/CodeGen/RegAllocBasic.cpp +++ b/lib/CodeGen/RegAllocBasic.cpp @@ -20,7 +20,6 @@ #include "RenderMachineFunction.h" #include "Spiller.h" #include "VirtRegMap.h" -#include "RegisterCoalescer.h" #include "llvm/ADT/OwningPtr.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" @@ -160,7 +159,7 @@ void RABasic::getAnalysisUsage(AnalysisUsage &AU) const { AU.addPreserved<LiveDebugVariables>(); if (StrongPHIElim) AU.addRequiredID(StrongPHIEliminationID); - AU.addRequiredTransitive<RegisterCoalescer>(); + AU.addRequiredTransitiveID(RegisterCoalescerPassID); AU.addRequired<CalculateSpillWeights>(); AU.addRequired<LiveStacks>(); AU.addPreserved<LiveStacks>(); @@ -439,6 +438,7 @@ void RegAllocBase::addMBBLiveIns(MachineFunction *MF) { LiveIntervalUnion &LiveUnion = PhysReg2LiveUnion[PhysReg]; if (LiveUnion.empty()) continue; + DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " live-in:"); MachineFunction::iterator MBB = llvm::next(MF->begin()); MachineFunction::iterator MFE = MF->end(); SlotIndex Start, Stop; @@ -449,6 +449,8 @@ void RegAllocBase::addMBBLiveIns(MachineFunction *MF) { if (SI.start() <= Start) { if (!MBB->isLiveIn(PhysReg)) MBB->addLiveIn(PhysReg); + DEBUG(dbgs() << "\tBB#" << MBB->getNumber() << ':' + << PrintReg(SI.value()->reg, TRI)); } else if (SI.start() > Stop) MBB = Indexes->getMBBFromIndex(SI.start().getPrevIndex()); if (++MBB == MFE) @@ -456,6 +458,7 @@ void RegAllocBase::addMBBLiveIns(MachineFunction *MF) { tie(Start, Stop) = Indexes->getMBBRange(MBB); SI.advanceTo(Start); } + DEBUG(dbgs() << '\n'); } } @@ -495,8 +498,9 @@ unsigned RABasic::selectOrSplit(LiveInterval &VirtReg, // Found an available register. return PhysReg; } + Queries[interfReg].collectInterferingVRegs(1); LiveInterval *interferingVirtReg = - Queries[interfReg].firstInterference().liveUnionPos().value(); + Queries[interfReg].interferingVRegs().front(); // The current VirtReg must either be spillable, or one of its interferences // must have less spill weight. diff --git a/lib/CodeGen/RegAllocGreedy.cpp b/lib/CodeGen/RegAllocGreedy.cpp index e235e87..f54a2c8 100644 --- a/lib/CodeGen/RegAllocGreedy.cpp +++ b/lib/CodeGen/RegAllocGreedy.cpp @@ -22,7 +22,6 @@ #include "SpillPlacement.h" #include "SplitKit.h" #include "VirtRegMap.h" -#include "RegisterCoalescer.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Function.h" @@ -38,6 +37,7 @@ #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/Target/TargetOptions.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" @@ -51,6 +51,15 @@ STATISTIC(NumGlobalSplits, "Number of split global live ranges"); STATISTIC(NumLocalSplits, "Number of split local live ranges"); STATISTIC(NumEvicted, "Number of interferences evicted"); +static cl::opt<SplitEditor::ComplementSpillMode> +SplitSpillMode("split-spill-mode", cl::Hidden, + cl::desc("Spill mode for splitting live ranges"), + cl::values(clEnumValN(SplitEditor::SM_Partition, "default", "Default"), + clEnumValN(SplitEditor::SM_Size, "size", "Optimize for size"), + clEnumValN(SplitEditor::SM_Speed, "speed", "Optimize for speed"), + clEnumValEnd), + cl::init(SplitEditor::SM_Partition)); + static RegisterRegAlloc greedyRegAlloc("greedy", "greedy register allocator", createGreedyRegisterAllocator); @@ -90,12 +99,26 @@ class RAGreedy : public MachineFunctionPass, // range splitting algorithm terminates, something that is otherwise hard to // ensure. enum LiveRangeStage { - RS_New, ///< Never seen before. - RS_First, ///< First time in the queue. - RS_Second, ///< Second time in the queue. - RS_Global, ///< Produced by global splitting. - RS_Local, ///< Produced by local splitting. - RS_Spill ///< Produced by spilling. + /// Newly created live range that has never been queued. + RS_New, + + /// Only attempt assignment and eviction. Then requeue as RS_Split. + RS_Assign, + + /// Attempt live range splitting if assignment is impossible. + RS_Split, + + /// Attempt more aggressive live range splitting that is guaranteed to make + /// progress. This is used for split products that may not be making + /// progress. + RS_Split2, + + /// Live range will be spilled. No more splitting will be attempted. + RS_Spill, + + /// There is nothing more we can do to this live range. Abort compilation + /// if it can't be assigned. + RS_Done }; static const char *const StageName[]; @@ -157,17 +180,38 @@ class RAGreedy : public MachineFunctionPass, /// Global live range splitting candidate info. struct GlobalSplitCandidate { + // Register intended for assignment, or 0. unsigned PhysReg; + + // SplitKit interval index for this candidate. + unsigned IntvIdx; + + // Interference for PhysReg. InterferenceCache::Cursor Intf; + + // Bundles where this candidate should be live. BitVector LiveBundles; SmallVector<unsigned, 8> ActiveBlocks; void reset(InterferenceCache &Cache, unsigned Reg) { PhysReg = Reg; + IntvIdx = 0; Intf.setPhysReg(Cache, Reg); LiveBundles.clear(); ActiveBlocks.clear(); } + + // Set B[i] = C for every live bundle where B[i] was NoCand. + unsigned getBundles(SmallVectorImpl<unsigned> &B, unsigned C) { + unsigned Count = 0; + for (int i = LiveBundles.find_first(); i >= 0; + i = LiveBundles.find_next(i)) + if (B[i] == NoCand) { + B[i] = C; + Count++; + } + return Count; + } }; /// Candidate info for for each PhysReg in AllocationOrder. @@ -175,6 +219,12 @@ class RAGreedy : public MachineFunctionPass, /// class. SmallVector<GlobalSplitCandidate, 32> GlobalCand; + enum { NoCand = ~0u }; + + /// Candidate map. Each edge bundle is assigned to a GlobalCand entry, or to + /// NoCand which indicates the stack interval. + SmallVector<unsigned, 32> BundleCand; + public: RAGreedy(); @@ -208,8 +258,8 @@ private: void addThroughConstraints(InterferenceCache::Cursor, ArrayRef<unsigned>); void growRegion(GlobalSplitCandidate &Cand); float calcGlobalSplitCost(GlobalSplitCandidate&); - void splitAroundRegion(LiveInterval&, GlobalSplitCandidate&, - SmallVectorImpl<LiveInterval*>&); + bool calcCompactRegion(GlobalSplitCandidate&); + void splitAroundRegion(LiveRangeEdit&, ArrayRef<unsigned>); void calcGapWeights(unsigned, SmallVectorImpl<float>&); bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool); bool canEvictInterference(LiveInterval&, unsigned, bool, EvictionCost&); @@ -222,6 +272,8 @@ private: SmallVectorImpl<LiveInterval*>&, unsigned = ~0u); unsigned tryRegionSplit(LiveInterval&, AllocationOrder&, SmallVectorImpl<LiveInterval*>&); + unsigned tryBlockSplit(LiveInterval&, AllocationOrder&, + SmallVectorImpl<LiveInterval*>&); unsigned tryLocalSplit(LiveInterval&, AllocationOrder&, SmallVectorImpl<LiveInterval*>&); unsigned trySplit(LiveInterval&, AllocationOrder&, @@ -233,12 +285,12 @@ char RAGreedy::ID = 0; #ifndef NDEBUG const char *const RAGreedy::StageName[] = { - "RS_New", - "RS_First", - "RS_Second", - "RS_Global", - "RS_Local", - "RS_Spill" + "RS_New", + "RS_Assign", + "RS_Split", + "RS_Split2", + "RS_Spill", + "RS_Done" }; #endif @@ -278,7 +330,7 @@ void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const { AU.addPreserved<LiveDebugVariables>(); if (StrongPHIElim) AU.addRequiredID(StrongPHIEliminationID); - AU.addRequiredTransitive<RegisterCoalescer>(); + AU.addRequiredTransitiveID(RegisterCoalescerPassID); AU.addRequired<CalculateSpillWeights>(); AU.addRequired<LiveStacks>(); AU.addPreserved<LiveStacks>(); @@ -325,9 +377,15 @@ void RAGreedy::LRE_WillShrinkVirtReg(unsigned VirtReg) { } void RAGreedy::LRE_DidCloneVirtReg(unsigned New, unsigned Old) { + // Cloning a register we haven't even heard about yet? Just ignore it. + if (!ExtraRegInfo.inBounds(Old)) + return; + // LRE may clone a virtual register because dead code elimination causes it to - // be split into connected components. Ensure that the new register gets the + // be split into connected components. The new components are much smaller + // than the original, so they should get a new chance at being assigned. // same stage as the parent. + ExtraRegInfo[Old].Stage = RS_Assign; ExtraRegInfo.grow(New); ExtraRegInfo[New] = ExtraRegInfo[Old]; } @@ -350,16 +408,15 @@ void RAGreedy::enqueue(LiveInterval *LI) { ExtraRegInfo.grow(Reg); if (ExtraRegInfo[Reg].Stage == RS_New) - ExtraRegInfo[Reg].Stage = RS_First; + ExtraRegInfo[Reg].Stage = RS_Assign; - if (ExtraRegInfo[Reg].Stage == RS_Second) + if (ExtraRegInfo[Reg].Stage == RS_Split) { // Unsplit ranges that couldn't be allocated immediately are deferred until - // everything else has been allocated. Long ranges are allocated last so - // they are split against realistic interference. - Prio = (1u << 31) - Size; - else { - // Everything else is allocated in long->short order. Long ranges that don't - // fit should be spilled ASAP so they don't create interference. + // everything else has been allocated. + Prio = Size; + } else { + // Everything is allocated in long->short order. Long ranges that don't fit + // should be spilled (or split) ASAP so they don't create interference. Prio = (1u << 31) + Size; // Boost ranges that have a physical register hint. @@ -442,7 +499,7 @@ unsigned RAGreedy::tryAssign(LiveInterval &VirtReg, /// @param BreaksHint True when B is already assigned to its preferred register. bool RAGreedy::shouldEvict(LiveInterval &A, bool IsHint, LiveInterval &B, bool BreaksHint) { - bool CanSplit = getStage(B) <= RS_Second; + bool CanSplit = getStage(B) < RS_Spill; // Be fairly aggressive about following hints as long as the evictee can be // split. @@ -487,7 +544,7 @@ bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, unsigned PhysReg, if (TargetRegisterInfo::isPhysicalRegister(Intf->reg)) return false; // Never evict spill products. They cannot split or spill. - if (getStage(*Intf) == RS_Spill) + if (getStage(*Intf) == RS_Done) return false; // Once a live range becomes small enough, it is urgent that we find a // register for it. This is indicated by an infinite spill weight. These @@ -627,6 +684,7 @@ bool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf, Intf.moveToBlock(BC.Number); BC.Entry = BI.LiveIn ? SpillPlacement::PrefReg : SpillPlacement::DontCare; BC.Exit = BI.LiveOut ? SpillPlacement::PrefReg : SpillPlacement::DontCare; + BC.ChangesValue = BI.FirstDef; if (!Intf.hasInterference()) continue; @@ -638,9 +696,9 @@ bool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf, if (BI.LiveIn) { if (Intf.first() <= Indexes->getMBBStartIdx(BC.Number)) BC.Entry = SpillPlacement::MustSpill, ++Ins; - else if (Intf.first() < BI.FirstUse) + else if (Intf.first() < BI.FirstInstr) BC.Entry = SpillPlacement::PrefSpill, ++Ins; - else if (Intf.first() < BI.LastUse) + else if (Intf.first() < BI.LastInstr) ++Ins; } @@ -648,9 +706,9 @@ bool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf, if (BI.LiveOut) { if (Intf.last() >= SA->getLastSplitPoint(BC.Number)) BC.Exit = SpillPlacement::MustSpill, ++Ins; - else if (Intf.last() > BI.LastUse) + else if (Intf.last() > BI.LastInstr) BC.Exit = SpillPlacement::PrefSpill, ++Ins; - else if (Intf.last() > BI.FirstUse) + else if (Intf.last() > BI.FirstInstr) ++Ins; } @@ -684,7 +742,7 @@ void RAGreedy::addThroughConstraints(InterferenceCache::Cursor Intf, assert(T < GroupSize && "Array overflow"); TBS[T] = Number; if (++T == GroupSize) { - SpillPlacer->addLinks(ArrayRef<unsigned>(TBS, T)); + SpillPlacer->addLinks(makeArrayRef(TBS, T)); T = 0; } continue; @@ -714,7 +772,7 @@ void RAGreedy::addThroughConstraints(InterferenceCache::Cursor Intf, ArrayRef<SpillPlacement::BlockConstraint> Array(BCS, B); SpillPlacer->addConstraints(Array); - SpillPlacer->addLinks(ArrayRef<unsigned>(TBS, T)); + SpillPlacer->addLinks(makeArrayRef(TBS, T)); } void RAGreedy::growRegion(GlobalSplitCandidate &Cand) { @@ -749,8 +807,16 @@ void RAGreedy::growRegion(GlobalSplitCandidate &Cand) { // Any new blocks to add? if (ActiveBlocks.size() == AddedTo) break; - addThroughConstraints(Cand.Intf, - ArrayRef<unsigned>(ActiveBlocks).slice(AddedTo)); + + // Compute through constraints from the interference, or assume that all + // through blocks prefer spilling when forming compact regions. + ArrayRef<unsigned> NewBlocks = makeArrayRef(ActiveBlocks).slice(AddedTo); + if (Cand.PhysReg) + addThroughConstraints(Cand.Intf, NewBlocks); + else + // Provide a strong negative bias on through blocks to prevent unwanted + // liveness on loop backedges. + SpillPlacer->addPrefSpill(NewBlocks, /* Strong= */ true); AddedTo = ActiveBlocks.size(); // Perhaps iterating can enable more bundles? @@ -759,11 +825,55 @@ void RAGreedy::growRegion(GlobalSplitCandidate &Cand) { DEBUG(dbgs() << ", v=" << Visited); } +/// calcCompactRegion - Compute the set of edge bundles that should be live +/// when splitting the current live range into compact regions. Compact +/// regions can be computed without looking at interference. They are the +/// regions formed by removing all the live-through blocks from the live range. +/// +/// Returns false if the current live range is already compact, or if the +/// compact regions would form single block regions anyway. +bool RAGreedy::calcCompactRegion(GlobalSplitCandidate &Cand) { + // Without any through blocks, the live range is already compact. + if (!SA->getNumThroughBlocks()) + return false; + + // Compact regions don't correspond to any physreg. + Cand.reset(IntfCache, 0); + + DEBUG(dbgs() << "Compact region bundles"); + + // Use the spill placer to determine the live bundles. GrowRegion pretends + // that all the through blocks have interference when PhysReg is unset. + SpillPlacer->prepare(Cand.LiveBundles); + + // The static split cost will be zero since Cand.Intf reports no interference. + float Cost; + if (!addSplitConstraints(Cand.Intf, Cost)) { + DEBUG(dbgs() << ", none.\n"); + return false; + } + + growRegion(Cand); + SpillPlacer->finish(); + + if (!Cand.LiveBundles.any()) { + DEBUG(dbgs() << ", none.\n"); + return false; + } + + DEBUG({ + for (int i = Cand.LiveBundles.find_first(); i>=0; + i = Cand.LiveBundles.find_next(i)) + dbgs() << " EB#" << i; + dbgs() << ".\n"; + }); + return true; +} + /// calcSpillCost - Compute how expensive it would be to split the live range in /// SA around all use blocks instead of forming bundle regions. float RAGreedy::calcSpillCost() { float Cost = 0; - const LiveInterval &LI = SA->getParent(); ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); for (unsigned i = 0; i != UseBlocks.size(); ++i) { const SplitAnalysis::BlockInfo &BI = UseBlocks[i]; @@ -772,16 +882,8 @@ float RAGreedy::calcSpillCost() { Cost += SpillPlacer->getBlockFrequency(Number); // Unless the value is redefined in the block. - if (BI.LiveIn && BI.LiveOut) { - SlotIndex Start, Stop; - tie(Start, Stop) = Indexes->getMBBRange(Number); - LiveInterval::const_iterator I = LI.find(Start); - assert(I != LI.end() && "Expected live-in value"); - // Is there a different live-out value? If so, we need an extra spill - // instruction. - if (I->end < Stop) - Cost += SpillPlacer->getBlockFrequency(Number); - } + if (BI.LiveIn && BI.LiveOut && BI.FirstDef) + Cost += SpillPlacer->getBlockFrequency(Number); } return Cost; } @@ -828,81 +930,115 @@ float RAGreedy::calcGlobalSplitCost(GlobalSplitCandidate &Cand) { return GlobalCost; } -/// splitAroundRegion - Split VirtReg around the region determined by -/// LiveBundles. Make an effort to avoid interference from PhysReg. +/// splitAroundRegion - Split the current live range around the regions +/// determined by BundleCand and GlobalCand. /// -/// The 'register' interval is going to contain as many uses as possible while -/// avoiding interference. The 'stack' interval is the complement constructed by -/// SplitEditor. It will contain the rest. +/// Before calling this function, GlobalCand and BundleCand must be initialized +/// so each bundle is assigned to a valid candidate, or NoCand for the +/// stack-bound bundles. The shared SA/SE SplitAnalysis and SplitEditor +/// objects must be initialized for the current live range, and intervals +/// created for the used candidates. /// -void RAGreedy::splitAroundRegion(LiveInterval &VirtReg, - GlobalSplitCandidate &Cand, - SmallVectorImpl<LiveInterval*> &NewVRegs) { - const BitVector &LiveBundles = Cand.LiveBundles; - - DEBUG({ - dbgs() << "Splitting around region for " << PrintReg(Cand.PhysReg, TRI) - << " with bundles"; - for (int i = LiveBundles.find_first(); i>=0; i = LiveBundles.find_next(i)) - dbgs() << " EB#" << i; - dbgs() << ".\n"; - }); - - InterferenceCache::Cursor &Intf = Cand.Intf; - LiveRangeEdit LREdit(VirtReg, NewVRegs, this); - SE->reset(LREdit); - - // Create the main cross-block interval. - const unsigned MainIntv = SE->openIntv(); +/// @param LREdit The LiveRangeEdit object handling the current split. +/// @param UsedCands List of used GlobalCand entries. Every BundleCand value +/// must appear in this list. +void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit, + ArrayRef<unsigned> UsedCands) { + // These are the intervals created for new global ranges. We may create more + // intervals for local ranges. + const unsigned NumGlobalIntvs = LREdit.size(); + DEBUG(dbgs() << "splitAroundRegion with " << NumGlobalIntvs << " globals.\n"); + assert(NumGlobalIntvs && "No global intervals configured"); + + // Isolate even single instructions when dealing with a proper sub-class. + // That guarantees register class inflation for the stack interval because it + // is all copies. + unsigned Reg = SA->getParent().reg; + bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg)); // First handle all the blocks with uses. ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); for (unsigned i = 0; i != UseBlocks.size(); ++i) { const SplitAnalysis::BlockInfo &BI = UseBlocks[i]; - bool RegIn = BI.LiveIn && - LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 0)]; - bool RegOut = BI.LiveOut && - LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 1)]; + unsigned Number = BI.MBB->getNumber(); + unsigned IntvIn = 0, IntvOut = 0; + SlotIndex IntfIn, IntfOut; + if (BI.LiveIn) { + unsigned CandIn = BundleCand[Bundles->getBundle(Number, 0)]; + if (CandIn != NoCand) { + GlobalSplitCandidate &Cand = GlobalCand[CandIn]; + IntvIn = Cand.IntvIdx; + Cand.Intf.moveToBlock(Number); + IntfIn = Cand.Intf.first(); + } + } + if (BI.LiveOut) { + unsigned CandOut = BundleCand[Bundles->getBundle(Number, 1)]; + if (CandOut != NoCand) { + GlobalSplitCandidate &Cand = GlobalCand[CandOut]; + IntvOut = Cand.IntvIdx; + Cand.Intf.moveToBlock(Number); + IntfOut = Cand.Intf.last(); + } + } // Create separate intervals for isolated blocks with multiple uses. - if (!RegIn && !RegOut) { + if (!IntvIn && !IntvOut) { DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " isolated.\n"); - if (!BI.isOneInstr()) { + if (SA->shouldSplitSingleBlock(BI, SingleInstrs)) SE->splitSingleBlock(BI); - SE->selectIntv(MainIntv); - } continue; } - Intf.moveToBlock(BI.MBB->getNumber()); - - if (RegIn && RegOut) - SE->splitLiveThroughBlock(BI.MBB->getNumber(), - MainIntv, Intf.first(), - MainIntv, Intf.last()); - else if (RegIn) - SE->splitRegInBlock(BI, MainIntv, Intf.first()); + if (IntvIn && IntvOut) + SE->splitLiveThroughBlock(Number, IntvIn, IntfIn, IntvOut, IntfOut); + else if (IntvIn) + SE->splitRegInBlock(BI, IntvIn, IntfIn); else - SE->splitRegOutBlock(BI, MainIntv, Intf.last()); + SE->splitRegOutBlock(BI, IntvOut, IntfOut); } - // Handle live-through blocks. - for (unsigned i = 0, e = Cand.ActiveBlocks.size(); i != e; ++i) { - unsigned Number = Cand.ActiveBlocks[i]; - bool RegIn = LiveBundles[Bundles->getBundle(Number, 0)]; - bool RegOut = LiveBundles[Bundles->getBundle(Number, 1)]; - if (!RegIn && !RegOut) - continue; - Intf.moveToBlock(Number); - SE->splitLiveThroughBlock(Number, RegIn ? MainIntv : 0, Intf.first(), - RegOut ? MainIntv : 0, Intf.last()); + // Handle live-through blocks. The relevant live-through blocks are stored in + // the ActiveBlocks list with each candidate. We need to filter out + // duplicates. + BitVector Todo = SA->getThroughBlocks(); + for (unsigned c = 0; c != UsedCands.size(); ++c) { + ArrayRef<unsigned> Blocks = GlobalCand[UsedCands[c]].ActiveBlocks; + for (unsigned i = 0, e = Blocks.size(); i != e; ++i) { + unsigned Number = Blocks[i]; + if (!Todo.test(Number)) + continue; + Todo.reset(Number); + + unsigned IntvIn = 0, IntvOut = 0; + SlotIndex IntfIn, IntfOut; + + unsigned CandIn = BundleCand[Bundles->getBundle(Number, 0)]; + if (CandIn != NoCand) { + GlobalSplitCandidate &Cand = GlobalCand[CandIn]; + IntvIn = Cand.IntvIdx; + Cand.Intf.moveToBlock(Number); + IntfIn = Cand.Intf.first(); + } + + unsigned CandOut = BundleCand[Bundles->getBundle(Number, 1)]; + if (CandOut != NoCand) { + GlobalSplitCandidate &Cand = GlobalCand[CandOut]; + IntvOut = Cand.IntvIdx; + Cand.Intf.moveToBlock(Number); + IntfOut = Cand.Intf.last(); + } + if (!IntvIn && !IntvOut) + continue; + SE->splitLiveThroughBlock(Number, IntvIn, IntfIn, IntvOut, IntfOut); + } } ++NumGlobalSplits; SmallVector<unsigned, 8> IntvMap; SE->finish(&IntvMap); - DebugVars->splitRegister(VirtReg.reg, LREdit.regs()); + DebugVars->splitRegister(Reg, LREdit.regs()); ExtraRegInfo.resize(MRI->getNumVirtRegs()); unsigned OrigBlocks = SA->getNumLiveBlocks(); @@ -922,18 +1058,18 @@ void RAGreedy::splitAroundRegion(LiveInterval &VirtReg, // Remainder interval. Don't try splitting again, spill if it doesn't // allocate. if (IntvMap[i] == 0) { - setStage(Reg, RS_Global); + setStage(Reg, RS_Spill); continue; } - // Main interval. Allow repeated splitting as long as the number of live + // Global intervals. Allow repeated splitting as long as the number of live // blocks is strictly decreasing. - if (IntvMap[i] == MainIntv) { + if (IntvMap[i] < NumGlobalIntvs) { if (SA->countLiveBlocks(&Reg) >= OrigBlocks) { DEBUG(dbgs() << "Main interval covers the same " << OrigBlocks << " blocks as original.\n"); // Don't allow repeated splitting as a safe guard against looping. - setStage(Reg, RS_Global); + setStage(Reg, RS_Split2); } continue; } @@ -948,11 +1084,23 @@ void RAGreedy::splitAroundRegion(LiveInterval &VirtReg, unsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order, SmallVectorImpl<LiveInterval*> &NewVRegs) { - float BestCost = Hysteresis * calcSpillCost(); - DEBUG(dbgs() << "Cost of isolating all blocks = " << BestCost << '\n'); - const unsigned NoCand = ~0u; - unsigned BestCand = NoCand; unsigned NumCands = 0; + unsigned BestCand = NoCand; + float BestCost; + SmallVector<unsigned, 8> UsedCands; + + // Check if we can split this live range around a compact region. + bool HasCompact = calcCompactRegion(GlobalCand.front()); + if (HasCompact) { + // Yes, keep GlobalCand[0] as the compact region candidate. + NumCands = 1; + BestCost = HUGE_VALF; + } else { + // No benefit from the compact region, our fallback will be per-block + // splitting. Make sure we find a solution that is cheaper than spilling. + BestCost = Hysteresis * calcSpillCost(); + DEBUG(dbgs() << "Cost of isolating all blocks = " << BestCost << '\n'); + } Order.rewind(); while (unsigned PhysReg = Order.next()) { @@ -962,7 +1110,7 @@ unsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order, unsigned WorstCount = ~0u; unsigned Worst = 0; for (unsigned i = 0; i != NumCands; ++i) { - if (i == BestCand) + if (i == BestCand || !GlobalCand[i].PhysReg) continue; unsigned Count = GlobalCand[i].LiveBundles.count(); if (Count < WorstCount) @@ -1019,15 +1167,94 @@ unsigned RAGreedy::tryRegionSplit(LiveInterval &VirtReg, AllocationOrder &Order, ++NumCands; } - if (BestCand == NoCand) + // No solutions found, fall back to single block splitting. + if (!HasCompact && BestCand == NoCand) return 0; - splitAroundRegion(VirtReg, GlobalCand[BestCand], NewVRegs); + // Prepare split editor. + LiveRangeEdit LREdit(VirtReg, NewVRegs, this); + SE->reset(LREdit, SplitSpillMode); + + // Assign all edge bundles to the preferred candidate, or NoCand. + BundleCand.assign(Bundles->getNumBundles(), NoCand); + + // Assign bundles for the best candidate region. + if (BestCand != NoCand) { + GlobalSplitCandidate &Cand = GlobalCand[BestCand]; + if (unsigned B = Cand.getBundles(BundleCand, BestCand)) { + UsedCands.push_back(BestCand); + Cand.IntvIdx = SE->openIntv(); + DEBUG(dbgs() << "Split for " << PrintReg(Cand.PhysReg, TRI) << " in " + << B << " bundles, intv " << Cand.IntvIdx << ".\n"); + (void)B; + } + } + + // Assign bundles for the compact region. + if (HasCompact) { + GlobalSplitCandidate &Cand = GlobalCand.front(); + assert(!Cand.PhysReg && "Compact region has no physreg"); + if (unsigned B = Cand.getBundles(BundleCand, 0)) { + UsedCands.push_back(0); + Cand.IntvIdx = SE->openIntv(); + DEBUG(dbgs() << "Split for compact region in " << B << " bundles, intv " + << Cand.IntvIdx << ".\n"); + (void)B; + } + } + + splitAroundRegion(LREdit, UsedCands); return 0; } //===----------------------------------------------------------------------===// +// Per-Block Splitting +//===----------------------------------------------------------------------===// + +/// tryBlockSplit - Split a global live range around every block with uses. This +/// creates a lot of local live ranges, that will be split by tryLocalSplit if +/// they don't allocate. +unsigned RAGreedy::tryBlockSplit(LiveInterval &VirtReg, AllocationOrder &Order, + SmallVectorImpl<LiveInterval*> &NewVRegs) { + assert(&SA->getParent() == &VirtReg && "Live range wasn't analyzed"); + unsigned Reg = VirtReg.reg; + bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg)); + LiveRangeEdit LREdit(VirtReg, NewVRegs, this); + SE->reset(LREdit, SplitSpillMode); + ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); + for (unsigned i = 0; i != UseBlocks.size(); ++i) { + const SplitAnalysis::BlockInfo &BI = UseBlocks[i]; + if (SA->shouldSplitSingleBlock(BI, SingleInstrs)) + SE->splitSingleBlock(BI); + } + // No blocks were split. + if (LREdit.empty()) + return 0; + + // We did split for some blocks. + SmallVector<unsigned, 8> IntvMap; + SE->finish(&IntvMap); + + // Tell LiveDebugVariables about the new ranges. + DebugVars->splitRegister(Reg, LREdit.regs()); + + ExtraRegInfo.resize(MRI->getNumVirtRegs()); + + // Sort out the new intervals created by splitting. The remainder interval + // goes straight to spilling, the new local ranges get to stay RS_New. + for (unsigned i = 0, e = LREdit.size(); i != e; ++i) { + LiveInterval &LI = *LREdit.get(i); + if (getStage(LI) == RS_New && IntvMap[i] == 0) + setStage(LI, RS_Spill); + } + + if (VerifyEnabled) + MF->verify(this, "After splitting live range around basic blocks"); + return 0; +} + +//===----------------------------------------------------------------------===// // Local Splitting //===----------------------------------------------------------------------===// @@ -1045,8 +1272,10 @@ void RAGreedy::calcGapWeights(unsigned PhysReg, const unsigned NumGaps = Uses.size()-1; // Start and end points for the interference check. - SlotIndex StartIdx = BI.LiveIn ? BI.FirstUse.getBaseIndex() : BI.FirstUse; - SlotIndex StopIdx = BI.LiveOut ? BI.LastUse.getBoundaryIndex() : BI.LastUse; + SlotIndex StartIdx = + BI.LiveIn ? BI.FirstInstr.getBaseIndex() : BI.FirstInstr; + SlotIndex StopIdx = + BI.LiveOut ? BI.LastInstr.getBoundaryIndex() : BI.LastInstr; GapWeight.assign(NumGaps, 0.0f); @@ -1056,8 +1285,8 @@ void RAGreedy::calcGapWeights(unsigned PhysReg, .checkInterference()) continue; - // We know that VirtReg is a continuous interval from FirstUse to LastUse, - // so we don't need InterferenceQuery. + // We know that VirtReg is a continuous interval from FirstInstr to + // LastInstr, so we don't need InterferenceQuery. // // Interference that overlaps an instruction is counted in both gaps // surrounding the instruction. The exception is interference before @@ -1097,8 +1326,8 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, // while only covering a single block - A phi-def can use undef values from // predecessors, and the block could be a single-block loop. // We don't bother doing anything clever about such a case, we simply assume - // that the interval is continuous from FirstUse to LastUse. We should make - // sure that we don't do anything illegal to such an interval, though. + // that the interval is continuous from FirstInstr to LastInstr. We should + // make sure that we don't do anything illegal to such an interval, though. const SmallVectorImpl<SlotIndex> &Uses = SA->UseSlots; if (Uses.size() <= 2) @@ -1120,17 +1349,17 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, // // Instead we use these rules: // - // 1. Allow any split for ranges with getStage() < RS_Local. (Except for the + // 1. Allow any split for ranges with getStage() < RS_Split2. (Except for the // noop split, of course). - // 2. Require progress be made for ranges with getStage() >= RS_Local. All + // 2. Require progress be made for ranges with getStage() == RS_Split2. All // the new ranges must have fewer instructions than before the split. - // 3. New ranges with the same number of instructions are marked RS_Local, + // 3. New ranges with the same number of instructions are marked RS_Split2, // smaller ranges are marked RS_New. // // These rules allow a 3 -> 2+3 split once, which we need. They also prevent // excessive splitting and infinite loops. // - bool ProgressRequired = getStage(VirtReg) >= RS_Local; + bool ProgressRequired = getStage(VirtReg) >= RS_Split2; // Best split candidate. unsigned BestBefore = NumGaps; @@ -1249,7 +1478,7 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, DebugVars->splitRegister(VirtReg.reg, LREdit.regs()); // If the new range has the same number of instructions as before, mark it as - // RS_Local so the next split will be forced to make progress. Otherwise, + // RS_Split2 so the next split will be forced to make progress. Otherwise, // leave the new intervals as RS_New so they can compete. bool LiveBefore = BestBefore != 0 || BI.LiveIn; bool LiveAfter = BestAfter != NumGaps || BI.LiveOut; @@ -1259,7 +1488,7 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, assert(!ProgressRequired && "Didn't make progress when it was required."); for (unsigned i = 0, e = IntvMap.size(); i != e; ++i) if (IntvMap[i] == 1) { - setStage(*LREdit.get(i), RS_Local); + setStage(*LREdit.get(i), RS_Split2); DEBUG(dbgs() << PrintReg(LREdit.get(i)->reg)); } DEBUG(dbgs() << '\n'); @@ -1278,6 +1507,10 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, /// @return Physreg when VirtReg may be assigned and/or new NewVRegs. unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order, SmallVectorImpl<LiveInterval*>&NewVRegs) { + // Ranges must be Split2 or less. + if (getStage(VirtReg) >= RS_Spill) + return 0; + // Local intervals are handled separately. if (LIS->intervalIsInOneMBB(VirtReg)) { NamedRegionTimer T("Local Splitting", TimerGroupName, TimePassesIsEnabled); @@ -1287,11 +1520,6 @@ unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order, NamedRegionTimer T("Global Splitting", TimerGroupName, TimePassesIsEnabled); - // Don't iterate global splitting. - // Move straight to spilling if this range was produced by a global split. - if (getStage(VirtReg) >= RS_Global) - return 0; - SA->analyze(&VirtReg); // FIXME: SplitAnalysis may repair broken live ranges coming from the @@ -1305,24 +1533,17 @@ unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order, return PhysReg; } - // First try to split around a region spanning multiple blocks. - unsigned PhysReg = tryRegionSplit(VirtReg, Order, NewVRegs); - if (PhysReg || !NewVRegs.empty()) - return PhysReg; - - // Then isolate blocks with multiple uses. - SplitAnalysis::BlockPtrSet Blocks; - if (SA->getMultiUseBlocks(Blocks)) { - LiveRangeEdit LREdit(VirtReg, NewVRegs, this); - SE->reset(LREdit); - SE->splitSingleBlocks(Blocks); - setStage(NewVRegs.begin(), NewVRegs.end(), RS_Global); - if (VerifyEnabled) - MF->verify(this, "After splitting live range around basic blocks"); + // First try to split around a region spanning multiple blocks. RS_Split2 + // ranges already made dubious progress with region splitting, so they go + // straight to single block splitting. + if (getStage(VirtReg) < RS_Split2) { + unsigned PhysReg = tryRegionSplit(VirtReg, Order, NewVRegs); + if (PhysReg || !NewVRegs.empty()) + return PhysReg; } - // Don't assign any physregs. - return 0; + // Then isolate blocks. + return tryBlockSplit(VirtReg, Order, NewVRegs); } @@ -1342,9 +1563,9 @@ unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg, << " Cascade " << ExtraRegInfo[VirtReg.reg].Cascade << '\n'); // Try to evict a less worthy live range, but only for ranges from the primary - // queue. The RS_Second ranges already failed to do this, and they should not + // queue. The RS_Split ranges already failed to do this, and they should not // get a second chance until they have been split. - if (Stage != RS_Second) + if (Stage != RS_Split) if (unsigned PhysReg = tryEvict(VirtReg, Order, NewVRegs)) return PhysReg; @@ -1353,8 +1574,8 @@ unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg, // The first time we see a live range, don't try to split or spill. // Wait until the second time, when all smaller ranges have been allocated. // This gives a better picture of the interference to split around. - if (Stage == RS_First) { - setStage(VirtReg, RS_Second); + if (Stage < RS_Split) { + setStage(VirtReg, RS_Split); DEBUG(dbgs() << "wait for second round\n"); NewVRegs.push_back(&VirtReg); return 0; @@ -1362,7 +1583,7 @@ unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg, // If we couldn't allocate a register from spilling, there is probably some // invalid inline assembly. The base class wil report it. - if (Stage >= RS_Spill || !VirtReg.isSpillable()) + if (Stage >= RS_Done || !VirtReg.isSpillable()) return ~0u; // Try splitting VirtReg or interferences. @@ -1374,7 +1595,7 @@ unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg, NamedRegionTimer T("Spiller", TimerGroupName, TimePassesIsEnabled); LiveRangeEdit LRE(VirtReg, NewVRegs, this); spiller().spill(LRE); - setStage(NewVRegs.begin(), NewVRegs.end(), RS_Spill); + setStage(NewVRegs.begin(), NewVRegs.end(), RS_Done); if (VerifyEnabled) MF->verify(this, "After spilling"); @@ -1408,6 +1629,7 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) { ExtraRegInfo.resize(MRI->getNumVirtRegs()); NextCascade = 1; IntfCache.init(MF, &PhysReg2LiveUnion[0], Indexes, TRI); + GlobalCand.resize(32); // This will grow as needed. allocatePhysRegs(); addMBBLiveIns(MF); @@ -1420,7 +1642,10 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) { } // Write out new DBG_VALUE instructions. - DebugVars->emitDebugValues(VRM); + { + NamedRegionTimer T("Emit Debug Info", TimerGroupName, TimePassesIsEnabled); + DebugVars->emitDebugValues(VRM); + } // The pass output is in VirtRegMap. Release all the transient data. releaseMemory(); diff --git a/lib/CodeGen/RegAllocLinearScan.cpp b/lib/CodeGen/RegAllocLinearScan.cpp index 0dd3c598..ce3fb90 100644 --- a/lib/CodeGen/RegAllocLinearScan.cpp +++ b/lib/CodeGen/RegAllocLinearScan.cpp @@ -18,7 +18,6 @@ #include "VirtRegRewriter.h" #include "RegisterClassInfo.h" #include "Spiller.h" -#include "RegisterCoalescer.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Function.h" #include "llvm/CodeGen/CalcSpillWeights.h" @@ -209,7 +208,7 @@ namespace { AU.addRequiredID(StrongPHIEliminationID); // Make sure PassManager knows which analyses to make available // to coalescing and which analyses coalescing invalidates. - AU.addRequiredTransitive<RegisterCoalescer>(); + AU.addRequiredTransitiveID(RegisterCoalescerPassID); AU.addRequired<CalculateSpillWeights>(); AU.addRequiredID(LiveStacksID); AU.addPreservedID(LiveStacksID); diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp index 72230d4..0d2cf2d 100644 --- a/lib/CodeGen/RegAllocPBQP.cpp +++ b/lib/CodeGen/RegAllocPBQP.cpp @@ -450,7 +450,7 @@ void RegAllocPBQP::getAnalysisUsage(AnalysisUsage &au) const { au.addPreserved<SlotIndexes>(); au.addRequired<LiveIntervals>(); //au.addRequiredID(SplitCriticalEdgesID); - au.addRequired<RegisterCoalescer>(); + au.addRequiredID(RegisterCoalescerPassID); if (customPassID) au.addRequiredID(*customPassID); au.addRequired<CalculateSpillWeights>(); diff --git a/lib/CodeGen/RegisterClassInfo.cpp b/lib/CodeGen/RegisterClassInfo.cpp index 5a77e47..786d279 100644 --- a/lib/CodeGen/RegisterClassInfo.cpp +++ b/lib/CodeGen/RegisterClassInfo.cpp @@ -99,11 +99,16 @@ void RegisterClassInfo::compute(const TargetRegisterClass *RC) const { // CSR aliases go after the volatile registers, preserve the target's order. std::copy(CSRAlias.begin(), CSRAlias.end(), &RCI.Order[N]); + // Check if RC is a proper sub-class. + if (const TargetRegisterClass *Super = TRI->getLargestLegalSuperClass(RC)) + if (Super != RC && getNumAllocatableRegs(Super) > RCI.NumRegs) + RCI.ProperSubClass = true; + DEBUG({ dbgs() << "AllocationOrder(" << RC->getName() << ") = ["; for (unsigned I = 0; I != RCI.NumRegs; ++I) dbgs() << ' ' << PrintReg(RCI.Order[I], TRI); - dbgs() << " ]\n"; + dbgs() << (RCI.ProperSubClass ? " ] (sub-class)\n" : " ]\n"); }); // RCI is now up-to-date. diff --git a/lib/CodeGen/RegisterClassInfo.h b/lib/CodeGen/RegisterClassInfo.h index d21fd67..2c14070 100644 --- a/lib/CodeGen/RegisterClassInfo.h +++ b/lib/CodeGen/RegisterClassInfo.h @@ -28,11 +28,12 @@ class RegisterClassInfo { struct RCInfo { unsigned Tag; unsigned NumRegs; + bool ProperSubClass; OwningArrayPtr<unsigned> Order; - RCInfo() : Tag(0), NumRegs(0) {} + RCInfo() : Tag(0), NumRegs(0), ProperSubClass(false) {} operator ArrayRef<unsigned>() const { - return ArrayRef<unsigned>(Order.get(), NumRegs); + return makeArrayRef(Order.get(), NumRegs); } }; @@ -87,6 +88,16 @@ public: return get(RC); } + /// isProperSubClass - Returns true if RC has a legal super-class with more + /// allocatable registers. + /// + /// Register classes like GR32_NOSP are not proper sub-classes because %esp + /// is not allocatable. Similarly, tGPR is not a proper sub-class in Thumb + /// mode because the GPR super-class is not legal. + bool isProperSubClass(const TargetRegisterClass *RC) const { + return get(RC).ProperSubClass; + } + /// getLastCalleeSavedAlias - Returns the last callee saved register that /// overlaps PhysReg, or 0 if Reg doesn't overlap a CSR. unsigned getLastCalleeSavedAlias(unsigned PhysReg) const { diff --git a/lib/CodeGen/RegisterCoalescer.cpp b/lib/CodeGen/RegisterCoalescer.cpp index b91f92c..9b414d6 100644 --- a/lib/CodeGen/RegisterCoalescer.cpp +++ b/lib/CodeGen/RegisterCoalescer.cpp @@ -15,8 +15,9 @@ #define DEBUG_TYPE "regcoalescing" #include "RegisterCoalescer.h" -#include "VirtRegMap.h" #include "LiveDebugVariables.h" +#include "RegisterClassInfo.h" +#include "VirtRegMap.h" #include "llvm/Pass.h" #include "llvm/Value.h" @@ -54,6 +55,7 @@ STATISTIC(numExtends , "Number of copies extended"); STATISTIC(NumReMats , "Number of instructions re-materialized"); STATISTIC(numPeep , "Number of identity moves eliminated after coalescing"); STATISTIC(numAborts , "Number of times interval joining aborted"); +STATISTIC(NumInflated , "Number of register classes inflated"); static cl::opt<bool> EnableJoining("join-liveintervals", @@ -75,6 +77,128 @@ VerifyCoalescing("verify-coalescing", cl::desc("Verify machine instrs before and after register coalescing"), cl::Hidden); +namespace { + class RegisterCoalescer : public MachineFunctionPass { + MachineFunction* MF; + MachineRegisterInfo* MRI; + const TargetMachine* TM; + const TargetRegisterInfo* TRI; + const TargetInstrInfo* TII; + LiveIntervals *LIS; + LiveDebugVariables *LDV; + const MachineLoopInfo* Loops; + AliasAnalysis *AA; + RegisterClassInfo RegClassInfo; + + /// JoinedCopies - Keep track of copies eliminated due to coalescing. + /// + SmallPtrSet<MachineInstr*, 32> JoinedCopies; + + /// ReMatCopies - Keep track of copies eliminated due to remat. + /// + SmallPtrSet<MachineInstr*, 32> ReMatCopies; + + /// ReMatDefs - Keep track of definition instructions which have + /// been remat'ed. + SmallPtrSet<MachineInstr*, 8> ReMatDefs; + + /// joinIntervals - join compatible live intervals + void joinIntervals(); + + /// CopyCoalesceInMBB - Coalesce copies in the specified MBB, putting + /// copies that cannot yet be coalesced into the "TryAgain" list. + void CopyCoalesceInMBB(MachineBasicBlock *MBB, + std::vector<MachineInstr*> &TryAgain); + + /// JoinCopy - Attempt to join intervals corresponding to SrcReg/DstReg, + /// which are the src/dst of the copy instruction CopyMI. This returns + /// true if the copy was successfully coalesced away. If it is not + /// currently possible to coalesce this interval, but it may be possible if + /// other things get coalesced, then it returns true by reference in + /// 'Again'. + bool JoinCopy(MachineInstr *TheCopy, bool &Again); + + /// JoinIntervals - Attempt to join these two intervals. On failure, this + /// returns false. The output "SrcInt" will not have been modified, so we + /// can use this information below to update aliases. + bool JoinIntervals(CoalescerPair &CP); + + /// AdjustCopiesBackFrom - We found a non-trivially-coalescable copy. If + /// the source value number is defined by a copy from the destination reg + /// see if we can merge these two destination reg valno# into a single + /// value number, eliminating a copy. + bool AdjustCopiesBackFrom(const CoalescerPair &CP, MachineInstr *CopyMI); + + /// HasOtherReachingDefs - Return true if there are definitions of IntB + /// other than BValNo val# that can reach uses of AValno val# of IntA. + bool HasOtherReachingDefs(LiveInterval &IntA, LiveInterval &IntB, + VNInfo *AValNo, VNInfo *BValNo); + + /// RemoveCopyByCommutingDef - We found a non-trivially-coalescable copy. + /// If the source value number is defined by a commutable instruction and + /// its other operand is coalesced to the copy dest register, see if we + /// can transform the copy into a noop by commuting the definition. + bool RemoveCopyByCommutingDef(const CoalescerPair &CP,MachineInstr *CopyMI); + + /// ReMaterializeTrivialDef - If the source of a copy is defined by a + /// trivial computation, replace the copy by rematerialize the definition. + /// If PreserveSrcInt is true, make sure SrcInt is valid after the call. + bool ReMaterializeTrivialDef(LiveInterval &SrcInt, bool PreserveSrcInt, + unsigned DstReg, MachineInstr *CopyMI); + + /// shouldJoinPhys - Return true if a physreg copy should be joined. + bool shouldJoinPhys(CoalescerPair &CP); + + /// isWinToJoinCrossClass - Return true if it's profitable to coalesce + /// two virtual registers from different register classes. + bool isWinToJoinCrossClass(unsigned SrcReg, + unsigned DstReg, + const TargetRegisterClass *SrcRC, + const TargetRegisterClass *DstRC, + const TargetRegisterClass *NewRC); + + /// UpdateRegDefsUses - Replace all defs and uses of SrcReg to DstReg and + /// update the subregister number if it is not zero. If DstReg is a + /// physical register and the existing subregister number of the def / use + /// being updated is not zero, make sure to set it to the correct physical + /// subregister. + void UpdateRegDefsUses(const CoalescerPair &CP); + + /// RemoveDeadDef - If a def of a live interval is now determined dead, + /// remove the val# it defines. If the live interval becomes empty, remove + /// it as well. + bool RemoveDeadDef(LiveInterval &li, MachineInstr *DefMI); + + /// RemoveCopyFlag - If DstReg is no longer defined by CopyMI, clear the + /// VNInfo copy flag for DstReg and all aliases. + void RemoveCopyFlag(unsigned DstReg, const MachineInstr *CopyMI); + + /// markAsJoined - Remember that CopyMI has already been joined. + void markAsJoined(MachineInstr *CopyMI); + + /// eliminateUndefCopy - Handle copies of undef values. + bool eliminateUndefCopy(MachineInstr *CopyMI, const CoalescerPair &CP); + + public: + static char ID; // Class identification, replacement for typeinfo + RegisterCoalescer() : MachineFunctionPass(ID) { + initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry()); + } + + virtual void getAnalysisUsage(AnalysisUsage &AU) const; + + virtual void releaseMemory(); + + /// runOnMachineFunction - pass entry point + virtual bool runOnMachineFunction(MachineFunction&); + + /// print - Implement the dump method. + virtual void print(raw_ostream &O, const Module* = 0) const; + }; +} /// end anonymous namespace + +char &llvm::RegisterCoalescerPassID = RegisterCoalescer::ID; + INITIALIZE_PASS_BEGIN(RegisterCoalescer, "simple-register-coalescing", "Simple Register Coalescing", false, false) INITIALIZE_PASS_DEPENDENCY(LiveIntervals) @@ -116,14 +240,14 @@ static bool isMoveInstr(const TargetRegisterInfo &tri, const MachineInstr *MI, } bool CoalescerPair::setRegisters(const MachineInstr *MI) { - srcReg_ = dstReg_ = subIdx_ = 0; - newRC_ = 0; - flipped_ = crossClass_ = false; + SrcReg = DstReg = SubIdx = 0; + NewRC = 0; + Flipped = CrossClass = false; unsigned Src, Dst, SrcSub, DstSub; - if (!isMoveInstr(tri_, MI, Src, Dst, SrcSub, DstSub)) + if (!isMoveInstr(TRI, MI, Src, Dst, SrcSub, DstSub)) return false; - partial_ = SrcSub || DstSub; + Partial = SrcSub || DstSub; // If one register is a physreg, it must be Dst. if (TargetRegisterInfo::isPhysicalRegister(Src)) { @@ -131,7 +255,7 @@ bool CoalescerPair::setRegisters(const MachineInstr *MI) { return false; std::swap(Src, Dst); std::swap(SrcSub, DstSub); - flipped_ = true; + Flipped = true; } const MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo(); @@ -139,14 +263,14 @@ bool CoalescerPair::setRegisters(const MachineInstr *MI) { if (TargetRegisterInfo::isPhysicalRegister(Dst)) { // Eliminate DstSub on a physreg. if (DstSub) { - Dst = tri_.getSubReg(Dst, DstSub); + Dst = TRI.getSubReg(Dst, DstSub); if (!Dst) return false; DstSub = 0; } // Eliminate SrcSub by picking a corresponding Dst superregister. if (SrcSub) { - Dst = tri_.getMatchingSuperReg(Dst, SrcSub, MRI.getRegClass(Src)); + Dst = TRI.getMatchingSuperReg(Dst, SrcSub, MRI.getRegClass(Src)); if (!Dst) return false; SrcSub = 0; } else if (!MRI.getRegClass(Src)->contains(Dst)) { @@ -164,7 +288,7 @@ bool CoalescerPair::setRegisters(const MachineInstr *MI) { return false; const TargetRegisterClass *SrcRC = MRI.getRegClass(Src); const TargetRegisterClass *DstRC = MRI.getRegClass(Dst); - if (!getCommonSubClass(DstRC, SrcRC)) + if (!TRI.getCommonSubClass(DstRC, SrcRC)) return false; SrcSub = DstSub = 0; } @@ -174,36 +298,36 @@ bool CoalescerPair::setRegisters(const MachineInstr *MI) { std::swap(Src, Dst); DstSub = SrcSub; SrcSub = 0; - assert(!flipped_ && "Unexpected flip"); - flipped_ = true; + assert(!Flipped && "Unexpected flip"); + Flipped = true; } // Find the new register class. const TargetRegisterClass *SrcRC = MRI.getRegClass(Src); const TargetRegisterClass *DstRC = MRI.getRegClass(Dst); if (DstSub) - newRC_ = tri_.getMatchingSuperRegClass(DstRC, SrcRC, DstSub); + NewRC = TRI.getMatchingSuperRegClass(DstRC, SrcRC, DstSub); else - newRC_ = getCommonSubClass(DstRC, SrcRC); - if (!newRC_) + NewRC = TRI.getCommonSubClass(DstRC, SrcRC); + if (!NewRC) return false; - crossClass_ = newRC_ != DstRC || newRC_ != SrcRC; + CrossClass = NewRC != DstRC || NewRC != SrcRC; } // Check our invariants assert(TargetRegisterInfo::isVirtualRegister(Src) && "Src must be virtual"); assert(!(TargetRegisterInfo::isPhysicalRegister(Dst) && DstSub) && "Cannot have a physical SubIdx"); - srcReg_ = Src; - dstReg_ = Dst; - subIdx_ = DstSub; + SrcReg = Src; + DstReg = Dst; + SubIdx = DstSub; return true; } bool CoalescerPair::flip() { - if (subIdx_ || TargetRegisterInfo::isPhysicalRegister(dstReg_)) + if (SubIdx || TargetRegisterInfo::isPhysicalRegister(DstReg)) return false; - std::swap(srcReg_, dstReg_); - flipped_ = !flipped_; + std::swap(SrcReg, DstReg); + Flipped = !Flipped; return true; } @@ -211,36 +335,36 @@ bool CoalescerPair::isCoalescable(const MachineInstr *MI) const { if (!MI) return false; unsigned Src, Dst, SrcSub, DstSub; - if (!isMoveInstr(tri_, MI, Src, Dst, SrcSub, DstSub)) + if (!isMoveInstr(TRI, MI, Src, Dst, SrcSub, DstSub)) return false; - // Find the virtual register that is srcReg_. - if (Dst == srcReg_) { + // Find the virtual register that is SrcReg. + if (Dst == SrcReg) { std::swap(Src, Dst); std::swap(SrcSub, DstSub); - } else if (Src != srcReg_) { + } else if (Src != SrcReg) { return false; } - // Now check that Dst matches dstReg_. - if (TargetRegisterInfo::isPhysicalRegister(dstReg_)) { + // Now check that Dst matches DstReg. + if (TargetRegisterInfo::isPhysicalRegister(DstReg)) { if (!TargetRegisterInfo::isPhysicalRegister(Dst)) return false; - assert(!subIdx_ && "Inconsistent CoalescerPair state."); + assert(!SubIdx && "Inconsistent CoalescerPair state."); // DstSub could be set for a physreg from INSERT_SUBREG. if (DstSub) - Dst = tri_.getSubReg(Dst, DstSub); + Dst = TRI.getSubReg(Dst, DstSub); // Full copy of Src. if (!SrcSub) - return dstReg_ == Dst; + return DstReg == Dst; // This is a partial register copy. Check that the parts match. - return tri_.getSubReg(dstReg_, SrcSub) == Dst; + return TRI.getSubReg(DstReg, SrcSub) == Dst; } else { - // dstReg_ is virtual. - if (dstReg_ != Dst) + // DstReg is virtual. + if (DstReg != Dst) return false; // Registers match, do the subregisters line up? - return compose(tri_, subIdx_, SrcSub) == DstSub; + return compose(TRI, SubIdx, SrcSub) == DstSub; } } @@ -292,14 +416,14 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP, MachineInstr *CopyMI) { // Bail if there is no dst interval - can happen when merging physical subreg // operations. - if (!li_->hasInterval(CP.getDstReg())) + if (!LIS->hasInterval(CP.getDstReg())) return false; LiveInterval &IntA = - li_->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); + LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); LiveInterval &IntB = - li_->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); - SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getDefIndex(); + LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); + SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI).getDefIndex(); // BValNo is a value number in B that is defined by a copy from A. 'B3' in // the example above. @@ -355,7 +479,7 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP, // Make sure that the end of the live range is inside the same block as // CopyMI. MachineInstr *ValLREndInst = - li_->getInstructionFromIndex(ValLR->end.getPrevSlot()); + LIS->getInstructionFromIndex(ValLR->end.getPrevSlot()); if (!ValLREndInst || ValLREndInst->getParent() != CopyMI->getParent()) return false; @@ -368,11 +492,11 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP, // of its aliases is overlapping the live interval of the virtual register. // If so, do not coalesce. if (TargetRegisterInfo::isPhysicalRegister(IntB.reg)) { - for (const unsigned *AS = tri_->getAliasSet(IntB.reg); *AS; ++AS) - if (li_->hasInterval(*AS) && IntA.overlaps(li_->getInterval(*AS))) { + for (const unsigned *AS = TRI->getAliasSet(IntB.reg); *AS; ++AS) + if (LIS->hasInterval(*AS) && IntA.overlaps(LIS->getInterval(*AS))) { DEBUG({ dbgs() << "\t\tInterfere with alias "; - li_->getInterval(*AS).print(dbgs(), tri_); + LIS->getInterval(*AS).print(dbgs(), TRI); }); return false; } @@ -380,7 +504,7 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP, DEBUG({ dbgs() << "Extending: "; - IntB.print(dbgs(), tri_); + IntB.print(dbgs(), TRI); }); SlotIndex FillerStart = ValLR->end, FillerEnd = BLR->start; @@ -398,13 +522,13 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP, // If the IntB live range is assigned to a physical register, and if that // physreg has sub-registers, update their live intervals as well. if (TargetRegisterInfo::isPhysicalRegister(IntB.reg)) { - for (const unsigned *SR = tri_->getSubRegisters(IntB.reg); *SR; ++SR) { - if (!li_->hasInterval(*SR)) + for (const unsigned *SR = TRI->getSubRegisters(IntB.reg); *SR; ++SR) { + if (!LIS->hasInterval(*SR)) continue; - LiveInterval &SRLI = li_->getInterval(*SR); + LiveInterval &SRLI = LIS->getInterval(*SR); SRLI.addRange(LiveRange(FillerStart, FillerEnd, SRLI.getNextValue(FillerStart, 0, - li_->getVNInfoAllocator()))); + LIS->getVNInfoAllocator()))); } } @@ -419,7 +543,7 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP, } DEBUG({ dbgs() << " result = "; - IntB.print(dbgs(), tri_); + IntB.print(dbgs(), TRI); dbgs() << "\n"; }); @@ -434,7 +558,7 @@ bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP, // merge, find the last use and trim the live range. That will also add the // isKill marker. if (ALR->end == CopyIdx) - li_->shrinkToUses(&IntA); + LIS->shrinkToUses(&IntA); ++numExtends; return true; @@ -498,15 +622,15 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP, return false; // Bail if there is no dst interval. - if (!li_->hasInterval(CP.getDstReg())) + if (!LIS->hasInterval(CP.getDstReg())) return false; - SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getDefIndex(); + SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI).getDefIndex(); LiveInterval &IntA = - li_->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); + LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); LiveInterval &IntB = - li_->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); + LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); // BValNo is a value number in B that is defined by a copy from A. 'B3' in // the example above. @@ -524,7 +648,7 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP, // the optimization. if (AValNo->isPHIDef() || AValNo->isUnused() || AValNo->hasPHIKill()) return false; - MachineInstr *DefMI = li_->getInstructionFromIndex(AValNo->def); + MachineInstr *DefMI = LIS->getInstructionFromIndex(AValNo->def); if (!DefMI) return false; const MCInstrDesc &MCID = DefMI->getDesc(); @@ -538,7 +662,7 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP, if (!DefMI->isRegTiedToUseOperand(DefIdx, &UseOpIdx)) return false; unsigned Op1, Op2, NewDstIdx; - if (!tii_->findCommutedOpIndices(DefMI, Op1, Op2)) + if (!TII->findCommutedOpIndices(DefMI, Op1, Op2)) return false; if (Op1 == UseOpIdx) NewDstIdx = Op2; @@ -560,18 +684,18 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP, // Abort if the aliases of IntB.reg have values that are not simply the // clobbers from the superreg. if (TargetRegisterInfo::isPhysicalRegister(IntB.reg)) - for (const unsigned *AS = tri_->getAliasSet(IntB.reg); *AS; ++AS) - if (li_->hasInterval(*AS) && - HasOtherReachingDefs(IntA, li_->getInterval(*AS), AValNo, 0)) + for (const unsigned *AS = TRI->getAliasSet(IntB.reg); *AS; ++AS) + if (LIS->hasInterval(*AS) && + HasOtherReachingDefs(IntA, LIS->getInterval(*AS), AValNo, 0)) return false; // If some of the uses of IntA.reg is already coalesced away, return false. // It's not possible to determine whether it's safe to perform the coalescing. - for (MachineRegisterInfo::use_nodbg_iterator UI = - mri_->use_nodbg_begin(IntA.reg), - UE = mri_->use_nodbg_end(); UI != UE; ++UI) { + for (MachineRegisterInfo::use_nodbg_iterator UI = + MRI->use_nodbg_begin(IntA.reg), + UE = MRI->use_nodbg_end(); UI != UE; ++UI) { MachineInstr *UseMI = &*UI; - SlotIndex UseIdx = li_->getInstructionIndex(UseMI); + SlotIndex UseIdx = LIS->getInstructionIndex(UseMI); LiveInterval::iterator ULR = IntA.FindLiveRangeContaining(UseIdx); if (ULR == IntA.end()) continue; @@ -585,15 +709,15 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP, // At this point we have decided that it is legal to do this // transformation. Start by commuting the instruction. MachineBasicBlock *MBB = DefMI->getParent(); - MachineInstr *NewMI = tii_->commuteInstruction(DefMI); + MachineInstr *NewMI = TII->commuteInstruction(DefMI); if (!NewMI) return false; if (TargetRegisterInfo::isVirtualRegister(IntA.reg) && TargetRegisterInfo::isVirtualRegister(IntB.reg) && - !mri_->constrainRegClass(IntB.reg, mri_->getRegClass(IntA.reg))) + !MRI->constrainRegClass(IntB.reg, MRI->getRegClass(IntA.reg))) return false; if (NewMI != DefMI) { - li_->ReplaceMachineInstrInMaps(DefMI, NewMI); + LIS->ReplaceMachineInstrInMaps(DefMI, NewMI); MBB->insert(DefMI, NewMI); MBB->erase(DefMI); } @@ -610,8 +734,8 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP, // = B // Update uses of IntA of the specific Val# with IntB. - for (MachineRegisterInfo::use_iterator UI = mri_->use_begin(IntA.reg), - UE = mri_->use_end(); UI != UE;) { + for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(IntA.reg), + UE = MRI->use_end(); UI != UE;) { MachineOperand &UseMO = UI.getOperand(); MachineInstr *UseMI = &*UI; ++UI; @@ -623,12 +747,12 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP, UseMO.setReg(NewReg); continue; } - SlotIndex UseIdx = li_->getInstructionIndex(UseMI).getUseIndex(); + SlotIndex UseIdx = LIS->getInstructionIndex(UseMI).getUseIndex(); LiveInterval::iterator ULR = IntA.FindLiveRangeContaining(UseIdx); if (ULR == IntA.end() || ULR->valno != AValNo) continue; if (TargetRegisterInfo::isPhysicalRegister(NewReg)) - UseMO.substPhysReg(NewReg, *tri_); + UseMO.substPhysReg(NewReg, *TRI); else UseMO.setReg(NewReg); if (UseMI == CopyMI) @@ -674,27 +798,24 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP, bool RegisterCoalescer::ReMaterializeTrivialDef(LiveInterval &SrcInt, bool preserveSrcInt, unsigned DstReg, - unsigned DstSubIdx, MachineInstr *CopyMI) { - SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getUseIndex(); + SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI).getUseIndex(); LiveInterval::iterator SrcLR = SrcInt.FindLiveRangeContaining(CopyIdx); assert(SrcLR != SrcInt.end() && "Live range not found!"); VNInfo *ValNo = SrcLR->valno; - // If other defs can reach uses of this def, then it's not safe to perform - // the optimization. - if (ValNo->isPHIDef() || ValNo->isUnused() || ValNo->hasPHIKill()) + if (ValNo->isPHIDef() || ValNo->isUnused()) return false; - MachineInstr *DefMI = li_->getInstructionFromIndex(ValNo->def); + MachineInstr *DefMI = LIS->getInstructionFromIndex(ValNo->def); if (!DefMI) return false; assert(DefMI && "Defining instruction disappeared"); const MCInstrDesc &MCID = DefMI->getDesc(); if (!MCID.isAsCheapAsAMove()) return false; - if (!tii_->isTriviallyReMaterializable(DefMI, AA)) + if (!TII->isTriviallyReMaterializable(DefMI, AA)) return false; bool SawStore = false; - if (!DefMI->isSafeToMove(tii_, AA, SawStore)) + if (!DefMI->isSafeToMove(TII, AA, SawStore)) return false; if (MCID.getNumDefs() != 1) return false; @@ -702,36 +823,20 @@ bool RegisterCoalescer::ReMaterializeTrivialDef(LiveInterval &SrcInt, // Make sure the copy destination register class fits the instruction // definition register class. The mismatch can happen as a result of earlier // extract_subreg, insert_subreg, subreg_to_reg coalescing. - const TargetRegisterClass *RC = tii_->getRegClass(MCID, 0, tri_); + const TargetRegisterClass *RC = TII->getRegClass(MCID, 0, TRI); if (TargetRegisterInfo::isVirtualRegister(DstReg)) { - if (mri_->getRegClass(DstReg) != RC) + if (MRI->getRegClass(DstReg) != RC) return false; } else if (!RC->contains(DstReg)) return false; } - // If destination register has a sub-register index on it, make sure it - // matches the instruction register class. - if (DstSubIdx) { - const MCInstrDesc &MCID = DefMI->getDesc(); - if (MCID.getNumDefs() != 1) - return false; - const TargetRegisterClass *DstRC = mri_->getRegClass(DstReg); - const TargetRegisterClass *DstSubRC = - DstRC->getSubRegisterRegClass(DstSubIdx); - const TargetRegisterClass *DefRC = tii_->getRegClass(MCID, 0, tri_); - if (DefRC == DstRC) - DstSubIdx = 0; - else if (DefRC != DstSubRC) - return false; - } - RemoveCopyFlag(DstReg, CopyMI); MachineBasicBlock *MBB = CopyMI->getParent(); MachineBasicBlock::iterator MII = llvm::next(MachineBasicBlock::iterator(CopyMI)); - tii_->reMaterialize(*MBB, MII, DstReg, DstSubIdx, DefMI, *tri_); + TII->reMaterialize(*MBB, MII, DstReg, 0, DefMI, *TRI); MachineInstr *NewMI = prior(MII); // CopyMI may have implicit operands, transfer them over to the newly @@ -746,7 +851,7 @@ bool RegisterCoalescer::ReMaterializeTrivialDef(LiveInterval &SrcInt, } NewMI->copyImplicitOps(CopyMI); - li_->ReplaceMachineInstrInMaps(CopyMI, NewMI); + LIS->ReplaceMachineInstrInMaps(CopyMI, NewMI); CopyMI->eraseFromParent(); ReMatCopies.insert(CopyMI); ReMatDefs.insert(DefMI); @@ -755,8 +860,51 @@ bool RegisterCoalescer::ReMaterializeTrivialDef(LiveInterval &SrcInt, // The source interval can become smaller because we removed a use. if (preserveSrcInt) - li_->shrinkToUses(&SrcInt); + LIS->shrinkToUses(&SrcInt); + + return true; +} + +/// eliminateUndefCopy - ProcessImpicitDefs may leave some copies of <undef> +/// values, it only removes local variables. When we have a copy like: +/// +/// %vreg1 = COPY %vreg2<undef> +/// +/// We delete the copy and remove the corresponding value number from %vreg1. +/// Any uses of that value number are marked as <undef>. +bool RegisterCoalescer::eliminateUndefCopy(MachineInstr *CopyMI, + const CoalescerPair &CP) { + SlotIndex Idx = LIS->getInstructionIndex(CopyMI); + LiveInterval *SrcInt = &LIS->getInterval(CP.getSrcReg()); + if (SrcInt->liveAt(Idx)) + return false; + LiveInterval *DstInt = &LIS->getInterval(CP.getDstReg()); + if (DstInt->liveAt(Idx)) + return false; + // No intervals are live-in to CopyMI - it is undef. + if (CP.isFlipped()) + DstInt = SrcInt; + SrcInt = 0; + + VNInfo *DeadVNI = DstInt->getVNInfoAt(Idx.getDefIndex()); + assert(DeadVNI && "No value defined in DstInt"); + DstInt->removeValNo(DeadVNI); + + // Find new undef uses. + for (MachineRegisterInfo::reg_nodbg_iterator + I = MRI->reg_nodbg_begin(DstInt->reg), E = MRI->reg_nodbg_end(); + I != E; ++I) { + MachineOperand &MO = I.getOperand(); + if (MO.isDef() || MO.isUndef()) + continue; + MachineInstr *MI = MO.getParent(); + SlotIndex Idx = LIS->getInstructionIndex(MI); + if (DstInt->liveAt(Idx)) + continue; + MO.setIsUndef(true); + DEBUG(dbgs() << "\tnew undef: " << Idx << '\t' << *MI); + } return true; } @@ -773,22 +921,20 @@ RegisterCoalescer::UpdateRegDefsUses(const CoalescerPair &CP) { unsigned SubIdx = CP.getSubIdx(); // Update LiveDebugVariables. - ldv_->renameRegister(SrcReg, DstReg, SubIdx); + LDV->renameRegister(SrcReg, DstReg, SubIdx); - for (MachineRegisterInfo::reg_iterator I = mri_->reg_begin(SrcReg); + for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(SrcReg); MachineInstr *UseMI = I.skipInstruction();) { // A PhysReg copy that won't be coalesced can perhaps be rematerialized // instead. if (DstIsPhys) { - if (UseMI->isCopy() && - !UseMI->getOperand(1).getSubReg() && - !UseMI->getOperand(0).getSubReg() && + if (UseMI->isFullCopy() && UseMI->getOperand(1).getReg() == SrcReg && UseMI->getOperand(0).getReg() != SrcReg && UseMI->getOperand(0).getReg() != DstReg && !JoinedCopies.count(UseMI) && - ReMaterializeTrivialDef(li_->getInterval(SrcReg), false, - UseMI->getOperand(0).getReg(), 0, UseMI)) + ReMaterializeTrivialDef(LIS->getInterval(SrcReg), false, + UseMI->getOperand(0).getReg(), UseMI)) continue; } @@ -803,10 +949,18 @@ RegisterCoalescer::UpdateRegDefsUses(const CoalescerPair &CP) { Kills |= MO.isKill(); Deads |= MO.isDead(); + // Make sure we don't create read-modify-write defs accidentally. We + // assume here that a SrcReg def cannot be joined into a live DstReg. If + // RegisterCoalescer starts tracking partially live registers, we will + // need to check the actual LiveInterval to determine if DstReg is live + // here. + if (SubIdx && !Reads) + MO.setIsUndef(); + if (DstIsPhys) - MO.substPhysReg(DstReg, *tri_); + MO.substPhysReg(DstReg, *TRI); else - MO.substVirtReg(DstReg, SubIdx, *tri_); + MO.substVirtReg(DstReg, SubIdx, *TRI); } // This instruction is a copy that will be removed. @@ -817,19 +971,19 @@ RegisterCoalescer::UpdateRegDefsUses(const CoalescerPair &CP) { // If UseMI was a simple SrcReg def, make sure we didn't turn it into a // read-modify-write of DstReg. if (Deads) - UseMI->addRegisterDead(DstReg, tri_); + UseMI->addRegisterDead(DstReg, TRI); else if (!Reads && Writes) - UseMI->addRegisterDefined(DstReg, tri_); + UseMI->addRegisterDefined(DstReg, TRI); // Kill flags apply to the whole physical register. if (DstIsPhys && Kills) - UseMI->addRegisterKilled(DstReg, tri_); + UseMI->addRegisterKilled(DstReg, TRI); } DEBUG({ dbgs() << "\t\tupdated: "; if (!UseMI->isDebugValue()) - dbgs() << li_->getInstructionIndex(UseMI) << "\t"; + dbgs() << LIS->getInstructionIndex(UseMI) << "\t"; dbgs() << *UseMI; }); } @@ -838,18 +992,18 @@ RegisterCoalescer::UpdateRegDefsUses(const CoalescerPair &CP) { /// removeIntervalIfEmpty - Check if the live interval of a physical register /// is empty, if so remove it and also remove the empty intervals of its /// sub-registers. Return true if live interval is removed. -static bool removeIntervalIfEmpty(LiveInterval &li, LiveIntervals *li_, - const TargetRegisterInfo *tri_) { +static bool removeIntervalIfEmpty(LiveInterval &li, LiveIntervals *LIS, + const TargetRegisterInfo *TRI) { if (li.empty()) { if (TargetRegisterInfo::isPhysicalRegister(li.reg)) - for (const unsigned* SR = tri_->getSubRegisters(li.reg); *SR; ++SR) { - if (!li_->hasInterval(*SR)) + for (const unsigned* SR = TRI->getSubRegisters(li.reg); *SR; ++SR) { + if (!LIS->hasInterval(*SR)) continue; - LiveInterval &sli = li_->getInterval(*SR); + LiveInterval &sli = LIS->getInterval(*SR); if (sli.empty()) - li_->removeInterval(*SR); + LIS->removeInterval(*SR); } - li_->removeInterval(li.reg); + LIS->removeInterval(li.reg); return true; } return false; @@ -859,29 +1013,29 @@ static bool removeIntervalIfEmpty(LiveInterval &li, LiveIntervals *li_, /// the val# it defines. If the live interval becomes empty, remove it as well. bool RegisterCoalescer::RemoveDeadDef(LiveInterval &li, MachineInstr *DefMI) { - SlotIndex DefIdx = li_->getInstructionIndex(DefMI).getDefIndex(); + SlotIndex DefIdx = LIS->getInstructionIndex(DefMI).getDefIndex(); LiveInterval::iterator MLR = li.FindLiveRangeContaining(DefIdx); if (DefIdx != MLR->valno->def) return false; li.removeValNo(MLR->valno); - return removeIntervalIfEmpty(li, li_, tri_); + return removeIntervalIfEmpty(li, LIS, TRI); } void RegisterCoalescer::RemoveCopyFlag(unsigned DstReg, const MachineInstr *CopyMI) { - SlotIndex DefIdx = li_->getInstructionIndex(CopyMI).getDefIndex(); - if (li_->hasInterval(DstReg)) { - LiveInterval &LI = li_->getInterval(DstReg); + SlotIndex DefIdx = LIS->getInstructionIndex(CopyMI).getDefIndex(); + if (LIS->hasInterval(DstReg)) { + LiveInterval &LI = LIS->getInterval(DstReg); if (const LiveRange *LR = LI.getLiveRangeContaining(DefIdx)) if (LR->valno->def == DefIdx) LR->valno->setCopy(0); } if (!TargetRegisterInfo::isPhysicalRegister(DstReg)) return; - for (const unsigned* AS = tri_->getAliasSet(DstReg); *AS; ++AS) { - if (!li_->hasInterval(*AS)) + for (const unsigned* AS = TRI->getAliasSet(DstReg); *AS; ++AS) { + if (!LIS->hasInterval(*AS)) continue; - LiveInterval &LI = li_->getInterval(*AS); + LiveInterval &LI = LIS->getInterval(*AS); if (const LiveRange *LR = LI.getLiveRangeContaining(DefIdx)) if (LR->valno->def == DefIdx) LR->valno->setCopy(0); @@ -894,8 +1048,8 @@ void RegisterCoalescer::RemoveCopyFlag(unsigned DstReg, /// are not spillable! If the destination interval uses are far away, think /// twice about coalescing them! bool RegisterCoalescer::shouldJoinPhys(CoalescerPair &CP) { - bool Allocatable = li_->isAllocatable(CP.getDstReg()); - LiveInterval &JoinVInt = li_->getInterval(CP.getSrcReg()); + bool Allocatable = LIS->isAllocatable(CP.getDstReg()); + LiveInterval &JoinVInt = LIS->getInterval(CP.getSrcReg()); /// Always join simple intervals that are defined by a single copy from a /// reserved register. This doesn't increase register pressure, so it is @@ -918,8 +1072,8 @@ bool RegisterCoalescer::shouldJoinPhys(CoalescerPair &CP) { // Don't join with physregs that have a ridiculous number of live // ranges. The data structure performance is really bad when that // happens. - if (li_->hasInterval(CP.getDstReg()) && - li_->getInterval(CP.getDstReg()).ranges.size() > 1000) { + if (LIS->hasInterval(CP.getDstReg()) && + LIS->getInterval(CP.getDstReg()).ranges.size() > 1000) { ++numAborts; DEBUG(dbgs() << "\tPhysical register live interval too complicated, abort!\n"); @@ -929,9 +1083,9 @@ bool RegisterCoalescer::shouldJoinPhys(CoalescerPair &CP) { // FIXME: Why are we skipping this test for partial copies? // CodeGen/X86/phys_subreg_coalesce-3.ll needs it. if (!CP.isPartial()) { - const TargetRegisterClass *RC = mri_->getRegClass(CP.getSrcReg()); + const TargetRegisterClass *RC = MRI->getRegClass(CP.getSrcReg()); unsigned Threshold = RegClassInfo.getNumAllocatableRegs(RC) * 2; - unsigned Length = li_->getApproximateInstructionCount(JoinVInt); + unsigned Length = LIS->getApproximateInstructionCount(JoinVInt); if (Length > Threshold) { ++numAborts; DEBUG(dbgs() << "\tMay tie down a physical register, abort!\n"); @@ -957,12 +1111,12 @@ RegisterCoalescer::isWinToJoinCrossClass(unsigned SrcReg, // Early exit if the function is fairly small, coalesce aggressively if // that's the case. For really special register classes with 3 or // fewer registers, be a bit more careful. - (li_->getFuncInstructionCount() / NewRCCount) < 8) + (LIS->getFuncInstructionCount() / NewRCCount) < 8) return true; - LiveInterval &SrcInt = li_->getInterval(SrcReg); - LiveInterval &DstInt = li_->getInterval(DstReg); - unsigned SrcSize = li_->getApproximateInstructionCount(SrcInt); - unsigned DstSize = li_->getApproximateInstructionCount(DstInt); + LiveInterval &SrcInt = LIS->getInterval(SrcReg); + LiveInterval &DstInt = LIS->getInterval(DstReg); + unsigned SrcSize = LIS->getApproximateInstructionCount(SrcInt); + unsigned DstSize = LIS->getApproximateInstructionCount(DstInt); // Coalesce aggressively if the intervals are small compared to the number of // registers in the new class. The number 4 is fairly arbitrary, chosen to be @@ -972,10 +1126,10 @@ RegisterCoalescer::isWinToJoinCrossClass(unsigned SrcReg, return true; // Estimate *register use density*. If it doubles or more, abort. - unsigned SrcUses = std::distance(mri_->use_nodbg_begin(SrcReg), - mri_->use_nodbg_end()); - unsigned DstUses = std::distance(mri_->use_nodbg_begin(DstReg), - mri_->use_nodbg_end()); + unsigned SrcUses = std::distance(MRI->use_nodbg_begin(SrcReg), + MRI->use_nodbg_end()); + unsigned DstUses = std::distance(MRI->use_nodbg_begin(DstReg), + MRI->use_nodbg_end()); unsigned NewUses = SrcUses + DstUses; unsigned NewSize = SrcSize + DstSize; if (SrcRC != NewRC && SrcSize > ThresSize) { @@ -1003,9 +1157,9 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) { if (JoinedCopies.count(CopyMI) || ReMatCopies.count(CopyMI)) return false; // Already done. - DEBUG(dbgs() << li_->getInstructionIndex(CopyMI) << '\t' << *CopyMI); + DEBUG(dbgs() << LIS->getInstructionIndex(CopyMI) << '\t' << *CopyMI); - CoalescerPair CP(*tii_, *tri_); + CoalescerPair CP(*TII, *TRI); if (!CP.setRegisters(CopyMI)) { DEBUG(dbgs() << "\tNot coalescable.\n"); return false; @@ -1018,8 +1172,15 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) { return false; // Not coalescable. } - DEBUG(dbgs() << "\tConsidering merging " << PrintReg(CP.getSrcReg(), tri_) - << " with " << PrintReg(CP.getDstReg(), tri_, CP.getSubIdx()) + // Eliminate undefs. + if (!CP.isPhys() && eliminateUndefCopy(CopyMI, CP)) { + markAsJoined(CopyMI); + DEBUG(dbgs() << "\tEliminated copy of <undef> value.\n"); + return false; // Not coalescable. + } + + DEBUG(dbgs() << "\tConsidering merging " << PrintReg(CP.getSrcReg(), TRI) + << " with " << PrintReg(CP.getDstReg(), TRI, CP.getSubIdx()) << "\n"); // Enforce policies. @@ -1028,8 +1189,8 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) { // Before giving up coalescing, if definition of source is defined by // trivial computation, try rematerializing it. if (!CP.isFlipped() && - ReMaterializeTrivialDef(li_->getInterval(CP.getSrcReg()), true, - CP.getDstReg(), 0, CopyMI)) + ReMaterializeTrivialDef(LIS->getInterval(CP.getSrcReg()), true, + CP.getDstReg(), CopyMI)) return true; return false; } @@ -1042,8 +1203,8 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) { return false; } if (!isWinToJoinCrossClass(CP.getSrcReg(), CP.getDstReg(), - mri_->getRegClass(CP.getSrcReg()), - mri_->getRegClass(CP.getDstReg()), + MRI->getRegClass(CP.getSrcReg()), + MRI->getRegClass(CP.getDstReg()), CP.getNewRC())) { DEBUG(dbgs() << "\tAvoid coalescing to constrained register class.\n"); Again = true; // May be possible to coalesce later. @@ -1052,8 +1213,8 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) { } // When possible, let DstReg be the larger interval. - if (!CP.getSubIdx() && li_->getInterval(CP.getSrcReg()).ranges.size() > - li_->getInterval(CP.getDstReg()).ranges.size()) + if (!CP.getSubIdx() && LIS->getInterval(CP.getSrcReg()).ranges.size() > + LIS->getInterval(CP.getDstReg()).ranges.size()) CP.flip(); } @@ -1067,8 +1228,8 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) { // If definition of source is defined by trivial computation, try // rematerializing it. if (!CP.isFlipped() && - ReMaterializeTrivialDef(li_->getInterval(CP.getSrcReg()), true, - CP.getDstReg(), 0, CopyMI)) + ReMaterializeTrivialDef(LIS->getInterval(CP.getSrcReg()), true, + CP.getDstReg(), CopyMI)) return true; // If we can eliminate the copy without merging the live ranges, do so now. @@ -1091,7 +1252,7 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) { // other. Make sure the resulting register is set to the right register class. if (CP.isCrossClass()) { ++numCrossRCs; - mri_->setRegClass(CP.getDstReg(), CP.getNewRC()); + MRI->setRegClass(CP.getDstReg(), CP.getNewRC()); } // Remember to delete the copy instruction. @@ -1105,10 +1266,10 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) { SmallVector<MachineBasicBlock*, 16> BlockSeq; // JoinIntervals invalidates the VNInfos in SrcInt, but we only need the // ranges for this, and they are preserved. - LiveInterval &SrcInt = li_->getInterval(CP.getSrcReg()); + LiveInterval &SrcInt = LIS->getInterval(CP.getSrcReg()); for (LiveInterval::const_iterator I = SrcInt.begin(), E = SrcInt.end(); I != E; ++I ) { - li_->findLiveInMBBs(I->start, I->end, BlockSeq); + LIS->findLiveInMBBs(I->start, I->end, BlockSeq); for (unsigned idx = 0, size = BlockSeq.size(); idx != size; ++idx) { MachineBasicBlock &block = *BlockSeq[idx]; if (!block.isLiveIn(CP.getDstReg())) @@ -1120,15 +1281,15 @@ bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) { // SrcReg is guarateed to be the register whose live interval that is // being merged. - li_->removeInterval(CP.getSrcReg()); + LIS->removeInterval(CP.getSrcReg()); // Update regalloc hint. - tri_->UpdateRegAllocHint(CP.getSrcReg(), CP.getDstReg(), *mf_); + TRI->UpdateRegAllocHint(CP.getSrcReg(), CP.getDstReg(), *MF); DEBUG({ - LiveInterval &DstInt = li_->getInterval(CP.getDstReg()); + LiveInterval &DstInt = LIS->getInterval(CP.getDstReg()); dbgs() << "\tJoined. Result = "; - DstInt.print(dbgs(), tri_); + DstInt.print(dbgs(), TRI); dbgs() << "\n"; }); @@ -1197,6 +1358,7 @@ static unsigned ComputeUltimateVN(VNInfo *VNI, // which allows us to coalesce A and B. // VNI is the definition of B. LR is the life range of A that includes // the slot just before B. If we return true, we add "B = X" to DupCopies. +// This implies that A dominates B. static bool RegistersDefinedFromSameValue(LiveIntervals &li, const TargetRegisterInfo &tri, CoalescerPair &CP, @@ -1248,7 +1410,9 @@ static bool RegistersDefinedFromSameValue(LiveIntervals &li, // If the copies use two different value numbers of X, we cannot merge // A and B. LiveInterval &SrcInt = li.getInterval(Src); - if (SrcInt.getVNInfoAt(Other->def) != SrcInt.getVNInfoAt(VNI->def)) + // getVNInfoBefore returns NULL for undef copies. In this case, the + // optimization is still safe. + if (SrcInt.getVNInfoBefore(Other->def) != SrcInt.getVNInfoBefore(VNI->def)) return false; DupCopies.push_back(MI); @@ -1259,18 +1423,18 @@ static bool RegistersDefinedFromSameValue(LiveIntervals &li, /// JoinIntervals - Attempt to join these two intervals. On failure, this /// returns false. bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) { - LiveInterval &RHS = li_->getInterval(CP.getSrcReg()); - DEBUG({ dbgs() << "\t\tRHS = "; RHS.print(dbgs(), tri_); dbgs() << "\n"; }); + LiveInterval &RHS = LIS->getInterval(CP.getSrcReg()); + DEBUG({ dbgs() << "\t\tRHS = "; RHS.print(dbgs(), TRI); dbgs() << "\n"; }); // If a live interval is a physical register, check for interference with any // aliases. The interference check implemented here is a bit more conservative // than the full interfeence check below. We allow overlapping live ranges // only when one is a copy of the other. if (CP.isPhys()) { - for (const unsigned *AS = tri_->getAliasSet(CP.getDstReg()); *AS; ++AS){ - if (!li_->hasInterval(*AS)) + for (const unsigned *AS = TRI->getAliasSet(CP.getDstReg()); *AS; ++AS){ + if (!LIS->hasInterval(*AS)) continue; - const LiveInterval &LHS = li_->getInterval(*AS); + const LiveInterval &LHS = LIS->getInterval(*AS); LiveInterval::const_iterator LI = LHS.begin(); for (LiveInterval::const_iterator RI = RHS.begin(), RE = RHS.end(); RI != RE; ++RI) { @@ -1278,10 +1442,10 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) { // Does LHS have an overlapping live range starting before RI? if ((LI != LHS.begin() && LI[-1].end > RI->start) && (RI->start != RI->valno->def || - !CP.isCoalescable(li_->getInstructionFromIndex(RI->start)))) { + !CP.isCoalescable(LIS->getInstructionFromIndex(RI->start)))) { DEBUG({ dbgs() << "\t\tInterference from alias: "; - LHS.print(dbgs(), tri_); + LHS.print(dbgs(), TRI); dbgs() << "\n\t\tOverlap at " << RI->start << " and no copy.\n"; }); return false; @@ -1290,10 +1454,10 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) { // Check that LHS ranges beginning in this range are copies. for (; LI != LHS.end() && LI->start < RI->end; ++LI) { if (LI->start != LI->valno->def || - !CP.isCoalescable(li_->getInstructionFromIndex(LI->start))) { + !CP.isCoalescable(LIS->getInstructionFromIndex(LI->start))) { DEBUG({ dbgs() << "\t\tInterference from alias: "; - LHS.print(dbgs(), tri_); + LHS.print(dbgs(), TRI); dbgs() << "\n\t\tDef at " << LI->start << " is not a copy.\n"; }); return false; @@ -1313,8 +1477,8 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) { SmallVector<MachineInstr*, 8> DupCopies; - LiveInterval &LHS = li_->getOrCreateInterval(CP.getDstReg()); - DEBUG({ dbgs() << "\t\tLHS = "; LHS.print(dbgs(), tri_); dbgs() << "\n"; }); + LiveInterval &LHS = LIS->getOrCreateInterval(CP.getDstReg()); + DEBUG({ dbgs() << "\t\tLHS = "; LHS.print(dbgs(), TRI); dbgs() << "\n"; }); // Loop over the value numbers of the LHS, seeing if any are defined from // the RHS. @@ -1337,7 +1501,7 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) { // from the RHS interval, we can use its value #. MachineInstr *MI = VNI->getCopy(); if (!CP.isCoalescable(MI) && - !RegistersDefinedFromSameValue(*li_, *tri_, CP, VNI, lr, DupCopies)) + !RegistersDefinedFromSameValue(*LIS, *TRI, CP, VNI, lr, DupCopies)) continue; LHSValsDefinedFromRHS[VNI] = lr->valno; @@ -1364,7 +1528,7 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) { // from the LHS interval, we can use its value #. MachineInstr *MI = VNI->getCopy(); if (!CP.isCoalescable(MI) && - !RegistersDefinedFromSameValue(*li_, *tri_, CP, VNI, lr, DupCopies)) + !RegistersDefinedFromSameValue(*LIS, *TRI, CP, VNI, lr, DupCopies)) continue; RHSValsDefinedFromLHS[VNI] = lr->valno; @@ -1486,7 +1650,7 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) { // and mark the X as coalesced to keep the illusion. unsigned Src = MI->getOperand(1).getReg(); SourceRegisters.push_back(Src); - MI->getOperand(0).substVirtReg(Src, 0, *tri_); + MI->getOperand(0).substVirtReg(Src, 0, *TRI); markAsJoined(MI); } @@ -1495,13 +1659,13 @@ bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) { // that B = X is gone. for (SmallVector<unsigned, 8>::iterator I = SourceRegisters.begin(), E = SourceRegisters.end(); I != E; ++I) { - li_->shrinkToUses(&li_->getInterval(*I)); + LIS->shrinkToUses(&LIS->getInterval(*I)); } // If we get here, we know that we can coalesce the live ranges. Ask the // intervals to coalesce themselves now. LHS.join(RHS, &LHSValNoAssignments[0], &RHSValNoAssignments[0], NewVNInfo, - mri_); + MRI); return true; } @@ -1552,7 +1716,7 @@ void RegisterCoalescer::CopyCoalesceInMBB(MachineBasicBlock *MBB, bool SrcIsPhys = TargetRegisterInfo::isPhysicalRegister(SrcReg); bool DstIsPhys = TargetRegisterInfo::isPhysicalRegister(DstReg); - if (li_->hasInterval(SrcReg) && li_->getInterval(SrcReg).empty()) + if (LIS->hasInterval(SrcReg) && LIS->getInterval(SrcReg).empty()) ImpDefCopies.push_back(Inst); else if (SrcIsPhys || DstIsPhys) PhysCopies.push_back(Inst); @@ -1590,9 +1754,9 @@ void RegisterCoalescer::joinIntervals() { DEBUG(dbgs() << "********** JOINING INTERVALS ***********\n"); std::vector<MachineInstr*> TryAgainList; - if (loopInfo->empty()) { + if (Loops->empty()) { // If there are no loops in the function, join intervals in function order. - for (MachineFunction::iterator I = mf_->begin(), E = mf_->end(); + for (MachineFunction::iterator I = MF->begin(), E = MF->end(); I != E; ++I) CopyCoalesceInMBB(I, TryAgainList); } else { @@ -1603,9 +1767,9 @@ void RegisterCoalescer::joinIntervals() { // Join intervals in the function prolog first. We want to join physical // registers with virtual registers before the intervals got too long. std::vector<std::pair<unsigned, MachineBasicBlock*> > MBBs; - for (MachineFunction::iterator I = mf_->begin(), E = mf_->end();I != E;++I){ + for (MachineFunction::iterator I = MF->begin(), E = MF->end();I != E;++I){ MachineBasicBlock *MBB = I; - MBBs.push_back(std::make_pair(loopInfo->getLoopDepth(MBB), I)); + MBBs.push_back(std::make_pair(Loops->getLoopDepth(MBB), I)); } // Sort by loop depth. @@ -1644,22 +1808,22 @@ void RegisterCoalescer::releaseMemory() { } bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { - mf_ = &fn; - mri_ = &fn.getRegInfo(); - tm_ = &fn.getTarget(); - tri_ = tm_->getRegisterInfo(); - tii_ = tm_->getInstrInfo(); - li_ = &getAnalysis<LiveIntervals>(); - ldv_ = &getAnalysis<LiveDebugVariables>(); + MF = &fn; + MRI = &fn.getRegInfo(); + TM = &fn.getTarget(); + TRI = TM->getRegisterInfo(); + TII = TM->getInstrInfo(); + LIS = &getAnalysis<LiveIntervals>(); + LDV = &getAnalysis<LiveDebugVariables>(); AA = &getAnalysis<AliasAnalysis>(); - loopInfo = &getAnalysis<MachineLoopInfo>(); + Loops = &getAnalysis<MachineLoopInfo>(); DEBUG(dbgs() << "********** SIMPLE REGISTER COALESCING **********\n" << "********** Function: " - << ((Value*)mf_->getFunction())->getName() << '\n'); + << ((Value*)MF->getFunction())->getName() << '\n'); if (VerifyCoalescing) - mf_->verify(this, "Before register coalescing"); + MF->verify(this, "Before register coalescing"); RegClassInfo.runOnMachineFunction(fn); @@ -1668,9 +1832,9 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { joinIntervals(); DEBUG({ dbgs() << "********** INTERVALS POST JOINING **********\n"; - for (LiveIntervals::iterator I = li_->begin(), E = li_->end(); + for (LiveIntervals::iterator I = LIS->begin(), E = LIS->end(); I != E; ++I){ - I->second->print(dbgs(), tri_); + I->second->print(dbgs(), TRI); dbgs() << "\n"; } }); @@ -1678,8 +1842,8 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { // Perform a final pass over the instructions and compute spill weights // and remove identity moves. - SmallVector<unsigned, 4> DeadDefs; - for (MachineFunction::iterator mbbi = mf_->begin(), mbbe = mf_->end(); + SmallVector<unsigned, 4> DeadDefs, InflateRegs; + for (MachineFunction::iterator mbbi = MF->begin(), mbbe = MF->end(); mbbi != mbbe; ++mbbi) { MachineBasicBlock* mbb = mbbi; for (MachineBasicBlock::iterator mii = mbb->begin(), mie = mbb->end(); @@ -1690,6 +1854,16 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { bool DoDelete = true; assert(MI->isCopyLike() && "Unrecognized copy instruction"); unsigned SrcReg = MI->getOperand(MI->isSubregToReg() ? 2 : 1).getReg(); + unsigned DstReg = MI->getOperand(0).getReg(); + + // Collect candidates for register class inflation. + if (TargetRegisterInfo::isVirtualRegister(SrcReg) && + RegClassInfo.isProperSubClass(MRI->getRegClass(SrcReg))) + InflateRegs.push_back(SrcReg); + if (TargetRegisterInfo::isVirtualRegister(DstReg) && + RegClassInfo.isProperSubClass(MRI->getRegClass(DstReg))) + InflateRegs.push_back(DstReg); + if (TargetRegisterInfo::isPhysicalRegister(SrcReg) && MI->getNumOperands() > 2) // Do not delete extract_subreg, insert_subreg of physical @@ -1701,8 +1875,8 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { if (MI->allDefsAreDead()) { if (TargetRegisterInfo::isVirtualRegister(SrcReg) && - li_->hasInterval(SrcReg)) - li_->shrinkToUses(&li_->getInterval(SrcReg)); + LIS->hasInterval(SrcReg)) + LIS->shrinkToUses(&LIS->getInterval(SrcReg)); DoDelete = true; } if (!DoDelete) { @@ -1711,10 +1885,10 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { MI->RemoveOperand(3); MI->RemoveOperand(1); } - MI->setDesc(tii_->get(TargetOpcode::KILL)); + MI->setDesc(TII->get(TargetOpcode::KILL)); mii = llvm::next(mii); } else { - li_->RemoveMachineInstrFromMaps(MI); + LIS->RemoveMachineInstrFromMaps(MI); mii = mbbi->erase(mii); ++numPeep; } @@ -1731,12 +1905,16 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { unsigned Reg = MO.getReg(); if (!Reg) continue; - if (TargetRegisterInfo::isVirtualRegister(Reg)) + if (TargetRegisterInfo::isVirtualRegister(Reg)) { DeadDefs.push_back(Reg); + // Remat may also enable register class inflation. + if (RegClassInfo.isProperSubClass(MRI->getRegClass(Reg))) + InflateRegs.push_back(Reg); + } if (MO.isDead()) continue; if (TargetRegisterInfo::isPhysicalRegister(Reg) || - !mri_->use_nodbg_empty(Reg)) { + !MRI->use_nodbg_empty(Reg)) { isDead = false; break; } @@ -1745,9 +1923,9 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { while (!DeadDefs.empty()) { unsigned DeadDef = DeadDefs.back(); DeadDefs.pop_back(); - RemoveDeadDef(li_->getInterval(DeadDef), MI); + RemoveDeadDef(LIS->getInterval(DeadDef), MI); } - li_->RemoveMachineInstrFromMaps(mii); + LIS->RemoveMachineInstrFromMaps(mii); mii = mbbi->erase(mii); continue; } else @@ -1757,14 +1935,14 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { ++mii; // Check for now unnecessary kill flags. - if (li_->isNotInMIMap(MI)) continue; - SlotIndex DefIdx = li_->getInstructionIndex(MI).getDefIndex(); + if (LIS->isNotInMIMap(MI)) continue; + SlotIndex DefIdx = LIS->getInstructionIndex(MI).getDefIndex(); for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); if (!MO.isReg() || !MO.isKill()) continue; unsigned reg = MO.getReg(); - if (!reg || !li_->hasInterval(reg)) continue; - if (!li_->getInterval(reg).killedAt(DefIdx)) { + if (!reg || !LIS->hasInterval(reg)) continue; + if (!LIS->getInterval(reg).killedAt(DefIdx)) { MO.setIsKill(false); continue; } @@ -1772,26 +1950,40 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { // remain alive. if (!TargetRegisterInfo::isPhysicalRegister(reg)) continue; - for (const unsigned *SR = tri_->getSubRegisters(reg); + for (const unsigned *SR = TRI->getSubRegisters(reg); unsigned S = *SR; ++SR) - if (li_->hasInterval(S) && li_->getInterval(S).liveAt(DefIdx)) - MI->addRegisterDefined(S, tri_); + if (LIS->hasInterval(S) && LIS->getInterval(S).liveAt(DefIdx)) + MI->addRegisterDefined(S, TRI); } } } + // After deleting a lot of copies, register classes may be less constrained. + // Removing sub-register opreands may alow GR32_ABCD -> GR32 and DPR_VFP2 -> + // DPR inflation. + array_pod_sort(InflateRegs.begin(), InflateRegs.end()); + InflateRegs.erase(std::unique(InflateRegs.begin(), InflateRegs.end()), + InflateRegs.end()); + DEBUG(dbgs() << "Trying to inflate " << InflateRegs.size() << " regs.\n"); + for (unsigned i = 0, e = InflateRegs.size(); i != e; ++i) { + unsigned Reg = InflateRegs[i]; + if (MRI->reg_nodbg_empty(Reg)) + continue; + if (MRI->recomputeRegClass(Reg, *TM)) { + DEBUG(dbgs() << PrintReg(Reg) << " inflated to " + << MRI->getRegClass(Reg)->getName() << '\n'); + ++NumInflated; + } + } + DEBUG(dump()); - DEBUG(ldv_->dump()); + DEBUG(LDV->dump()); if (VerifyCoalescing) - mf_->verify(this, "After register coalescing"); + MF->verify(this, "After register coalescing"); return true; } /// print - Implement the dump method. void RegisterCoalescer::print(raw_ostream &O, const Module* m) const { - li_->print(O, m); -} - -RegisterCoalescer *llvm::createRegisterCoalescer() { - return new RegisterCoalescer(); + LIS->print(O, m); } diff --git a/lib/CodeGen/RegisterCoalescer.h b/lib/CodeGen/RegisterCoalescer.h index 4131d91..472c483 100644 --- a/lib/CodeGen/RegisterCoalescer.h +++ b/lib/CodeGen/RegisterCoalescer.h @@ -12,198 +12,60 @@ // //===----------------------------------------------------------------------===// -#include "RegisterClassInfo.h" -#include "llvm/Support/IncludeFile.h" -#include "llvm/CodeGen/LiveInterval.h" -#include "llvm/ADT/SmallPtrSet.h" - #ifndef LLVM_CODEGEN_REGISTER_COALESCER_H #define LLVM_CODEGEN_REGISTER_COALESCER_H namespace llvm { - class MachineFunction; - class RegallocQuery; - class AnalysisUsage; class MachineInstr; class TargetRegisterInfo; class TargetRegisterClass; class TargetInstrInfo; - class LiveDebugVariables; - class VirtRegMap; - class MachineLoopInfo; - - class CoalescerPair; - - /// An abstract interface for register coalescers. Coalescers must - /// implement this interface to be part of the coalescer analysis - /// group. - class RegisterCoalescer : public MachineFunctionPass { - MachineFunction* mf_; - MachineRegisterInfo* mri_; - const TargetMachine* tm_; - const TargetRegisterInfo* tri_; - const TargetInstrInfo* tii_; - LiveIntervals *li_; - LiveDebugVariables *ldv_; - const MachineLoopInfo* loopInfo; - AliasAnalysis *AA; - RegisterClassInfo RegClassInfo; - - /// JoinedCopies - Keep track of copies eliminated due to coalescing. - /// - SmallPtrSet<MachineInstr*, 32> JoinedCopies; - - /// ReMatCopies - Keep track of copies eliminated due to remat. - /// - SmallPtrSet<MachineInstr*, 32> ReMatCopies; - - /// ReMatDefs - Keep track of definition instructions which have - /// been remat'ed. - SmallPtrSet<MachineInstr*, 8> ReMatDefs; - - /// joinIntervals - join compatible live intervals - void joinIntervals(); - - /// CopyCoalesceInMBB - Coalesce copies in the specified MBB, putting - /// copies that cannot yet be coalesced into the "TryAgain" list. - void CopyCoalesceInMBB(MachineBasicBlock *MBB, - std::vector<MachineInstr*> &TryAgain); - - /// JoinCopy - Attempt to join intervals corresponding to SrcReg/DstReg, - /// which are the src/dst of the copy instruction CopyMI. This returns true - /// if the copy was successfully coalesced away. If it is not currently - /// possible to coalesce this interval, but it may be possible if other - /// things get coalesced, then it returns true by reference in 'Again'. - bool JoinCopy(MachineInstr *TheCopy, bool &Again); - - /// JoinIntervals - Attempt to join these two intervals. On failure, this - /// returns false. The output "SrcInt" will not have been modified, so we can - /// use this information below to update aliases. - bool JoinIntervals(CoalescerPair &CP); - - /// AdjustCopiesBackFrom - We found a non-trivially-coalescable copy. If - /// the source value number is defined by a copy from the destination reg - /// see if we can merge these two destination reg valno# into a single - /// value number, eliminating a copy. - bool AdjustCopiesBackFrom(const CoalescerPair &CP, MachineInstr *CopyMI); - - /// HasOtherReachingDefs - Return true if there are definitions of IntB - /// other than BValNo val# that can reach uses of AValno val# of IntA. - bool HasOtherReachingDefs(LiveInterval &IntA, LiveInterval &IntB, - VNInfo *AValNo, VNInfo *BValNo); - - /// RemoveCopyByCommutingDef - We found a non-trivially-coalescable copy. - /// If the source value number is defined by a commutable instruction and - /// its other operand is coalesced to the copy dest register, see if we - /// can transform the copy into a noop by commuting the definition. - bool RemoveCopyByCommutingDef(const CoalescerPair &CP,MachineInstr *CopyMI); - - /// ReMaterializeTrivialDef - If the source of a copy is defined by a trivial - /// computation, replace the copy by rematerialize the definition. - /// If PreserveSrcInt is true, make sure SrcInt is valid after the call. - bool ReMaterializeTrivialDef(LiveInterval &SrcInt, bool PreserveSrcInt, - unsigned DstReg, unsigned DstSubIdx, - MachineInstr *CopyMI); - - /// shouldJoinPhys - Return true if a physreg copy should be joined. - bool shouldJoinPhys(CoalescerPair &CP); - - /// isWinToJoinCrossClass - Return true if it's profitable to coalesce - /// two virtual registers from different register classes. - bool isWinToJoinCrossClass(unsigned SrcReg, - unsigned DstReg, - const TargetRegisterClass *SrcRC, - const TargetRegisterClass *DstRC, - const TargetRegisterClass *NewRC); - - /// UpdateRegDefsUses - Replace all defs and uses of SrcReg to DstReg and - /// update the subregister number if it is not zero. If DstReg is a - /// physical register and the existing subregister number of the def / use - /// being updated is not zero, make sure to set it to the correct physical - /// subregister. - void UpdateRegDefsUses(const CoalescerPair &CP); - - /// RemoveDeadDef - If a def of a live interval is now determined dead, - /// remove the val# it defines. If the live interval becomes empty, remove - /// it as well. - bool RemoveDeadDef(LiveInterval &li, MachineInstr *DefMI); - - /// RemoveCopyFlag - If DstReg is no longer defined by CopyMI, clear the - /// VNInfo copy flag for DstReg and all aliases. - void RemoveCopyFlag(unsigned DstReg, const MachineInstr *CopyMI); - - /// markAsJoined - Remember that CopyMI has already been joined. - void markAsJoined(MachineInstr *CopyMI); - - public: - static char ID; // Class identification, replacement for typeinfo - RegisterCoalescer() : MachineFunctionPass(ID) { - initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry()); - } - - /// Register allocators must call this from their own - /// getAnalysisUsage to cover the case where the coalescer is not - /// a Pass in the proper sense and isn't managed by PassManager. - /// PassManager needs to know which analyses to make available and - /// which to invalidate when running the register allocator or any - /// pass that might call coalescing. The long-term solution is to - /// allow hierarchies of PassManagers. - virtual void getAnalysisUsage(AnalysisUsage &AU) const; - - virtual void releaseMemory(); - - /// runOnMachineFunction - pass entry point - virtual bool runOnMachineFunction(MachineFunction&); - - /// print - Implement the dump method. - virtual void print(raw_ostream &O, const Module* = 0) const; - }; /// CoalescerPair - A helper class for register coalescers. When deciding if /// two registers can be coalesced, CoalescerPair can determine if a copy /// instruction would become an identity copy after coalescing. class CoalescerPair { - const TargetInstrInfo &tii_; - const TargetRegisterInfo &tri_; + const TargetInstrInfo &TII; + const TargetRegisterInfo &TRI; - /// dstReg_ - The register that will be left after coalescing. It can be a + /// DstReg - The register that will be left after coalescing. It can be a /// virtual or physical register. - unsigned dstReg_; + unsigned DstReg; - /// srcReg_ - the virtual register that will be coalesced into dstReg. - unsigned srcReg_; + /// SrcReg - the virtual register that will be coalesced into dstReg. + unsigned SrcReg; - /// subReg_ - The subregister index of srcReg in dstReg_. It is possible the - /// coalesce srcReg_ into a subreg of the larger dstReg_ when dstReg_ is a + /// subReg_ - The subregister index of srcReg in DstReg. It is possible the + /// coalesce SrcReg into a subreg of the larger DstReg when DstReg is a /// virtual register. - unsigned subIdx_; + unsigned SubIdx; - /// partial_ - True when the original copy was a partial subregister copy. - bool partial_; + /// Partial - True when the original copy was a partial subregister copy. + bool Partial; - /// crossClass_ - True when both regs are virtual, and newRC is constrained. - bool crossClass_; + /// CrossClass - True when both regs are virtual, and newRC is constrained. + bool CrossClass; - /// flipped_ - True when DstReg and SrcReg are reversed from the oriignal copy - /// instruction. - bool flipped_; + /// Flipped - True when DstReg and SrcReg are reversed from the oriignal + /// copy instruction. + bool Flipped; - /// newRC_ - The register class of the coalesced register, or NULL if dstReg_ + /// NewRC - The register class of the coalesced register, or NULL if DstReg /// is a physreg. - const TargetRegisterClass *newRC_; + const TargetRegisterClass *NewRC; public: CoalescerPair(const TargetInstrInfo &tii, const TargetRegisterInfo &tri) - : tii_(tii), tri_(tri), dstReg_(0), srcReg_(0), subIdx_(0), - partial_(false), crossClass_(false), flipped_(false), newRC_(0) {} + : TII(tii), TRI(tri), DstReg(0), SrcReg(0), SubIdx(0), + Partial(false), CrossClass(false), Flipped(false), NewRC(0) {} /// setRegisters - set registers to match the copy instruction MI. Return /// false if MI is not a coalescable copy instruction. bool setRegisters(const MachineInstr*); - /// flip - Swap srcReg_ and dstReg_. Return false if swapping is impossible - /// because dstReg_ is a physical register, or subIdx_ is set. + /// flip - Swap SrcReg and DstReg. Return false if swapping is impossible + /// because DstReg is a physical register, or SubIdx is set. bool flip(); /// isCoalescable - Return true if MI is a copy instruction that will become @@ -211,32 +73,33 @@ namespace llvm { bool isCoalescable(const MachineInstr*) const; /// isPhys - Return true if DstReg is a physical register. - bool isPhys() const { return !newRC_; } + bool isPhys() const { return !NewRC; } - /// isPartial - Return true if the original copy instruction did not copy the - /// full register, but was a subreg operation. - bool isPartial() const { return partial_; } + /// isPartial - Return true if the original copy instruction did not copy + /// the full register, but was a subreg operation. + bool isPartial() const { return Partial; } - /// isCrossClass - Return true if DstReg is virtual and NewRC is a smaller register class than DstReg's. - bool isCrossClass() const { return crossClass_; } + /// isCrossClass - Return true if DstReg is virtual and NewRC is a smaller + /// register class than DstReg's. + bool isCrossClass() const { return CrossClass; } /// isFlipped - Return true when getSrcReg is the register being defined by /// the original copy instruction. - bool isFlipped() const { return flipped_; } + bool isFlipped() const { return Flipped; } /// getDstReg - Return the register (virtual or physical) that will remain /// after coalescing. - unsigned getDstReg() const { return dstReg_; } + unsigned getDstReg() const { return DstReg; } /// getSrcReg - Return the virtual register that will be coalesced away. - unsigned getSrcReg() const { return srcReg_; } + unsigned getSrcReg() const { return SrcReg; } /// getSubIdx - Return the subregister index in DstReg that SrcReg will be /// coalesced into, or 0. - unsigned getSubIdx() const { return subIdx_; } + unsigned getSubIdx() const { return SubIdx; } /// getNewRC - Return the register class of the coalesced register. - const TargetRegisterClass *getNewRC() const { return newRC_; } + const TargetRegisterClass *getNewRC() const { return NewRC; } }; } // End llvm namespace diff --git a/lib/CodeGen/RegisterScavenging.cpp b/lib/CodeGen/RegisterScavenging.cpp index 9e9a145..ca02aa1 100644 --- a/lib/CodeGen/RegisterScavenging.cpp +++ b/lib/CodeGen/RegisterScavenging.cpp @@ -206,6 +206,7 @@ void RegScavenger::forward() { break; } assert(SubUsed && "Using an undefined register!"); + (void)SubUsed; } assert((!EarlyClobberRegs.test(Reg) || MI->isRegTiedToDefOperand(i)) && "Using an early clobbered register!"); diff --git a/lib/CodeGen/ScheduleDAG.cpp b/lib/CodeGen/ScheduleDAG.cpp index 21375b2..1e9b5c8 100644 --- a/lib/CodeGen/ScheduleDAG.cpp +++ b/lib/CodeGen/ScheduleDAG.cpp @@ -26,7 +26,7 @@ using namespace llvm; #ifndef NDEBUG -cl::opt<bool> StressSchedOpt( +static cl::opt<bool> StressSchedOpt( "stress-sched", cl::Hidden, cl::init(false), cl::desc("Stress test instruction scheduling")); #endif @@ -140,6 +140,7 @@ void SUnit::removePred(const SDep &D) { break; } assert(FoundSucc && "Mismatching preds / succs lists!"); + (void)FoundSucc; Preds.erase(I); // Update the bookkeeping. if (P.getKind() == SDep::Data) { diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index 446adfc..34b8ab0 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -36,7 +36,7 @@ ScheduleDAGInstrs::ScheduleDAGInstrs(MachineFunction &mf, const MachineDominatorTree &mdt) : ScheduleDAG(mf), MLI(mli), MDT(mdt), MFI(mf.getFrameInfo()), InstrItins(mf.getTarget().getInstrItineraryData()), - Defs(TRI->getNumRegs()), Uses(TRI->getNumRegs()), + Defs(TRI->getNumRegs()), Uses(TRI->getNumRegs()), LoopRegs(MLI, MDT), FirstDbgValue(0) { DbgValues.clear(); } @@ -134,6 +134,7 @@ static const Value *getUnderlyingObjectForInstr(const MachineInstr *MI, } void ScheduleDAGInstrs::StartBlock(MachineBasicBlock *BB) { + LoopRegs.Deps.clear(); if (MachineLoop *ML = MLI.getLoopFor(BB)) if (BB == ML->getLoopLatch()) { MachineBasicBlock *Header = ML->getHeader(); diff --git a/lib/CodeGen/ScheduleDAGInstrs.h b/lib/CodeGen/ScheduleDAGInstrs.h index 8a4ea85..666bdf5 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.h +++ b/lib/CodeGen/ScheduleDAGInstrs.h @@ -48,7 +48,8 @@ namespace llvm { /// VisitLoop - Clear out any previous state and analyze the given loop. /// void VisitLoop(const MachineLoop *Loop) { - Deps.clear(); + assert(Deps.empty() && "stale loop dependencies"); + MachineBasicBlock *Header = Loop->getHeader(); SmallSet<unsigned, 8> LoopLiveIns; for (MachineBasicBlock::livein_iterator LI = Header->livein_begin(), @@ -109,7 +110,7 @@ namespace llvm { /// initialized and destructed for each block. std::vector<std::vector<SUnit *> > Defs; std::vector<std::vector<SUnit *> > Uses; - + /// PendingLoads - Remember where unknown loads are after the most recent /// unknown store, as we iterate. As with Defs and Uses, this is here /// to minimize construction/destruction. @@ -127,7 +128,7 @@ namespace llvm { protected: /// DbgValues - Remember instruction that preceeds DBG_VALUE. - typedef std::vector<std::pair<MachineInstr *, MachineInstr *> > + typedef std::vector<std::pair<MachineInstr *, MachineInstr *> > DbgValueVector; DbgValueVector DbgValues; MachineInstr *FirstDbgValue; diff --git a/lib/CodeGen/ScoreboardHazardRecognizer.cpp b/lib/CodeGen/ScoreboardHazardRecognizer.cpp index 0e005d3..b80c01e 100644 --- a/lib/CodeGen/ScoreboardHazardRecognizer.cpp +++ b/lib/CodeGen/ScoreboardHazardRecognizer.cpp @@ -213,7 +213,6 @@ void ScoreboardHazardRecognizer::EmitInstruction(SUnit *SU) { freeUnits = freeUnit & (freeUnit - 1); } while (freeUnits); - assert(freeUnit && "No function unit available!"); if (IS->getReservationKind() == InstrStage::Required) RequiredScoreboard[cycle + i] |= freeUnit; else diff --git a/lib/CodeGen/SelectionDAG/CMakeLists.txt b/lib/CodeGen/SelectionDAG/CMakeLists.txt index 15932c0..2282f0e 100644 --- a/lib/CodeGen/SelectionDAG/CMakeLists.txt +++ b/lib/CodeGen/SelectionDAG/CMakeLists.txt @@ -21,3 +21,13 @@ add_llvm_library(LLVMSelectionDAG TargetLowering.cpp TargetSelectionDAGInfo.cpp ) + +add_llvm_library_dependencies(LLVMSelectionDAG + LLVMAnalysis + LLVMCodeGen + LLVMCore + LLVMMC + LLVMSupport + LLVMTarget + LLVMTransformUtils + ) diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 4f0d2ca..7b87868 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -216,6 +216,7 @@ namespace { SDValue visitEXTRACT_VECTOR_ELT(SDNode *N); SDValue visitBUILD_VECTOR(SDNode *N); SDValue visitCONCAT_VECTORS(SDNode *N); + SDValue visitEXTRACT_SUBVECTOR(SDNode *N); SDValue visitVECTOR_SHUFFLE(SDNode *N); SDValue visitMEMBARRIER(SDNode *N); @@ -1105,6 +1106,7 @@ SDValue DAGCombiner::visit(SDNode *N) { case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N); case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N); case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N); + case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N); case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N); case ISD::MEMBARRIER: return visitMEMBARRIER(N); } @@ -1526,12 +1528,6 @@ SDValue DAGCombiner::visitADDE(SDNode *N) { ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); - // If both operands are null we know that carry out will always be false. - if (N0C && N0C->isNullValue() && N0 == N1) - DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), DAG.getNode(ISD::CARRY_FALSE, - N->getDebugLoc(), - MVT::Glue)); - // canonicalize constant to RHS if (N0C && !N1C) return DAG.getNode(ISD::ADDE, N->getDebugLoc(), N->getVTList(), @@ -3763,7 +3759,7 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { if (VT.isInteger() && (VT0 == MVT::i1 || (VT0.isInteger() && - TLI.getBooleanContents() == TargetLowering::ZeroOrOneBooleanContent)) && + TLI.getBooleanContents(false) == TargetLowering::ZeroOrOneBooleanContent)) && N1C && N2C && N1C->isNullValue() && N2C->getAPIntValue() == 1) { SDValue XORNode; if (VT == VT0) @@ -4118,7 +4114,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) { // we know that the element size of the sext'd result matches the // element size of the compare operands. if (VT.getSizeInBits() == N0VT.getSizeInBits()) - return DAG.getVSetCC(N->getDebugLoc(), VT, N0.getOperand(0), + return DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0), N0.getOperand(1), cast<CondCodeSDNode>(N0.getOperand(2))->get()); // If the desired elements are smaller or larger than the source @@ -4132,7 +4128,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) { EVT::getVectorVT(*DAG.getContext(), MatchingElementType, N0VT.getVectorNumElements()); SDValue VsetCC = - DAG.getVSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0), + DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0), N0.getOperand(1), cast<CondCodeSDNode>(N0.getOperand(2))->get()); return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT); @@ -4348,7 +4344,7 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) { // we know that the element size of the sext'd result matches the // element size of the compare operands. return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, - DAG.getVSetCC(N->getDebugLoc(), VT, N0.getOperand(0), + DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0), N0.getOperand(1), cast<CondCodeSDNode>(N0.getOperand(2))->get()), DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT, @@ -4364,7 +4360,7 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) { EVT::getVectorVT(*DAG.getContext(), MatchingElementType, N0VT.getVectorNumElements()); SDValue VsetCC = - DAG.getVSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0), + DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0), N0.getOperand(1), cast<CondCodeSDNode>(N0.getOperand(2))->get()); return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, @@ -4532,7 +4528,7 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) { // we know that the element size of the sext'd result matches the // element size of the compare operands. if (VT.getSizeInBits() == N0VT.getSizeInBits()) - return DAG.getVSetCC(N->getDebugLoc(), VT, N0.getOperand(0), + return DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0), N0.getOperand(1), cast<CondCodeSDNode>(N0.getOperand(2))->get()); // If the desired elements are smaller or larger than the source @@ -4546,7 +4542,7 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) { EVT::getVectorVT(*DAG.getContext(), MatchingElementType, N0VT.getVectorNumElements()); SDValue VsetCC = - DAG.getVSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0), + DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0), N0.getOperand(1), cast<CondCodeSDNode>(N0.getOperand(2))->get()); return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT); @@ -6479,7 +6475,7 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) { PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff; unsigned NewAlign = MinAlign(LD->getAlignment(), PtrOff); - const Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext()); + Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext()); if (NewAlign < TLI.getTargetData()->getABITypeAlignment(NewVTTy)) return SDValue(); @@ -6542,7 +6538,7 @@ SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) { unsigned LDAlign = LD->getAlignment(); unsigned STAlign = ST->getAlignment(); - const Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext()); + Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext()); unsigned ABIAlign = TLI.getTargetData()->getABITypeAlignment(IntVTTy); if (LDAlign < ABIAlign || STAlign < ABIAlign) return SDValue(); @@ -6776,6 +6772,7 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) { SDValue InVec = N->getOperand(0); SDValue InVal = N->getOperand(1); SDValue EltNo = N->getOperand(2); + DebugLoc dl = N->getDebugLoc(); // If the inserted element is an UNDEF, just use the input vector. if (InVal.getOpcode() == ISD::UNDEF) @@ -6787,32 +6784,40 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) { if (LegalOperations && !TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) return SDValue(); - // If the invec is a BUILD_VECTOR and if EltNo is a constant, build a new - // vector with the inserted element. - if (InVec.getOpcode() == ISD::BUILD_VECTOR && isa<ConstantSDNode>(EltNo)) { - unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); - SmallVector<SDValue, 8> Ops(InVec.getNode()->op_begin(), - InVec.getNode()->op_end()); - if (Elt < Ops.size()) - Ops[Elt] = InVal; - return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), - VT, &Ops[0], Ops.size()); - } - // If the invec is an UNDEF and if EltNo is a constant, create a new - // BUILD_VECTOR with undef elements and the inserted element. - if (InVec.getOpcode() == ISD::UNDEF && - isa<ConstantSDNode>(EltNo)) { - EVT EltVT = VT.getVectorElementType(); + // Check that we know which element is being inserted + if (!isa<ConstantSDNode>(EltNo)) + return SDValue(); + unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); + + // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially + // be converted to a BUILD_VECTOR). Fill in the Ops vector with the + // vector elements. + SmallVector<SDValue, 8> Ops; + if (InVec.getOpcode() == ISD::BUILD_VECTOR) { + Ops.append(InVec.getNode()->op_begin(), + InVec.getNode()->op_end()); + } else if (InVec.getOpcode() == ISD::UNDEF) { unsigned NElts = VT.getVectorNumElements(); - SmallVector<SDValue, 8> Ops(NElts, DAG.getUNDEF(EltVT)); + Ops.append(NElts, DAG.getUNDEF(InVal.getValueType())); + } else { + return SDValue(); + } - unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); - if (Elt < Ops.size()) - Ops[Elt] = InVal; - return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), - VT, &Ops[0], Ops.size()); + // Insert the element + if (Elt < Ops.size()) { + // All the operands of BUILD_VECTOR must have the same type; + // we enforce that here. + EVT OpVT = Ops[0].getValueType(); + if (InVal.getValueType() != OpVT) + InVal = OpVT.bitsGT(InVal.getValueType()) ? + DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) : + DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal); + Ops[Elt] = InVal; } - return SDValue(); + + // Return the new vector + return DAG.getNode(ISD::BUILD_VECTOR, dl, + VT, &Ops[0], Ops.size()); } SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { @@ -6896,7 +6901,7 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { // If Idx was -1 above, Elt is going to be -1, so just return undef. if (Elt == -1) - return DAG.getUNDEF(LN0->getBasePtr().getValueType()); + return DAG.getUNDEF(LVT); unsigned Align = LN0->getAlignment(); if (NewLoad) { @@ -7028,6 +7033,36 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { return SDValue(); } +SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { + EVT NVT = N->getValueType(0); + SDValue V = N->getOperand(0); + + if (V->getOpcode() == ISD::INSERT_SUBVECTOR) { + // Handle only simple case where vector being inserted and vector + // being extracted are of same type, and are half size of larger vectors. + EVT BigVT = V->getOperand(0).getValueType(); + EVT SmallVT = V->getOperand(1).getValueType(); + if (NVT != SmallVT || NVT.getSizeInBits()*2 != BigVT.getSizeInBits()) + return SDValue(); + + // Combine: + // (extract_subvec (insert_subvec V1, V2, InsIdx), ExtIdx) + // Into: + // indicies are equal => V1 + // otherwise => (extract_subvec V1, ExtIdx) + // + SDValue InsIdx = N->getOperand(1); + SDValue ExtIdx = V->getOperand(2); + + if (InsIdx == ExtIdx) + return V->getOperand(1); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), NVT, + V->getOperand(0), N->getOperand(1)); + } + + return SDValue(); +} + SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { EVT VT = N->getValueType(0); unsigned NumElts = VT.getVectorNumElements(); @@ -7447,7 +7482,7 @@ SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1, const_cast<ConstantFP*>(FV->getConstantFPValue()), const_cast<ConstantFP*>(TV->getConstantFPValue()) }; - const Type *FPTy = Elts[0]->getType(); + Type *FPTy = Elts[0]->getType(); const TargetData &TD = *TLI.getTargetData(); // Create a ConstantArray of the two constants. @@ -7465,10 +7500,13 @@ SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1, SDValue Cond = DAG.getSetCC(DL, TLI.getSetCCResultType(N0.getValueType()), N0, N1, CC); + AddToWorkList(Cond.getNode()); SDValue CstOffset = DAG.getNode(ISD::SELECT, DL, Zero.getValueType(), Cond, One, Zero); + AddToWorkList(CstOffset.getNode()); CPIdx = DAG.getNode(ISD::ADD, DL, TLI.getPointerTy(), CPIdx, CstOffset); + AddToWorkList(CPIdx.getNode()); return DAG.getLoad(TV->getValueType(0), DL, DAG.getEntryNode(), CPIdx, MachinePointerInfo::getConstantPool(), false, false, Alignment); @@ -7553,7 +7591,8 @@ SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1, // fold select C, 16, 0 -> shl C, 4 if (N2C && N3C && N3C->isNullValue() && N2C->getAPIntValue().isPowerOf2() && - TLI.getBooleanContents() == TargetLowering::ZeroOrOneBooleanContent) { + TLI.getBooleanContents(N0.getValueType().isVector()) == + TargetLowering::ZeroOrOneBooleanContent) { // If the caller doesn't want us to simplify this into a zext of a compare, // don't do it. diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp index 54a7d43..e8f8c73 100644 --- a/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -66,17 +66,22 @@ using namespace llvm; void FastISel::startNewBlock() { LocalValueMap.clear(); - // Start out as null, meaining no local-value instructions have - // been emitted. - LastLocalValue = 0; + EmitStartPt = 0; - // Advance the last local value past any EH_LABEL instructions. + // Advance the emit start point past any EH_LABEL instructions. MachineBasicBlock::iterator I = FuncInfo.MBB->begin(), E = FuncInfo.MBB->end(); while (I != E && I->getOpcode() == TargetOpcode::EH_LABEL) { - LastLocalValue = I; + EmitStartPt = I; ++I; } + LastLocalValue = EmitStartPt; +} + +void FastISel::flushLocalValueMap() { + LocalValueMap.clear(); + LastLocalValue = EmitStartPt; + recomputeInsertPt(); } bool FastISel::hasTrivialKill(const Value *V) const { @@ -183,7 +188,7 @@ unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) { (void) Flt.convertToInteger(x, IntBitWidth, /*isSigned=*/true, APFloat::rmTowardZero, &isExact); if (isExact) { - APInt IntVal(IntBitWidth, 2, x); + APInt IntVal(IntBitWidth, x); unsigned IntegerReg = getRegForValue(ConstantInt::get(V->getContext(), IntVal)); @@ -422,12 +427,12 @@ bool FastISel::SelectGetElementPtr(const User *I) { bool NIsKill = hasTrivialKill(I->getOperand(0)); - const Type *Ty = I->getOperand(0)->getType(); + Type *Ty = I->getOperand(0)->getType(); MVT VT = TLI.getPointerTy(); for (GetElementPtrInst::const_op_iterator OI = I->op_begin()+1, E = I->op_end(); OI != E; ++OI) { const Value *Idx = *OI; - if (const StructType *StTy = dyn_cast<StructType>(Ty)) { + if (StructType *StTy = dyn_cast<StructType>(Ty)) { unsigned Field = cast<ConstantInt>(Idx)->getZExtValue(); if (Field) { // N = N + Offset @@ -489,7 +494,7 @@ bool FastISel::SelectCall(const User *I) { const CallInst *Call = cast<CallInst>(I); // Handle simple inline asms. - if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getArgOperand(0))) { + if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) { // Don't attempt to handle constraints. if (!IA->getConstraintString().empty()) return false; @@ -526,13 +531,10 @@ bool FastISel::SelectCall(const User *I) { unsigned Reg = 0; unsigned Offset = 0; if (const Argument *Arg = dyn_cast<Argument>(Address)) { - if (Arg->hasByValAttr()) { - // Byval arguments' frame index is recorded during argument lowering. - // Use this info directly. - Offset = FuncInfo.getByValArgumentFrameIndex(Arg); - if (Offset) - Reg = TRI.getFrameRegister(*FuncInfo.MF); - } + // Some arguments' frame index is recorded during argument lowering. + Offset = FuncInfo.getArgumentFrameIndex(Arg); + if (Offset) + Reg = TRI.getFrameRegister(*FuncInfo.MF); } if (!Reg) Reg = getRegForValue(Address); @@ -645,6 +647,16 @@ bool FastISel::SelectCall(const User *I) { } } + // Usually, it does not make sense to initialize a value, + // make an unrelated function call and use the value, because + // it tends to be spilled on the stack. So, we move the pointer + // to the last local value to the beginning of the block, so that + // all the values which have already been materialized, + // appear after the call. It also makes sense to skip intrinsics + // since they tend to be inlined. + if (!isa<IntrinsicInst>(F)) + flushLocalValueMap(); + // An arbitrary call. Bail. return false; } @@ -839,7 +851,7 @@ FastISel::SelectExtractValue(const User *U) { return false; const Value *Op0 = EVI->getOperand(0); - const Type *AggTy = Op0->getType(); + Type *AggTy = Op0->getType(); // Get the base result register. unsigned ResultReg; @@ -1074,7 +1086,7 @@ unsigned FastISel::FastEmit_ri_(MVT VT, unsigned Opcode, if (MaterialReg == 0) { // This is a bit ugly/slow, but failing here means falling out of // fast-isel, which would be very slow. - const IntegerType *ITy = IntegerType::get(FuncInfo.Fn->getContext(), + IntegerType *ITy = IntegerType::get(FuncInfo.Fn->getContext(), VT.getSizeInBits()); MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm)); } diff --git a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp index d518b5d..b052740 100644 --- a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -78,7 +78,7 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf) { for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E; ++I) if (const AllocaInst *AI = dyn_cast<AllocaInst>(I)) if (const ConstantInt *CUI = dyn_cast<ConstantInt>(AI->getArraySize())) { - const Type *Ty = AI->getAllocatedType(); + Type *Ty = AI->getAllocatedType(); uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(Ty); unsigned Align = std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), @@ -216,7 +216,7 @@ unsigned FunctionLoweringInfo::CreateReg(EVT VT) { /// In the case that the given value has struct or array type, this function /// will assign registers for each member or element. /// -unsigned FunctionLoweringInfo::CreateRegs(const Type *Ty) { +unsigned FunctionLoweringInfo::CreateRegs(Type *Ty) { SmallVector<EVT, 4> ValueVTs; ComputeValueVTs(TLI, Ty, ValueVTs); @@ -260,7 +260,7 @@ FunctionLoweringInfo::GetLiveOutRegInfo(unsigned Reg, unsigned BitWidth) { /// ComputePHILiveOutRegInfo - Compute LiveOutInfo for a PHI's destination /// register based on the LiveOutInfo of its operands. void FunctionLoweringInfo::ComputePHILiveOutRegInfo(const PHINode *PN) { - const Type *Ty = PN->getType(); + Type *Ty = PN->getType(); if (!Ty->isIntegerTy() || Ty->isVectorTy()) return; @@ -351,20 +351,18 @@ void FunctionLoweringInfo::ComputePHILiveOutRegInfo(const PHINode *PN) { } } -/// setByValArgumentFrameIndex - Record frame index for the byval +/// setArgumentFrameIndex - Record frame index for the byval /// argument. This overrides previous frame index entry for this argument, /// if any. -void FunctionLoweringInfo::setByValArgumentFrameIndex(const Argument *A, +void FunctionLoweringInfo::setArgumentFrameIndex(const Argument *A, int FI) { - assert (A->hasByValAttr() && "Argument does not have byval attribute!"); ByValArgFrameIndexMap[A] = FI; } -/// getByValArgumentFrameIndex - Get frame index for the byval argument. +/// getArgumentFrameIndex - Get frame index for the byval argument. /// If the argument does not have any assigned frame index then 0 is /// returned. -int FunctionLoweringInfo::getByValArgumentFrameIndex(const Argument *A) { - assert (A->hasByValAttr() && "Argument does not have byval attribute!"); +int FunctionLoweringInfo::getArgumentFrameIndex(const Argument *A) { DenseMap<const Argument *, int>::iterator I = ByValArgFrameIndexMap.find(A); if (I != ByValArgFrameIndexMap.end()) @@ -454,3 +452,34 @@ void llvm::CopyCatchInfo(const BasicBlock *SuccBB, const BasicBlock *LPad, break; } } + +/// AddLandingPadInfo - Extract the exception handling information from the +/// landingpad instruction and add them to the specified machine module info. +void llvm::AddLandingPadInfo(const LandingPadInst &I, MachineModuleInfo &MMI, + MachineBasicBlock *MBB) { + MMI.addPersonality(MBB, + cast<Function>(I.getPersonalityFn()->stripPointerCasts())); + + if (I.isCleanup()) + MMI.addCleanup(MBB); + + // FIXME: New EH - Add the clauses in reverse order. This isn't 100% correct, + // but we need to do it this way because of how the DWARF EH emitter + // processes the clauses. + for (unsigned i = I.getNumClauses(); i != 0; --i) { + Value *Val = I.getClause(i - 1); + if (I.isCatch(i - 1)) { + MMI.addCatchTypeInfo(MBB, + dyn_cast<GlobalVariable>(Val->stripPointerCasts())); + } else { + // Add filters in a list. + Constant *CVal = cast<Constant>(Val); + SmallVector<const GlobalVariable*, 4> FilterList; + for (User::op_iterator + II = CVal->op_begin(), IE = CVal->op_end(); II != IE; ++II) + FilterList.push_back(cast<GlobalVariable>((*II)->stripPointerCasts())); + + MMI.addFilterTypeInfo(MBB, FilterList); + } + } +} diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index f0f4743..2ff66f8 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -30,6 +30,12 @@ #include "llvm/Support/MathExtras.h" using namespace llvm; +/// MinRCSize - Smallest register class we allow when constraining virtual +/// registers. If satisfying all register class constraints would require +/// using a smaller register class, emit a COPY to a new virtual register +/// instead. +const unsigned MinRCSize = 4; + /// CountResults - The results of target nodes have register or immediate /// operands first, then an optional chain, and optional glue operands (which do /// not go into the resulting MachineInstr). @@ -87,7 +93,7 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned, UI != E; ++UI) { SDNode *User = *UI; bool Match = true; - if (User->getOpcode() == ISD::CopyToReg && + if (User->getOpcode() == ISD::CopyToReg && User->getOperand(2).getNode() == Node && User->getOperand(2).getResNo() == ResNo) { unsigned DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg(); @@ -113,7 +119,8 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned, if (!UseRC) UseRC = RC; else if (RC) { - const TargetRegisterClass *ComRC = getCommonSubClass(UseRC, RC); + const TargetRegisterClass *ComRC = + TRI->getCommonSubClass(UseRC, RC); // If multiple uses expect disjoint register classes, we emit // copies in AddRegisterOperand. if (ComRC) @@ -139,7 +146,7 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned, } else { DstRC = TLI->getRegClassFor(VT); } - + // If all uses are reading from the src physical register and copying the // register is either impossible or very expensive, then don't create a copy. if (MatchReg && SrcRC->getCopyCost() < 0) { @@ -167,7 +174,7 @@ unsigned InstrEmitter::getDstOfOnlyCopyToRegUse(SDNode *Node, return 0; SDNode *User = *Node->use_begin(); - if (User->getOpcode() == ISD::CopyToReg && + if (User->getOpcode() == ISD::CopyToReg && User->getOperand(2).getNode() == Node && User->getOperand(2).getResNo() == ResNo) { unsigned Reg = cast<RegisterSDNode>(User->getOperand(1))->getReg(); @@ -202,7 +209,7 @@ void InstrEmitter::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); UI != E; ++UI) { SDNode *User = *UI; - if (User->getOpcode() == ISD::CopyToReg && + if (User->getOpcode() == ISD::CopyToReg && User->getOperand(2).getNode() == Node && User->getOperand(2).getResNo() == i) { unsigned Reg = cast<RegisterSDNode>(User->getOperand(1))->getReg(); @@ -280,15 +287,16 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, MCID.OpInfo[IIOpNum].isOptionalDef(); // If the instruction requires a register in a different class, create - // a new virtual register and copy the value into it. + // a new virtual register and copy the value into it, but first attempt to + // shrink VReg's register class within reason. For example, if VReg == GR32 + // and II requires a GR32_NOSP, just constrain VReg to GR32_NOSP. if (II) { - const TargetRegisterClass *SrcRC = MRI->getRegClass(VReg); const TargetRegisterClass *DstRC = 0; if (IIOpNum < II->getNumOperands()) DstRC = TII->getRegClass(*II, IIOpNum, TRI); assert((DstRC || (MCID.isVariadic() && IIOpNum >= MCID.getNumOperands())) && "Don't have operand info for this instruction!"); - if (DstRC && !SrcRC->hasSuperClassEq(DstRC)) { + if (DstRC && !MRI->constrainRegClass(VReg, DstRC, MinRCSize)) { unsigned NewVReg = MRI->createVirtualRegister(DstRC); BuildMI(*MBB, InsertPos, Op.getNode()->getDebugLoc(), TII->get(TargetOpcode::COPY), NewVReg).addReg(VReg); @@ -326,7 +334,7 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, /// AddOperand - Add the specified operand to the specified machine instr. II /// specifies the instruction information for the node, and IIOpNum is the -/// operand number (in the II) that we are adding. IIOpNum and II are used for +/// operand number (in the II) that we are adding. IIOpNum and II are used for /// assertions only. void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op, unsigned IIOpNum, @@ -356,7 +364,7 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op, } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op)) { int Offset = CP->getOffset(); unsigned Align = CP->getAlignment(); - const Type *Type = CP->getType(); + Type *Type = CP->getType(); // MachineConstantPool wants an explicit alignment. if (Align == 0) { Align = TM->getTargetData()->getPrefTypeAlignment(Type); @@ -365,7 +373,7 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op, Align = TM->getTargetData()->getTypeAllocSize(Type); } } - + unsigned Idx; MachineConstantPool *MCP = MF->getConstantPool(); if (CP->isMachineConstantPoolEntry()) @@ -389,35 +397,44 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op, } } -/// getSuperRegisterRegClass - Returns the register class of a superreg A whose -/// "SubIdx"'th sub-register class is the specified register class and whose -/// type matches the specified type. -static const TargetRegisterClass* -getSuperRegisterRegClass(const TargetRegisterClass *TRC, - unsigned SubIdx, EVT VT) { - // Pick the register class of the superegister for this type - for (TargetRegisterInfo::regclass_iterator I = TRC->superregclasses_begin(), - E = TRC->superregclasses_end(); I != E; ++I) - if ((*I)->hasType(VT) && (*I)->getSubRegisterRegClass(SubIdx) == TRC) - return *I; - assert(false && "Couldn't find the register class"); - return 0; +unsigned InstrEmitter::ConstrainForSubReg(unsigned VReg, unsigned SubIdx, + EVT VT, DebugLoc DL) { + const TargetRegisterClass *VRC = MRI->getRegClass(VReg); + const TargetRegisterClass *RC = TRI->getSubClassWithSubReg(VRC, SubIdx); + + // RC is a sub-class of VRC that supports SubIdx. Try to constrain VReg + // within reason. + if (RC && RC != VRC) + RC = MRI->constrainRegClass(VReg, RC, MinRCSize); + + // VReg has been adjusted. It can be used with SubIdx operands now. + if (RC) + return VReg; + + // VReg couldn't be reasonably constrained. Emit a COPY to a new virtual + // register instead. + RC = TRI->getSubClassWithSubReg(TLI->getRegClassFor(VT), SubIdx); + assert(RC && "No legal register class for VT supports that SubIdx"); + unsigned NewReg = MRI->createVirtualRegister(RC); + BuildMI(*MBB, InsertPos, DL, TII->get(TargetOpcode::COPY), NewReg) + .addReg(VReg); + return NewReg; } /// EmitSubregNode - Generate machine code for subreg nodes. /// -void InstrEmitter::EmitSubregNode(SDNode *Node, +void InstrEmitter::EmitSubregNode(SDNode *Node, DenseMap<SDValue, unsigned> &VRBaseMap, bool IsClone, bool IsCloned) { unsigned VRBase = 0; unsigned Opc = Node->getMachineOpcode(); - + // If the node is only used by a CopyToReg and the dest reg is a vreg, use // the CopyToReg'd destination register instead of creating a new vreg. for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); UI != E; ++UI) { SDNode *User = *UI; - if (User->getOpcode() == ISD::CopyToReg && + if (User->getOpcode() == ISD::CopyToReg && User->getOperand(2).getNode() == Node) { unsigned DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg(); if (TargetRegisterInfo::isVirtualRegister(DestReg)) { @@ -426,12 +443,14 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, } } } - + if (Opc == TargetOpcode::EXTRACT_SUBREG) { - // EXTRACT_SUBREG is lowered as %dst = COPY %src:sub + // EXTRACT_SUBREG is lowered as %dst = COPY %src:sub. There are no + // constraints on the %dst register, COPY can target all legal register + // classes. unsigned SubIdx = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); + const TargetRegisterClass *TRC = TLI->getRegClassFor(Node->getValueType(0)); - // Figure out the register class to create for the destreg. unsigned VReg = getVR(Node->getOperand(0), VRBaseMap); MachineInstr *DefMI = MRI->getVRegDef(VReg); unsigned SrcReg, DstReg, DefSubIdx; @@ -443,62 +462,57 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, // r1026 = extract_subreg r1025, 4 // to a copy // r1026 = copy r1024 - const TargetRegisterClass *TRC = MRI->getRegClass(SrcReg); VRBase = MRI->createVirtualRegister(TRC); BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TargetOpcode::COPY), VRBase).addReg(SrcReg); } else { - const TargetRegisterClass *TRC = MRI->getRegClass(VReg); - const TargetRegisterClass *SRC = TRC->getSubRegisterRegClass(SubIdx); - assert(SRC && "Invalid subregister index in EXTRACT_SUBREG"); - - // Figure out the register class to create for the destreg. - // Note that if we're going to directly use an existing register, - // it must be precisely the required class, and not a subclass - // thereof. - if (VRBase == 0 || SRC != MRI->getRegClass(VRBase)) { - // Create the reg - assert(SRC && "Couldn't find source register class"); - VRBase = MRI->createVirtualRegister(SRC); - } + // VReg may not support a SubIdx sub-register, and we may need to + // constrain its register class or issue a COPY to a compatible register + // class. + VReg = ConstrainForSubReg(VReg, SubIdx, + Node->getOperand(0).getValueType(), + Node->getDebugLoc()); - // Create the extract_subreg machine instruction. - MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), - TII->get(TargetOpcode::COPY), VRBase); + // Create the destreg if it is missing. + if (VRBase == 0) + VRBase = MRI->createVirtualRegister(TRC); - // Add source, and subreg index - AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap, /*IsDebug=*/false, - IsClone, IsCloned); - assert(TargetRegisterInfo::isVirtualRegister(MI->getOperand(1).getReg())&& - "Cannot yet extract from physregs"); - MI->getOperand(1).setSubReg(SubIdx); - MBB->insert(InsertPos, MI); + // Create the extract_subreg machine instruction. + BuildMI(*MBB, InsertPos, Node->getDebugLoc(), + TII->get(TargetOpcode::COPY), VRBase).addReg(VReg, 0, SubIdx); } } else if (Opc == TargetOpcode::INSERT_SUBREG || Opc == TargetOpcode::SUBREG_TO_REG) { SDValue N0 = Node->getOperand(0); SDValue N1 = Node->getOperand(1); SDValue N2 = Node->getOperand(2); - unsigned SubReg = getVR(N1, VRBaseMap); unsigned SubIdx = cast<ConstantSDNode>(N2)->getZExtValue(); - const TargetRegisterClass *TRC = MRI->getRegClass(SubReg); - const TargetRegisterClass *SRC = - getSuperRegisterRegClass(TRC, SubIdx, Node->getValueType(0)); - - // Figure out the register class to create for the destreg. - // Note that if we're going to directly use an existing register, - // it must be precisely the required class, and not a subclass - // thereof. - if (VRBase == 0 || SRC != MRI->getRegClass(VRBase)) { - // Create the reg - assert(SRC && "Couldn't find source register class"); + + // Figure out the register class to create for the destreg. It should be + // the largest legal register class supporting SubIdx sub-registers. + // RegisterCoalescer will constrain it further if it decides to eliminate + // the INSERT_SUBREG instruction. + // + // %dst = INSERT_SUBREG %src, %sub, SubIdx + // + // is lowered by TwoAddressInstructionPass to: + // + // %dst = COPY %src + // %dst:SubIdx = COPY %sub + // + // There is no constraint on the %src register class. + // + const TargetRegisterClass *SRC = TLI->getRegClassFor(Node->getValueType(0)); + SRC = TRI->getSubClassWithSubReg(SRC, SubIdx); + assert(SRC && "No register class supports VT and SubIdx for INSERT_SUBREG"); + + if (VRBase == 0 || !SRC->hasSubClassEq(MRI->getRegClass(VRBase))) VRBase = MRI->createVirtualRegister(SRC); - } // Create the insert_subreg or subreg_to_reg machine instruction. MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), TII->get(Opc)); MI->addOperand(MachineOperand::CreateReg(VRBase, true)); - + // If creating a subreg_to_reg, then the first input operand // is an implicit value immediate, otherwise it's a register if (Opc == TargetOpcode::SUBREG_TO_REG) { @@ -514,7 +528,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, MBB->insert(InsertPos, MI); } else llvm_unreachable("Node is not insert_subreg, extract_subreg, or subreg_to_reg"); - + SDValue Op(Node, 0); bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second; (void)isNew; // Silence compiler warning. @@ -643,9 +657,9 @@ void InstrEmitter:: EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, DenseMap<SDValue, unsigned> &VRBaseMap) { unsigned Opc = Node->getMachineOpcode(); - + // Handle subreg insert/extract specially - if (Opc == TargetOpcode::EXTRACT_SUBREG || + if (Opc == TargetOpcode::EXTRACT_SUBREG || Opc == TargetOpcode::INSERT_SUBREG || Opc == TargetOpcode::SUBREG_TO_REG) { EmitSubregNode(Node, VRBaseMap, IsClone, IsCloned); @@ -667,7 +681,7 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, if (Opc == TargetOpcode::IMPLICIT_DEF) // We want a unique VR for each IMPLICIT_DEF use. return; - + const MCInstrDesc &II = TII->get(Opc); unsigned NumResults = CountResults(Node); unsigned NodeOperands = CountOperands(Node); @@ -712,12 +726,12 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, // Then mark unused registers as dead. MI->setPhysRegsDeadExcept(UsedRegs, *TRI); } - + // Add result register values for things that are defined by this // instruction. if (NumResults) CreateVirtualRegisters(Node, MI, II, IsClone, IsCloned, VRBaseMap); - + // Emit all of the actual operands of this instruction, adding them to the // instruction as appropriate. bool HasOptPRefs = II.getNumDefs() > NumResults; @@ -751,7 +765,7 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, MI->addRegisterDead(Reg, TRI); } } - + // If the instruction has implicit defs and the node doesn't, mark the // implicit def as dead. If the node has any glue outputs, we don't do this // because we don't know what implicit defs are being used by glued nodes. @@ -761,6 +775,12 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, i != e; ++i) MI->addRegisterDead(IDList[i-II.getNumDefs()], TRI); } + + // Run post-isel target hook to adjust this instruction if needed. +#ifdef NDEBUG + if (II.hasPostISelHook()) +#endif + TLI->AdjustInstrPostInstrSelection(MI, Node); } /// EmitSpecialNode - Generate machine code for a target-independent node and @@ -788,7 +808,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, SrcReg = R->getReg(); else SrcReg = getVR(SrcVal, VRBaseMap); - + unsigned DestReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg(); if (SrcReg == DestReg) // Coalesced away the copy? Ignore. break; @@ -808,12 +828,12 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, TII->get(TargetOpcode::EH_LABEL)).addSym(S); break; } - + case ISD::INLINEASM: { unsigned NumOps = Node->getNumOperands(); if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue) --NumOps; // Ignore the glue operand. - + // Create the inline asm machine instruction. MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), TII->get(TargetOpcode::INLINEASM)); @@ -822,7 +842,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, SDValue AsmStrV = Node->getOperand(InlineAsm::Op_AsmString); const char *AsmStr = cast<ExternalSymbolSDNode>(AsmStrV)->getSymbol(); MI->addOperand(MachineOperand::CreateES(AsmStr)); - + // Add the HasSideEffect and isAlignStack bits. int64_t ExtraInfo = cast<ConstantSDNode>(Node->getOperand(InlineAsm::Op_ExtraInfo))-> @@ -834,10 +854,10 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, unsigned Flags = cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue(); unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); - + MI->addOperand(MachineOperand::CreateImm(Flags)); ++i; // Skip the ID value. - + switch (InlineAsm::getKind(Flags)) { default: llvm_unreachable("Bad flags!"); case InlineAsm::Kind_RegDef: @@ -873,13 +893,13 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, break; } } - + // Get the mdnode from the asm if it exists and add it to the instruction. SDValue MDV = Node->getOperand(InlineAsm::Op_MDNode); const MDNode *MD = cast<MDNodeSDNode>(MDV)->getMD(); if (MD) MI->addOperand(MachineOperand::CreateMetadata(MD)); - + MBB->insert(InsertPos, MI); break; } diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.h b/lib/CodeGen/SelectionDAG/InstrEmitter.h index 19fc044..c081f38 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.h +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.h @@ -77,6 +77,12 @@ class InstrEmitter { DenseMap<SDValue, unsigned> &VRBaseMap, bool IsDebug, bool IsClone, bool IsCloned); + /// ConstrainForSubReg - Try to constrain VReg to a register class that + /// supports SubIdx sub-registers. Emit a copy if that isn't possible. + /// Return the virtual register to use. + unsigned ConstrainForSubReg(unsigned VReg, unsigned SubIdx, + EVT VT, DebugLoc DL); + /// EmitSubregNode - Generate machine code for subreg nodes. /// void EmitSubregNode(SDNode *Node, DenseMap<SDValue, unsigned> &VRBaseMap, diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index d06e2bd..63255ae 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -53,10 +53,15 @@ class SelectionDAGLegalize { // Libcall insertion helpers. - /// LastCALLSEQ - This keeps track of the CALLSEQ_END node that has been + /// LastCALLSEQ_END - This keeps track of the CALLSEQ_END node that has been /// legalized. We use this to ensure that calls are properly serialized /// against each other, including inserted libcalls. - SmallVector<SDValue, 8> LastCALLSEQ; + SDValue LastCALLSEQ_END; + + /// IsLegalizingCall - This member is used *only* for purposes of providing + /// helpful assertions that a libcall isn't created while another call is + /// being legalized (which could lead to non-serialized call sequences). + bool IsLegalizingCall; /// LegalizedNodes - For nodes that are of legal width, and that have more /// than one use, this map indicates what regularized operand to use. This @@ -149,15 +154,6 @@ private: void ExpandNode(SDNode *Node, SmallVectorImpl<SDValue> &Results); void PromoteNode(SDNode *Node, SmallVectorImpl<SDValue> &Results); - - SDValue getLastCALLSEQ() { return LastCALLSEQ.back(); } - void setLastCALLSEQ(const SDValue s) { LastCALLSEQ.back() = s; } - void pushLastCALLSEQ(SDValue s) { - LastCALLSEQ.push_back(s); - } - void popLastCALLSEQ() { - LastCALLSEQ.pop_back(); - } }; } @@ -199,7 +195,8 @@ SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag) } void SelectionDAGLegalize::LegalizeDAG() { - pushLastCALLSEQ(DAG.getEntryNode()); + LastCALLSEQ_END = DAG.getEntryNode(); + IsLegalizingCall = false; // The legalize process is inherently a bottom-up recursive process (users // legalize their uses before themselves). Given infinite stack space, we @@ -227,15 +224,14 @@ void SelectionDAGLegalize::LegalizeDAG() { /// FindCallEndFromCallStart - Given a chained node that is part of a call /// sequence, find the CALLSEQ_END node that terminates the call sequence. static SDNode *FindCallEndFromCallStart(SDNode *Node, int depth = 0) { - int next_depth = depth; + // Nested CALLSEQ_START/END constructs aren't yet legal, + // but we can DTRT and handle them correctly here. if (Node->getOpcode() == ISD::CALLSEQ_START) - next_depth = depth + 1; - if (Node->getOpcode() == ISD::CALLSEQ_END) { - assert(depth > 0 && "negative depth!"); - if (depth == 1) + depth++; + else if (Node->getOpcode() == ISD::CALLSEQ_END) { + depth--; + if (depth == 0) return Node; - else - next_depth = depth - 1; } if (Node->use_empty()) return 0; // No CallSeqEnd @@ -266,7 +262,7 @@ static SDNode *FindCallEndFromCallStart(SDNode *Node, int depth = 0) { SDNode *User = *UI; for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) if (User->getOperand(i) == TheChain) - if (SDNode *Result = FindCallEndFromCallStart(User, next_depth)) + if (SDNode *Result = FindCallEndFromCallStart(User, depth)) return Result; } return 0; @@ -287,7 +283,6 @@ static SDNode *FindCallStartFromCallEnd(SDNode *Node) { case ISD::CALLSEQ_START: if (!nested) return Node; - Node = Node->getOperand(0).getNode(); nested--; break; case ISD::CALLSEQ_END: @@ -295,7 +290,7 @@ static SDNode *FindCallStartFromCallEnd(SDNode *Node) { break; } } - return (Node->getOpcode() == ISD::CALLSEQ_START) ? Node : 0; + return 0; } /// LegalizeAllNodesNotLeadingTo - Recursively walk the uses of N, looking to @@ -365,7 +360,7 @@ static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, // smaller type. TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) && TLI.ShouldShrinkFPConstant(OrigVT)) { - const Type *SType = SVT.getTypeForEVT(*DAG.getContext()); + Type *SType = SVT.getTypeForEVT(*DAG.getContext()); LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); VT = SVT; Extend = true; @@ -819,6 +814,11 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Action = TLI.getOperationAction(Node->getOpcode(), InnerType); break; } + case ISD::ATOMIC_STORE: { + Action = TLI.getOperationAction(Node->getOpcode(), + Node->getOperand(2).getValueType()); + break; + } case ISD::SELECT_CC: case ISD::SETCC: case ISD::BR_CC: { @@ -872,7 +872,8 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { if (Action == TargetLowering::Legal) Action = TargetLowering::Expand; break; - case ISD::TRAMPOLINE: + case ISD::INIT_TRAMPOLINE: + case ISD::ADJUST_TRAMPOLINE: case ISD::FRAMEADDR: case ISD::RETURNADDR: // These operations lie about being legal: when they claim to be legal, @@ -912,12 +913,11 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { case ISD::BR_JT: case ISD::BR_CC: case ISD::BRCOND: - assert(LastCALLSEQ.size() == 1 && "branch inside CALLSEQ_BEGIN/END?"); - // Branches tweak the chain to include LastCALLSEQ + // Branches tweak the chain to include LastCALLSEQ_END Ops[0] = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ops[0], - getLastCALLSEQ()); + LastCALLSEQ_END); Ops[0] = LegalizeOp(Ops[0]); - setLastCALLSEQ(DAG.getEntryNode()); + LastCALLSEQ_END = DAG.getEntryNode(); break; case ISD::SHL: case ISD::SRL: @@ -989,6 +989,31 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { #endif assert(0 && "Do not know how to legalize this operator!"); + case ISD::SRA: + case ISD::SRL: + case ISD::SHL: { + // Scalarize vector SRA/SRL/SHL. + EVT VT = Node->getValueType(0); + assert(VT.isVector() && "Unable to legalize non-vector shift"); + assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); + unsigned NumElem = VT.getVectorNumElements(); + + SmallVector<SDValue, 8> Scalars; + for (unsigned Idx = 0; Idx < NumElem; Idx++) { + SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, + VT.getScalarType(), + Node->getOperand(0), DAG.getIntPtrConstant(Idx)); + SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, + VT.getScalarType(), + Node->getOperand(1), DAG.getIntPtrConstant(Idx)); + Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, + VT.getScalarType(), Ex, Sh)); + } + Result = DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), + &Scalars[0], Scalars.size()); + break; + } + case ISD::BUILD_VECTOR: switch (TLI.getOperationAction(ISD::BUILD_VECTOR, Node->getValueType(0))) { default: assert(0 && "This action is not supported yet!"); @@ -1006,7 +1031,6 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { break; case ISD::CALLSEQ_START: { SDNode *CallEnd = FindCallEndFromCallStart(Node); - assert(CallEnd && "didn't find CALLSEQ_END!"); // Recursively Legalize all of the inputs of the call end that do not lead // to this call start. This ensures that any libcalls that need be inserted @@ -1023,9 +1047,9 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // Merge in the last call to ensure that this call starts after the last // call ended. - if (getLastCALLSEQ().getOpcode() != ISD::EntryToken) { + if (LastCALLSEQ_END.getOpcode() != ISD::EntryToken) { Tmp1 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, - Tmp1, getLastCALLSEQ()); + Tmp1, LastCALLSEQ_END); Tmp1 = LegalizeOp(Tmp1); } @@ -1046,29 +1070,25 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // sequence have been legalized, legalize the call itself. During this // process, no libcalls can/will be inserted, guaranteeing that no calls // can overlap. + assert(!IsLegalizingCall && "Inconsistent sequentialization of calls!"); // Note that we are selecting this call! - setLastCALLSEQ(SDValue(CallEnd, 0)); + LastCALLSEQ_END = SDValue(CallEnd, 0); + IsLegalizingCall = true; // Legalize the call, starting from the CALLSEQ_END. - LegalizeOp(getLastCALLSEQ()); + LegalizeOp(LastCALLSEQ_END); + assert(!IsLegalizingCall && "CALLSEQ_END should have cleared this!"); return Result; } case ISD::CALLSEQ_END: - { - SDNode *myCALLSEQ_BEGIN = FindCallStartFromCallEnd(Node); - - // If the CALLSEQ_START node hasn't been legalized first, legalize it. - // This will cause this node to be legalized as well as handling libcalls - // right. - if (getLastCALLSEQ().getNode() != Node) { - LegalizeOp(SDValue(myCALLSEQ_BEGIN, 0)); - DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op); - assert(I != LegalizedNodes.end() && - "Legalizing the call start should have legalized this node!"); - return I->second; - } - - pushLastCALLSEQ(SDValue(myCALLSEQ_BEGIN, 0)); + // If the CALLSEQ_START node hasn't been legalized first, legalize it. This + // will cause this node to be legalized as well as handling libcalls right. + if (LastCALLSEQ_END.getNode() != Node) { + LegalizeOp(SDValue(FindCallStartFromCallEnd(Node), 0)); + DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op); + assert(I != LegalizedNodes.end() && + "Legalizing the call start should have legalized this node!"); + return I->second; } // Otherwise, the call start has been legalized and everything is going @@ -1096,8 +1116,9 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Result.getResNo()); } } + assert(IsLegalizingCall && "Call sequence imbalance between start/end?"); // This finishes up call legalization. - popLastCALLSEQ(); + IsLegalizingCall = false; // If the CALLSEQ_END node has a flag, remember that we legalized it. AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); @@ -1124,7 +1145,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // If this is an unaligned load and the target doesn't support it, // expand it. if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) { - const Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); + Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); unsigned ABIAlignment = TLI.getTargetData()->getABITypeAlignment(Ty); if (LD->getAlignment() < ABIAlignment){ Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.getNode()), @@ -1311,7 +1332,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // If this is an unaligned load and the target doesn't support it, // expand it. if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) { - const Type *Ty = + Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); unsigned ABIAlignment = TLI.getTargetData()->getABITypeAlignment(Ty); @@ -1491,7 +1512,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // If this is an unaligned store and the target doesn't support it, // expand it. if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) { - const Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); + Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); unsigned ABIAlignment= TLI.getTargetData()->getABITypeAlignment(Ty); if (ST->getAlignment() < ABIAlignment) Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()), @@ -1596,7 +1617,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // If this is an unaligned store and the target doesn't support it, // expand it. if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) { - const Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); + Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); unsigned ABIAlignment= TLI.getTargetData()->getABITypeAlignment(Ty); if (ST->getAlignment() < ABIAlignment) Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.getNode()), @@ -1611,82 +1632,101 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { EVT WideScalarVT = Tmp3.getValueType().getScalarType(); EVT NarrowScalarVT = StVT.getScalarType(); - // The Store type is illegal, must scalarize the vector store. - SmallVector<SDValue, 8> Stores; - bool ScalarLegal = TLI.isTypeLegal(WideScalarVT); - if (!TLI.isTypeLegal(StVT) && StVT.isVector() && ScalarLegal) { + if (StVT.isVector()) { unsigned NumElem = StVT.getVectorNumElements(); + // The type of the data we want to save + EVT RegVT = Tmp3.getValueType(); + EVT RegSclVT = RegVT.getScalarType(); + // The type of data as saved in memory. + EVT MemSclVT = StVT.getScalarType(); + + bool RegScalarLegal = TLI.isTypeLegal(RegSclVT); + bool MemScalarLegal = TLI.isTypeLegal(MemSclVT); + + // We need to expand this store. If the register element type + // is legal then we can scalarize the vector and use + // truncating stores. + if (RegScalarLegal) { + // Cast floats into integers + unsigned ScalarSize = MemSclVT.getSizeInBits(); + EVT EltVT = EVT::getIntegerVT(*DAG.getContext(), ScalarSize); + + // Round odd types to the next pow of two. + if (!isPowerOf2_32(ScalarSize)) + ScalarSize = NextPowerOf2(ScalarSize); + + // Store Stride in bytes + unsigned Stride = ScalarSize/8; + // Extract each of the elements from the original vector + // and save them into memory individually. + SmallVector<SDValue, 8> Stores; + for (unsigned Idx = 0; Idx < NumElem; Idx++) { + SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, + RegSclVT, Tmp3, DAG.getIntPtrConstant(Idx)); + + Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, + DAG.getIntPtrConstant(Stride)); + + // This scalar TruncStore may be illegal, but we lehalize it + // later. + SDValue Store = DAG.getTruncStore(Tmp1, dl, Ex, Tmp2, + ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT, + isVolatile, isNonTemporal, Alignment); - unsigned ScalarSize = StVT.getScalarType().getSizeInBits(); - // Round odd types to the next pow of two. - if (!isPowerOf2_32(ScalarSize)) - ScalarSize = NextPowerOf2(ScalarSize); - // Types smaller than 8 bits are promoted to 8 bits. - ScalarSize = std::max<unsigned>(ScalarSize, 8); - // Store stride - unsigned Stride = ScalarSize/8; - assert(isPowerOf2_32(Stride) && "Stride must be a power of two"); - - for (unsigned Idx=0; Idx<NumElem; Idx++) { - SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, - WideScalarVT, Tmp3, DAG.getIntPtrConstant(Idx)); - - - EVT NVT = EVT::getIntegerVT(*DAG.getContext(), ScalarSize); + Stores.push_back(Store); + } - Ex = DAG.getNode(ISD::TRUNCATE, dl, NVT, Ex); - Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, - DAG.getIntPtrConstant(Stride)); - SDValue Store = DAG.getStore(Tmp1, dl, Ex, Tmp2, - ST->getPointerInfo().getWithOffset(Idx*Stride), - isVolatile, isNonTemporal, Alignment); - Stores.push_back(Store); + Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, + &Stores[0], Stores.size()); + break; } - Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, - &Stores[0], Stores.size()); - break; - } - // The Store type is illegal, must scalarize the vector store. - // However, the scalar type is illegal. Must bitcast the result - // and store it in smaller parts. - if (!TLI.isTypeLegal(StVT) && StVT.isVector()) { - unsigned WideNumElem = StVT.getVectorNumElements(); - unsigned Stride = NarrowScalarVT.getSizeInBits()/8; - - unsigned SizeRatio = - (WideScalarVT.getSizeInBits() / NarrowScalarVT.getSizeInBits()); - - EVT CastValueVT = EVT::getVectorVT(*DAG.getContext(), NarrowScalarVT, - SizeRatio*WideNumElem); - - // Cast the wide elem vector to wider vec with smaller elem type. - // Example <2 x i64> -> <4 x i32> - Tmp3 = DAG.getNode(ISD::BITCAST, dl, CastValueVT, Tmp3); - - for (unsigned Idx=0; Idx<WideNumElem*SizeRatio; Idx++) { - // Extract elment i - SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, - NarrowScalarVT, Tmp3, DAG.getIntPtrConstant(Idx)); - // bump pointer. - Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, - DAG.getIntPtrConstant(Stride)); - - // Store if, this element is: - // - First element on big endian, or - // - Last element on little endian - if (( TLI.isBigEndian() && (Idx%SizeRatio == 0)) || - ((!TLI.isBigEndian() && (Idx%SizeRatio == SizeRatio-1)))) { - SDValue Store = DAG.getStore(Tmp1, dl, Ex, Tmp2, - ST->getPointerInfo().getWithOffset(Idx*Stride), - isVolatile, isNonTemporal, Alignment); - Stores.push_back(Store); + // The scalar register type is illegal. + // For example saving <2 x i64> -> <2 x i32> on a x86. + // In here we bitcast the value into a vector of smaller parts and + // save it using smaller scalars. + if (!RegScalarLegal && MemScalarLegal) { + // Store Stride in bytes + unsigned Stride = MemSclVT.getSizeInBits()/8; + + unsigned SizeRatio = + (RegSclVT.getSizeInBits() / MemSclVT.getSizeInBits()); + + EVT CastValueVT = EVT::getVectorVT(*DAG.getContext(), + MemSclVT, + SizeRatio * NumElem); + + // Cast the wide elem vector to wider vec with smaller elem type. + // Example <2 x i64> -> <4 x i32> + Tmp3 = DAG.getNode(ISD::BITCAST, dl, CastValueVT, Tmp3); + + SmallVector<SDValue, 8> Stores; + for (unsigned Idx=0; Idx < NumElem * SizeRatio; Idx++) { + // Extract the Ith element. + SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, + NarrowScalarVT, Tmp3, DAG.getIntPtrConstant(Idx)); + // Bump pointer. + Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, + DAG.getIntPtrConstant(Stride)); + + // Store if, this element is: + // - First element on big endian, or + // - Last element on little endian + if (( TLI.isBigEndian() && (Idx % SizeRatio == 0)) || + ((!TLI.isBigEndian() && (Idx % SizeRatio == SizeRatio-1)))) { + SDValue Store = DAG.getStore(Tmp1, dl, Ex, Tmp2, + ST->getPointerInfo().getWithOffset(Idx*Stride), + isVolatile, isNonTemporal, Alignment); + Stores.push_back(Store); + } } + Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, + &Stores[0], Stores.size()); + break; } - Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, - &Stores[0], Stores.size()); - break; - } + + assert(false && "Unable to legalize the vector trunc store!"); + }// is vector // TRUNCSTORE:i16 i32 -> STORE i16 @@ -1999,7 +2039,7 @@ SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); unsigned SlotSize = SlotVT.getSizeInBits(); unsigned DestSize = DestVT.getSizeInBits(); - const Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); + Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); unsigned DestAlign = TLI.getTargetData()->getPrefTypeAlignment(DestType); // Emit a store to the stack slot. Use a truncstore if the input value is @@ -2106,7 +2146,7 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { } } else { assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); - const Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); + Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); CV.push_back(UndefValue::get(OpNTy)); } } @@ -2150,6 +2190,7 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { // and leave the Hi part unset. SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned) { + assert(!IsLegalizingCall && "Cannot overlap legalization of calls!"); // The input chain to this libcall is the entry node of the function. // Legalizing the call will automatically add the previous call to the // dependence. @@ -2159,7 +2200,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, TargetLowering::ArgListEntry Entry; for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { EVT ArgVT = Node->getOperand(i).getValueType(); - const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); + Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; Entry.isSExt = isSigned; Entry.isZExt = !isSigned; @@ -2169,7 +2210,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, TLI.getPointerTy()); // Splice the libcall in wherever FindInputOutputChains tells us to. - const Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); + Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); // isTailCall may be true since the callee does not reference caller stack // frame. Check if it's in the right position. @@ -2185,7 +2226,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, return DAG.getRoot(); // Legalize the call sequence, starting with the chain. This will advance - // the LastCALLSEQ to the legalized version of the CALLSEQ_END node that + // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that // was added by LowerCallTo (guaranteeing proper serialization of calls). LegalizeOp(CallInfo.second); return CallInfo.first; @@ -2210,7 +2251,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), TLI.getPointerTy()); - const Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); + Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false, false, 0, TLI.getLibcallCallingConv(LC), false, @@ -2231,13 +2272,14 @@ std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned) { + assert(!IsLegalizingCall && "Cannot overlap legalization of calls!"); SDValue InChain = Node->getOperand(0); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) { EVT ArgVT = Node->getOperand(i).getValueType(); - const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); + Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; Entry.isSExt = isSigned; @@ -2248,7 +2290,7 @@ SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, TLI.getPointerTy()); // Splice the libcall in wherever FindInputOutputChains tells us to. - const Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); + Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false, 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, @@ -2256,7 +2298,7 @@ SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, Callee, Args, DAG, Node->getDebugLoc()); // Legalize the call sequence, starting with the chain. This will advance - // the LastCALLSEQ to the legalized version of the CALLSEQ_END node that + // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that // was added by LowerCallTo (guaranteeing proper serialization of calls). LegalizeOp(CallInfo.second); return CallInfo; @@ -2360,13 +2402,13 @@ SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, SDValue InChain = DAG.getEntryNode(); EVT RetVT = Node->getValueType(0); - const Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); + Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { EVT ArgVT = Node->getOperand(i).getValueType(); - const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); + Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; Entry.isSExt = isSigned; Entry.isZExt = !isSigned; @@ -2397,7 +2439,7 @@ SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, LegalizeOp(CallInfo.second); // Remainder is loaded back from the stack frame. - SDValue Rem = DAG.getLoad(RetVT, dl, getLastCALLSEQ(), FIPtr, + SDValue Rem = DAG.getLoad(RetVT, dl, LastCALLSEQ_END, FIPtr, MachinePointerInfo(), false, false, 0); Results.push_back(CallInfo.first); Results.push_back(Rem); @@ -2955,8 +2997,10 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, Results.push_back(DAG.getConstant(0, MVT::i32)); Results.push_back(Node->getOperand(0)); break; + case ISD::ATOMIC_FENCE: case ISD::MEMBARRIER: { // If the target didn't lower this, lower it to '__sync_synchronize()' call + // FIXME: handle "fence singlethread" more efficiently. TargetLowering::ArgListTy Args; std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(Node->getOperand(0), Type::getVoidTy(*DAG.getContext()), @@ -2969,6 +3013,32 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, Results.push_back(CallResult.second); break; } + case ISD::ATOMIC_LOAD: { + // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. + SDValue Zero = DAG.getConstant(0, Node->getValueType(0)); + SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, + cast<AtomicSDNode>(Node)->getMemoryVT(), + Node->getOperand(0), + Node->getOperand(1), Zero, Zero, + cast<AtomicSDNode>(Node)->getMemOperand(), + cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getSynchScope()); + Results.push_back(Swap.getValue(0)); + Results.push_back(Swap.getValue(1)); + break; + } + case ISD::ATOMIC_STORE: { + // There is no libcall for atomic store; fake it with ATOMIC_SWAP. + SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, + cast<AtomicSDNode>(Node)->getMemoryVT(), + Node->getOperand(0), + Node->getOperand(1), Node->getOperand(2), + cast<AtomicSDNode>(Node)->getMemOperand(), + cast<AtomicSDNode>(Node)->getOrdering(), + cast<AtomicSDNode>(Node)->getSynchScope()); + Results.push_back(Swap.getValue(1)); + break; + } // By default, atomic intrinsics are marked Legal and lowered. Targets // which don't support them directly, however, may want libcalls, in which // case they mark them Expand, and we get here. @@ -3727,8 +3797,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, LegalizeSetCCCondCode(TLI.getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, dl); - assert(LastCALLSEQ.size() == 1 && "branch inside CALLSEQ_BEGIN/END?"); - setLastCALLSEQ(DAG.getEntryNode()); + LastCALLSEQ_END = DAG.getEntryNode(); assert(!Tmp3.getNode() && "Can't legalize BR_CC with legal condition!"); Tmp3 = DAG.getConstant(0, Tmp2.getValueType()); diff --git a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index e6835d8..7c1cc69 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -55,6 +55,7 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { #endif llvm_unreachable("Do not know how to soften the result of this operator!"); + case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break; case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break; case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break; case ISD::ConstantFP: @@ -107,6 +108,12 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) { return BitConvertToInteger(N->getOperand(0)); } +SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N, + unsigned ResNo) { + SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); + return BitConvertToInteger(Op); +} + SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) { // Convert the inputs to integers, and build a new pair out of them. return DAG.getNode(ISD::BUILD_PAIR, N->getDebugLoc(), @@ -827,11 +834,11 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { #endif llvm_unreachable("Do not know how to expand the result of this operator!"); - case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; + case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; @@ -879,10 +886,10 @@ void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, assert(NVT.getSizeInBits() == integerPartWidth && "Do not know how to expand this float constant!"); APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt(); - Lo = DAG.getConstantFP(APFloat(APInt(integerPartWidth, 1, - &C.getRawData()[1])), NVT); - Hi = DAG.getConstantFP(APFloat(APInt(integerPartWidth, 1, - &C.getRawData()[0])), NVT); + Lo = DAG.getConstantFP(APFloat(APInt(integerPartWidth, C.getRawData()[1])), + NVT); + Hi = DAG.getConstantFP(APFloat(APInt(integerPartWidth, C.getRawData()[0])), + NVT); } void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo, @@ -1201,7 +1208,7 @@ void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 }; - const uint64_t *Parts = 0; + ArrayRef<uint64_t> Parts; switch (SrcVT.getSimpleVT().SimpleTy) { default: @@ -1218,7 +1225,7 @@ void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, } Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, - DAG.getConstantFP(APFloat(APInt(128, 2, Parts)), + DAG.getConstantFP(APFloat(APInt(128, Parts)), MVT::ppcf128)); Lo = DAG.getNode(ISD::SELECT_CC, dl, VT, Src, DAG.getConstant(0, SrcVT), Lo, Hi, DAG.getCondCode(ISD::SETLT)); @@ -1291,8 +1298,7 @@ void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS, GetExpandedFloat(NewLHS, LHSLo, LHSHi); GetExpandedFloat(NewRHS, RHSLo, RHSHi); - EVT VT = NewLHS.getValueType(); - assert(VT == MVT::ppcf128 && "Unsupported setcc type!"); + assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!"); // FIXME: This generated code sucks. We want to generate // FCMPU crN, hi1, hi2 @@ -1373,7 +1379,7 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_UINT(SDNode *N) { assert(N->getOperand(0).getValueType() == MVT::ppcf128 && "Logic only correct for ppcf128!"); const uint64_t TwoE31[] = {0x41e0000000000000LL, 0}; - APFloat APF = APFloat(APInt(128, 2, TwoE31)); + APFloat APF = APFloat(APInt(128, TwoE31)); SDValue Tmp = DAG.getConstantFP(APF, MVT::ppcf128); // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X // FIXME: generated code sucks. @@ -1445,6 +1451,7 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { ST->getValue().getValueType()); assert(NVT.isByteSized() && "Expanded type not byte sized!"); assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?"); + (void)NVT; SDValue Lo, Hi; GetExpandedOp(ST->getValue(), Lo, Hi); diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index e7c77dd..a5c4c2d 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -48,6 +48,7 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { N->dump(&DAG); dbgs() << "\n"; #endif llvm_unreachable("Do not know how to promote this operator!"); + case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; @@ -63,6 +64,7 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break; case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; + case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; case ISD::SHL: Res = PromoteIntRes_SHL(N); break; @@ -84,6 +86,8 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { Res = PromoteIntRes_BUILD_VECTOR(N); break; case ISD::SCALAR_TO_VECTOR: Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; + case ISD::CONCAT_VECTORS: + Res = PromoteIntRes_CONCAT_VECTORS(N); break; case ISD::SIGN_EXTEND: case ISD::ZERO_EXTEND: @@ -114,6 +118,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { case ISD::SMULO: case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; + case ISD::ATOMIC_LOAD: + Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; + case ISD::ATOMIC_LOAD_ADD: case ISD::ATOMIC_LOAD_SUB: case ISD::ATOMIC_LOAD_AND: @@ -136,6 +143,12 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { SetPromotedInteger(SDValue(N, ResNo), Res); } +SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, + unsigned ResNo) { + SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); + return GetPromotedInteger(Op); +} + SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { // Sign-extend the new bits, and continue the assertion. SDValue Op = SExtPromotedInteger(N->getOperand(0)); @@ -150,12 +163,26 @@ SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { Op.getValueType(), Op, N->getOperand(1)); } +SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { + EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), + N->getMemoryVT(), ResVT, + N->getChain(), N->getBasePtr(), + N->getMemOperand(), N->getOrdering(), + N->getSynchScope()); + // Legalized the chain result - switch anything that used the old chain to + // use the new one. + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + return Res; +} + SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { SDValue Op2 = GetPromotedInteger(N->getOperand(2)); SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), N->getChain(), N->getBasePtr(), - Op2, N->getMemOperand()); + Op2, N->getMemOperand(), N->getOrdering(), + N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); @@ -167,7 +194,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) { SDValue Op3 = GetPromotedInteger(N->getOperand(3)); SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), N->getChain(), N->getBasePtr(), - Op2, Op3, N->getMemOperand()); + Op2, Op3, N->getMemOperand(), N->getOrdering(), + N->getSynchScope()); // Legalized the chain result - switch anything that used the old chain to // use the new one. ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); @@ -457,6 +485,14 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { LHS.getValueType(), N->getOperand(0),LHS,RHS); } +SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) { + SDValue Mask = GetPromotedInteger(N->getOperand(0)); + SDValue LHS = GetPromotedInteger(N->getOperand(1)); + SDValue RHS = GetPromotedInteger(N->getOperand(2)); + return DAG.getNode(ISD::VSELECT, N->getDebugLoc(), + LHS.getValueType(), Mask, LHS, RHS); +} + SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { SDValue LHS = GetPromotedInteger(N->getOperand(2)); SDValue RHS = GetPromotedInteger(N->getOperand(3)); @@ -467,16 +503,24 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { EVT SVT = TLI.getSetCCResultType(N->getOperand(0).getValueType()); - assert(isTypeLegal(SVT) && "Illegal SetCC type!"); + + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + + // Only use the result of getSetCCResultType if it is legal, + // otherwise just use the promoted result type (NVT). + if (!TLI.isTypeLegal(SVT)) + SVT = NVT; + DebugLoc dl = N->getDebugLoc(); + assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() && + "Vector compare must return a vector result!"); // Get the SETCC result using the canonical SETCC type. - SDValue SetCC = DAG.getNode(ISD::SETCC, dl, SVT, N->getOperand(0), + SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), N->getOperand(1), N->getOperand(2)); - // Convert to the expected type. - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); assert(NVT.bitsLE(SVT) && "Integer type overpromoted?"); + // Convert to the expected type. return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC); } @@ -707,6 +751,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { llvm_unreachable("Do not know how to promote this operator's operand!"); case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; + case ISD::ATOMIC_STORE: + Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); + break; case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; @@ -721,6 +768,7 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { case ISD::MEMBARRIER: Res = PromoteIntOp_MEMBARRIER(N); break; case ISD::SCALAR_TO_VECTOR: Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; + case ISD::VSELECT: case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; @@ -791,6 +839,13 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), Op); } +SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { + SDValue Op2 = GetPromotedInteger(N->getOperand(2)); + return DAG.getAtomic(N->getOpcode(), N->getDebugLoc(), N->getMemoryVT(), + N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(), + N->getOrdering(), N->getSynchScope()); +} + SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { // This should only occur in unusual situations like bitcasting to an // x86_fp80, so just turn it into a store+load @@ -913,14 +968,17 @@ SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { } SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { - assert(OpNo == 0 && "Only know how to promote condition"); + assert(OpNo == 0 && "Only know how to promote the condition!"); + SDValue Cond = N->getOperand(0); + EVT OpTy = N->getOperand(1).getValueType(); // Promote all the way up to the canonical SetCC type. - EVT SVT = TLI.getSetCCResultType(N->getOperand(1).getValueType()); - SDValue Cond = PromoteTargetBoolean(N->getOperand(0), SVT); + EVT SVT = TLI.getSetCCResultType(N->getOpcode() == ISD::SELECT ? + OpTy.getScalarType() : OpTy); + Cond = PromoteTargetBoolean(Cond, SVT); - return SDValue(DAG.UpdateNodeOperands(N, Cond, - N->getOperand(1), N->getOperand(2)), 0); + return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), + N->getOperand(2)), 0); } SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { @@ -1024,7 +1082,7 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { #endif llvm_unreachable("Do not know how to expand the result of this operator!"); - case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; + case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; @@ -1055,6 +1113,7 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; + case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; case ISD::ATOMIC_LOAD_ADD: case ISD::ATOMIC_LOAD_SUB: @@ -1546,6 +1605,12 @@ void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); } +void DAGTypeLegalizer::ExpandIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo, + SDValue &Lo, SDValue &Hi) { + SDValue Res = DisintegrateMERGE_VALUES(N, ResNo); + SplitInteger(Res, Lo, Hi); +} + void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, SDValue &Lo, SDValue &Hi) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); @@ -2176,9 +2241,9 @@ void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, SDValue &Lo, SDValue &Hi) { EVT VT = N->getValueType(0); - const Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); + Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); EVT PtrVT = TLI.getPointerTy(); - const Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); + Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); DebugLoc dl = N->getDebugLoc(); // A divide for UMULO should be faster than a function call. @@ -2222,7 +2287,7 @@ void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, TargetLowering::ArgListEntry Entry; for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { EVT ArgVT = N->getOperand(i).getValueType(); - const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); + Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); Entry.Node = N->getOperand(i); Entry.Ty = ArgTy; Entry.isSExt = true; @@ -2321,6 +2386,20 @@ void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, } } +void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, + SDValue &Lo, SDValue &Hi) { + DebugLoc dl = N->getDebugLoc(); + EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); + SDValue Zero = DAG.getConstant(0, VT); + SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT, + N->getOperand(0), + N->getOperand(1), Zero, Zero, + cast<AtomicSDNode>(N)->getMemOperand(), + cast<AtomicSDNode>(N)->getOrdering(), + cast<AtomicSDNode>(N)->getSynchScope()); + ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); + ReplaceValueWith(SDValue(N, 1), Swap.getValue(1)); +} //===----------------------------------------------------------------------===// // Integer Operand Expansion @@ -2365,6 +2444,8 @@ bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; case ISD::RETURNADDR: case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; + + case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; } // If the result is null, the sub-method took care of registering results etc. @@ -2742,6 +2823,19 @@ SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { return MakeLibCall(LC, DstVT, &Op, 1, true, dl); } +SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { + DebugLoc dl = N->getDebugLoc(); + SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, + cast<AtomicSDNode>(N)->getMemoryVT(), + N->getOperand(0), + N->getOperand(1), N->getOperand(2), + cast<AtomicSDNode>(N)->getMemOperand(), + cast<AtomicSDNode>(N)->getOrdering(), + cast<AtomicSDNode>(N)->getSynchScope()); + return Swap.getValue(1); +} + + SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { SDValue InOp0 = N->getOperand(0); EVT InVT = InOp0.getValueType(); @@ -2775,7 +2869,6 @@ SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { - ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); EVT VT = N->getValueType(0); DebugLoc dl = N->getDebugLoc(); @@ -2830,6 +2923,46 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); } +SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { + DebugLoc dl = N->getDebugLoc(); + + SDValue Op0 = N->getOperand(1); + SDValue Op1 = N->getOperand(1); + assert(Op0.getValueType() == Op1.getValueType() && + "Invalid input vector types"); + + EVT OutVT = N->getValueType(0); + EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); + assert(NOutVT.isVector() && "This type must be promoted to a vector type"); + + EVT OutElemTy = NOutVT.getVectorElementType(); + + unsigned NumElem0 = Op0.getValueType().getVectorNumElements(); + unsigned NumElem1 = Op1.getValueType().getVectorNumElements(); + unsigned NumOutElem = NOutVT.getVectorNumElements(); + assert(NumElem0 + NumElem1 == NumOutElem && + "Invalid number of incoming elements"); + + // Take the elements from the first vector. + SmallVector<SDValue, 8> Ops(NumOutElem); + for (unsigned i = 0; i < NumElem0; ++i) { + SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, + Op0.getValueType().getScalarType(), Op0, + DAG.getIntPtrConstant(i)); + Ops[i] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext); + } + + // Take the elements from the second vector + for (unsigned i = 0; i < NumElem1; ++i) { + SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, + Op1.getValueType().getScalarType(), Op1, + DAG.getIntPtrConstant(i)); + Ops[i + NumElem0] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext); + } + + return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, &Ops[0], Ops.size()); +} + SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { EVT OutVT = N->getValueType(0); EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); @@ -2838,14 +2971,12 @@ SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { EVT NOutVTElem = NOutVT.getVectorElementType(); DebugLoc dl = N->getDebugLoc(); - - SDValue ConvertedVector = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, - N->getOperand(0)); + SDValue V0 = GetPromotedInteger(N->getOperand(0)); SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(1)); - return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,NOutVT, - ConvertedVector, ConvElem, N->getOperand(2)); + return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, + V0, ConvElem, N->getOperand(2)); } SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { @@ -2855,20 +2986,23 @@ SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, V0->getValueType(0).getScalarType(), V0, V1); - return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); - + // EXTRACT_VECTOR_ELT can return types which are wider than the incoming + // element types. If this is the case then we need to expand the outgoing + // value and not truncate it. + return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); } SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { - DebugLoc dl = N->getDebugLoc(); + unsigned NumElems = N->getNumOperands(); EVT RetSclrTy = N->getValueType(0).getVectorElementType(); SmallVector<SDValue, 8> NewOps; + NewOps.reserve(NumElems); // For each incoming vector - for (unsigned VecIdx = 0, E = N->getNumOperands(); VecIdx!= E; ++VecIdx) { + for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp index ba658b0..a4bb577 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp @@ -946,6 +946,13 @@ bool DAGTypeLegalizer::CustomWidenLowerNode(SDNode *N, EVT VT) { return true; } +SDValue DAGTypeLegalizer::DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo) { + for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) + if (i != ResNo) + ReplaceValueWith(SDValue(N, i), SDValue(N->getOperand(i))); + return SDValue(N, ResNo); +} + /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type /// which is split into two not necessarily identical pieces. void DAGTypeLegalizer::GetSplitDestVTs(EVT InVT, EVT &LoVT, EVT &HiVT) { @@ -1046,7 +1053,7 @@ SDValue DAGTypeLegalizer::MakeLibCall(RTLIB::Libcall LC, EVT RetVT, SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), TLI.getPointerTy()); - const Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); + Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false, false, 0, TLI.getLibcallCallingConv(LC), false, @@ -1067,7 +1074,7 @@ DAGTypeLegalizer::ExpandChainLibCall(RTLIB::Libcall LC, TargetLowering::ArgListEntry Entry; for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) { EVT ArgVT = Node->getOperand(i).getValueType(); - const Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); + Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; Entry.isSExt = isSigned; @@ -1078,7 +1085,7 @@ DAGTypeLegalizer::ExpandChainLibCall(RTLIB::Libcall LC, TLI.getPointerTy()); // Splice the libcall in wherever FindInputOutputChains tells us to. - const Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); + Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false, 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, @@ -1093,24 +1100,8 @@ DAGTypeLegalizer::ExpandChainLibCall(RTLIB::Libcall LC, /// type i1, the bits of which conform to getBooleanContents. SDValue DAGTypeLegalizer::PromoteTargetBoolean(SDValue Bool, EVT VT) { DebugLoc dl = Bool.getDebugLoc(); - ISD::NodeType ExtendCode; - switch (TLI.getBooleanContents()) { - default: - assert(false && "Unknown BooleanContent!"); - case TargetLowering::UndefinedBooleanContent: - // Extend to VT by adding rubbish bits. - ExtendCode = ISD::ANY_EXTEND; - break; - case TargetLowering::ZeroOrOneBooleanContent: - // Extend to VT by adding zero bits. - ExtendCode = ISD::ZERO_EXTEND; - break; - case TargetLowering::ZeroOrNegativeOneBooleanContent: { - // Extend to VT by copying the sign bit. - ExtendCode = ISD::SIGN_EXTEND; - break; - } - } + ISD::NodeType ExtendCode = + TargetLowering::getExtendForContent(TLI.getBooleanContents(VT.isVector())); return DAG.getNode(ExtendCode, dl, VT, Bool); } diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 952797d..abacdac 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -148,15 +148,22 @@ private: SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT); bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult); bool CustomWidenLowerNode(SDNode *N, EVT VT); + + /// DisintegrateMERGE_VALUES - Replace each result of the given MERGE_VALUES + /// node with the corresponding input operand, except for the result 'ResNo', + /// which is returned. + SDValue DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo); + SDValue GetVectorElementPointer(SDValue VecPtr, EVT EltVT, SDValue Index); SDValue JoinIntegers(SDValue Lo, SDValue Hi); SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned); SDValue MakeLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, unsigned NumOps, bool isSigned, DebugLoc dl); - std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, - SDNode *Node, bool isSigned); - std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node); + + std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, + SDNode *Node, bool isSigned); + std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node); SDValue PromoteTargetBoolean(SDValue Bool, EVT VT); void ReplaceValueWith(SDValue From, SDValue To); @@ -206,8 +213,10 @@ private: // Integer Result Promotion. void PromoteIntegerResult(SDNode *N, unsigned ResNo); + SDValue PromoteIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo); SDValue PromoteIntRes_AssertSext(SDNode *N); SDValue PromoteIntRes_AssertZext(SDNode *N); + SDValue PromoteIntRes_Atomic0(AtomicSDNode *N); SDValue PromoteIntRes_Atomic1(AtomicSDNode *N); SDValue PromoteIntRes_Atomic2(AtomicSDNode *N); SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N); @@ -215,6 +224,7 @@ private: SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N); SDValue PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N); SDValue PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N); + SDValue PromoteIntRes_CONCAT_VECTORS(SDNode *N); SDValue PromoteIntRes_BITCAST(SDNode *N); SDValue PromoteIntRes_BSWAP(SDNode *N); SDValue PromoteIntRes_BUILD_PAIR(SDNode *N); @@ -232,6 +242,7 @@ private: SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo); SDValue PromoteIntRes_SDIV(SDNode *N); SDValue PromoteIntRes_SELECT(SDNode *N); + SDValue PromoteIntRes_VSELECT(SDNode *N); SDValue PromoteIntRes_SELECT_CC(SDNode *N); SDValue PromoteIntRes_SETCC(SDNode *N); SDValue PromoteIntRes_SHL(SDNode *N); @@ -249,6 +260,7 @@ private: // Integer Operand Promotion. bool PromoteIntegerOperand(SDNode *N, unsigned OperandNo); SDValue PromoteIntOp_ANY_EXTEND(SDNode *N); + SDValue PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N); SDValue PromoteIntOp_BITCAST(SDNode *N); SDValue PromoteIntOp_BUILD_PAIR(SDNode *N); SDValue PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo); @@ -264,6 +276,7 @@ private: SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo); + SDValue PromoteIntOp_VSETCC(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_Shift(SDNode *N); SDValue PromoteIntOp_SIGN_EXTEND(SDNode *N); SDValue PromoteIntOp_SINT_TO_FP(SDNode *N); @@ -289,6 +302,8 @@ private: // Integer Result Expansion. void ExpandIntegerResult(SDNode *N, unsigned ResNo); + void ExpandIntRes_MERGE_VALUES (SDNode *N, unsigned ResNo, + SDValue &Lo, SDValue &Hi); void ExpandIntRes_ANY_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_AssertSext (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi); @@ -320,6 +335,8 @@ private: void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_ATOMIC_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandShiftByConstant(SDNode *N, unsigned Amt, SDValue &Lo, SDValue &Hi); bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi); @@ -339,6 +356,7 @@ private: SDValue ExpandIntOp_TRUNCATE(SDNode *N); SDValue ExpandIntOp_UINT_TO_FP(SDNode *N); SDValue ExpandIntOp_RETURNADDR(SDNode *N); + SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N); void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, DebugLoc dl); @@ -362,6 +380,7 @@ private: // Result Float to Integer Conversion. void SoftenFloatResult(SDNode *N, unsigned OpNo); + SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo); SDValue SoftenFloatRes_BITCAST(SDNode *N); SDValue SoftenFloatRes_BUILD_PAIR(SDNode *N); SDValue SoftenFloatRes_ConstantFP(ConstantFPSDNode *N); @@ -488,6 +507,7 @@ private: // Vector Result Scalarization: <1 x ty> -> ty. void ScalarizeVectorResult(SDNode *N, unsigned OpNo); + SDValue ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo); SDValue ScalarizeVecRes_BinOp(SDNode *N); SDValue ScalarizeVecRes_UnaryOp(SDNode *N); SDValue ScalarizeVecRes_InregOp(SDNode *N); @@ -559,6 +579,7 @@ private: SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N); SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo); SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N); + SDValue SplitVecOp_VSETCC(SDNode *N); SDValue SplitVecOp_FP_ROUND(SDNode *N); //===--------------------------------------------------------------------===// @@ -581,6 +602,7 @@ private: // Widen Vector Result Promotion. void WidenVectorResult(SDNode *N, unsigned ResNo); + SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo); SDValue WidenVecRes_BITCAST(SDNode* N); SDValue WidenVecRes_BUILD_VECTOR(SDNode* N); SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N); @@ -677,7 +699,8 @@ private: void GetPairElements(SDValue Pair, SDValue &Lo, SDValue &Hi); // Generic Result Splitting. - void SplitRes_MERGE_VALUES(SDNode *N, SDValue &Lo, SDValue &Hi); + void SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo, + SDValue &Lo, SDValue &Hi); void SplitRes_SELECT (SDNode *N, SDValue &Lo, SDValue &Hi); void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi); void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi); @@ -699,6 +722,8 @@ private: } // Generic Result Expansion. + void ExpandRes_MERGE_VALUES (SDNode *N, unsigned ResNo, + SDValue &Lo, SDValue &Hi); void ExpandRes_BITCAST (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandRes_BUILD_PAIR (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandRes_EXTRACT_ELEMENT (SDNode *N, SDValue &Lo, SDValue &Hi); diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp index 85ea6b6..8e7e498 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp @@ -31,6 +31,11 @@ using namespace llvm; // These routines assume that the Lo/Hi part is stored first in memory on // little/big-endian machines, followed by the Hi/Lo part. This means that // they cannot be used as is on vectors, for which Lo is always stored first. +void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo, + SDValue &Lo, SDValue &Hi) { + SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); + GetExpandedOp(Op, Lo, Hi); +} void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) { EVT OutVT = N->getValueType(0); @@ -426,37 +431,34 @@ SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) { // bytes; for integers and floats it is Lo first if and only if the machine is // little-endian). -void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, +void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo, SDValue &Lo, SDValue &Hi) { - // A MERGE_VALUES node can produce any number of values. We know that the - // first illegal one needs to be expanded into Lo/Hi. - unsigned i; - - // The string of legal results gets turned into input operands, which have - // the same type. - for (i = 0; isTypeLegal(N->getValueType(i)); ++i) - ReplaceValueWith(SDValue(N, i), SDValue(N->getOperand(i))); - - // The first illegal result must be the one that needs to be expanded. - GetSplitOp(N->getOperand(i), Lo, Hi); - - // Legalize the rest of the results into the input operands whether they are - // legal or not. - unsigned e = N->getNumValues(); - for (++i; i != e; ++i) - ReplaceValueWith(SDValue(N, i), SDValue(N->getOperand(i))); + SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); + GetSplitOp(Op, Lo, Hi); } void DAGTypeLegalizer::SplitRes_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi) { - SDValue LL, LH, RL, RH; + SDValue LL, LH, RL, RH, CL, CH; DebugLoc dl = N->getDebugLoc(); GetSplitOp(N->getOperand(1), LL, LH); GetSplitOp(N->getOperand(2), RL, RH); SDValue Cond = N->getOperand(0); - Lo = DAG.getNode(ISD::SELECT, dl, LL.getValueType(), Cond, LL, RL); - Hi = DAG.getNode(ISD::SELECT, dl, LH.getValueType(), Cond, LH, RH); + CL = CH = Cond; + if (Cond.getValueType().isVector()) { + assert(Cond.getValueType().getVectorElementType() == MVT::i1 && + "Condition legalized before result?"); + unsigned NumElements = Cond.getValueType().getVectorNumElements(); + EVT VCondTy = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElements / 2); + CL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VCondTy, Cond, + DAG.getIntPtrConstant(0)); + CH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VCondTy, Cond, + DAG.getIntPtrConstant(NumElements / 2)); + } + + Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), CL, LL, RL); + Hi = DAG.getNode(N->getOpcode(), dl, LH.getValueType(), CH, LH, RH); } void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo, diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index ffff10c..f815b00 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -61,6 +61,9 @@ class VectorLegalizer { // Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if // SINT_TO_FLOAT and SHR on vectors isn't legal. SDValue ExpandUINT_TO_FLOAT(SDValue Op); + // Implement vselect in terms of XOR, AND, OR when blend is not supported + // by the target. + SDValue ExpandVSELECT(SDValue Op); SDValue ExpandFNEG(SDValue Op); // Implements vector promotion; this is essentially just bitcasting the // operands to a different type and bitcasting the result back to the @@ -157,8 +160,9 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { case ISD::CTLZ: case ISD::CTPOP: case ISD::SELECT: + case ISD::VSELECT: case ISD::SELECT_CC: - case ISD::VSETCC: + case ISD::SETCC: case ISD::ZERO_EXTEND: case ISD::ANY_EXTEND: case ISD::TRUNCATE: @@ -210,11 +214,13 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { // FALL THROUGH } case TargetLowering::Expand: - if (Node->getOpcode() == ISD::UINT_TO_FP) + if (Node->getOpcode() == ISD::VSELECT) + Result = ExpandVSELECT(Op); + else if (Node->getOpcode() == ISD::UINT_TO_FP) Result = ExpandUINT_TO_FLOAT(Op); else if (Node->getOpcode() == ISD::FNEG) Result = ExpandFNEG(Op); - else if (Node->getOpcode() == ISD::VSETCC) + else if (Node->getOpcode() == ISD::SETCC) Result = UnrollVSETCC(Op); else Result = DAG.UnrollVectorOp(Op.getNode()); @@ -256,9 +262,41 @@ SDValue VectorLegalizer::PromoteVectorOp(SDValue Op) { return DAG.getNode(ISD::BITCAST, dl, VT, Op); } -SDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) { +SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) { + // Implement VSELECT in terms of XOR, AND, OR + // on platforms which do not support blend natively. + EVT VT = Op.getOperand(0).getValueType(); + DebugLoc DL = Op.getDebugLoc(); + SDValue Mask = Op.getOperand(0); + SDValue Op1 = Op.getOperand(1); + SDValue Op2 = Op.getOperand(2); + + // If we can't even use the basic vector operations of + // AND,OR,XOR, we will have to scalarize the op. + if (!TLI.isOperationLegalOrCustom(ISD::AND, VT) || + !TLI.isOperationLegalOrCustom(ISD::XOR, VT) || + !TLI.isOperationLegalOrCustom(ISD::OR, VT)) + return DAG.UnrollVectorOp(Op.getNode()); + + assert(VT.getSizeInBits() == Op.getOperand(1).getValueType().getSizeInBits() + && "Invalid mask size"); + // Bitcast the operands to be the same type as the mask. + // This is needed when we select between FP types because + // the mask is a vector of integers. + Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1); + Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2); + + SDValue AllOnes = DAG.getConstant( + APInt::getAllOnesValue(VT.getScalarType().getSizeInBits()), VT); + SDValue NotMask = DAG.getNode(ISD::XOR, DL, VT, Mask, AllOnes); + + Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask); + Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask); + return DAG.getNode(ISD::OR, DL, VT, Op1, Op2); +} +SDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) { EVT VT = Op.getOperand(0).getValueType(); DebugLoc DL = Op.getDebugLoc(); diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index b5698f9..107a42b 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -44,8 +44,10 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { N->dump(&DAG); dbgs() << "\n"; #endif - llvm_unreachable("Do not know how to scalarize the result of this operator!"); + report_fatal_error("Do not know how to scalarize the result of this " + "operator!\n"); + case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; case ISD::BUILD_VECTOR: R = N->getOperand(0); break; case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; @@ -62,8 +64,6 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; - case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break; - case ISD::ANY_EXTEND: case ISD::CTLZ: case ISD::CTPOP: @@ -129,6 +129,12 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { LHS.getValueType(), LHS, RHS); } +SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, + unsigned ResNo) { + SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); + return GetScalarizedVector(Op); +} + SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { EVT NewVT = N->getValueType(0).getVectorElementType(); return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), @@ -237,6 +243,12 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { } SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { + assert(N->getValueType(0).isVector() == + N->getOperand(0).getValueType().isVector() && + "Scalar/Vector type mismatch"); + + if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N); + SDValue LHS = GetScalarizedVector(N->getOperand(0)); SDValue RHS = GetScalarizedVector(N->getOperand(1)); DebugLoc DL = N->getDebugLoc(); @@ -259,35 +271,23 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { } SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { + assert(N->getValueType(0).isVector() && + N->getOperand(0).getValueType().isVector() && + "Operand types must be vectors"); + SDValue LHS = GetScalarizedVector(N->getOperand(0)); SDValue RHS = GetScalarizedVector(N->getOperand(1)); EVT NVT = N->getValueType(0).getVectorElementType(); - EVT SVT = TLI.getSetCCResultType(LHS.getValueType()); DebugLoc DL = N->getDebugLoc(); // Turn it into a scalar SETCC. - SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2)); - - // VSETCC always returns a sign-extended value, while SETCC may not. The - // SETCC result type may not match the vector element type. Correct these. - if (NVT.bitsLE(SVT)) { - // The SETCC result type is bigger than the vector element type. - // Ensure the SETCC result is sign-extended. - if (TLI.getBooleanContents() != - TargetLowering::ZeroOrNegativeOneBooleanContent) - Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res, - DAG.getValueType(MVT::i1)); - // Truncate to the final type. - return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res); - } - - // The SETCC result type is smaller than the vector element type. - // If the SetCC result is not sign-extended, chop it down to MVT::i1. - if (TLI.getBooleanContents() != - TargetLowering::ZeroOrNegativeOneBooleanContent) - Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res); - // Sign extend to the final type. - return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res); + SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, + N->getOperand(2)); + // Vectors may have a different boolean contents to scalars. Promote the + // value appropriately. + ISD::NodeType ExtendCode = + TargetLowering::getExtendForContent(TLI.getBooleanContents(true)); + return DAG.getNode(ExtendCode, DL, NVT, Res); } @@ -415,7 +415,8 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { #endif llvm_unreachable("Do not know how to split the result of this operator!"); - case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break; + case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; + case ISD::VSELECT: case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; @@ -432,7 +433,6 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; case ISD::SETCC: - case ISD::VSETCC: SplitVecRes_SETCC(N, Lo, Hi); break; case ISD::VECTOR_SHUFFLE: @@ -524,12 +524,11 @@ void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, // Handle some special cases efficiently. switch (getTypeAction(InVT)) { - default: - assert(false && "Unknown type action!"); case TargetLowering::TypeLegal: case TargetLowering::TypePromoteInteger: case TargetLowering::TypeSoftenFloat: case TargetLowering::TypeScalarizeVector: + case TargetLowering::TypeWidenVector: break; case TargetLowering::TypeExpandInteger: case TargetLowering::TypeExpandFloat: @@ -670,7 +669,7 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, // Store the new element. This may be larger than the vector element type, // so use a truncating store. SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); - const Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); + Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); unsigned Alignment = TLI.getTargetData()->getPrefTypeAlignment(VecType); Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT, @@ -740,6 +739,10 @@ void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, } void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { + assert(N->getValueType(0).isVector() && + N->getOperand(0).getValueType().isVector() && + "Operand types must be vectors"); + EVT LoVT, HiVT; DebugLoc DL = N->getDebugLoc(); GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); @@ -965,7 +968,7 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { dbgs() << "\n"; #endif llvm_unreachable("Do not know how to split this operator's operand!"); - + case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; @@ -1163,6 +1166,26 @@ SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { &Elts[0], Elts.size()); } +SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { + assert(N->getValueType(0).isVector() && + N->getOperand(0).getValueType().isVector() && + "Operand types must be vectors"); + // The result has a legal vector type, but the input needs splitting. + SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; + DebugLoc DL = N->getDebugLoc(); + GetSplitVector(N->getOperand(0), Lo0, Hi0); + GetSplitVector(N->getOperand(1), Lo1, Hi1); + unsigned PartElements = Lo0.getValueType().getVectorNumElements(); + EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements); + EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements); + + LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); + HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); + SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); + return PromoteTargetBoolean(Con, N->getValueType(0)); +} + + SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { // The result has a legal vector type, but the input needs splitting. EVT ResVT = N->getValueType(0); @@ -1205,6 +1228,7 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { #endif llvm_unreachable("Do not know how to widen the result of this operator!"); + case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; @@ -1222,10 +1246,6 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { case ISD::VECTOR_SHUFFLE: Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); break; - case ISD::VSETCC: - Res = WidenVecRes_VSETCC(N); - break; - case ISD::ADD: case ISD::AND: case ISD::BSWAP: @@ -1557,6 +1577,11 @@ SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { WidenVT, WidenLHS, DAG.getValueType(ExtVT)); } +SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { + SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); + return GetWidenedVector(WidenVec); +} + SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { SDValue InOp = N->getOperand(0); EVT InVT = InOp.getValueType(); @@ -1661,6 +1686,7 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); DebugLoc dl = N->getDebugLoc(); unsigned WidenNumElts = WidenVT.getVectorNumElements(); + unsigned NumInElts = InVT.getVectorNumElements(); unsigned NumOperands = N->getNumOperands(); bool InputWidened = false; // Indicates we need to widen the input. @@ -1686,17 +1712,17 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { if (N->getOperand(i).getOpcode() != ISD::UNDEF) break; - if (i > NumOperands) + if (i == NumOperands) // Everything but the first operand is an UNDEF so just return the // widened first operand. return GetWidenedVector(N->getOperand(0)); if (NumOperands == 2) { // Replace concat of two operands with a shuffle. - SmallVector<int, 16> MaskOps(WidenNumElts); - for (unsigned i=0; i < WidenNumElts/2; ++i) { + SmallVector<int, 16> MaskOps(WidenNumElts, -1); + for (unsigned i = 0; i < NumInElts; ++i) { MaskOps[i] = i; - MaskOps[i+WidenNumElts/2] = i+WidenNumElts; + MaskOps[i + NumInElts] = i + WidenNumElts; } return DAG.getVectorShuffle(WidenVT, dl, GetWidenedVector(N->getOperand(0)), @@ -1708,7 +1734,6 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { // Fall back to use extracts and build vector. EVT EltVT = WidenVT.getVectorElementType(); - unsigned NumInElts = InVT.getVectorNumElements(); SmallVector<SDValue, 16> Ops(WidenNumElts); unsigned Idx = 0; for (unsigned i=0; i < NumOperands; ++i) { @@ -1916,6 +1941,11 @@ SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { } SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { + assert(N->getValueType(0).isVector() == + N->getOperand(0).getValueType().isVector() && + "Scalar/Vector type mismatch"); + if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N); + EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue InOp1 = GetWidenedVector(N->getOperand(0)); SDValue InOp2 = GetWidenedVector(N->getOperand(1)); @@ -1954,6 +1984,9 @@ SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { } SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { + assert(N->getValueType(0).isVector() && + N->getOperand(0).getValueType().isVector() && + "Operands must be vectors"); EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); unsigned WidenNumElts = WidenVT.getVectorNumElements(); @@ -1970,7 +2003,8 @@ SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { assert(InOp1.getValueType() == WidenInVT && InOp2.getValueType() == WidenInVT && "Input not widened to expected type!"); - return DAG.getNode(ISD::VSETCC, N->getDebugLoc(), + (void)WidenInVT; + return DAG.getNode(ISD::SETCC, N->getDebugLoc(), WidenVT, InOp1, InOp2, N->getOperand(2)); } diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index 12b1838..e757def 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -2621,6 +2621,39 @@ bool RegReductionPQBase::canClobber(const SUnit *SU, const SUnit *Op) { return false; } +/// canClobberReachingPhysRegUse - True if SU would clobber one of it's +/// successor's explicit physregs whose definition can reach DepSU. +/// i.e. DepSU should not be scheduled above SU. +static bool canClobberReachingPhysRegUse(const SUnit *DepSU, const SUnit *SU, + ScheduleDAGRRList *scheduleDAG, + const TargetInstrInfo *TII, + const TargetRegisterInfo *TRI) { + const unsigned *ImpDefs + = TII->get(SU->getNode()->getMachineOpcode()).getImplicitDefs(); + if(!ImpDefs) + return false; + + for (SUnit::const_succ_iterator SI = SU->Succs.begin(), SE = SU->Succs.end(); + SI != SE; ++SI) { + SUnit *SuccSU = SI->getSUnit(); + for (SUnit::const_pred_iterator PI = SuccSU->Preds.begin(), + PE = SuccSU->Preds.end(); PI != PE; ++PI) { + if (!PI->isAssignedRegDep()) + continue; + + for (const unsigned *ImpDef = ImpDefs; *ImpDef; ++ImpDef) { + // Return true if SU clobbers this physical register use and the + // definition of the register reaches from DepSU. IsReachable queries a + // topological forward sort of the DAG (following the successors). + if (TRI->regsOverlap(*ImpDef, PI->getReg()) && + scheduleDAG->IsReachable(DepSU, PI->getSUnit())) + return true; + } + } + } + return false; +} + /// canClobberPhysRegDefs - True if SU would clobber one of SuccSU's /// physical register defs. static bool canClobberPhysRegDefs(const SUnit *SuccSU, const SUnit *SU, @@ -2837,7 +2870,8 @@ void RegReductionPQBase::AddPseudoTwoAddrDeps() { SuccOpc == TargetOpcode::INSERT_SUBREG || SuccOpc == TargetOpcode::SUBREG_TO_REG) continue; - if ((!canClobber(SuccSU, DUSU) || + if (!canClobberReachingPhysRegUse(SuccSU, SU, scheduleDAG, TII, TRI) && + (!canClobber(SuccSU, DUSU) || (isLiveOut && !hasOnlyLiveOutUses(SuccSU)) || (!SU->isCommutable && SuccSU->isCommutable)) && !scheduleDAG->IsReachable(SuccSU, SU)) { diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 35ea0bb..20bea8e 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -403,7 +403,7 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { ID.AddInteger(CP->getAlignment()); ID.AddInteger(CP->getOffset()); if (CP->isMachineConstantPoolEntry()) - CP->getMachineCPVal()->AddSelectionDAGCSEId(ID); + CP->getMachineCPVal()->addSelectionDAGCSEId(ID); else ID.AddPointer(CP->getConstVal()); ID.AddInteger(CP->getTargetFlags()); @@ -432,7 +432,9 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { case ISD::ATOMIC_LOAD_MIN: case ISD::ATOMIC_LOAD_MAX: case ISD::ATOMIC_LOAD_UMIN: - case ISD::ATOMIC_LOAD_UMAX: { + case ISD::ATOMIC_LOAD_UMAX: + case ISD::ATOMIC_LOAD: + case ISD::ATOMIC_STORE: { const AtomicSDNode *AT = cast<AtomicSDNode>(N); ID.AddInteger(AT->getMemoryVT().getRawBits()); ID.AddInteger(AT->getRawSubclassData()); @@ -769,11 +771,14 @@ static void VerifyNodeCommon(SDNode *N) { assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && "Wrong number of operands!"); EVT EltVT = N->getValueType(0).getVectorElementType(); - for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) + for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { assert((I->getValueType() == EltVT || (EltVT.isInteger() && I->getValueType().isInteger() && EltVT.bitsLE(I->getValueType()))) && "Wrong operand type!"); + assert(I->getValueType() == N->getOperand(0).getValueType() && + "Operands must all have the same type"); + } break; } } @@ -821,7 +826,7 @@ static void VerifyMachineNode(SDNode *N) { /// given type. /// unsigned SelectionDAG::getEVTAlignment(EVT VT) const { - const Type *Ty = VT == MVT::iPTR ? + Type *Ty = VT == MVT::iPTR ? PointerType::get(Type::getInt8Ty(*getContext()), 0) : VT.getTypeForEVT(*getContext()); @@ -876,6 +881,12 @@ void SelectionDAG::clear() { DbgInfo->clear(); } +SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { + return VT.bitsGT(Op.getValueType()) ? + getNode(ISD::ANY_EXTEND, DL, VT, Op) : + getNode(ISD::TRUNCATE, DL, VT, Op); +} + SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { return VT.bitsGT(Op.getValueType()) ? getNode(ISD::SIGN_EXTEND, DL, VT, Op) : @@ -925,13 +936,25 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { assert(VT.isInteger() && "Cannot create FP integer constant!"); EVT EltVT = VT.getScalarType(); - assert(Val.getBitWidth() == EltVT.getSizeInBits() && - "APInt size does not match type size!"); + const ConstantInt *Elt = &Val; + // In some cases the vector type is legal but the element type is illegal and + // needs to be promoted, for example v8i8 on ARM. In this case, promote the + // inserted value (the type does not need to match the vector element type). + // Any extra bits introduced will be truncated away. + if (VT.isVector() && TLI.getTypeAction(*getContext(), EltVT) == + TargetLowering::TypePromoteInteger) { + EltVT = TLI.getTypeToTransformTo(*getContext(), EltVT); + APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits()); + Elt = ConstantInt::get(*getContext(), NewVal); + } + + assert(Elt->getBitWidth() == EltVT.getSizeInBits() && + "APInt size does not match type size!"); unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; FoldingSetNodeID ID; AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); - ID.AddPointer(&Val); + ID.AddPointer(Elt); void *IP = 0; SDNode *N = NULL; if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) @@ -939,7 +962,7 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { return SDValue(N, 0); if (!N) { - N = new (NodeAllocator) ConstantSDNode(isT, &Val, EltVT); + N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); } @@ -1131,7 +1154,7 @@ SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); ID.AddInteger(Alignment); ID.AddInteger(Offset); - C->AddSelectionDAGCSEId(ID); + C->addSelectionDAGCSEId(ID); ID.AddInteger(TargetFlags); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) @@ -1432,7 +1455,7 @@ SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); unsigned ByteSize = VT.getStoreSize(); - const Type *Ty = VT.getTypeForEVT(*getContext()); + Type *Ty = VT.getTypeForEVT(*getContext()); unsigned StackAlign = std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), minAlign); @@ -1445,8 +1468,8 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { unsigned Bytes = std::max(VT1.getStoreSizeInBits(), VT2.getStoreSizeInBits())/8; - const Type *Ty1 = VT1.getTypeForEVT(*getContext()); - const Type *Ty2 = VT2.getTypeForEVT(*getContext()); + Type *Ty1 = VT1.getTypeForEVT(*getContext()); + Type *Ty2 = VT2.getTypeForEVT(*getContext()); const TargetData *TD = TLI.getTargetData(); unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1), TD->getPrefTypeAlignment(Ty2)); @@ -1718,8 +1741,8 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, // The boolean result conforms to getBooleanContents. Fall through. case ISD::SETCC: // If we know the result of a setcc has the top bits zero, use this info. - if (TLI.getBooleanContents() == TargetLowering::ZeroOrOneBooleanContent && - BitWidth > 1) + if (TLI.getBooleanContents(Op.getValueType().isVector()) == + TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); return; case ISD::SHL: @@ -2153,7 +2176,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ // The boolean result conforms to getBooleanContents. Fall through. case ISD::SETCC: // If setcc returns 0/-1, all bits are sign bits. - if (TLI.getBooleanContents() == + if (TLI.getBooleanContents(Op.getValueType().isVector()) == TargetLowering::ZeroOrNegativeOneBooleanContent) return VTBits; break; @@ -2437,7 +2460,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, APFloat::rmTowardZero, &ignored); if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual break; - APInt api(VT.getSizeInBits(), 2, x); + APInt api(VT.getSizeInBits(), x); return getConstant(api, VT); } case ISD::BITCAST: @@ -2777,6 +2800,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, EVT.getVectorNumElements() == VT.getVectorNumElements()) && "Vector element counts must match in FP_ROUND_INREG"); assert(EVT.bitsLE(VT) && "Not rounding down!"); + (void)EVT; if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. break; } @@ -2884,6 +2908,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); assert(!N1.getValueType().isVector() && !VT.isVector() && (N1.getValueType().isInteger() == VT.isInteger()) && + N1.getValueType() != VT && "Wrong types for EXTRACT_ELEMENT!"); // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding @@ -3425,7 +3450,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, return SDValue(); if (DstAlignCanChange) { - const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); + Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); if (NewAlign > Align) { // Give the stack frame object a larger alignment if needed. @@ -3514,7 +3539,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, return SDValue(); if (DstAlignCanChange) { - const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); + Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); if (NewAlign > Align) { // Give the stack frame object a larger alignment if needed. @@ -3589,7 +3614,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, return SDValue(); if (DstAlignCanChange) { - const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); + Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); if (NewAlign > Align) { // Give the stack frame object a larger alignment if needed. @@ -3782,7 +3807,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, return Result; // Emit a library call. - const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext()); + Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext()); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; Entry.Node = Dst; Entry.Ty = IntPtrTy; @@ -3815,7 +3840,9 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, - unsigned Alignment) { + unsigned Alignment, + AtomicOrdering Ordering, + SynchronizationScope SynchScope) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); @@ -3823,18 +3850,23 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; // For now, atomics are considered to be volatile always. + // FIXME: Volatile isn't really correct; we should keep track of atomic + // orderings in the memoperand. Flags |= MachineMemOperand::MOVolatile; MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); - return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO); + return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, + Ordering, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, - SDValue Swp, MachineMemOperand *MMO) { + SDValue Swp, MachineMemOperand *MMO, + AtomicOrdering Ordering, + SynchronizationScope SynchScope) { assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); @@ -3851,7 +3883,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, return SDValue(E, 0); } SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, - Ptr, Cmp, Swp, MMO); + Ptr, Cmp, Swp, MMO, Ordering, + SynchScope); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3861,27 +3894,39 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value* PtrVal, - unsigned Alignment) { + unsigned Alignment, + AtomicOrdering Ordering, + SynchronizationScope SynchScope) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); MachineFunction &MF = getMachineFunction(); - unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + // A monotonic store does not load; a release store "loads" in the sense + // that other stores cannot be sunk past it. + // (An atomicrmw obviously both loads and stores.) + unsigned Flags = MachineMemOperand::MOStore; + if (Opcode != ISD::ATOMIC_STORE || Ordering > Monotonic) + Flags |= MachineMemOperand::MOLoad; // For now, atomics are considered to be volatile always. + // FIXME: Volatile isn't really correct; we should keep track of atomic + // orderings in the memoperand. Flags |= MachineMemOperand::MOVolatile; MachineMemOperand *MMO = MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, MemVT.getStoreSize(), Alignment); - return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO); + return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, + Ordering, SynchScope); } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, - MachineMemOperand *MMO) { + MachineMemOperand *MMO, + AtomicOrdering Ordering, + SynchronizationScope SynchScope) { assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB || Opcode == ISD::ATOMIC_LOAD_AND || @@ -3892,12 +3937,14 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || - Opcode == ISD::ATOMIC_SWAP) && + Opcode == ISD::ATOMIC_SWAP || + Opcode == ISD::ATOMIC_STORE) && "Invalid Atomic Op"); EVT VT = Val.getValueType(); - SDVTList VTs = getVTList(VT, MVT::Other); + SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : + getVTList(VT, MVT::Other); FoldingSetNodeID ID; ID.AddInteger(MemVT.getRawBits()); SDValue Ops[] = {Chain, Ptr, Val}; @@ -3908,7 +3955,63 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, return SDValue(E, 0); } SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, - Ptr, Val, MMO); + Ptr, Val, MMO, + Ordering, SynchScope); + CSEMap.InsertNode(N, IP); + AllNodes.push_back(N); + return SDValue(N, 0); +} + +SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, + EVT VT, SDValue Chain, + SDValue Ptr, + const Value* PtrVal, + unsigned Alignment, + AtomicOrdering Ordering, + SynchronizationScope SynchScope) { + if (Alignment == 0) // Ensure that codegen never sees alignment 0 + Alignment = getEVTAlignment(MemVT); + + MachineFunction &MF = getMachineFunction(); + // A monotonic load does not store; an acquire load "stores" in the sense + // that other loads cannot be hoisted past it. + unsigned Flags = MachineMemOperand::MOLoad; + if (Ordering > Monotonic) + Flags |= MachineMemOperand::MOStore; + + // For now, atomics are considered to be volatile always. + // FIXME: Volatile isn't really correct; we should keep track of atomic + // orderings in the memoperand. + Flags |= MachineMemOperand::MOVolatile; + + MachineMemOperand *MMO = + MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, + MemVT.getStoreSize(), Alignment); + + return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, + Ordering, SynchScope); +} + +SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, + EVT VT, SDValue Chain, + SDValue Ptr, + MachineMemOperand *MMO, + AtomicOrdering Ordering, + SynchronizationScope SynchScope) { + assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); + + SDVTList VTs = getVTList(VT, MVT::Other); + FoldingSetNodeID ID; + ID.AddInteger(MemVT.getRawBits()); + SDValue Ops[] = {Chain, Ptr}; + AddNodeIDNode(ID, Opcode, VTs, Ops, 2); + void* IP = 0; + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { + cast<AtomicSDNode>(E)->refineAlignment(MMO); + return SDValue(E, 0); + } + SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, + Ptr, MMO, Ordering, SynchScope); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -5769,6 +5872,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { #endif case ISD::PREFETCH: return "Prefetch"; case ISD::MEMBARRIER: return "MemBarrier"; + case ISD::ATOMIC_FENCE: return "AtomicFence"; case ISD::ATOMIC_CMP_SWAP: return "AtomicCmpSwap"; case ISD::ATOMIC_SWAP: return "AtomicSwap"; case ISD::ATOMIC_LOAD_ADD: return "AtomicLoadAdd"; @@ -5781,6 +5885,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::ATOMIC_LOAD_MAX: return "AtomicLoadMax"; case ISD::ATOMIC_LOAD_UMIN: return "AtomicLoadUMin"; case ISD::ATOMIC_LOAD_UMAX: return "AtomicLoadUMax"; + case ISD::ATOMIC_LOAD: return "AtomicLoad"; + case ISD::ATOMIC_STORE: return "AtomicStore"; case ISD::PCMARKER: return "PCMarker"; case ISD::READCYCLECOUNTER: return "ReadCycleCounter"; case ISD::SRCVALUE: return "SrcValue"; @@ -5896,8 +6002,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::FPOWI: return "fpowi"; case ISD::SETCC: return "setcc"; - case ISD::VSETCC: return "vsetcc"; case ISD::SELECT: return "select"; + case ISD::VSELECT: return "vselect"; case ISD::SELECT_CC: return "select_cc"; case ISD::INSERT_VECTOR_ELT: return "insert_vector_elt"; case ISD::EXTRACT_VECTOR_ELT: return "extract_vector_elt"; @@ -5985,7 +6091,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::CTLZ: return "ctlz"; // Trampolines - case ISD::TRAMPOLINE: return "trampoline"; + case ISD::INIT_TRAMPOLINE: return "init_trampoline"; + case ISD::ADJUST_TRAMPOLINE: return "adjust_trampoline"; case ISD::CONDCODE: switch (cast<CondCodeSDNode>(this)->get()) { @@ -6245,8 +6352,7 @@ void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const { static void printrWithDepthHelper(raw_ostream &OS, const SDNode *N, const SelectionDAG *G, unsigned depth, - unsigned indent) -{ + unsigned indent) { if (depth == 0) return; @@ -6340,6 +6446,10 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, &Operands[0], Operands.size())); break; + case ISD::VSELECT: + Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, + &Operands[0], Operands.size())); + break; case ISD::SHL: case ISD::SRA: case ISD::SRL: @@ -6427,6 +6537,8 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { Align = TD->getPreferredAlignment(GVar); } } + if (!Align) + Align = TLI.getTargetData()->getABITypeAlignment(GV->getType()); } return MinAlign(Align, GVOffset); } @@ -6528,7 +6640,7 @@ unsigned GlobalAddressSDNode::getAddressSpace() const { } -const Type *ConstantPoolSDNode::getType() const { +Type *ConstantPoolSDNode::getType() const { if (isMachineConstantPoolEntry()) return Val.MachineCPVal->getType(); return Val.ConstVal->getType(); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 81b03ee..7ed46a6 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -578,7 +578,7 @@ namespace { : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {} RegsForValue(LLVMContext &Context, const TargetLowering &tli, - unsigned Reg, const Type *Ty) { + unsigned Reg, Type *Ty) { ComputeValueVTs(tli, Ty, ValueVTs); for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { @@ -788,6 +788,18 @@ void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size()); if (HasMatching) Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx); + else if (!Regs.empty() && + TargetRegisterInfo::isVirtualRegister(Regs.front())) { + // Put the register class of the virtual registers in the flag word. That + // way, later passes can recompute register class constraints for inline + // assembly as well as normal instructions. + // Don't do this for tied operands that can use the regclass information + // from the def. + const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); + const TargetRegisterClass *RC = MRI.getRegClass(Regs.front()); + Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID()); + } + SDValue Res = DAG.getTargetConstant(Flag, MVT::i32); Ops.push_back(Res); @@ -805,6 +817,7 @@ void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa) { AA = &aa; GFI = gfi; TD = DAG.getTarget().getTargetData(); + LPadToCallSiteMap.clear(); } /// clear - Clear out the current SelectionDAG and the associated @@ -956,7 +969,7 @@ void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, } } -// getValue - Return an SDValue for the given Value. +/// getValue - Return an SDValue for the given Value. SDValue SelectionDAGBuilder::getValue(const Value *V) { // If we already have an SDValue for this value, use it. It's important // to do this first, so that we don't create a CopyFromReg if we already @@ -971,7 +984,7 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { unsigned InReg = It->second; RegsForValue RFV(*DAG.getContext(), TLI, InReg, V->getType()); SDValue Chain = DAG.getEntryNode(); - N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurDebugLoc(), Chain,NULL); + N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurDebugLoc(), Chain, NULL); resolveDanglingDebugInfo(V, N); return N; } @@ -1069,7 +1082,7 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) { if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) return DAG.getBlockAddress(BA, VT); - const VectorType *VecTy = cast<VectorType>(V->getType()); + VectorType *VecTy = cast<VectorType>(V->getType()); unsigned NumElements = VecTy->getNumElements(); // Now that we know the number and type of the elements, get that number of @@ -1277,15 +1290,17 @@ uint32_t SelectionDAGBuilder::getEdgeWeight(MachineBasicBlock *Src, BranchProbabilityInfo *BPI = FuncInfo.BPI; if (!BPI) return 0; - BasicBlock *SrcBB = const_cast<BasicBlock*>(Src->getBasicBlock()); - BasicBlock *DstBB = const_cast<BasicBlock*>(Dst->getBasicBlock()); + const BasicBlock *SrcBB = Src->getBasicBlock(); + const BasicBlock *DstBB = Dst->getBasicBlock(); return BPI->getEdgeWeight(SrcBB, DstBB); } -void SelectionDAGBuilder::addSuccessorWithWeight(MachineBasicBlock *Src, - MachineBasicBlock *Dst) { - uint32_t weight = getEdgeWeight(Src, Dst); - Src->addSuccessor(Dst, weight); +void SelectionDAGBuilder:: +addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst, + uint32_t Weight /* = 0 */) { + if (!Weight) + Weight = getEdgeWeight(Src, Dst); + Src->addSuccessor(Dst, Weight); } @@ -1558,8 +1573,8 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, } // Update successor info - addSuccessorWithWeight(SwitchBB, CB.TrueBB); - addSuccessorWithWeight(SwitchBB, CB.FalseBB); + addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight); + addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight); // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. @@ -1677,7 +1692,7 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, UsePtrType = true; else { for (unsigned i = 0, e = B.Cases.size(); i != e; ++i) - if ((uint64_t)((int64_t)B.Cases[i].Mask >> VT.getSizeInBits()) + 1 >= 2) { + if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) { // Switch table case range are encoded into series of masks. // Just use pointer type, it's guaranteed to fit. UsePtrType = true; @@ -1808,6 +1823,49 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { void SelectionDAGBuilder::visitUnwind(const UnwindInst &I) { } +void SelectionDAGBuilder::visitResume(const ResumeInst &RI) { + llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!"); +} + +void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) { + assert(FuncInfo.MBB->isLandingPad() && + "Call to landingpad not in landing pad!"); + + MachineBasicBlock *MBB = FuncInfo.MBB; + MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); + AddLandingPadInfo(LP, MMI, MBB); + + SmallVector<EVT, 2> ValueVTs; + ComputeValueVTs(TLI, LP.getType(), ValueVTs); + + // Insert the EXCEPTIONADDR instruction. + assert(FuncInfo.MBB->isLandingPad() && + "Call to eh.exception not in landing pad!"); + SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other); + SDValue Ops[2]; + Ops[0] = DAG.getRoot(); + SDValue Op1 = DAG.getNode(ISD::EXCEPTIONADDR, getCurDebugLoc(), VTs, Ops, 1); + SDValue Chain = Op1.getValue(1); + + // Insert the EHSELECTION instruction. + VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other); + Ops[0] = Op1; + Ops[1] = Chain; + SDValue Op2 = DAG.getNode(ISD::EHSELECTION, getCurDebugLoc(), VTs, Ops, 2); + Chain = Op2.getValue(1); + Op2 = DAG.getSExtOrTrunc(Op2, getCurDebugLoc(), MVT::i32); + + Ops[0] = Op1; + Ops[1] = Op2; + SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(), + DAG.getVTList(&ValueVTs[0], ValueVTs.size()), + &Ops[0], 2); + + std::pair<SDValue, SDValue> RetPair = std::make_pair(Res, Chain); + setValue(&LP, RetPair.first); + DAG.setRoot(RetPair.second); +} + /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for /// small case ranges). bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, @@ -1866,8 +1924,8 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, ISD::SETEQ); // Update successor info. - SwitchBB->addSuccessor(Small.BB); - SwitchBB->addSuccessor(Default); + addSuccessorWithWeight(SwitchBB, Small.BB); + addSuccessorWithWeight(SwitchBB, Default); // Insert the true branch. SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other, @@ -1923,7 +1981,11 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, CC = ISD::SETLE; LHS = I->Low; MHS = SV; RHS = I->High; } - CaseBlock CB(CC, LHS, RHS, MHS, I->BB, FallThrough, CurBlock); + + uint32_t ExtraWeight = I->ExtraWeight; + CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough, + /* me */ CurBlock, + /* trueweight */ ExtraWeight / 2, /* falseweight */ ExtraWeight / 2); // If emitting the first comparison, just call visitSwitchCase to emit the // code into the current block. Otherwise, push the CaseBlock onto the @@ -1953,10 +2015,10 @@ static APInt ComputeRange(const APInt &First, const APInt &Last) { } /// handleJTSwitchCase - Emit jumptable for current switch case range -bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec& CR, - CaseRecVector& WorkList, - const Value* SV, - MachineBasicBlock* Default, +bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR, + CaseRecVector &WorkList, + const Value *SV, + MachineBasicBlock *Default, MachineBasicBlock *SwitchBB) { Case& FrontCase = *CR.Range.first; Case& BackCase = *(CR.Range.second-1); @@ -1965,8 +2027,7 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec& CR, const APInt &Last = cast<ConstantInt>(BackCase.High)->getValue(); APInt TSize(First.getBitWidth(), 0); - for (CaseItr I = CR.Range.first, E = CR.Range.second; - I!=E; ++I) + for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) TSize += I->size(); if (!areJTsAllowed(TLI) || TSize.ult(4)) @@ -2044,7 +2105,6 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec& CR, visitJumpTableHeader(JT, JTH, SwitchBB); JTCases.push_back(JumpTableBlock(JTH, JT)); - return true; } @@ -2318,12 +2378,17 @@ size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases, const SwitchInst& SI) { size_t numCmps = 0; + BranchProbabilityInfo *BPI = FuncInfo.BPI; // Start with "simple" cases for (size_t i = 1; i < SI.getNumSuccessors(); ++i) { - MachineBasicBlock *SMBB = FuncInfo.MBBMap[SI.getSuccessor(i)]; + BasicBlock *SuccBB = SI.getSuccessor(i); + MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB]; + + uint32_t ExtraWeight = BPI ? BPI->getEdgeWeight(SI.getParent(), SuccBB) : 0; + Cases.push_back(Case(SI.getSuccessorValue(i), SI.getSuccessorValue(i), - SMBB)); + SMBB, ExtraWeight)); } std::sort(Cases.begin(), Cases.end(), CaseCmp()); @@ -2343,6 +2408,16 @@ size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases, if ((nextValue - currentValue == 1) && (currentBB == nextBB)) { I->High = J->High; J = Cases.erase(J); + + if (BranchProbabilityInfo *BPI = FuncInfo.BPI) { + uint32_t CurWeight = currentBB->getBasicBlock() ? + BPI->getEdgeWeight(SI.getParent(), currentBB->getBasicBlock()) : 16; + uint32_t NextWeight = nextBB->getBasicBlock() ? + BPI->getEdgeWeight(SI.getParent(), nextBB->getBasicBlock()) : 16; + + BPI->setEdgeWeight(SI.getParent(), currentBB->getBasicBlock(), + CurWeight + NextWeight); + } } else { I = J++; } @@ -2379,7 +2454,7 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { // If there is only the default destination, branch to it if it is not the // next basic block. Otherwise, just fall through. - if (SI.getNumOperands() == 2) { + if (SI.getNumCases() == 1) { // Update machine-CFG edges. // If this is not a fall-through branch, emit the branch. @@ -2399,12 +2474,12 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { size_t numCmps = Clusterify(Cases, SI); DEBUG(dbgs() << "Clusterify finished. Total clusters: " << Cases.size() << ". Total compares: " << numCmps << '\n'); - numCmps = 0; + (void)numCmps; // Get the Value to be switched on and default basic blocks, which will be // inserted into CaseBlock records, representing basic blocks in the binary // search tree. - const Value *SV = SI.getOperand(0); + const Value *SV = SI.getCondition(); // Push the initial CaseRec onto the worklist CaseRecVector WorkList; @@ -2458,7 +2533,7 @@ void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) { void SelectionDAGBuilder::visitFSub(const User &I) { // -0.0 - X --> fneg - const Type *Ty = I.getType(); + Type *Ty = I.getType(); if (isa<Constant>(I.getOperand(0)) && I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) { SDValue Op2 = getValue(I.getOperand(1)); @@ -2562,10 +2637,12 @@ void SelectionDAGBuilder::visitSelect(const User &I) { SDValue Cond = getValue(I.getOperand(0)); SDValue TrueVal = getValue(I.getOperand(1)); SDValue FalseVal = getValue(I.getOperand(2)); + ISD::NodeType OpCode = Cond.getValueType().isVector() ? + ISD::VSELECT : ISD::SELECT; for (unsigned i = 0; i != NumValues; ++i) - Values[i] = DAG.getNode(ISD::SELECT, getCurDebugLoc(), - TrueVal.getNode()->getValueType(TrueVal.getResNo()+i), + Values[i] = DAG.getNode(OpCode, getCurDebugLoc(), + TrueVal.getNode()->getValueType(TrueVal.getResNo()+i), Cond, SDValue(TrueVal.getNode(), TrueVal.getResNo() + i), @@ -2778,7 +2855,8 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) { // Analyze the access pattern of the vector to see if we can extract // two subvectors and do the shuffle. The analysis is done by calculating // the range of elements the mask access on both vectors. - int MinRange[2] = { SrcNumElts+1, SrcNumElts+1}; + int MinRange[2] = { static_cast<int>(SrcNumElts+1), + static_cast<int>(SrcNumElts+1)}; int MaxRange[2] = {-1, -1}; for (unsigned i = 0; i != MaskNumElts; ++i) { @@ -2886,8 +2964,8 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) { void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { const Value *Op0 = I.getOperand(0); const Value *Op1 = I.getOperand(1); - const Type *AggTy = I.getType(); - const Type *ValTy = Op1->getType(); + Type *AggTy = I.getType(); + Type *ValTy = Op1->getType(); bool IntoUndef = isa<UndefValue>(Op0); bool FromUndef = isa<UndefValue>(Op1); @@ -2927,8 +3005,8 @@ void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) { const Value *Op0 = I.getOperand(0); - const Type *AggTy = Op0->getType(); - const Type *ValTy = I.getType(); + Type *AggTy = Op0->getType(); + Type *ValTy = I.getType(); bool OutOfUndef = isa<UndefValue>(Op0); unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices()); @@ -2961,12 +3039,12 @@ void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) { void SelectionDAGBuilder::visitGetElementPtr(const User &I) { SDValue N = getValue(I.getOperand(0)); - const Type *Ty = I.getOperand(0)->getType(); + Type *Ty = I.getOperand(0)->getType(); for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end(); OI != E; ++OI) { const Value *Idx = *OI; - if (const StructType *StTy = dyn_cast<StructType>(Ty)) { + if (StructType *StTy = dyn_cast<StructType>(Ty)) { unsigned Field = cast<ConstantInt>(Idx)->getZExtValue(); if (Field) { // N = N + Offset @@ -3037,7 +3115,7 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { if (FuncInfo.StaticAllocaMap.count(&I)) return; // getValue will auto-populate this. - const Type *Ty = I.getAllocatedType(); + Type *Ty = I.getAllocatedType(); uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(Ty); unsigned Align = std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), @@ -3084,10 +3162,13 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { } void SelectionDAGBuilder::visitLoad(const LoadInst &I) { + if (I.isAtomic()) + return visitAtomicLoad(I); + const Value *SV = I.getOperand(0); SDValue Ptr = getValue(SV); - const Type *Ty = I.getType(); + Type *Ty = I.getType(); bool isVolatile = I.isVolatile(); bool isNonTemporal = I.getMetadata("nontemporal") != 0; @@ -3161,6 +3242,9 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { } void SelectionDAGBuilder::visitStore(const StoreInst &I) { + if (I.isAtomic()) + return visitAtomicStore(I); + const Value *SrcV = I.getOperand(0); const Value *PtrV = I.getOperand(1); @@ -3211,6 +3295,179 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) { DAG.setRoot(StoreNode); } +static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order, + SynchronizationScope Scope, + bool Before, DebugLoc dl, + SelectionDAG &DAG, + const TargetLowering &TLI) { + // Fence, if necessary + if (Before) { + if (Order == AcquireRelease || Order == SequentiallyConsistent) + Order = Release; + else if (Order == Acquire || Order == Monotonic) + return Chain; + } else { + if (Order == AcquireRelease) + Order = Acquire; + else if (Order == Release || Order == Monotonic) + return Chain; + } + SDValue Ops[3]; + Ops[0] = Chain; + Ops[1] = DAG.getConstant(Order, TLI.getPointerTy()); + Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy()); + return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3); +} + +void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { + DebugLoc dl = getCurDebugLoc(); + AtomicOrdering Order = I.getOrdering(); + SynchronizationScope Scope = I.getSynchScope(); + + SDValue InChain = getRoot(); + + if (TLI.getInsertFencesForAtomic()) + InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, + DAG, TLI); + + SDValue L = + DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, + getValue(I.getCompareOperand()).getValueType().getSimpleVT(), + InChain, + getValue(I.getPointerOperand()), + getValue(I.getCompareOperand()), + getValue(I.getNewValOperand()), + MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */, + TLI.getInsertFencesForAtomic() ? Monotonic : Order, + Scope); + + SDValue OutChain = L.getValue(1); + + if (TLI.getInsertFencesForAtomic()) + OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, + DAG, TLI); + + setValue(&I, L); + DAG.setRoot(OutChain); +} + +void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { + DebugLoc dl = getCurDebugLoc(); + ISD::NodeType NT; + switch (I.getOperation()) { + default: llvm_unreachable("Unknown atomicrmw operation"); return; + case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break; + case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break; + case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break; + case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break; + case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break; + case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break; + case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break; + case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break; + case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break; + case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break; + case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; + } + AtomicOrdering Order = I.getOrdering(); + SynchronizationScope Scope = I.getSynchScope(); + + SDValue InChain = getRoot(); + + if (TLI.getInsertFencesForAtomic()) + InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, + DAG, TLI); + + SDValue L = + DAG.getAtomic(NT, dl, + getValue(I.getValOperand()).getValueType().getSimpleVT(), + InChain, + getValue(I.getPointerOperand()), + getValue(I.getValOperand()), + I.getPointerOperand(), 0 /* Alignment */, + TLI.getInsertFencesForAtomic() ? Monotonic : Order, + Scope); + + SDValue OutChain = L.getValue(1); + + if (TLI.getInsertFencesForAtomic()) + OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, + DAG, TLI); + + setValue(&I, L); + DAG.setRoot(OutChain); +} + +void SelectionDAGBuilder::visitFence(const FenceInst &I) { + DebugLoc dl = getCurDebugLoc(); + SDValue Ops[3]; + Ops[0] = getRoot(); + Ops[1] = DAG.getConstant(I.getOrdering(), TLI.getPointerTy()); + Ops[2] = DAG.getConstant(I.getSynchScope(), TLI.getPointerTy()); + DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3)); +} + +void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { + DebugLoc dl = getCurDebugLoc(); + AtomicOrdering Order = I.getOrdering(); + SynchronizationScope Scope = I.getSynchScope(); + + SDValue InChain = getRoot(); + + EVT VT = EVT::getEVT(I.getType()); + + if (I.getAlignment() * 8 < VT.getSizeInBits()) + report_fatal_error("Cannot generate unaligned atomic load"); + + SDValue L = + DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain, + getValue(I.getPointerOperand()), + I.getPointerOperand(), I.getAlignment(), + TLI.getInsertFencesForAtomic() ? Monotonic : Order, + Scope); + + SDValue OutChain = L.getValue(1); + + if (TLI.getInsertFencesForAtomic()) + OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, + DAG, TLI); + + setValue(&I, L); + DAG.setRoot(OutChain); +} + +void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { + DebugLoc dl = getCurDebugLoc(); + + AtomicOrdering Order = I.getOrdering(); + SynchronizationScope Scope = I.getSynchScope(); + + SDValue InChain = getRoot(); + + EVT VT = EVT::getEVT(I.getValueOperand()->getType()); + + if (I.getAlignment() * 8 < VT.getSizeInBits()) + report_fatal_error("Cannot generate unaligned atomic store"); + + if (TLI.getInsertFencesForAtomic()) + InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl, + DAG, TLI); + + SDValue OutChain = + DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, + InChain, + getValue(I.getPointerOperand()), + getValue(I.getValueOperand()), + I.getPointerOperand(), I.getAlignment(), + TLI.getInsertFencesForAtomic() ? Monotonic : Order, + Scope); + + if (TLI.getInsertFencesForAtomic()) + OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl, + DAG, TLI); + + DAG.setRoot(OutChain); +} + /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC /// node. void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, @@ -3290,7 +3547,7 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, } if (!I.getType()->isVoidTy()) { - if (const VectorType *PTy = dyn_cast<VectorType>(I.getType())) { + if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) { EVT VT = TLI.getValueType(PTy); Result = DAG.getNode(ISD::BITCAST, getCurDebugLoc(), VT, Result); } @@ -3337,25 +3594,6 @@ getF32Constant(SelectionDAG &DAG, unsigned Flt) { return DAG.getConstantFP(APFloat(APInt(32, Flt)), MVT::f32); } -/// Inlined utility function to implement binary input atomic intrinsics for -/// visitIntrinsicCall: I is a call instruction -/// Op is the associated NodeType for I -const char * -SelectionDAGBuilder::implVisitBinaryAtomic(const CallInst& I, - ISD::NodeType Op) { - SDValue Root = getRoot(); - SDValue L = - DAG.getAtomic(Op, getCurDebugLoc(), - getValue(I.getArgOperand(1)).getValueType().getSimpleVT(), - Root, - getValue(I.getArgOperand(0)), - getValue(I.getArgOperand(1)), - I.getArgOperand(0)); - setValue(&I, L); - DAG.setRoot(L.getValue(1)); - return 0; -} - // implVisitAluOverflow - Lower arithmetic overflow instrinsics. const char * SelectionDAGBuilder::implVisitAluOverflow(const CallInst &I, ISD::NodeType Op) { @@ -4154,17 +4392,12 @@ SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable, return false; unsigned Reg = 0; - if (Arg->hasByValAttr()) { - // Byval arguments' frame index is recorded during argument lowering. - // Use this info directly. - Reg = TRI->getFrameRegister(MF); - Offset = FuncInfo.getByValArgumentFrameIndex(Arg); - // If byval argument ofset is not recorded then ignore this. - if (!Offset) - Reg = 0; - } + // Some arguments' frame index is recorded during argument lowering. + Offset = FuncInfo.getArgumentFrameIndex(Arg); + if (Offset) + Reg = TRI->getFrameRegister(MF); - if (N.getNode()) { + if (!Reg && N.getNode()) { if (N.getOpcode() == ISD::CopyFromReg) Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg(); else @@ -4295,7 +4528,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { const DbgDeclareInst &DI = cast<DbgDeclareInst>(I); MDNode *Variable = DI.getVariable(); const Value *Address = DI.getAddress(); - if (!Address || !DIVariable(DI.getVariable()).Verify()) + if (!Address || !DIVariable(Variable).Verify()) return 0; // Build an entry in DbgOrdering. Debug info input nodes get an SDNodeOrder @@ -4385,7 +4618,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { // debug info exists. ++SDNodeOrder; SDDbgValue *SDV; - if (isa<ConstantInt>(V) || isa<ConstantFP>(V)) { + if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) { SDV = DAG.getDbgValue(Variable, V, Offset, dl, SDNodeOrder); DAG.AddDbgValue(SDV, 0, false); } else { @@ -4514,9 +4747,24 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { MMI.setCurrentCallSite(CI->getZExtValue()); return 0; } + case Intrinsic::eh_sjlj_functioncontext: { + // Get and store the index of the function context. + MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); + AllocaInst *FnCtx = + cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts()); + int FI = FuncInfo.StaticAllocaMap[FnCtx]; + MFI->setFunctionContextIndex(FI); + return 0; + } case Intrinsic::eh_sjlj_setjmp: { - setValue(&I, DAG.getNode(ISD::EH_SJLJ_SETJMP, dl, MVT::i32, getRoot(), - getValue(I.getArgOperand(0)))); + SDValue Ops[2]; + Ops[0] = getRoot(); + Ops[1] = getValue(I.getArgOperand(0)); + SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, dl, + DAG.getVTList(MVT::i32, MVT::Other), + Ops, 2); + setValue(&I, Op.getValue(0)); + DAG.setRoot(Op.getValue(1)); return 0; } case Intrinsic::eh_sjlj_longjmp: { @@ -4778,12 +5026,15 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { Ops[4] = DAG.getSrcValue(I.getArgOperand(0)); Ops[5] = DAG.getSrcValue(F); - Res = DAG.getNode(ISD::TRAMPOLINE, dl, - DAG.getVTList(TLI.getPointerTy(), MVT::Other), - Ops, 6); + Res = DAG.getNode(ISD::INIT_TRAMPOLINE, dl, MVT::Other, Ops, 6); - setValue(&I, Res); - DAG.setRoot(Res.getValue(1)); + DAG.setRoot(Res); + return 0; + } + case Intrinsic::adjust_trampoline: { + setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, dl, + TLI.getPointerTy(), + getValue(I.getArgOperand(0)))); return 0; } case Intrinsic::gcroot: @@ -4857,51 +5108,6 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { rw==1)); /* write */ return 0; } - case Intrinsic::memory_barrier: { - SDValue Ops[6]; - Ops[0] = getRoot(); - for (int x = 1; x < 6; ++x) - Ops[x] = getValue(I.getArgOperand(x - 1)); - - DAG.setRoot(DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, &Ops[0], 6)); - return 0; - } - case Intrinsic::atomic_cmp_swap: { - SDValue Root = getRoot(); - SDValue L = - DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, getCurDebugLoc(), - getValue(I.getArgOperand(1)).getValueType().getSimpleVT(), - Root, - getValue(I.getArgOperand(0)), - getValue(I.getArgOperand(1)), - getValue(I.getArgOperand(2)), - MachinePointerInfo(I.getArgOperand(0))); - setValue(&I, L); - DAG.setRoot(L.getValue(1)); - return 0; - } - case Intrinsic::atomic_load_add: - return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_ADD); - case Intrinsic::atomic_load_sub: - return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_SUB); - case Intrinsic::atomic_load_or: - return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_OR); - case Intrinsic::atomic_load_xor: - return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_XOR); - case Intrinsic::atomic_load_and: - return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_AND); - case Intrinsic::atomic_load_nand: - return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_NAND); - case Intrinsic::atomic_load_max: - return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MAX); - case Intrinsic::atomic_load_min: - return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MIN); - case Intrinsic::atomic_load_umin: - return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMIN); - case Intrinsic::atomic_load_umax: - return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMAX); - case Intrinsic::atomic_swap: - return implVisitBinaryAtomic(I, ISD::ATOMIC_SWAP); case Intrinsic::invariant_start: case Intrinsic::lifetime_start: @@ -4918,9 +5124,9 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, bool isTailCall, MachineBasicBlock *LandingPad) { - const PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType()); - const FunctionType *FTy = cast<FunctionType>(PT->getElementType()); - const Type *RetTy = FTy->getReturnType(); + PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType()); + FunctionType *FTy = cast<FunctionType>(PT->getElementType()); + Type *RetTy = FTy->getReturnType(); MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); MCSymbol *BeginLabel = 0; @@ -4949,7 +5155,7 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, FTy->getReturnType()); MachineFunction &MF = DAG.getMachineFunction(); DemoteStackIdx = MF.getFrameInfo()->CreateStackObject(TySize, Align, false); - const Type *StackSlotPtrType = PointerType::getUnqual(FTy->getReturnType()); + Type *StackSlotPtrType = PointerType::getUnqual(FTy->getReturnType()); DemoteStackSlot = DAG.getFrameIndex(DemoteStackIdx, TLI.getPointerTy()); Entry.Node = DemoteStackSlot; @@ -4997,6 +5203,8 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, unsigned CallSiteIndex = MMI.getCurrentCallSite(); if (CallSiteIndex) { MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex); + LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex); + // Now that the call site is handled, stop tracking it. MMI.setCurrentCallSite(0); } @@ -5037,7 +5245,7 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, // The instruction result is the result of loading from the // hidden sret parameter. SmallVector<EVT, 1> PVTs; - const Type *PtrRetTy = PointerType::getUnqual(FTy->getReturnType()); + Type *PtrRetTy = PointerType::getUnqual(FTy->getReturnType()); ComputeValueVTs(TLI, PtrRetTy, PVTs); assert(PVTs.size() == 1 && "Pointers should fit in one register"); @@ -5130,7 +5338,7 @@ static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) { } static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, - const Type *LoadTy, + Type *LoadTy, SelectionDAGBuilder &Builder) { // Check to see if this load can be trivially constant folded, e.g. if the @@ -5193,7 +5401,7 @@ bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) { if (Size && IsOnlyUsedInZeroEqualityComparison(&I)) { bool ActuallyDoIt = true; MVT LoadVT; - const Type *LoadTy; + Type *LoadTy; switch (Size->getZExtValue()) { default: LoadVT = MVT::Other; @@ -5261,14 +5469,14 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { // See if any floating point values are being passed to this function. This is // used to emit an undefined reference to fltused on Windows. - const FunctionType *FT = + FunctionType *FT = cast<FunctionType>(I.getCalledValue()->getType()->getContainedType(0)); MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); if (FT->isVarArg() && !MMI.callsExternalVAFunctionWithFloatingPointArguments()) { for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) { - const Type* T = I.getArgOperand(i)->getType(); - for (po_iterator<const Type*> i = po_begin(T), e = po_end(T); + Type* T = I.getArgOperand(i)->getType(); + for (po_iterator<Type*> i = po_begin(T), e = po_end(T); i != e; ++i) { if (!i->isFloatingPointTy()) continue; MMI.setCallsExternalVAFunctionWithFloatingPointArguments(true); @@ -5412,20 +5620,20 @@ public: if (isa<BasicBlock>(CallOperandVal)) return TLI.getPointerTy(); - const llvm::Type *OpTy = CallOperandVal->getType(); + llvm::Type *OpTy = CallOperandVal->getType(); // FIXME: code duplicated from TargetLowering::ParseConstraints(). // If this is an indirect operand, the operand is a pointer to the // accessed type. if (isIndirect) { - const llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy); + llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy); if (!PtrTy) report_fatal_error("Indirect operand for inline asm not a pointer!"); OpTy = PtrTy->getElementType(); } // Look for vector wrapped in a struct. e.g. { <16 x i8> }. - if (const StructType *STy = dyn_cast<StructType>(OpTy)) + if (StructType *STy = dyn_cast<StructType>(OpTy)) if (STy->getNumElements() == 1) OpTy = STy->getElementType(0); @@ -5637,9 +5845,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { // The return value of the call is this value. As such, there is no // corresponding argument. - assert(!CS.getType()->isVoidTy() && - "Bad inline asm!"); - if (const StructType *STy = dyn_cast<StructType>(CS.getType())) { + assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); + if (StructType *STy = dyn_cast<StructType>(CS.getType())) { OpVT = TLI.getValueType(STy->getElementType(ResNo)); } else { assert(ResNo == 0 && "Asm only has one result!"); @@ -5707,9 +5914,11 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { if (OpInfo.ConstraintVT != Input.ConstraintVT) { std::pair<unsigned, const TargetRegisterClass*> MatchRC = - TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode, OpInfo.ConstraintVT); + TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode, + OpInfo.ConstraintVT); std::pair<unsigned, const TargetRegisterClass*> InputRC = - TLI.getRegForInlineAsmConstraint(Input.ConstraintCode, Input.ConstraintVT); + TLI.getRegForInlineAsmConstraint(Input.ConstraintCode, + Input.ConstraintVT); if ((OpInfo.ConstraintVT.isInteger() != Input.ConstraintVT.isInteger()) || (MatchRC.second != InputRC.second)) { @@ -5750,7 +5959,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { } else { // Otherwise, create a stack slot and emit a store to it before the // asm. - const Type *Ty = OpVal->getType(); + Type *Ty = OpVal->getType(); uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(Ty); unsigned Align = TLI.getTargetData()->getPrefTypeAlignment(Ty); MachineFunction &MF = DAG.getMachineFunction(); @@ -6111,7 +6320,7 @@ void SelectionDAGBuilder::visitVACopy(const CallInst &I) { /// FIXME: When all targets are /// migrated to using LowerCall, this hook should be integrated into SDISel. std::pair<SDValue, SDValue> -TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, +TargetLowering::LowerCallTo(SDValue Chain, Type *RetTy, bool RetSExt, bool RetZExt, bool isVarArg, bool isInreg, unsigned NumFixedArgs, CallingConv::ID CallConv, bool isTailCall, @@ -6128,7 +6337,7 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues; ++Value) { EVT VT = ValueVTs[Value]; - const Type *ArgTy = VT.getTypeForEVT(RetTy->getContext()); + Type *ArgTy = VT.getTypeForEVT(RetTy->getContext()); SDValue Op = SDValue(Args[i].Node.getNode(), Args[i].Node.getResNo() + Value); ISD::ArgFlagsTy Flags; @@ -6145,8 +6354,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, Flags.setSRet(); if (Args[i].isByVal) { Flags.setByVal(); - const PointerType *Ty = cast<PointerType>(Args[i].Ty); - const Type *ElementTy = Ty->getElementType(); + PointerType *Ty = cast<PointerType>(Args[i].Ty); + Type *ElementTy = Ty->getElementType(); Flags.setByValSize(getTargetData()->getTypeAllocSize(ElementTy)); // For ByVal, alignment should come from FE. BE will guess if this // info is not there but there are cases it cannot get right. @@ -6356,7 +6565,7 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues; ++Value) { EVT VT = ValueVTs[Value]; - const Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); + Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); ISD::ArgFlagsTy Flags; unsigned OriginalAlignment = TD->getABITypeAlignment(ArgTy); @@ -6371,8 +6580,8 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { Flags.setSRet(); if (F.paramHasAttr(Idx, Attribute::ByVal)) { Flags.setByVal(); - const PointerType *Ty = cast<PointerType>(I->getType()); - const Type *ElementTy = Ty->getElementType(); + PointerType *Ty = cast<PointerType>(I->getType()); + Type *ElementTy = Ty->getElementType(); Flags.setByValSize(TD->getTypeAllocSize(ElementTy)); // For ByVal, alignment should be passed from FE. BE will guess if // this info is not there but there are cases it cannot get right. @@ -6487,15 +6696,22 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { if (ArgValues.empty()) continue; - // Note down frame index for byval arguments. - if (I->hasByValAttr()) - if (FrameIndexSDNode *FI = - dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode())) - FuncInfo->setByValArgumentFrameIndex(I, FI->getIndex()); + // Note down frame index. + if (FrameIndexSDNode *FI = + dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode())) + FuncInfo->setArgumentFrameIndex(I, FI->getIndex()); SDValue Res = DAG.getMergeValues(&ArgValues[0], NumValues, SDB->getCurDebugLoc()); + SDB->setValue(I, Res); + if (!EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) { + if (LoadSDNode *LNode = + dyn_cast<LoadSDNode>(Res.getOperand(0).getNode())) + if (FrameIndexSDNode *FI = + dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) + FuncInfo->setArgumentFrameIndex(I, FI->getIndex()); + } // If this argument is live outside of the entry block, insert a copy from // wherever we got it to the vreg that other BB's will reference it as. diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index a0884eb..0a21ca3 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -132,10 +132,13 @@ private: Constant* Low; Constant* High; MachineBasicBlock* BB; + uint32_t ExtraWeight; + + Case() : Low(0), High(0), BB(0), ExtraWeight(0) { } + Case(Constant* low, Constant* high, MachineBasicBlock* bb, + uint32_t extraweight) : Low(low), High(high), BB(bb), + ExtraWeight(extraweight) { } - Case() : Low(0), High(0), BB(0) { } - Case(Constant* low, Constant* high, MachineBasicBlock* bb) : - Low(low), High(high), BB(bb) { } APInt size() const { const APInt &rHigh = cast<ConstantInt>(High)->getValue(); const APInt &rLow = cast<ConstantInt>(Low)->getValue(); @@ -203,20 +206,30 @@ private: CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs, const Value *cmpmiddle, MachineBasicBlock *truebb, MachineBasicBlock *falsebb, - MachineBasicBlock *me) + MachineBasicBlock *me, + uint32_t trueweight = 0, uint32_t falseweight = 0) : CC(cc), CmpLHS(cmplhs), CmpMHS(cmpmiddle), CmpRHS(cmprhs), - TrueBB(truebb), FalseBB(falsebb), ThisBB(me) {} + TrueBB(truebb), FalseBB(falsebb), ThisBB(me), + TrueWeight(trueweight), FalseWeight(falseweight) { } + // CC - the condition code to use for the case block's setcc node ISD::CondCode CC; + // CmpLHS/CmpRHS/CmpMHS - The LHS/MHS/RHS of the comparison to emit. // Emit by default LHS op RHS. MHS is used for range comparisons: // If MHS is not null: (LHS <= MHS) and (MHS <= RHS). const Value *CmpLHS, *CmpMHS, *CmpRHS; + // TrueBB/FalseBB - the block to branch to if the setcc is true/false. MachineBasicBlock *TrueBB, *FalseBB; + // ThisBB - the block into which to emit the code for the setcc and branches MachineBasicBlock *ThisBB; + + // TrueWeight/FalseWeight - branch weights. + uint32_t TrueWeight, FalseWeight; }; + struct JumpTable { JumpTable(unsigned R, unsigned J, MachineBasicBlock *M, MachineBasicBlock *D): Reg(R), JTI(J), MBB(M), Default(D) {} @@ -307,6 +320,9 @@ public: /// GFI - Garbage collection metadata for the function. GCFunctionInfo *GFI; + /// LPadToCallSiteMap - Map a landing pad to the call site indexes. + DenseMap<MachineBasicBlock*, SmallVector<unsigned, 4> > LPadToCallSiteMap; + /// HasTailCall - This is set to true if a call in the current /// block has been translated as a tail call. In this case, /// no subsequent DAG nodes should be created. @@ -436,7 +452,8 @@ private: MachineBasicBlock *SwitchBB); uint32_t getEdgeWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst); - void addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst); + void addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst, + uint32_t Weight = 0); public: void visitSwitchCase(CaseBlock &CB, MachineBasicBlock *SwitchBB); @@ -453,6 +470,7 @@ public: private: // These all get lowered before this pass. void visitInvoke(const InvokeInst &I); + void visitResume(const ResumeInst &I); void visitUnwind(const UnwindInst &I); void visitBinary(const User &I, unsigned OpCode); @@ -497,6 +515,7 @@ private: void visitExtractValue(const ExtractValueInst &I); void visitInsertValue(const InsertValueInst &I); + void visitLandingPad(const LandingPadInst &I); void visitGetElementPtr(const User &I); void visitSelect(const User &I); @@ -504,10 +523,15 @@ private: void visitAlloca(const AllocaInst &I); void visitLoad(const LoadInst &I); void visitStore(const StoreInst &I); + void visitAtomicCmpXchg(const AtomicCmpXchgInst &I); + void visitAtomicRMW(const AtomicRMWInst &I); + void visitFence(const FenceInst &I); void visitPHI(const PHINode &I); void visitCall(const CallInst &I); bool visitMemCmpCall(const CallInst &I); - + void visitAtomicLoad(const LoadInst &I); + void visitAtomicStore(const StoreInst &I); + void visitInlineAsm(ImmutableCallSite CS); const char *visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic); @@ -531,7 +555,6 @@ private: llvm_unreachable("UserOp2 should not exist at instruction selection time!"); } - const char *implVisitBinaryAtomic(const CallInst& I, ISD::NodeType Op); const char *implVisitAluOverflow(const CallInst &I, ISD::NodeType Op); void HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 87bb296..68b9146 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -177,6 +177,13 @@ TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, return 0; } +void TargetLowering::AdjustInstrPostInstrSelection(MachineInstr *MI, + SDNode *Node) const { + assert(!MI->getDesc().hasPostISelHook() && + "If a target marks an instruction with 'hasPostISelHook', " + "it must implement TargetLowering::AdjustInstrPostInstrSelection!"); +} + //===----------------------------------------------------------------------===// // SelectionDAGISel code //===----------------------------------------------------------------------===// @@ -463,6 +470,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { GroupName = "Instruction Selection and Scheduling"; std::string BlockName; int BlockNumber = -1; + (void)BlockNumber; #ifdef NDEBUG if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs || ViewDAGCombine2 || ViewDAGCombineLT || ViewISelDAGs || ViewSchedDAGs || @@ -677,21 +685,26 @@ void SelectionDAGISel::DoInstructionSelection() { /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and /// do other setup for EH landing-pad blocks. void SelectionDAGISel::PrepareEHLandingPad() { + MachineBasicBlock *MBB = FuncInfo->MBB; + // Add a label to mark the beginning of the landing pad. Deletion of the // landing pad can thus be detected via the MachineModuleInfo. - MCSymbol *Label = MF->getMMI().addLandingPad(FuncInfo->MBB); + MCSymbol *Label = MF->getMMI().addLandingPad(MBB); + // Assign the call site to the landing pad's begin label. + MF->getMMI().setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]); + const MCInstrDesc &II = TM.getInstrInfo()->get(TargetOpcode::EH_LABEL); - BuildMI(*FuncInfo->MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II) + BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II) .addSym(Label); // Mark exception register as live in. unsigned Reg = TLI.getExceptionAddressRegister(); - if (Reg) FuncInfo->MBB->addLiveIn(Reg); + if (Reg) MBB->addLiveIn(Reg); // Mark exception selector register as live in. Reg = TLI.getExceptionSelectorRegister(); - if (Reg) FuncInfo->MBB->addLiveIn(Reg); + if (Reg) MBB->addLiveIn(Reg); // FIXME: Hack around an exception handling flaw (PR1508): the personality // function and list of typeids logically belong to the invoke (or, if you @@ -704,7 +717,7 @@ void SelectionDAGISel::PrepareEHLandingPad() { // in exceptions not being caught because no typeids are associated with // the invoke. This may not be the only way things can go wrong, but it // is the only way we try to work around for the moment. - const BasicBlock *LLVMBB = FuncInfo->MBB->getBasicBlock(); + const BasicBlock *LLVMBB = MBB->getBasicBlock(); const BranchInst *Br = dyn_cast<BranchInst>(LLVMBB->getTerminator()); if (Br && Br->isUnconditional()) { // Critical edge? @@ -719,8 +732,6 @@ void SelectionDAGISel::PrepareEHLandingPad() { } } - - /// TryToFoldFastISelLoad - We're checking to see if we can fold the specified /// load into the specified FoldInst. Note that we could have a sequence where /// multiple LLVM IR instructions are folded into the same machineinstr. For @@ -741,7 +752,7 @@ bool SelectionDAGISel::TryToFoldFastISelLoad(const LoadInst *LI, // isn't one of the folded instructions, then we can't succeed here. Handle // this by scanning the single-use users of the load until we get to FoldInst. unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs. - + const Instruction *TheUser = LI->use_back(); while (TheUser != FoldInst && // Scan up until we find FoldInst. // Stay in the right block. @@ -750,10 +761,15 @@ bool SelectionDAGISel::TryToFoldFastISelLoad(const LoadInst *LI, // If there are multiple or no uses of this instruction, then bail out. if (!TheUser->hasOneUse()) return false; - + TheUser = TheUser->use_back(); } - + + // If we didn't find the fold instruction, then we failed to collapse the + // sequence. + if (TheUser != FoldInst) + return false; + // Don't try to fold volatile loads. Target has to deal with alignment // constraints. if (LI->isVolatile()) return false; @@ -802,6 +818,7 @@ static bool isFoldedOrDeadInstruction(const Instruction *I, return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded. !isa<TerminatorInst>(I) && // Terminators aren't folded. !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded. + !isa<LandingPadInst>(I) && // Landingpad instructions aren't folded. !FuncInfo->isExportedInst(I); // Exported instrs must be computed. } diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 2626ac3..907d8d9 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -317,7 +317,7 @@ static void InitLibcallNames(const char **Names) { Names[RTLIB::SYNC_FETCH_AND_OR_8] = "__sync_fetch_and_or_8"; Names[RTLIB::SYNC_FETCH_AND_XOR_1] = "__sync_fetch_and_xor_1"; Names[RTLIB::SYNC_FETCH_AND_XOR_2] = "__sync_fetch_and_xor_2"; - Names[RTLIB::SYNC_FETCH_AND_XOR_4] = "__sync_fetch_and-xor_4"; + Names[RTLIB::SYNC_FETCH_AND_XOR_4] = "__sync_fetch_and_xor_4"; Names[RTLIB::SYNC_FETCH_AND_XOR_8] = "__sync_fetch_and_xor_8"; Names[RTLIB::SYNC_FETCH_AND_NAND_1] = "__sync_fetch_and_nand_1"; Names[RTLIB::SYNC_FETCH_AND_NAND_2] = "__sync_fetch_and_nand_2"; @@ -609,6 +609,7 @@ TargetLowering::TargetLowering(const TargetMachine &tm, ExceptionPointerRegister = 0; ExceptionSelectorRegister = 0; BooleanContents = UndefinedBooleanContent; + BooleanVectorContents = UndefinedBooleanContent; SchedPreferenceInfo = Sched::Latency; JumpBufSize = 0; JumpBufAlignment = 0; @@ -617,6 +618,7 @@ TargetLowering::TargetLowering(const TargetMachine &tm, PrefLoopAlignment = 0; MinStackArgumentAlignment = 1; ShouldFoldAtomicFences = false; + InsertFencesForAtomic = false; InitLibcallNames(LibcallRoutineNames); InitCmpLibcallCCs(CmpLibcallCCs); @@ -914,7 +916,8 @@ const char *TargetLowering::getTargetNodeName(unsigned Opcode) const { } -MVT::SimpleValueType TargetLowering::getSetCCResultType(EVT VT) const { +EVT TargetLowering::getSetCCResultType(EVT VT) const { + assert(!VT.isVector() && "No default SetCC type for vectors!"); return PointerTy.SimpleTy; } @@ -996,7 +999,7 @@ unsigned TargetLowering::getVectorTypeBreakdown(LLVMContext &Context, EVT VT, /// type of the given function. This does not require a DAG or a return value, /// and is suitable for use before any DAGs for the function are constructed. /// TODO: Move this out of TargetLowering.cpp. -void llvm::GetReturnInfo(const Type* ReturnType, Attributes attr, +void llvm::GetReturnInfo(Type* ReturnType, Attributes attr, SmallVectorImpl<ISD::OutputArg> &Outs, const TargetLowering &TLI, SmallVectorImpl<uint64_t> *Offsets) { @@ -1054,7 +1057,7 @@ void llvm::GetReturnInfo(const Type* ReturnType, Attributes attr, /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate /// function arguments in the caller parameter area. This is the actual /// alignment, not its logarithm. -unsigned TargetLowering::getByValTypeAlignment(const Type *Ty) const { +unsigned TargetLowering::getByValTypeAlignment(Type *Ty) const { return TD->getCallFrameTypeAlignment(Ty); } @@ -1764,17 +1767,16 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op, break; } case ISD::AssertZext: { - // Demand all the bits of the input that are demanded in the output. - // The low bits are obvious; the high bits are demanded because we're - // asserting that they're zero here. - if (SimplifyDemandedBits(Op.getOperand(0), NewMask, + // AssertZext demands all of the high bits, plus any of the low bits + // demanded by its users. + EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); + APInt InMask = APInt::getLowBitsSet(BitWidth, + VT.getSizeInBits()); + if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | NewMask, KnownZero, KnownOne, TLO, Depth+1)) return true; assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); - EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); - APInt InMask = APInt::getLowBitsSet(BitWidth, - VT.getSizeInBits()); KnownZero |= ~InMask & NewMask; break; } @@ -2191,7 +2193,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, } } else if (N1C->getAPIntValue() == 1 && (VT == MVT::i1 || - getBooleanContents() == ZeroOrOneBooleanContent)) { + getBooleanContents(false) == ZeroOrOneBooleanContent)) { SDValue Op0 = N0; if (Op0.getOpcode() == ISD::TRUNCATE) Op0 = Op0.getOperand(0); @@ -2758,16 +2760,8 @@ getRegForInlineAsmConstraint(const std::string &Constraint, // If none of the value types for this register class are valid, we // can't use it. For example, 64-bit reg classes on 32-bit targets. - bool isLegal = false; - for (TargetRegisterClass::vt_iterator I = RC->vt_begin(), E = RC->vt_end(); - I != E; ++I) { - if (isTypeLegal(*I)) { - isLegal = true; - break; - } - } - - if (!isLegal) continue; + if (!isLegalRC(RC)) + continue; for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end(); I != E; ++I) { @@ -2840,7 +2834,7 @@ TargetLowering::AsmOperandInfoVector TargetLowering::ParseConstraints( // corresponding argument. assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); - if (const StructType *STy = dyn_cast<StructType>(CS.getType())) { + if (StructType *STy = dyn_cast<StructType>(CS.getType())) { OpInfo.ConstraintVT = getValueType(STy->getElementType(ResNo)); } else { assert(ResNo == 0 && "Asm only has one result!"); @@ -2857,16 +2851,16 @@ TargetLowering::AsmOperandInfoVector TargetLowering::ParseConstraints( } if (OpInfo.CallOperandVal) { - const llvm::Type *OpTy = OpInfo.CallOperandVal->getType(); + llvm::Type *OpTy = OpInfo.CallOperandVal->getType(); if (OpInfo.isIndirect) { - const llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy); + llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy); if (!PtrTy) report_fatal_error("Indirect operand for inline asm not a pointer!"); OpTy = PtrTy->getElementType(); } // Look for vector wrapped in a struct. e.g. { <16 x i8> }. - if (const StructType *STy = dyn_cast<StructType>(OpTy)) + if (StructType *STy = dyn_cast<StructType>(OpTy)) if (STy->getNumElements() == 1) OpTy = STy->getElementType(0); @@ -3187,7 +3181,7 @@ void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo, /// isLegalAddressingMode - Return true if the addressing mode represented /// by AM is legal for this target, for a load/store of the specified type. bool TargetLowering::isLegalAddressingMode(const AddrMode &AM, - const Type *Ty) const { + Type *Ty) const { // The default implementation of this implements a conservative RISCy, r+r and // r+i addr mode. diff --git a/lib/CodeGen/ShadowStackGC.cpp b/lib/CodeGen/ShadowStackGC.cpp index 5a253a4..2609256 100644 --- a/lib/CodeGen/ShadowStackGC.cpp +++ b/lib/CodeGen/ShadowStackGC.cpp @@ -61,7 +61,7 @@ namespace { private: bool IsNullValue(Value *V); Constant *GetFrameMap(Function &F); - const Type* GetConcreteStackEntryType(Function &F); + Type* GetConcreteStackEntryType(Function &F); void CollectRoots(Function &F); static GetElementPtrInst *CreateGEP(LLVMContext &Context, IRBuilder<> &B, Value *BasePtr, @@ -109,13 +109,15 @@ namespace { State = 1; case 1: - // Find all 'return' and 'unwind' instructions. + // Find all 'return', 'resume', and 'unwind' instructions. while (StateBB != StateE) { BasicBlock *CurBB = StateBB++; - // Branches and invokes do not escape, only unwind and return do. + // Branches and invokes do not escape, only unwind, resume, and return + // do. TerminatorInst *TI = CurBB->getTerminator(); - if (!isa<UnwindInst>(TI) && !isa<ReturnInst>(TI)) + if (!isa<UnwindInst>(TI) && !isa<ReturnInst>(TI) && + !isa<ResumeInst>(TI)) continue; Builder.SetInsertPoint(TI->getParent(), TI); @@ -139,9 +141,19 @@ namespace { return 0; // Create a cleanup block. - BasicBlock *CleanupBB = BasicBlock::Create(F.getContext(), - CleanupBBName, &F); - UnwindInst *UI = new UnwindInst(F.getContext(), CleanupBB); + LLVMContext &C = F.getContext(); + BasicBlock *CleanupBB = BasicBlock::Create(C, CleanupBBName, &F); + Type *ExnTy = StructType::get(Type::getInt8PtrTy(C), + Type::getInt32Ty(C), NULL); + Constant *PersFn = + F.getParent()-> + getOrInsertFunction("__gcc_personality_v0", + FunctionType::get(Type::getInt32Ty(C), true)); + LandingPadInst *LPad = LandingPadInst::Create(ExnTy, PersFn, 1, + "cleanup.lpad", + CleanupBB); + LPad->setCleanup(true); + ResumeInst *RI = ResumeInst::Create(LPad, CleanupBB); // Transform the 'call' instructions into 'invoke's branching to the // cleanup block. Go in reverse order to make prettier BB names. @@ -172,7 +184,7 @@ namespace { delete CI; } - Builder.SetInsertPoint(UI->getParent(), UI); + Builder.SetInsertPoint(RI->getParent(), RI); return &Builder; } } @@ -190,7 +202,7 @@ ShadowStackGC::ShadowStackGC() : Head(0), StackEntryTy(0) { Constant *ShadowStackGC::GetFrameMap(Function &F) { // doInitialization creates the abstract type of this value. - const Type *VoidPtr = Type::getInt8PtrTy(F.getContext()); + Type *VoidPtr = Type::getInt8PtrTy(F.getContext()); // Truncate the ShadowStackDescriptor if some metadata is null. unsigned NumMeta = 0; @@ -203,7 +215,7 @@ Constant *ShadowStackGC::GetFrameMap(Function &F) { } Metadata.resize(NumMeta); - const Type *Int32Ty = Type::getInt32Ty(F.getContext()); + Type *Int32Ty = Type::getInt32Ty(F.getContext()); Constant *BaseElts[] = { ConstantInt::get(Int32Ty, Roots.size(), false), @@ -216,7 +228,7 @@ Constant *ShadowStackGC::GetFrameMap(Function &F) { }; Type *EltTys[] = { DescriptorElts[0]->getType(),DescriptorElts[1]->getType()}; - StructType *STy = StructType::createNamed("gc_map."+utostr(NumMeta), EltTys); + StructType *STy = StructType::create(EltTys, "gc_map."+utostr(NumMeta)); Constant *FrameMap = ConstantStruct::get(STy, DescriptorElts); @@ -241,17 +253,17 @@ Constant *ShadowStackGC::GetFrameMap(Function &F) { ConstantInt::get(Type::getInt32Ty(F.getContext()), 0), ConstantInt::get(Type::getInt32Ty(F.getContext()), 0) }; - return ConstantExpr::getGetElementPtr(GV, GEPIndices, 2); + return ConstantExpr::getGetElementPtr(GV, GEPIndices); } -const Type* ShadowStackGC::GetConcreteStackEntryType(Function &F) { +Type* ShadowStackGC::GetConcreteStackEntryType(Function &F) { // doInitialization creates the generic version of this type. std::vector<Type*> EltTys; EltTys.push_back(StackEntryTy); for (size_t I = 0; I != Roots.size(); I++) EltTys.push_back(Roots[I].second->getAllocatedType()); - return StructType::createNamed("gc_stackentry."+F.getName().str(), EltTys); + return StructType::create(EltTys, "gc_stackentry."+F.getName().str()); } /// doInitialization - If this module uses the GC intrinsics, find them now. If @@ -267,7 +279,7 @@ bool ShadowStackGC::initializeCustomLowering(Module &M) { EltTys.push_back(Type::getInt32Ty(M.getContext())); // Specifies length of variable length array. EltTys.push_back(Type::getInt32Ty(M.getContext())); - FrameMapTy = StructType::createNamed("gc_map", EltTys); + FrameMapTy = StructType::create(EltTys, "gc_map"); PointerType *FrameMapPtrTy = PointerType::getUnqual(FrameMapTy); // struct StackEntry { @@ -276,13 +288,13 @@ bool ShadowStackGC::initializeCustomLowering(Module &M) { // void *Roots[]; // Stack roots (in-place array, so we pretend). // }; - StackEntryTy = StructType::createNamed(M.getContext(), "gc_stackentry"); + StackEntryTy = StructType::create(M.getContext(), "gc_stackentry"); EltTys.clear(); EltTys.push_back(PointerType::getUnqual(StackEntryTy)); EltTys.push_back(FrameMapPtrTy); StackEntryTy->setBody(EltTys); - const PointerType *StackEntryPtrTy = PointerType::getUnqual(StackEntryTy); + PointerType *StackEntryPtrTy = PointerType::getUnqual(StackEntryTy); // Get the root chain if it already exists. Head = M.getGlobalVariable("llvm_gc_root_chain"); @@ -340,7 +352,7 @@ ShadowStackGC::CreateGEP(LLVMContext &Context, IRBuilder<> &B, Value *BasePtr, Value *Indices[] = { ConstantInt::get(Type::getInt32Ty(Context), 0), ConstantInt::get(Type::getInt32Ty(Context), Idx), ConstantInt::get(Type::getInt32Ty(Context), Idx2) }; - Value* Val = B.CreateGEP(BasePtr, Indices, Indices + 3, Name); + Value* Val = B.CreateGEP(BasePtr, Indices, Name); assert(isa<GetElementPtrInst>(Val) && "Unexpected folded constant"); @@ -352,7 +364,7 @@ ShadowStackGC::CreateGEP(LLVMContext &Context, IRBuilder<> &B, Value *BasePtr, int Idx, const char *Name) { Value *Indices[] = { ConstantInt::get(Type::getInt32Ty(Context), 0), ConstantInt::get(Type::getInt32Ty(Context), Idx) }; - Value *Val = B.CreateGEP(BasePtr, Indices, Indices + 2, Name); + Value *Val = B.CreateGEP(BasePtr, Indices, Name); assert(isa<GetElementPtrInst>(Val) && "Unexpected folded constant"); @@ -373,7 +385,7 @@ bool ShadowStackGC::performCustomLowering(Function &F) { // Build the constant map and figure the type of the shadow stack entry. Value *FrameMap = GetFrameMap(F); - const Type *ConcreteStackEntryTy = GetConcreteStackEntryType(F); + Type *ConcreteStackEntryTy = GetConcreteStackEntryType(F); // Build the shadow stack entry at the very start of the function. BasicBlock::iterator IP = F.getEntryBlock().begin(); diff --git a/lib/CodeGen/SjLjEHPrepare.cpp b/lib/CodeGen/SjLjEHPrepare.cpp index 65a33da..ded2459d 100644 --- a/lib/CodeGen/SjLjEHPrepare.cpp +++ b/lib/CodeGen/SjLjEHPrepare.cpp @@ -21,26 +21,31 @@ #include "llvm/LLVMContext.h" #include "llvm/Module.h" #include "llvm/Pass.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/Support/Debug.h" +#include "llvm/Target/TargetData.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Local.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/IRBuilder.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" #include <set> using namespace llvm; +static cl::opt<bool> DisableOldSjLjEH("disable-old-sjlj-eh", cl::Hidden, + cl::desc("Disable the old SjLj EH preparation pass")); + STATISTIC(NumInvokes, "Number of invokes replaced"); STATISTIC(NumUnwinds, "Number of unwinds replaced"); STATISTIC(NumSpilled, "Number of registers live across unwind edges"); namespace { class SjLjEHPass : public FunctionPass { - const TargetLowering *TLI; - - const Type *FunctionContextTy; + Type *FunctionContextTy; Constant *RegisterFn; Constant *UnregisterFn; Constant *BuiltinSetjmpFn; @@ -53,8 +58,9 @@ namespace { Constant *ExceptionFn; Constant *CallSiteFn; Constant *DispatchSetupFn; - + Constant *FuncCtxFn; Value *CallSite; + DenseMap<InvokeInst*, BasicBlock*> LPadSuccMap; public: static char ID; // Pass identification, replacement for typeid explicit SjLjEHPass(const TargetLowering *tli = NULL) @@ -62,16 +68,22 @@ namespace { bool doInitialization(Module &M); bool runOnFunction(Function &F); - virtual void getAnalysisUsage(AnalysisUsage &AU) const { } + virtual void getAnalysisUsage(AnalysisUsage &AU) const {} const char *getPassName() const { return "SJLJ Exception Handling preparation"; } private: + bool setupEntryBlockAndCallSites(Function &F); + Value *setupFunctionContext(Function &F, ArrayRef<LandingPadInst*> LPads); + void lowerIncomingArguments(Function &F); + void lowerAcrossUnwindEdges(Function &F, ArrayRef<InvokeInst*> Invokes); + void insertCallSiteStore(Instruction *I, int Number, Value *CallSite); void markInvokeCallSite(InvokeInst *II, int InvokeNo, Value *CallSite, SwitchInst *CatchSwitch); void splitLiveRangesAcrossInvokes(SmallVector<InvokeInst*,16> &Invokes); + void splitLandingPad(InvokeInst *II); bool insertSjLjEHSupport(Function &F); }; } // end anonymous namespace @@ -116,6 +128,7 @@ bool SjLjEHPass::doInitialization(Module &M) { CallSiteFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_callsite); DispatchSetupFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_dispatch_setup); + FuncCtxFn = Intrinsic::getDeclaration(&M, Intrinsic::eh_sjlj_functioncontext); PersonalityFn = 0; return true; @@ -131,6 +144,42 @@ void SjLjEHPass::insertCallSiteStore(Instruction *I, int Number, new StoreInst(CallSiteNoC, CallSite, true, I); // volatile } +/// splitLandingPad - Split a landing pad. This takes considerable care because +/// of PHIs and other nasties. The problem is that the jump table needs to jump +/// to the landing pad block. However, the landing pad block can be jumped to +/// only by an invoke instruction. So we clone the landingpad instruction into +/// its own basic block, have the invoke jump to there. The landingpad +/// instruction's basic block's successor is now the target for the jump table. +/// +/// But because of PHI nodes, we need to create another basic block for the jump +/// table to jump to. This is definitely a hack, because the values for the PHI +/// nodes may not be defined on the edge from the jump table. But that's okay, +/// because the jump table is simply a construct to mimic what is happening in +/// the CFG. So the values are mysteriously there, even though there is no value +/// for the PHI from the jump table's edge (hence calling this a hack). +void SjLjEHPass::splitLandingPad(InvokeInst *II) { + SmallVector<BasicBlock*, 2> NewBBs; + SplitLandingPadPredecessors(II->getUnwindDest(), II->getParent(), + ".1", ".2", this, NewBBs); + + // Create an empty block so that the jump table has something to jump to + // which doesn't have any PHI nodes. + BasicBlock *LPad = NewBBs[0]; + BasicBlock *Succ = *succ_begin(LPad); + BasicBlock *JumpTo = BasicBlock::Create(II->getContext(), "jt.land", + LPad->getParent(), Succ); + LPad->getTerminator()->eraseFromParent(); + BranchInst::Create(JumpTo, LPad); + BranchInst::Create(Succ, JumpTo); + LPadSuccMap[II] = JumpTo; + + for (BasicBlock::iterator I = Succ->begin(); isa<PHINode>(I); ++I) { + PHINode *PN = cast<PHINode>(I); + Value *Val = PN->removeIncomingValue(LPad, false); + PN->addIncoming(Val, JumpTo); + } +} + /// markInvokeCallSite - Insert code to mark the call_site for this invoke void SjLjEHPass::markInvokeCallSite(InvokeInst *II, int InvokeNo, Value *CallSite, @@ -140,11 +189,15 @@ void SjLjEHPass::markInvokeCallSite(InvokeInst *II, int InvokeNo, // The runtime comes back to the dispatcher with the call_site - 1 in // the context. Odd, but there it is. ConstantInt *SwitchValC = ConstantInt::get(Type::getInt32Ty(II->getContext()), - InvokeNo - 1); + InvokeNo - 1); // If the unwind edge has phi nodes, split the edge. if (isa<PHINode>(II->getUnwindDest()->begin())) { - SplitCriticalEdge(II, 1, this); + // FIXME: New EH - This if-condition will be always true in the new scheme. + if (II->getUnwindDest()->isLandingPad()) + splitLandingPad(II); + else + SplitCriticalEdge(II, 1, this); // If there are any phi nodes left, they must have a single predecessor. while (PHINode *PN = dyn_cast<PHINode>(II->getUnwindDest()->begin())) { @@ -161,7 +214,12 @@ void SjLjEHPass::markInvokeCallSite(InvokeInst *II, int InvokeNo, CallInst::Create(CallSiteFn, CallSiteNoC, "", II); // Add a switch case to our unwind block. - CatchSwitch->addCase(SwitchValC, II->getUnwindDest()); + if (BasicBlock *SuccBB = LPadSuccMap[II]) { + CatchSwitch->addCase(SwitchValC, SuccBB); + } else { + CatchSwitch->addCase(SwitchValC, II->getUnwindDest()); + } + // We still want this to look like an invoke so we emit the LSDA properly, // so we don't transform the invoke into a call here. } @@ -187,10 +245,16 @@ splitLiveRangesAcrossInvokes(SmallVector<InvokeInst*,16> &Invokes) { for (unsigned i = 0, e = Invokes.size(); i != e; ++i) { InvokeInst *II = Invokes[i]; SplitCriticalEdge(II, 0, this); - SplitCriticalEdge(II, 1, this); + + // FIXME: New EH - This if-condition will be always true in the new scheme. + if (II->getUnwindDest()->isLandingPad()) + splitLandingPad(II); + else + SplitCriticalEdge(II, 1, this); + assert(!isa<PHINode>(II->getNormalDest()) && !isa<PHINode>(II->getUnwindDest()) && - "critical edge splitting left single entry phi nodes?"); + "Critical edge splitting left single entry phi nodes?"); } Function *F = Invokes.back()->getParent()->getParent(); @@ -204,7 +268,7 @@ splitLiveRangesAcrossInvokes(SmallVector<InvokeInst*,16> &Invokes) { ++AfterAllocaInsertPt; for (Function::arg_iterator AI = F->arg_begin(), E = F->arg_end(); AI != E; ++AI) { - const Type *Ty = AI->getType(); + Type *Ty = AI->getType(); // Aggregate types can't be cast, but are legal argument types, so we have // to handle them differently. We use an extract/insert pair as a // lightweight method to achieve the same goal. @@ -283,9 +347,8 @@ splitLiveRangesAcrossInvokes(SmallVector<InvokeInst*,16> &Invokes) { bool NeedsSpill = false; for (unsigned i = 0, e = Invokes.size(); i != e; ++i) { BasicBlock *UnwindBlock = Invokes[i]->getUnwindDest(); - if (UnwindBlock != BB && LiveBBs.count(UnwindBlock)) { + if (UnwindBlock != BB && LiveBBs.count(UnwindBlock)) NeedsSpill = true; - } } // If we decided we need a spill, do it. @@ -299,6 +362,44 @@ splitLiveRangesAcrossInvokes(SmallVector<InvokeInst*,16> &Invokes) { } } +/// CreateLandingPadLoad - Load the exception handling values and insert them +/// into a structure. +static Instruction *CreateLandingPadLoad(Function &F, Value *ExnAddr, + Value *SelAddr, + BasicBlock::iterator InsertPt) { + Value *Exn = new LoadInst(ExnAddr, "exn", false, + InsertPt); + Type *Ty = Type::getInt8PtrTy(F.getContext()); + Exn = CastInst::Create(Instruction::IntToPtr, Exn, Ty, "", InsertPt); + Value *Sel = new LoadInst(SelAddr, "sel", false, InsertPt); + + Ty = StructType::get(Exn->getType(), Sel->getType(), NULL); + InsertValueInst *LPadVal = InsertValueInst::Create(llvm::UndefValue::get(Ty), + Exn, 0, + "lpad.val", InsertPt); + return InsertValueInst::Create(LPadVal, Sel, 1, "lpad.val", InsertPt); +} + +/// ReplaceLandingPadVal - Replace the landingpad instruction's value with a +/// load from the stored values (via CreateLandingPadLoad). This looks through +/// PHI nodes, and removes them if they are dead. +static void ReplaceLandingPadVal(Function &F, Instruction *Inst, Value *ExnAddr, + Value *SelAddr) { + if (Inst->use_empty()) return; + + while (!Inst->use_empty()) { + Instruction *I = cast<Instruction>(Inst->use_back()); + + if (PHINode *PN = dyn_cast<PHINode>(I)) { + ReplaceLandingPadVal(F, PN, ExnAddr, SelAddr); + if (PN->use_empty()) PN->eraseFromParent(); + continue; + } + + I->replaceUsesOfWith(Inst, CreateLandingPadLoad(F, ExnAddr, SelAddr, I)); + } +} + bool SjLjEHPass::insertSjLjEHSupport(Function &F) { SmallVector<ReturnInst*,16> Returns; SmallVector<UnwindInst*,16> Unwinds; @@ -337,10 +438,23 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { SmallVector<CallInst*,16> EH_Exceptions; SmallVector<Instruction*,16> JmpbufUpdatePoints; - // Note: Skip the entry block since there's nothing there that interests - // us. eh.selector and eh.exception shouldn't ever be there, and we - // want to disregard any allocas that are there. - for (Function::iterator BB = F.begin(), E = F.end(); ++BB != E;) { + for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) { + // Note: Skip the entry block since there's nothing there that interests + // us. eh.selector and eh.exception shouldn't ever be there, and we + // want to disregard any allocas that are there. + // + // FIXME: This is awkward. The new EH scheme won't need to skip the entry + // block. + if (BB == F.begin()) { + if (InvokeInst *II = dyn_cast<InvokeInst>(F.begin()->getTerminator())) { + // FIXME: This will be always non-NULL in the new EH. + if (LandingPadInst *LPI = II->getUnwindDest()->getLandingPadInst()) + if (!PersonalityFn) PersonalityFn = LPI->getPersonalityFn(); + } + + continue; + } + for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) { if (CallInst *CI = dyn_cast<CallInst>(I)) { if (CI->getCalledFunction() == SelectorFn) { @@ -353,6 +467,10 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { } } else if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) { JmpbufUpdatePoints.push_back(AI); + } else if (InvokeInst *II = dyn_cast<InvokeInst>(I)) { + // FIXME: This will be always non-NULL in the new EH. + if (LandingPadInst *LPI = II->getUnwindDest()->getLandingPadInst()) + if (!PersonalityFn) PersonalityFn = LPI->getPersonalityFn(); } } } @@ -371,6 +489,16 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { // invoke's. splitLiveRangesAcrossInvokes(Invokes); + + SmallVector<LandingPadInst*, 16> LandingPads; + for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) { + if (InvokeInst *II = dyn_cast<InvokeInst>(BB->getTerminator())) + // FIXME: This will be always non-NULL in the new EH. + if (LandingPadInst *LPI = II->getUnwindDest()->getLandingPadInst()) + LandingPads.push_back(LPI); + } + + BasicBlock *EntryBB = F.begin(); // Create an alloca for the incoming jump buffer ptr and the new jump buffer // that needs to be restored on all exits from the function. This is an @@ -381,27 +509,25 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { "fcn_context", F.begin()->begin()); Value *Idxs[2]; - const Type *Int32Ty = Type::getInt32Ty(F.getContext()); + Type *Int32Ty = Type::getInt32Ty(F.getContext()); Value *Zero = ConstantInt::get(Int32Ty, 0); // We need to also keep around a reference to the call_site field Idxs[0] = Zero; Idxs[1] = ConstantInt::get(Int32Ty, 1); - CallSite = GetElementPtrInst::Create(FunctionContext, Idxs, Idxs+2, - "call_site", + CallSite = GetElementPtrInst::Create(FunctionContext, Idxs, "call_site", EntryBB->getTerminator()); // The exception selector comes back in context->data[1] Idxs[1] = ConstantInt::get(Int32Ty, 2); - Value *FCData = GetElementPtrInst::Create(FunctionContext, Idxs, Idxs+2, - "fc_data", + Value *FCData = GetElementPtrInst::Create(FunctionContext, Idxs, "fc_data", EntryBB->getTerminator()); Idxs[1] = ConstantInt::get(Int32Ty, 1); - Value *SelectorAddr = GetElementPtrInst::Create(FCData, Idxs, Idxs+2, + Value *SelectorAddr = GetElementPtrInst::Create(FCData, Idxs, "exc_selector_gep", EntryBB->getTerminator()); // The exception value comes back in context->data[0] Idxs[1] = Zero; - Value *ExceptionAddr = GetElementPtrInst::Create(FCData, Idxs, Idxs+2, + Value *ExceptionAddr = GetElementPtrInst::Create(FCData, Idxs, "exception_gep", EntryBB->getTerminator()); @@ -423,13 +549,16 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { // instruction hasn't already been removed. if (!I->getParent()) continue; Value *Val = new LoadInst(ExceptionAddr, "exception", true, I); - const Type *Ty = Type::getInt8PtrTy(F.getContext()); + Type *Ty = Type::getInt8PtrTy(F.getContext()); Val = CastInst::Create(Instruction::IntToPtr, Val, Ty, "", I); I->replaceAllUsesWith(Val); I->eraseFromParent(); } + for (unsigned i = 0, e = LandingPads.size(); i != e; ++i) + ReplaceLandingPadVal(F, LandingPads[i], ExceptionAddr, SelectorAddr); + // The entry block changes to have the eh.sjlj.setjmp, with a conditional // branch to a dispatch block for non-zero returns. If we return normally, // we're not handling an exception and just register the function context and @@ -466,8 +595,7 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { Idxs[0] = Zero; Idxs[1] = ConstantInt::get(Int32Ty, 4); Value *LSDAFieldPtr = - GetElementPtrInst::Create(FunctionContext, Idxs, Idxs+2, - "lsda_gep", + GetElementPtrInst::Create(FunctionContext, Idxs, "lsda_gep", EntryBB->getTerminator()); Value *LSDA = CallInst::Create(LSDAAddrFn, "lsda_addr", EntryBB->getTerminator()); @@ -475,8 +603,7 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { Idxs[1] = ConstantInt::get(Int32Ty, 3); Value *PersonalityFieldPtr = - GetElementPtrInst::Create(FunctionContext, Idxs, Idxs+2, - "lsda_gep", + GetElementPtrInst::Create(FunctionContext, Idxs, "lsda_gep", EntryBB->getTerminator()); new StoreInst(PersonalityFn, PersonalityFieldPtr, true, EntryBB->getTerminator()); @@ -484,12 +611,11 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { // Save the frame pointer. Idxs[1] = ConstantInt::get(Int32Ty, 5); Value *JBufPtr - = GetElementPtrInst::Create(FunctionContext, Idxs, Idxs+2, - "jbuf_gep", + = GetElementPtrInst::Create(FunctionContext, Idxs, "jbuf_gep", EntryBB->getTerminator()); Idxs[1] = ConstantInt::get(Int32Ty, 0); Value *FramePtr = - GetElementPtrInst::Create(JBufPtr, Idxs, Idxs+2, "jbuf_fp_gep", + GetElementPtrInst::Create(JBufPtr, Idxs, "jbuf_fp_gep", EntryBB->getTerminator()); Value *Val = CallInst::Create(FrameAddrFn, @@ -501,7 +627,7 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { // Save the stack pointer. Idxs[1] = ConstantInt::get(Int32Ty, 2); Value *StackPtr = - GetElementPtrInst::Create(JBufPtr, Idxs, Idxs+2, "jbuf_sp_gep", + GetElementPtrInst::Create(JBufPtr, Idxs, "jbuf_sp_gep", EntryBB->getTerminator()); Val = CallInst::Create(StackAddrFn, "sp", EntryBB->getTerminator()); @@ -513,7 +639,7 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { Type::getInt8PtrTy(F.getContext()), "", EntryBB->getTerminator()); Value *DispatchVal = CallInst::Create(BuiltinSetjmpFn, SetjmpArg, - "dispatch", + "", EntryBB->getTerminator()); // Add a call to dispatch_setup after the setjmp call. This is expanded to any @@ -554,6 +680,8 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { if (Callee != SelectorFn && Callee != ExceptionFn && !CI->doesNotThrow()) insertCallSiteStore(CI, -1, CallSite); + } else if (ResumeInst *RI = dyn_cast<ResumeInst>(I)) { + insertCallSiteStore(RI, -1, CallSite); } } @@ -582,7 +710,317 @@ bool SjLjEHPass::insertSjLjEHSupport(Function &F) { return true; } +/// setupFunctionContext - Allocate the function context on the stack and fill +/// it with all of the data that we know at this point. +Value *SjLjEHPass:: +setupFunctionContext(Function &F, ArrayRef<LandingPadInst*> LPads) { + BasicBlock *EntryBB = F.begin(); + + // Create an alloca for the incoming jump buffer ptr and the new jump buffer + // that needs to be restored on all exits from the function. This is an alloca + // because the value needs to be added to the global context list. + unsigned Align = + TLI->getTargetData()->getPrefTypeAlignment(FunctionContextTy); + AllocaInst *FuncCtx = + new AllocaInst(FunctionContextTy, 0, Align, "fn_context", EntryBB->begin()); + + // Fill in the function context structure. + Value *Idxs[2]; + Type *Int32Ty = Type::getInt32Ty(F.getContext()); + Value *Zero = ConstantInt::get(Int32Ty, 0); + Value *One = ConstantInt::get(Int32Ty, 1); + + // Keep around a reference to the call_site field. + Idxs[0] = Zero; + Idxs[1] = One; + CallSite = GetElementPtrInst::Create(FuncCtx, Idxs, "call_site", + EntryBB->getTerminator()); + + // Reference the __data field. + Idxs[1] = ConstantInt::get(Int32Ty, 2); + Value *FCData = GetElementPtrInst::Create(FuncCtx, Idxs, "__data", + EntryBB->getTerminator()); + + // The exception value comes back in context->__data[0]. + Idxs[1] = Zero; + Value *ExceptionAddr = GetElementPtrInst::Create(FCData, Idxs, + "exception_gep", + EntryBB->getTerminator()); + + // The exception selector comes back in context->__data[1]. + Idxs[1] = One; + Value *SelectorAddr = GetElementPtrInst::Create(FCData, Idxs, + "exn_selector_gep", + EntryBB->getTerminator()); + + for (unsigned I = 0, E = LPads.size(); I != E; ++I) { + LandingPadInst *LPI = LPads[I]; + IRBuilder<> Builder(LPI->getParent()->getFirstInsertionPt()); + + Value *ExnVal = Builder.CreateLoad(ExceptionAddr, true, "exn_val"); + ExnVal = Builder.CreateIntToPtr(ExnVal, Type::getInt8PtrTy(F.getContext())); + Value *SelVal = Builder.CreateLoad(SelectorAddr, true, "exn_selector_val"); + + Type *LPadType = LPI->getType(); + Value *LPadVal = UndefValue::get(LPadType); + LPadVal = Builder.CreateInsertValue(LPadVal, ExnVal, 0, "lpad.val"); + LPadVal = Builder.CreateInsertValue(LPadVal, SelVal, 1, "lpad.val"); + + LPI->replaceAllUsesWith(LPadVal); + } + + // Personality function + Idxs[1] = ConstantInt::get(Int32Ty, 3); + if (!PersonalityFn) + PersonalityFn = LPads[0]->getPersonalityFn(); + Value *PersonalityFieldPtr = + GetElementPtrInst::Create(FuncCtx, Idxs, "pers_fn_gep", + EntryBB->getTerminator()); + new StoreInst(PersonalityFn, PersonalityFieldPtr, true, + EntryBB->getTerminator()); + + // LSDA address + Idxs[1] = ConstantInt::get(Int32Ty, 4); + Value *LSDAFieldPtr = GetElementPtrInst::Create(FuncCtx, Idxs, "lsda_gep", + EntryBB->getTerminator()); + Value *LSDA = CallInst::Create(LSDAAddrFn, "lsda_addr", + EntryBB->getTerminator()); + new StoreInst(LSDA, LSDAFieldPtr, true, EntryBB->getTerminator()); + + return FuncCtx; +} + +/// lowerIncomingArguments - To avoid having to handle incoming arguments +/// specially, we lower each arg to a copy instruction in the entry block. This +/// ensures that the argument value itself cannot be live out of the entry +/// block. +void SjLjEHPass::lowerIncomingArguments(Function &F) { + BasicBlock::iterator AfterAllocaInsPt = F.begin()->begin(); + while (isa<AllocaInst>(AfterAllocaInsPt) && + isa<ConstantInt>(cast<AllocaInst>(AfterAllocaInsPt)->getArraySize())) + ++AfterAllocaInsPt; + + for (Function::arg_iterator + AI = F.arg_begin(), AE = F.arg_end(); AI != AE; ++AI) { + Type *Ty = AI->getType(); + + // Aggregate types can't be cast, but are legal argument types, so we have + // to handle them differently. We use an extract/insert pair as a + // lightweight method to achieve the same goal. + if (isa<StructType>(Ty) || isa<ArrayType>(Ty) || isa<VectorType>(Ty)) { + Instruction *EI = ExtractValueInst::Create(AI, 0, "", AfterAllocaInsPt); + Instruction *NI = InsertValueInst::Create(AI, EI, 0); + NI->insertAfter(EI); + AI->replaceAllUsesWith(NI); + + // Set the operand of the instructions back to the AllocaInst. + EI->setOperand(0, AI); + NI->setOperand(0, AI); + } else { + // This is always a no-op cast because we're casting AI to AI->getType() + // so src and destination types are identical. BitCast is the only + // possibility. + CastInst *NC = + new BitCastInst(AI, AI->getType(), AI->getName() + ".tmp", + AfterAllocaInsPt); + AI->replaceAllUsesWith(NC); + + // Set the operand of the cast instruction back to the AllocaInst. + // Normally it's forbidden to replace a CastInst's operand because it + // could cause the opcode to reflect an illegal conversion. However, we're + // replacing it here with the same value it was constructed with. We do + // this because the above replaceAllUsesWith() clobbered the operand, but + // we want this one to remain. + NC->setOperand(0, AI); + } + } +} + +/// lowerAcrossUnwindEdges - Find all variables which are alive across an unwind +/// edge and spill them. +void SjLjEHPass::lowerAcrossUnwindEdges(Function &F, + ArrayRef<InvokeInst*> Invokes) { + // Finally, scan the code looking for instructions with bad live ranges. + for (Function::iterator + BB = F.begin(), BBE = F.end(); BB != BBE; ++BB) { + for (BasicBlock::iterator + II = BB->begin(), IIE = BB->end(); II != IIE; ++II) { + // Ignore obvious cases we don't have to handle. In particular, most + // instructions either have no uses or only have a single use inside the + // current block. Ignore them quickly. + Instruction *Inst = II; + if (Inst->use_empty()) continue; + if (Inst->hasOneUse() && + cast<Instruction>(Inst->use_back())->getParent() == BB && + !isa<PHINode>(Inst->use_back())) continue; + + // If this is an alloca in the entry block, it's not a real register + // value. + if (AllocaInst *AI = dyn_cast<AllocaInst>(Inst)) + if (isa<ConstantInt>(AI->getArraySize()) && BB == F.begin()) + continue; + + // Avoid iterator invalidation by copying users to a temporary vector. + SmallVector<Instruction*, 16> Users; + for (Value::use_iterator + UI = Inst->use_begin(), E = Inst->use_end(); UI != E; ++UI) { + Instruction *User = cast<Instruction>(*UI); + if (User->getParent() != BB || isa<PHINode>(User)) + Users.push_back(User); + } + + // Find all of the blocks that this value is live in. + std::set<BasicBlock*> LiveBBs; + LiveBBs.insert(Inst->getParent()); + while (!Users.empty()) { + Instruction *U = Users.back(); + Users.pop_back(); + + if (!isa<PHINode>(U)) { + MarkBlocksLiveIn(U->getParent(), LiveBBs); + } else { + // Uses for a PHI node occur in their predecessor block. + PHINode *PN = cast<PHINode>(U); + for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) + if (PN->getIncomingValue(i) == Inst) + MarkBlocksLiveIn(PN->getIncomingBlock(i), LiveBBs); + } + } + + // Now that we know all of the blocks that this thing is live in, see if + // it includes any of the unwind locations. + bool NeedsSpill = false; + for (unsigned i = 0, e = Invokes.size(); i != e; ++i) { + BasicBlock *UnwindBlock = Invokes[i]->getUnwindDest(); + if (UnwindBlock != BB && LiveBBs.count(UnwindBlock)) { + NeedsSpill = true; + } + } + + // If we decided we need a spill, do it. + // FIXME: Spilling this way is overkill, as it forces all uses of + // the value to be reloaded from the stack slot, even those that aren't + // in the unwind blocks. We should be more selective. + if (NeedsSpill) { + ++NumSpilled; + DemoteRegToStack(*Inst, true); + } + } + } +} + +/// setupEntryBlockAndCallSites - Setup the entry block by creating and filling +/// the function context and marking the call sites with the appropriate +/// values. These values are used by the DWARF EH emitter. +bool SjLjEHPass::setupEntryBlockAndCallSites(Function &F) { + SmallVector<ReturnInst*, 16> Returns; + SmallVector<InvokeInst*, 16> Invokes; + SmallVector<LandingPadInst*, 16> LPads; + + // Look through the terminators of the basic blocks to find invokes. + for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) + if (InvokeInst *II = dyn_cast<InvokeInst>(BB->getTerminator())) { + Invokes.push_back(II); + LPads.push_back(II->getUnwindDest()->getLandingPadInst()); + } else if (ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator())) { + Returns.push_back(RI); + } + + if (Invokes.empty()) return false; + + lowerIncomingArguments(F); + lowerAcrossUnwindEdges(F, Invokes); + + Value *FuncCtx = setupFunctionContext(F, LPads); + BasicBlock *EntryBB = F.begin(); + Type *Int32Ty = Type::getInt32Ty(F.getContext()); + + Value *Idxs[2] = { + ConstantInt::get(Int32Ty, 0), 0 + }; + + // Get a reference to the jump buffer. + Idxs[1] = ConstantInt::get(Int32Ty, 5); + Value *JBufPtr = GetElementPtrInst::Create(FuncCtx, Idxs, "jbuf_gep", + EntryBB->getTerminator()); + + // Save the frame pointer. + Idxs[1] = ConstantInt::get(Int32Ty, 0); + Value *FramePtr = GetElementPtrInst::Create(JBufPtr, Idxs, "jbuf_fp_gep", + EntryBB->getTerminator()); + + Value *Val = CallInst::Create(FrameAddrFn, + ConstantInt::get(Int32Ty, 0), + "fp", + EntryBB->getTerminator()); + new StoreInst(Val, FramePtr, true, EntryBB->getTerminator()); + + // Save the stack pointer. + Idxs[1] = ConstantInt::get(Int32Ty, 2); + Value *StackPtr = GetElementPtrInst::Create(JBufPtr, Idxs, "jbuf_sp_gep", + EntryBB->getTerminator()); + + Val = CallInst::Create(StackAddrFn, "sp", EntryBB->getTerminator()); + new StoreInst(Val, StackPtr, true, EntryBB->getTerminator()); + + // Call the setjmp instrinsic. It fills in the rest of the jmpbuf. + Value *SetjmpArg = CastInst::Create(Instruction::BitCast, JBufPtr, + Type::getInt8PtrTy(F.getContext()), "", + EntryBB->getTerminator()); + CallInst::Create(BuiltinSetjmpFn, SetjmpArg, "", EntryBB->getTerminator()); + + // Store a pointer to the function context so that the back-end will know + // where to look for it. + Value *FuncCtxArg = CastInst::Create(Instruction::BitCast, FuncCtx, + Type::getInt8PtrTy(F.getContext()), "", + EntryBB->getTerminator()); + CallInst::Create(FuncCtxFn, FuncCtxArg, "", EntryBB->getTerminator()); + + // At this point, we are all set up, update the invoke instructions to mark + // their call_site values. + for (unsigned I = 0, E = Invokes.size(); I != E; ++I) { + insertCallSiteStore(Invokes[I], I + 1, CallSite); + + ConstantInt *CallSiteNum = + ConstantInt::get(Type::getInt32Ty(F.getContext()), I + 1); + + // Record the call site value for the back end so it stays associated with + // the invoke. + CallInst::Create(CallSiteFn, CallSiteNum, "", Invokes[I]); + } + + // Mark call instructions that aren't nounwind as no-action (call_site == + // -1). Skip the entry block, as prior to then, no function context has been + // created for this function and any unexpected exceptions thrown will go + // directly to the caller's context, which is what we want anyway, so no need + // to do anything here. + for (Function::iterator BB = F.begin(), E = F.end(); ++BB != E;) + for (BasicBlock::iterator I = BB->begin(), end = BB->end(); I != end; ++I) + if (CallInst *CI = dyn_cast<CallInst>(I)) { + if (!CI->doesNotThrow()) + insertCallSiteStore(CI, -1, CallSite); + } else if (ResumeInst *RI = dyn_cast<ResumeInst>(I)) { + insertCallSiteStore(RI, -1, CallSite); + } + + // Register the function context and make sure it's known to not throw + CallInst *Register = CallInst::Create(RegisterFn, FuncCtx, "", + EntryBB->getTerminator()); + Register->setDoesNotThrow(); + + // Finally, for any returns from this function, if this function contains an + // invoke, add a call to unregister the function context. + for (unsigned I = 0, E = Returns.size(); I != E; ++I) + CallInst::Create(UnregisterFn, FuncCtx, "", Returns[I]); + + return true; +} + bool SjLjEHPass::runOnFunction(Function &F) { - bool Res = insertSjLjEHSupport(F); + bool Res = false; + if (!DisableOldSjLjEH) + Res = insertSjLjEHSupport(F); + else + Res = setupEntryBlockAndCallSites(F); return Res; } diff --git a/lib/CodeGen/SpillPlacement.cpp b/lib/CodeGen/SpillPlacement.cpp index 6949618..6f33f54 100644 --- a/lib/CodeGen/SpillPlacement.cpp +++ b/lib/CodeGen/SpillPlacement.cpp @@ -220,6 +220,7 @@ void SpillPlacement::addConstraints(ArrayRef<BlockConstraint> LiveBlocks) { 0, // DontCare, 1, // PrefReg, -1, // PrefSpill + 0, // PrefBoth -HUGE_VALF // MustSpill }; @@ -239,6 +240,22 @@ void SpillPlacement::addConstraints(ArrayRef<BlockConstraint> LiveBlocks) { } } +/// addPrefSpill - Same as addConstraints(PrefSpill) +void SpillPlacement::addPrefSpill(ArrayRef<unsigned> Blocks, bool Strong) { + for (ArrayRef<unsigned>::iterator I = Blocks.begin(), E = Blocks.end(); + I != E; ++I) { + float Freq = getBlockFrequency(*I); + if (Strong) + Freq += Freq; + unsigned ib = bundles->getBundle(*I, 0); + unsigned ob = bundles->getBundle(*I, 1); + activate(ib); + activate(ob); + nodes[ib].addBias(-Freq, 1); + nodes[ob].addBias(-Freq, 0); + } +} + void SpillPlacement::addLinks(ArrayRef<unsigned> Links) { for (ArrayRef<unsigned>::iterator I = Links.begin(), E = Links.end(); I != E; ++I) { diff --git a/lib/CodeGen/SpillPlacement.h b/lib/CodeGen/SpillPlacement.h index 6952ad8..fc412f8 100644 --- a/lib/CodeGen/SpillPlacement.h +++ b/lib/CodeGen/SpillPlacement.h @@ -71,6 +71,7 @@ public: DontCare, ///< Block doesn't care / variable not live. PrefReg, ///< Block entry/exit prefers a register. PrefSpill, ///< Block entry/exit prefers a stack slot. + PrefBoth, ///< Block entry prefers both register and stack. MustSpill ///< A register is impossible, variable must be spilled. }; @@ -79,6 +80,11 @@ public: unsigned Number; ///< Basic block number (from MBB::getNumber()). BorderConstraint Entry : 8; ///< Constraint on block entry. BorderConstraint Exit : 8; ///< Constraint on block exit. + + /// True when this block changes the value of the live range. This means + /// the block has a non-PHI def. When this is false, a live-in value on + /// the stack can be live-out on the stack without inserting a spill. + bool ChangesValue; }; /// prepare - Reset state and prepare for a new spill placement computation. @@ -96,6 +102,14 @@ public: /// live out. void addConstraints(ArrayRef<BlockConstraint> LiveBlocks); + /// addPrefSpill - Add PrefSpill constraints to all blocks listed. This is + /// equivalent to calling addConstraint with identical BlockConstraints with + /// Entry = Exit = PrefSpill, and ChangesValue = false. + /// + /// @param Blocks Array of block numbers that prefer to spill in and out. + /// @param Strong When true, double the negative bias for these blocks. + void addPrefSpill(ArrayRef<unsigned> Blocks, bool Strong); + /// addLinks - Add transparent blocks with the given numbers. void addLinks(ArrayRef<unsigned> Links); diff --git a/lib/CodeGen/SplitKit.cpp b/lib/CodeGen/SplitKit.cpp index 761cab7..6362780 100644 --- a/lib/CodeGen/SplitKit.cpp +++ b/lib/CodeGen/SplitKit.cpp @@ -20,6 +20,7 @@ #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -178,45 +179,55 @@ bool SplitAnalysis::calcLiveBlockInfo() { return false; } else { // This block has uses. Find the first and last uses in the block. - BI.FirstUse = *UseI; - assert(BI.FirstUse >= Start); + BI.FirstInstr = *UseI; + assert(BI.FirstInstr >= Start); do ++UseI; while (UseI != UseE && *UseI < Stop); - BI.LastUse = UseI[-1]; - assert(BI.LastUse < Stop); + BI.LastInstr = UseI[-1]; + assert(BI.LastInstr < Stop); // LVI is the first live segment overlapping MBB. BI.LiveIn = LVI->start <= Start; + // When not live in, the first use should be a def. + if (!BI.LiveIn) { + assert(LVI->start == LVI->valno->def && "Dangling LiveRange start"); + assert(LVI->start == BI.FirstInstr && "First instr should be a def"); + BI.FirstDef = BI.FirstInstr; + } + // Look for gaps in the live range. BI.LiveOut = true; while (LVI->end < Stop) { SlotIndex LastStop = LVI->end; if (++LVI == LVE || LVI->start >= Stop) { BI.LiveOut = false; - BI.LastUse = LastStop; + BI.LastInstr = LastStop; break; } + if (LastStop < LVI->start) { // There is a gap in the live range. Create duplicate entries for the // live-in snippet and the live-out snippet. ++NumGapBlocks; // Push the Live-in part. - BI.LiveThrough = false; BI.LiveOut = false; UseBlocks.push_back(BI); - UseBlocks.back().LastUse = LastStop; + UseBlocks.back().LastInstr = LastStop; // Set up BI for the live-out part. BI.LiveIn = false; BI.LiveOut = true; - BI.FirstUse = LVI->start; + BI.FirstInstr = BI.FirstDef = LVI->start; } + + // A LiveRange that starts in the middle of the block must be a def. + assert(LVI->start == LVI->valno->def && "Dangling LiveRange start"); + if (!BI.FirstDef) + BI.FirstDef = LVI->start; } - // Don't set LiveThrough when the block has a gap. - BI.LiveThrough = BI.LiveIn && BI.LiveOut; UseBlocks.push_back(BI); // LVI is now at LVE or LVI->end >= Stop. @@ -299,17 +310,21 @@ SplitEditor::SplitEditor(SplitAnalysis &sa, TRI(*vrm.getMachineFunction().getTarget().getRegisterInfo()), Edit(0), OpenIdx(0), + SpillMode(SM_Partition), RegAssign(Allocator) {} -void SplitEditor::reset(LiveRangeEdit &lre) { - Edit = &lre; +void SplitEditor::reset(LiveRangeEdit &LRE, ComplementSpillMode SM) { + Edit = &LRE; + SpillMode = SM; OpenIdx = 0; RegAssign.clear(); Values.clear(); - // We don't need to clear LiveOutCache, only LiveOutSeen entries are read. - LiveOutSeen.clear(); + // Reset the LiveRangeCalc instances needed for this spill mode. + LRCalc[0].reset(&VRM.getMachineFunction()); + if (SpillMode) + LRCalc[1].reset(&VRM.getMachineFunction()); // We don't need an AliasAnalysis since we will only be performing // cheap-as-a-copy remats anyway. @@ -340,7 +355,8 @@ VNInfo *SplitEditor::defValue(unsigned RegIdx, // Use insert for lookup, so we can add missing values with a second lookup. std::pair<ValueMap::iterator, bool> InsP = - Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id), VNI)); + Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id), + ValueForcePair(VNI, false))); // This was the first time (RegIdx, ParentVNI) was mapped. // Keep it as a simple def without any liveness. @@ -348,11 +364,11 @@ VNInfo *SplitEditor::defValue(unsigned RegIdx, return VNI; // If the previous value was a simple mapping, add liveness for it now. - if (VNInfo *OldVNI = InsP.first->second) { + if (VNInfo *OldVNI = InsP.first->second.getPointer()) { SlotIndex Def = OldVNI->def; LI->addRange(LiveRange(Def, Def.getNextSlot(), OldVNI)); - // No longer a simple mapping. - InsP.first->second = 0; + // No longer a simple mapping. Switch to a complex, non-forced mapping. + InsP.first->second = ValueForcePair(); } // This is a complex mapping, add liveness for VNI @@ -362,230 +378,24 @@ VNInfo *SplitEditor::defValue(unsigned RegIdx, return VNI; } -void SplitEditor::markComplexMapped(unsigned RegIdx, const VNInfo *ParentVNI) { +void SplitEditor::forceRecompute(unsigned RegIdx, const VNInfo *ParentVNI) { assert(ParentVNI && "Mapping NULL value"); - VNInfo *&VNI = Values[std::make_pair(RegIdx, ParentVNI->id)]; + ValueForcePair &VFP = Values[std::make_pair(RegIdx, ParentVNI->id)]; + VNInfo *VNI = VFP.getPointer(); - // ParentVNI was either unmapped or already complex mapped. Either way. - if (!VNI) + // ParentVNI was either unmapped or already complex mapped. Either way, just + // set the force bit. + if (!VNI) { + VFP.setInt(true); return; + } // This was previously a single mapping. Make sure the old def is represented // by a trivial live range. SlotIndex Def = VNI->def; Edit->get(RegIdx)->addRange(LiveRange(Def, Def.getNextSlot(), VNI)); - VNI = 0; -} - -// extendRange - Extend the live range to reach Idx. -// Potentially create phi-def values. -void SplitEditor::extendRange(unsigned RegIdx, SlotIndex Idx) { - assert(Idx.isValid() && "Invalid SlotIndex"); - MachineBasicBlock *IdxMBB = LIS.getMBBFromIndex(Idx); - assert(IdxMBB && "No MBB at Idx"); - LiveInterval *LI = Edit->get(RegIdx); - - // Is there a def in the same MBB we can extend? - if (LI->extendInBlock(LIS.getMBBStartIdx(IdxMBB), Idx)) - return; - - // Now for the fun part. We know that ParentVNI potentially has multiple defs, - // and we may need to create even more phi-defs to preserve VNInfo SSA form. - // Perform a search for all predecessor blocks where we know the dominating - // VNInfo. - VNInfo *VNI = findReachingDefs(LI, IdxMBB, Idx.getNextSlot()); - - // When there were multiple different values, we may need new PHIs. - if (!VNI) - return updateSSA(); - - // Poor man's SSA update for the single-value case. - LiveOutPair LOP(VNI, MDT[LIS.getMBBFromIndex(VNI->def)]); - for (SmallVectorImpl<LiveInBlock>::iterator I = LiveInBlocks.begin(), - E = LiveInBlocks.end(); I != E; ++I) { - MachineBasicBlock *MBB = I->DomNode->getBlock(); - SlotIndex Start = LIS.getMBBStartIdx(MBB); - if (I->Kill.isValid()) - LI->addRange(LiveRange(Start, I->Kill, VNI)); - else { - LiveOutCache[MBB] = LOP; - LI->addRange(LiveRange(Start, LIS.getMBBEndIdx(MBB), VNI)); - } - } -} - -/// findReachingDefs - Search the CFG for known live-out values. -/// Add required live-in blocks to LiveInBlocks. -VNInfo *SplitEditor::findReachingDefs(LiveInterval *LI, - MachineBasicBlock *KillMBB, - SlotIndex Kill) { - // Initialize the live-out cache the first time it is needed. - if (LiveOutSeen.empty()) { - unsigned N = VRM.getMachineFunction().getNumBlockIDs(); - LiveOutSeen.resize(N); - LiveOutCache.resize(N); - } - - // Blocks where LI should be live-in. - SmallVector<MachineBasicBlock*, 16> WorkList(1, KillMBB); - - // Remember if we have seen more than one value. - bool UniqueVNI = true; - VNInfo *TheVNI = 0; - - // Using LiveOutCache as a visited set, perform a BFS for all reaching defs. - for (unsigned i = 0; i != WorkList.size(); ++i) { - MachineBasicBlock *MBB = WorkList[i]; - assert(!MBB->pred_empty() && "Value live-in to entry block?"); - for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), - PE = MBB->pred_end(); PI != PE; ++PI) { - MachineBasicBlock *Pred = *PI; - LiveOutPair &LOP = LiveOutCache[Pred]; - - // Is this a known live-out block? - if (LiveOutSeen.test(Pred->getNumber())) { - if (VNInfo *VNI = LOP.first) { - if (TheVNI && TheVNI != VNI) - UniqueVNI = false; - TheVNI = VNI; - } - continue; - } - - // First time. LOP is garbage and must be cleared below. - LiveOutSeen.set(Pred->getNumber()); - - // Does Pred provide a live-out value? - SlotIndex Start, Last; - tie(Start, Last) = LIS.getSlotIndexes()->getMBBRange(Pred); - Last = Last.getPrevSlot(); - VNInfo *VNI = LI->extendInBlock(Start, Last); - LOP.first = VNI; - if (VNI) { - LOP.second = MDT[LIS.getMBBFromIndex(VNI->def)]; - if (TheVNI && TheVNI != VNI) - UniqueVNI = false; - TheVNI = VNI; - continue; - } - LOP.second = 0; - - // No, we need a live-in value for Pred as well - if (Pred != KillMBB) - WorkList.push_back(Pred); - else - // Loopback to KillMBB, so value is really live through. - Kill = SlotIndex(); - } - } - - // Transfer WorkList to LiveInBlocks in reverse order. - // This ordering works best with updateSSA(). - LiveInBlocks.clear(); - LiveInBlocks.reserve(WorkList.size()); - while(!WorkList.empty()) - LiveInBlocks.push_back(MDT[WorkList.pop_back_val()]); - - // The kill block may not be live-through. - assert(LiveInBlocks.back().DomNode->getBlock() == KillMBB); - LiveInBlocks.back().Kill = Kill; - - return UniqueVNI ? TheVNI : 0; -} - -void SplitEditor::updateSSA() { - // This is essentially the same iterative algorithm that SSAUpdater uses, - // except we already have a dominator tree, so we don't have to recompute it. - unsigned Changes; - do { - Changes = 0; - // Propagate live-out values down the dominator tree, inserting phi-defs - // when necessary. - for (SmallVectorImpl<LiveInBlock>::iterator I = LiveInBlocks.begin(), - E = LiveInBlocks.end(); I != E; ++I) { - MachineDomTreeNode *Node = I->DomNode; - // Skip block if the live-in value has already been determined. - if (!Node) - continue; - MachineBasicBlock *MBB = Node->getBlock(); - MachineDomTreeNode *IDom = Node->getIDom(); - LiveOutPair IDomValue; - - // We need a live-in value to a block with no immediate dominator? - // This is probably an unreachable block that has survived somehow. - bool needPHI = !IDom || !LiveOutSeen.test(IDom->getBlock()->getNumber()); - - // IDom dominates all of our predecessors, but it may not be their - // immediate dominator. Check if any of them have live-out values that are - // properly dominated by IDom. If so, we need a phi-def here. - if (!needPHI) { - IDomValue = LiveOutCache[IDom->getBlock()]; - for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), - PE = MBB->pred_end(); PI != PE; ++PI) { - LiveOutPair Value = LiveOutCache[*PI]; - if (!Value.first || Value.first == IDomValue.first) - continue; - // This predecessor is carrying something other than IDomValue. - // It could be because IDomValue hasn't propagated yet, or it could be - // because MBB is in the dominance frontier of that value. - if (MDT.dominates(IDom, Value.second)) { - needPHI = true; - break; - } - } - } - - // The value may be live-through even if Kill is set, as can happen when - // we are called from extendRange. In that case LiveOutSeen is true, and - // LiveOutCache indicates a foreign or missing value. - LiveOutPair &LOP = LiveOutCache[MBB]; - - // Create a phi-def if required. - if (needPHI) { - ++Changes; - SlotIndex Start = LIS.getMBBStartIdx(MBB); - unsigned RegIdx = RegAssign.lookup(Start); - LiveInterval *LI = Edit->get(RegIdx); - VNInfo *VNI = LI->getNextValue(Start, 0, LIS.getVNInfoAllocator()); - VNI->setIsPHIDef(true); - I->Value = VNI; - // This block is done, we know the final value. - I->DomNode = 0; - if (I->Kill.isValid()) - LI->addRange(LiveRange(Start, I->Kill, VNI)); - else { - LI->addRange(LiveRange(Start, LIS.getMBBEndIdx(MBB), VNI)); - LOP = LiveOutPair(VNI, Node); - } - } else if (IDomValue.first) { - // No phi-def here. Remember incoming value. - I->Value = IDomValue.first; - if (I->Kill.isValid()) - continue; - // Propagate IDomValue if needed: - // MBB is live-out and doesn't define its own value. - if (LOP.second != Node && LOP.first != IDomValue.first) { - ++Changes; - LOP = IDomValue; - } - } - } - } while (Changes); - - // The values in LiveInBlocks are now accurate. No more phi-defs are needed - // for these blocks, so we can color the live ranges. - for (SmallVectorImpl<LiveInBlock>::iterator I = LiveInBlocks.begin(), - E = LiveInBlocks.end(); I != E; ++I) { - if (!I->DomNode) - continue; - assert(I->Value && "No live-in value found"); - MachineBasicBlock *MBB = I->DomNode->getBlock(); - SlotIndex Start = LIS.getMBBStartIdx(MBB); - unsigned RegIdx = RegAssign.lookup(Start); - LiveInterval *LI = Edit->get(RegIdx); - LI->addRange(LiveRange(Start, I->Kill.isValid() ? - I->Kill : LIS.getMBBEndIdx(MBB), I->Value)); - } + // Mark as complex mapped, forced. + VFP = ValueForcePair(0, true); } VNInfo *SplitEditor::defFromParent(unsigned RegIdx, @@ -710,17 +520,28 @@ SlotIndex SplitEditor::leaveIntvAfter(SlotIndex Idx) { DEBUG(dbgs() << " leaveIntvAfter " << Idx); // The interval must be live beyond the instruction at Idx. - Idx = Idx.getBoundaryIndex(); - VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx); + SlotIndex Boundary = Idx.getBoundaryIndex(); + VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Boundary); if (!ParentVNI) { DEBUG(dbgs() << ": not live\n"); - return Idx.getNextSlot(); + return Boundary.getNextSlot(); } DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n'); - - MachineInstr *MI = LIS.getInstructionFromIndex(Idx); + MachineInstr *MI = LIS.getInstructionFromIndex(Boundary); assert(MI && "No instruction at index"); - VNInfo *VNI = defFromParent(0, ParentVNI, Idx, *MI->getParent(), + + // In spill mode, make live ranges as short as possible by inserting the copy + // before MI. This is only possible if that instruction doesn't redefine the + // value. The inserted COPY is not a kill, and we don't need to recompute + // the source live range. The spiller also won't try to hoist this copy. + if (SpillMode && !SlotIndex::isSameInstr(ParentVNI->def, Idx) && + MI->readsVirtualRegister(Edit->getReg())) { + forceRecompute(0, ParentVNI); + defFromParent(0, ParentVNI, Idx, *MI->getParent(), MI); + return Idx; + } + + VNInfo *VNI = defFromParent(0, ParentVNI, Boundary, *MI->getParent(), llvm::next(MachineBasicBlock::iterator(MI))); return VNI->def; } @@ -730,7 +551,7 @@ SlotIndex SplitEditor::leaveIntvBefore(SlotIndex Idx) { DEBUG(dbgs() << " leaveIntvBefore " << Idx); // The interval must be live into the instruction at Idx. - Idx = Idx.getBoundaryIndex(); + Idx = Idx.getBaseIndex(); VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx); if (!ParentVNI) { DEBUG(dbgs() << ": not live\n"); @@ -770,19 +591,219 @@ void SplitEditor::overlapIntv(SlotIndex Start, SlotIndex End) { assert(LIS.getMBBFromIndex(Start) == LIS.getMBBFromIndex(End) && "Range cannot span basic blocks"); - // The complement interval will be extended as needed by extendRange(). + // The complement interval will be extended as needed by LRCalc.extend(). if (ParentVNI) - markComplexMapped(0, ParentVNI); + forceRecompute(0, ParentVNI); DEBUG(dbgs() << " overlapIntv [" << Start << ';' << End << "):"); RegAssign.insert(Start, End, OpenIdx); DEBUG(dump()); } +//===----------------------------------------------------------------------===// +// Spill modes +//===----------------------------------------------------------------------===// + +void SplitEditor::removeBackCopies(SmallVectorImpl<VNInfo*> &Copies) { + LiveInterval *LI = Edit->get(0); + DEBUG(dbgs() << "Removing " << Copies.size() << " back-copies.\n"); + RegAssignMap::iterator AssignI; + AssignI.setMap(RegAssign); + + for (unsigned i = 0, e = Copies.size(); i != e; ++i) { + VNInfo *VNI = Copies[i]; + SlotIndex Def = VNI->def; + MachineInstr *MI = LIS.getInstructionFromIndex(Def); + assert(MI && "No instruction for back-copy"); + + MachineBasicBlock *MBB = MI->getParent(); + MachineBasicBlock::iterator MBBI(MI); + bool AtBegin; + do AtBegin = MBBI == MBB->begin(); + while (!AtBegin && (--MBBI)->isDebugValue()); + + DEBUG(dbgs() << "Removing " << Def << '\t' << *MI); + LI->removeValNo(VNI); + LIS.RemoveMachineInstrFromMaps(MI); + MI->eraseFromParent(); + + // Adjust RegAssign if a register assignment is killed at VNI->def. We + // want to avoid calculating the live range of the source register if + // possible. + AssignI.find(VNI->def.getPrevSlot()); + if (!AssignI.valid() || AssignI.start() >= Def) + continue; + // If MI doesn't kill the assigned register, just leave it. + if (AssignI.stop() != Def) + continue; + unsigned RegIdx = AssignI.value(); + if (AtBegin || !MBBI->readsVirtualRegister(Edit->getReg())) { + DEBUG(dbgs() << " cannot find simple kill of RegIdx " << RegIdx << '\n'); + forceRecompute(RegIdx, Edit->getParent().getVNInfoAt(Def)); + } else { + SlotIndex Kill = LIS.getInstructionIndex(MBBI).getDefIndex(); + DEBUG(dbgs() << " move kill to " << Kill << '\t' << *MBBI); + AssignI.setStop(Kill); + } + } +} + +MachineBasicBlock* +SplitEditor::findShallowDominator(MachineBasicBlock *MBB, + MachineBasicBlock *DefMBB) { + if (MBB == DefMBB) + return MBB; + assert(MDT.dominates(DefMBB, MBB) && "MBB must be dominated by the def."); + + const MachineLoopInfo &Loops = SA.Loops; + const MachineLoop *DefLoop = Loops.getLoopFor(DefMBB); + MachineDomTreeNode *DefDomNode = MDT[DefMBB]; + + // Best candidate so far. + MachineBasicBlock *BestMBB = MBB; + unsigned BestDepth = UINT_MAX; + + for (;;) { + const MachineLoop *Loop = Loops.getLoopFor(MBB); + + // MBB isn't in a loop, it doesn't get any better. All dominators have a + // higher frequency by definition. + if (!Loop) { + DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#" + << MBB->getNumber() << " at depth 0\n"); + return MBB; + } + + // We'll never be able to exit the DefLoop. + if (Loop == DefLoop) { + DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#" + << MBB->getNumber() << " in the same loop\n"); + return MBB; + } + + // Least busy dominator seen so far. + unsigned Depth = Loop->getLoopDepth(); + if (Depth < BestDepth) { + BestMBB = MBB; + BestDepth = Depth; + DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#" + << MBB->getNumber() << " at depth " << Depth << '\n'); + } + + // Leave loop by going to the immediate dominator of the loop header. + // This is a bigger stride than simply walking up the dominator tree. + MachineDomTreeNode *IDom = MDT[Loop->getHeader()]->getIDom(); + + // Too far up the dominator tree? + if (!IDom || !MDT.dominates(DefDomNode, IDom)) + return BestMBB; + + MBB = IDom->getBlock(); + } +} + +void SplitEditor::hoistCopiesForSize() { + // Get the complement interval, always RegIdx 0. + LiveInterval *LI = Edit->get(0); + LiveInterval *Parent = &Edit->getParent(); + + // Track the nearest common dominator for all back-copies for each ParentVNI, + // indexed by ParentVNI->id. + typedef std::pair<MachineBasicBlock*, SlotIndex> DomPair; + SmallVector<DomPair, 8> NearestDom(Parent->getNumValNums()); + + // Find the nearest common dominator for parent values with multiple + // back-copies. If a single back-copy dominates, put it in DomPair.second. + for (LiveInterval::vni_iterator VI = LI->vni_begin(), VE = LI->vni_end(); + VI != VE; ++VI) { + VNInfo *VNI = *VI; + VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def); + assert(ParentVNI && "Parent not live at complement def"); + + // Don't hoist remats. The complement is probably going to disappear + // completely anyway. + if (Edit->didRematerialize(ParentVNI)) + continue; + + MachineBasicBlock *ValMBB = LIS.getMBBFromIndex(VNI->def); + DomPair &Dom = NearestDom[ParentVNI->id]; + + // Keep directly defined parent values. This is either a PHI or an + // instruction in the complement range. All other copies of ParentVNI + // should be eliminated. + if (VNI->def == ParentVNI->def) { + DEBUG(dbgs() << "Direct complement def at " << VNI->def << '\n'); + Dom = DomPair(ValMBB, VNI->def); + continue; + } + // Skip the singly mapped values. There is nothing to gain from hoisting a + // single back-copy. + if (Values.lookup(std::make_pair(0, ParentVNI->id)).getPointer()) { + DEBUG(dbgs() << "Single complement def at " << VNI->def << '\n'); + continue; + } + + if (!Dom.first) { + // First time we see ParentVNI. VNI dominates itself. + Dom = DomPair(ValMBB, VNI->def); + } else if (Dom.first == ValMBB) { + // Two defs in the same block. Pick the earlier def. + if (!Dom.second.isValid() || VNI->def < Dom.second) + Dom.second = VNI->def; + } else { + // Different basic blocks. Check if one dominates. + MachineBasicBlock *Near = + MDT.findNearestCommonDominator(Dom.first, ValMBB); + if (Near == ValMBB) + // Def ValMBB dominates. + Dom = DomPair(ValMBB, VNI->def); + else if (Near != Dom.first) + // None dominate. Hoist to common dominator, need new def. + Dom = DomPair(Near, SlotIndex()); + } + + DEBUG(dbgs() << "Multi-mapped complement " << VNI->id << '@' << VNI->def + << " for parent " << ParentVNI->id << '@' << ParentVNI->def + << " hoist to BB#" << Dom.first->getNumber() << ' ' + << Dom.second << '\n'); + } + + // Insert the hoisted copies. + for (unsigned i = 0, e = Parent->getNumValNums(); i != e; ++i) { + DomPair &Dom = NearestDom[i]; + if (!Dom.first || Dom.second.isValid()) + continue; + // This value needs a hoisted copy inserted at the end of Dom.first. + VNInfo *ParentVNI = Parent->getValNumInfo(i); + MachineBasicBlock *DefMBB = LIS.getMBBFromIndex(ParentVNI->def); + // Get a less loopy dominator than Dom.first. + Dom.first = findShallowDominator(Dom.first, DefMBB); + SlotIndex Last = LIS.getMBBEndIdx(Dom.first).getPrevSlot(); + Dom.second = + defFromParent(0, ParentVNI, Last, *Dom.first, + LIS.getLastSplitPoint(Edit->getParent(), Dom.first))->def; + } + + // Remove redundant back-copies that are now known to be dominated by another + // def with the same value. + SmallVector<VNInfo*, 8> BackCopies; + for (LiveInterval::vni_iterator VI = LI->vni_begin(), VE = LI->vni_end(); + VI != VE; ++VI) { + VNInfo *VNI = *VI; + VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def); + const DomPair &Dom = NearestDom[ParentVNI->id]; + if (!Dom.first || Dom.second == VNI->def) + continue; + BackCopies.push_back(VNI); + forceRecompute(0, ParentVNI); + } + removeBackCopies(BackCopies); +} + + /// transferValues - Transfer all possible values to the new live ranges. -/// Values that were rematerialized are left alone, they need extendRange(). +/// Values that were rematerialized are left alone, they need LRCalc.extend(). bool SplitEditor::transferValues() { bool Skipped = false; - LiveInBlocks.clear(); RegAssignMap::const_iterator AssignI = RegAssign.begin(); for (LiveInterval::const_iterator ParentI = Edit->getParent().begin(), ParentE = Edit->getParent().end(); ParentI != ParentE; ++ParentI) { @@ -812,28 +833,23 @@ bool SplitEditor::transferValues() { LiveInterval *LI = Edit->get(RegIdx); // Check for a simply defined value that can be blitted directly. - if (VNInfo *VNI = Values.lookup(std::make_pair(RegIdx, ParentVNI->id))) { + ValueForcePair VFP = Values.lookup(std::make_pair(RegIdx, ParentVNI->id)); + if (VNInfo *VNI = VFP.getPointer()) { DEBUG(dbgs() << ':' << VNI->id); LI->addRange(LiveRange(Start, End, VNI)); Start = End; continue; } - // Skip rematerialized values, we need to use extendRange() and - // extendPHIKillRanges() to completely recompute the live ranges. - if (Edit->didRematerialize(ParentVNI)) { - DEBUG(dbgs() << "(remat)"); + // Skip values with forced recomputation. + if (VFP.getInt()) { + DEBUG(dbgs() << "(recalc)"); Skipped = true; Start = End; continue; } - // Initialize the live-out cache the first time it is needed. - if (LiveOutSeen.empty()) { - unsigned N = VRM.getMachineFunction().getNumBlockIDs(); - LiveOutSeen.resize(N); - LiveOutCache.resize(N); - } + LiveRangeCalc &LRC = getLRCalc(RegIdx); // This value has multiple defs in RegIdx, but it wasn't rematerialized, // so the live range is accurate. Add live-in blocks in [Start;End) to the @@ -844,15 +860,13 @@ bool SplitEditor::transferValues() { // The first block may be live-in, or it may have its own def. if (Start != BlockStart) { - VNInfo *VNI = LI->extendInBlock(BlockStart, - std::min(BlockEnd, End).getPrevSlot()); + VNInfo *VNI = LI->extendInBlock(BlockStart, std::min(BlockEnd, End)); assert(VNI && "Missing def for complex mapped value"); DEBUG(dbgs() << ':' << VNI->id << "*BB#" << MBB->getNumber()); // MBB has its own def. Is it also live-out? - if (BlockEnd <= End) { - LiveOutSeen.set(MBB->getNumber()); - LiveOutCache[MBB] = LiveOutPair(VNI, MDT[MBB]); - } + if (BlockEnd <= End) + LRC.setLiveOutValue(MBB, VNI); + // Skip to the next block for live-in. ++MBB; BlockStart = BlockEnd; @@ -866,25 +880,19 @@ bool SplitEditor::transferValues() { if (BlockStart == ParentVNI->def) { // This block has the def of a parent PHI, so it isn't live-in. assert(ParentVNI->isPHIDef() && "Non-phi defined at block start?"); - VNInfo *VNI = LI->extendInBlock(BlockStart, - std::min(BlockEnd, End).getPrevSlot()); + VNInfo *VNI = LI->extendInBlock(BlockStart, std::min(BlockEnd, End)); assert(VNI && "Missing def for complex mapped parent PHI"); - if (End >= BlockEnd) { - // Live-out as well. - LiveOutSeen.set(MBB->getNumber()); - LiveOutCache[MBB] = LiveOutPair(VNI, MDT[MBB]); - } + if (End >= BlockEnd) + LRC.setLiveOutValue(MBB, VNI); // Live-out as well. } else { - // This block needs a live-in value. - LiveInBlocks.push_back(MDT[MBB]); - // The last block covered may not be live-out. + // This block needs a live-in value. The last block covered may not + // be live-out. if (End < BlockEnd) - LiveInBlocks.back().Kill = End; + LRC.addLiveInBlock(LI, MDT[MBB], End); else { - // Live-out, but we need updateSSA to tell us the value. - LiveOutSeen.set(MBB->getNumber()); - LiveOutCache[MBB] = LiveOutPair((VNInfo*)0, - (MachineDomTreeNode*)0); + // Live-through, and we don't know the value. + LRC.addLiveInBlock(LI, MDT[MBB]); + LRC.setLiveOutValue(MBB, 0); } } BlockStart = BlockEnd; @@ -895,8 +903,11 @@ bool SplitEditor::transferValues() { DEBUG(dbgs() << '\n'); } - if (!LiveInBlocks.empty()) - updateSSA(); + LRCalc[0].calculateValues(LIS.getSlotIndexes(), &MDT, + &LIS.getVNInfoAllocator()); + if (SpillMode) + LRCalc[1].calculateValues(LIS.getSlotIndexes(), &MDT, + &LIS.getVNInfoAllocator()); return Skipped; } @@ -909,16 +920,20 @@ void SplitEditor::extendPHIKillRanges() { if (PHIVNI->isUnused() || !PHIVNI->isPHIDef()) continue; unsigned RegIdx = RegAssign.lookup(PHIVNI->def); + LiveInterval *LI = Edit->get(RegIdx); + LiveRangeCalc &LRC = getLRCalc(RegIdx); MachineBasicBlock *MBB = LIS.getMBBFromIndex(PHIVNI->def); for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { - SlotIndex End = LIS.getMBBEndIdx(*PI).getPrevSlot(); + SlotIndex End = LIS.getMBBEndIdx(*PI); + SlotIndex LastUse = End.getPrevSlot(); // The predecessor may not have a live-out value. That is OK, like an // undef PHI operand. - if (Edit->getParent().liveAt(End)) { - assert(RegAssign.lookup(End) == RegIdx && + if (Edit->getParent().liveAt(LastUse)) { + assert(RegAssign.lookup(LastUse) == RegIdx && "Different register assignment in phi predecessor"); - extendRange(RegIdx, End); + LRC.extend(LI, End, + LIS.getSlotIndexes(), &MDT, &LIS.getVNInfoAllocator()); } } } @@ -938,25 +953,22 @@ void SplitEditor::rewriteAssigned(bool ExtendRanges) { continue; } - // <undef> operands don't really read the register, so just assign them to - // the complement. - if (MO.isUse() && MO.isUndef()) { - MO.setReg(Edit->get(0)->reg); - continue; - } - + // <undef> operands don't really read the register, so it doesn't matter + // which register we choose. When the use operand is tied to a def, we must + // use the same register as the def, so just do that always. SlotIndex Idx = LIS.getInstructionIndex(MI); - if (MO.isDef()) + if (MO.isDef() || MO.isUndef()) Idx = MO.isEarlyClobber() ? Idx.getUseIndex() : Idx.getDefIndex(); // Rewrite to the mapped register at Idx. unsigned RegIdx = RegAssign.lookup(Idx); - MO.setReg(Edit->get(RegIdx)->reg); + LiveInterval *LI = Edit->get(RegIdx); + MO.setReg(LI->reg); DEBUG(dbgs() << " rewr BB#" << MI->getParent()->getNumber() << '\t' << Idx << ':' << RegIdx << '\t' << *MI); // Extend liveness to Idx if the instruction reads reg. - if (!ExtendRanges) + if (!ExtendRanges || MO.isUndef()) continue; // Skip instructions that don't read Reg. @@ -971,7 +983,8 @@ void SplitEditor::rewriteAssigned(bool ExtendRanges) { } else Idx = Idx.getUseIndex(); - extendRange(RegIdx, Idx); + getLRCalc(RegIdx).extend(LI, Idx.getNextSlot(), LIS.getSlotIndexes(), + &MDT, &LIS.getVNInfoAllocator()); } } @@ -1019,11 +1032,24 @@ void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) { VNI->setIsPHIDef(ParentVNI->isPHIDef()); VNI->setCopy(ParentVNI->getCopy()); - // Mark rematted values as complex everywhere to force liveness computation. + // Force rematted values to be recomputed everywhere. // The new live ranges may be truncated. if (Edit->didRematerialize(ParentVNI)) for (unsigned i = 0, e = Edit->size(); i != e; ++i) - markComplexMapped(i, ParentVNI); + forceRecompute(i, ParentVNI); + } + + // Hoist back-copies to the complement interval when in spill mode. + switch (SpillMode) { + case SM_Partition: + // Leave all back-copies as is. + break; + case SM_Size: + hoistCopiesForSize(); + break; + case SM_Speed: + llvm_unreachable("Spill mode 'speed' not implemented yet"); + break; } // Transfer the simply mapped values, check if any are skipped. @@ -1081,50 +1107,39 @@ void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) { // Single Block Splitting //===----------------------------------------------------------------------===// -/// getMultiUseBlocks - if CurLI has more than one use in a basic block, it -/// may be an advantage to split CurLI for the duration of the block. -bool SplitAnalysis::getMultiUseBlocks(BlockPtrSet &Blocks) { - // If CurLI is local to one block, there is no point to splitting it. - if (UseBlocks.size() <= 1) +bool SplitAnalysis::shouldSplitSingleBlock(const BlockInfo &BI, + bool SingleInstrs) const { + // Always split for multiple instructions. + if (!BI.isOneInstr()) + return true; + // Don't split for single instructions unless explicitly requested. + if (!SingleInstrs) return false; - // Add blocks with multiple uses. - for (unsigned i = 0, e = UseBlocks.size(); i != e; ++i) { - const BlockInfo &BI = UseBlocks[i]; - if (BI.FirstUse == BI.LastUse) - continue; - Blocks.insert(BI.MBB); - } - return !Blocks.empty(); + // Splitting a live-through range always makes progress. + if (BI.LiveIn && BI.LiveOut) + return true; + // No point in isolating a copy. It has no register class constraints. + if (LIS.getInstructionFromIndex(BI.FirstInstr)->isCopyLike()) + return false; + // Finally, don't isolate an end point that was created by earlier splits. + return isOriginalEndpoint(BI.FirstInstr); } void SplitEditor::splitSingleBlock(const SplitAnalysis::BlockInfo &BI) { openIntv(); SlotIndex LastSplitPoint = SA.getLastSplitPoint(BI.MBB->getNumber()); - SlotIndex SegStart = enterIntvBefore(std::min(BI.FirstUse, + SlotIndex SegStart = enterIntvBefore(std::min(BI.FirstInstr, LastSplitPoint)); - if (!BI.LiveOut || BI.LastUse < LastSplitPoint) { - useIntv(SegStart, leaveIntvAfter(BI.LastUse)); + if (!BI.LiveOut || BI.LastInstr < LastSplitPoint) { + useIntv(SegStart, leaveIntvAfter(BI.LastInstr)); } else { // The last use is after the last valid split point. SlotIndex SegStop = leaveIntvBefore(LastSplitPoint); useIntv(SegStart, SegStop); - overlapIntv(SegStop, BI.LastUse); + overlapIntv(SegStop, BI.LastInstr); } } -/// splitSingleBlocks - Split CurLI into a separate live interval inside each -/// basic block in Blocks. -void SplitEditor::splitSingleBlocks(const SplitAnalysis::BlockPtrSet &Blocks) { - DEBUG(dbgs() << " splitSingleBlocks for " << Blocks.size() << " blocks.\n"); - ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA.getUseBlocks(); - for (unsigned i = 0; i != UseBlocks.size(); ++i) { - const SplitAnalysis::BlockInfo &BI = UseBlocks[i]; - if (Blocks.count(BI.MBB)) - splitSingleBlock(BI); - } - finish(); -} - //===----------------------------------------------------------------------===// // Global Live Range Splitting Support @@ -1149,6 +1164,12 @@ void SplitEditor::splitLiveThroughBlock(unsigned MBBNum, assert((IntvIn || IntvOut) && "Use splitSingleBlock for isolated blocks"); + assert((!LeaveBefore || LeaveBefore < Stop) && "Interference after block"); + assert((!IntvIn || !LeaveBefore || LeaveBefore > Start) && "Impossible intf"); + assert((!EnterAfter || EnterAfter >= Start) && "Interference before block"); + + MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum); + if (!IntvOut) { DEBUG(dbgs() << ", spill on entry.\n"); // @@ -1157,7 +1178,6 @@ void SplitEditor::splitLiveThroughBlock(unsigned MBBNum, // -____________ Spill on entry. // selectIntv(IntvIn); - MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum); SlotIndex Idx = leaveIntvAtTop(*MBB); assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); (void)Idx; @@ -1172,7 +1192,6 @@ void SplitEditor::splitLiveThroughBlock(unsigned MBBNum, // ___________-- Reload on exit. // selectIntv(IntvOut); - MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum); SlotIndex Idx = enterIntvAtEnd(*MBB); assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); (void)Idx; @@ -1192,6 +1211,7 @@ void SplitEditor::splitLiveThroughBlock(unsigned MBBNum, // We cannot legally insert splits after LSP. SlotIndex LSP = SA.getLastSplitPoint(MBBNum); + assert((!IntvOut || !EnterAfter || EnterAfter < LSP) && "Impossible intf"); if (IntvIn != IntvOut && (!LeaveBefore || !EnterAfter || LeaveBefore.getBaseIndex() > EnterAfter.getBoundaryIndex())) { @@ -1201,10 +1221,14 @@ void SplitEditor::splitLiveThroughBlock(unsigned MBBNum, // |-----------| Live through. // ------======= Switch intervals between interference. // - SlotIndex Cut = (LeaveBefore && LeaveBefore < LSP) ? LeaveBefore : LSP; selectIntv(IntvOut); - SlotIndex Idx = enterIntvBefore(Cut); - useIntv(Idx, Stop); + SlotIndex Idx; + if (LeaveBefore && LeaveBefore < LSP) { + Idx = enterIntvBefore(LeaveBefore); + useIntv(Idx, Stop); + } else { + Idx = enterIntvAtEnd(*MBB); + } selectIntv(IntvIn); useIntv(Start, Idx); assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); @@ -1238,7 +1262,7 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI, tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB); DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " [" << Start << ';' << Stop - << "), uses " << BI.FirstUse << '-' << BI.LastUse + << "), uses " << BI.FirstInstr << '-' << BI.LastInstr << ", reg-in " << IntvIn << ", leave before " << LeaveBefore << (BI.LiveOut ? ", stack-out" : ", killed in block")); @@ -1246,7 +1270,7 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI, assert(BI.LiveIn && "Must be live-in"); assert((!LeaveBefore || LeaveBefore > Start) && "Bad interference"); - if (!BI.LiveOut && (!LeaveBefore || LeaveBefore >= BI.LastUse)) { + if (!BI.LiveOut && (!LeaveBefore || LeaveBefore >= BI.LastInstr)) { DEBUG(dbgs() << " before interference.\n"); // // <<< Interference after kill. @@ -1254,13 +1278,13 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI, // ========= Use IntvIn everywhere. // selectIntv(IntvIn); - useIntv(Start, BI.LastUse); + useIntv(Start, BI.LastInstr); return; } SlotIndex LSP = SA.getLastSplitPoint(BI.MBB->getNumber()); - if (!LeaveBefore || LeaveBefore > BI.LastUse.getBoundaryIndex()) { + if (!LeaveBefore || LeaveBefore > BI.LastInstr.getBoundaryIndex()) { // // <<< Possible interference after last use. // |---o---o---| Live-out on stack. @@ -1271,17 +1295,17 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI, // ============ Copy to stack after LSP, overlap IntvIn. // \_____ Stack interval is live-out. // - if (BI.LastUse < LSP) { + if (BI.LastInstr < LSP) { DEBUG(dbgs() << ", spill after last use before interference.\n"); selectIntv(IntvIn); - SlotIndex Idx = leaveIntvAfter(BI.LastUse); + SlotIndex Idx = leaveIntvAfter(BI.LastInstr); useIntv(Start, Idx); assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); } else { DEBUG(dbgs() << ", spill before last split point.\n"); selectIntv(IntvIn); SlotIndex Idx = leaveIntvBefore(LSP); - overlapIntv(Idx, BI.LastUse); + overlapIntv(Idx, BI.LastInstr); useIntv(Start, Idx); assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); } @@ -1295,13 +1319,13 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI, (void)LocalIntv; DEBUG(dbgs() << ", creating local interval " << LocalIntv << ".\n"); - if (!BI.LiveOut || BI.LastUse < LSP) { + if (!BI.LiveOut || BI.LastInstr < LSP) { // // <<<<<<< Interference overlapping uses. // |---o---o---| Live-out on stack. // =====----____ Leave IntvIn before interference, then spill. // - SlotIndex To = leaveIntvAfter(BI.LastUse); + SlotIndex To = leaveIntvAfter(BI.LastInstr); SlotIndex From = enterIntvBefore(LeaveBefore); useIntv(From, To); selectIntv(IntvIn); @@ -1316,7 +1340,7 @@ void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI, // \_____ Stack interval is live-out. // SlotIndex To = leaveIntvBefore(LSP); - overlapIntv(To, BI.LastUse); + overlapIntv(To, BI.LastInstr); SlotIndex From = enterIntvBefore(std::min(To, LeaveBefore)); useIntv(From, To); selectIntv(IntvIn); @@ -1330,7 +1354,7 @@ void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI, tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB); DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " [" << Start << ';' << Stop - << "), uses " << BI.FirstUse << '-' << BI.LastUse + << "), uses " << BI.FirstInstr << '-' << BI.LastInstr << ", reg-out " << IntvOut << ", enter after " << EnterAfter << (BI.LiveIn ? ", stack-in" : ", defined in block")); @@ -1340,7 +1364,7 @@ void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI, assert(BI.LiveOut && "Must be live-out"); assert((!EnterAfter || EnterAfter < LSP) && "Bad interference"); - if (!BI.LiveIn && (!EnterAfter || EnterAfter <= BI.FirstUse)) { + if (!BI.LiveIn && (!EnterAfter || EnterAfter <= BI.FirstInstr)) { DEBUG(dbgs() << " after interference.\n"); // // >>>> Interference before def. @@ -1348,11 +1372,11 @@ void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI, // ========= Use IntvOut everywhere. // selectIntv(IntvOut); - useIntv(BI.FirstUse, Stop); + useIntv(BI.FirstInstr, Stop); return; } - if (!EnterAfter || EnterAfter < BI.FirstUse.getBaseIndex()) { + if (!EnterAfter || EnterAfter < BI.FirstInstr.getBaseIndex()) { DEBUG(dbgs() << ", reload after interference.\n"); // // >>>> Interference before def. @@ -1360,7 +1384,7 @@ void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI, // ____========= Enter IntvOut before first use. // selectIntv(IntvOut); - SlotIndex Idx = enterIntvBefore(std::min(LSP, BI.FirstUse)); + SlotIndex Idx = enterIntvBefore(std::min(LSP, BI.FirstInstr)); useIntv(Idx, Stop); assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); return; @@ -1381,6 +1405,6 @@ void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI, assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); openIntv(); - SlotIndex From = enterIntvBefore(std::min(Idx, BI.FirstUse)); + SlotIndex From = enterIntvBefore(std::min(Idx, BI.FirstInstr)); useIntv(From, Idx); } diff --git a/lib/CodeGen/SplitKit.h b/lib/CodeGen/SplitKit.h index 7948b72..d8fc212 100644 --- a/lib/CodeGen/SplitKit.h +++ b/lib/CodeGen/SplitKit.h @@ -15,13 +15,11 @@ #ifndef LLVM_CODEGEN_SPLITKIT_H #define LLVM_CODEGEN_SPLITKIT_H +#include "LiveRangeCalc.h" #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/IntervalMap.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/CodeGen/SlotIndexes.h" namespace llvm { @@ -38,12 +36,6 @@ class VirtRegMap; class VNInfo; class raw_ostream; -/// At some point we should just include MachineDominators.h: -class MachineDominatorTree; -template <class NodeT> class DomTreeNodeBase; -typedef DomTreeNodeBase<MachineBasicBlock> MachineDomTreeNode; - - /// SplitAnalysis - Analyze a LiveInterval, looking for live range splitting /// opportunities. class SplitAnalysis { @@ -76,16 +68,16 @@ public: /// struct BlockInfo { MachineBasicBlock *MBB; - SlotIndex FirstUse; ///< First instr using current reg. - SlotIndex LastUse; ///< Last instr using current reg. - bool LiveThrough; ///< Live in whole block (Templ 5. above). + SlotIndex FirstInstr; ///< First instr accessing current reg. + SlotIndex LastInstr; ///< Last instr accessing current reg. + SlotIndex FirstDef; ///< First non-phi valno->def, or SlotIndex(). bool LiveIn; ///< Current reg is live in. bool LiveOut; ///< Current reg is live out. /// isOneInstr - Returns true when this BlockInfo describes a single /// instruction. bool isOneInstr() const { - return SlotIndex::isSameInstr(FirstUse, LastUse); + return SlotIndex::isSameInstr(FirstInstr, LastInstr); } }; @@ -185,10 +177,15 @@ public: typedef SmallPtrSet<const MachineBasicBlock*, 16> BlockPtrSet; - /// getMultiUseBlocks - Add basic blocks to Blocks that may benefit from - /// having CurLI split to a new live interval. Return true if Blocks can be - /// passed to SplitEditor::splitSingleBlocks. - bool getMultiUseBlocks(BlockPtrSet &Blocks); + /// shouldSplitSingleBlock - Returns true if it would help to create a local + /// live range for the instructions in BI. There is normally no benefit to + /// creating a live range for a single instruction, but it does enable + /// register class inflation if the instruction has a restricted register + /// class. + /// + /// @param BI The block to be isolated. + /// @param SingleInstrs True when single instructions should be isolated. + bool shouldSplitSingleBlock(const BlockInfo &BI, bool SingleInstrs) const; }; @@ -212,6 +209,36 @@ class SplitEditor { const TargetInstrInfo &TII; const TargetRegisterInfo &TRI; +public: + + /// ComplementSpillMode - Select how the complement live range should be + /// created. SplitEditor automatically creates interval 0 to contain + /// anything that isn't added to another interval. This complement interval + /// can get quite complicated, and it can sometimes be an advantage to allow + /// it to overlap the other intervals. If it is going to spill anyway, no + /// registers are wasted by keeping a value in two places at the same time. + enum ComplementSpillMode { + /// SM_Partition(Default) - Try to create the complement interval so it + /// doesn't overlap any other intervals, and the original interval is + /// partitioned. This may require a large number of back copies and extra + /// PHI-defs. Only segments marked with overlapIntv will be overlapping. + SM_Partition, + + /// SM_Size - Overlap intervals to minimize the number of inserted COPY + /// instructions. Copies to the complement interval are hoisted to their + /// common dominator, so only one COPY is required per value in the + /// complement interval. This also means that no extra PHI-defs need to be + /// inserted in the complement interval. + SM_Size, + + /// SM_Speed - Overlap intervals to minimize the expected execution + /// frequency of the inserted copies. This is very similar to SM_Size, but + /// the complement interval may get some extra PHI-defs. + SM_Speed + }; + +private: + /// Edit - The current parent register and new intervals created. LiveRangeEdit *Edit; @@ -220,6 +247,9 @@ class SplitEditor { /// openIntv will be 1. unsigned OpenIdx; + /// The current spill mode, selected by reset(). + ComplementSpillMode SpillMode; + typedef IntervalMap<SlotIndex, unsigned> RegAssignMap; /// Allocator for the interval map. This will eventually be shared with @@ -231,65 +261,34 @@ class SplitEditor { /// Idx. RegAssignMap RegAssign; - typedef DenseMap<std::pair<unsigned, unsigned>, VNInfo*> ValueMap; + typedef PointerIntPair<VNInfo*, 1> ValueForcePair; + typedef DenseMap<std::pair<unsigned, unsigned>, ValueForcePair> ValueMap; /// Values - keep track of the mapping from parent values to values in the new /// intervals. Given a pair (RegIdx, ParentVNI->id), Values contains: /// /// 1. No entry - the value is not mapped to Edit.get(RegIdx). - /// 2. Null - the value is mapped to multiple values in Edit.get(RegIdx). - /// Each value is represented by a minimal live range at its def. - /// 3. A non-null VNInfo - the value is mapped to a single new value. + /// 2. (Null, false) - the value is mapped to multiple values in + /// Edit.get(RegIdx). Each value is represented by a minimal live range at + /// its def. The full live range can be inferred exactly from the range + /// of RegIdx in RegAssign. + /// 3. (Null, true). As above, but the ranges in RegAssign are too large, and + /// the live range must be recomputed using LiveRangeCalc::extend(). + /// 4. (VNI, false) The value is mapped to a single new value. /// The new value has no live ranges anywhere. ValueMap Values; - typedef std::pair<VNInfo*, MachineDomTreeNode*> LiveOutPair; - typedef IndexedMap<LiveOutPair, MBB2NumberFunctor> LiveOutMap; - - // LiveOutCache - Map each basic block where a new register is live out to the - // live-out value and its defining block. - // One of these conditions shall be true: - // - // 1. !LiveOutCache.count(MBB) - // 2. LiveOutCache[MBB].second.getNode() == MBB - // 3. forall P in preds(MBB): LiveOutCache[P] == LiveOutCache[MBB] - // - // This is only a cache, the values can be computed as: - // - // VNI = Edit.get(RegIdx)->getVNInfoAt(LIS.getMBBEndIdx(MBB)) - // Node = mbt_[LIS.getMBBFromIndex(VNI->def)] - // - // The cache is also used as a visited set by extendRange(). It can be shared - // by all the new registers because at most one is live out of each block. - LiveOutMap LiveOutCache; - - // LiveOutSeen - Indexed by MBB->getNumber(), a bit is set for each valid - // entry in LiveOutCache. - BitVector LiveOutSeen; - - /// LiveInBlock - Info for updateSSA() about a block where a register is - /// live-in. - /// The updateSSA caller provides DomNode and Kill inside MBB, updateSSA() - /// adds the computed live-in value. - struct LiveInBlock { - // Dominator tree node for the block. - // Cleared by updateSSA when the final value has been determined. - MachineDomTreeNode *DomNode; - - // Live-in value filled in by updateSSA once it is known. - VNInfo *Value; - - // Position in block where the live-in range ends, or SlotIndex() if the - // range passes through the block. - SlotIndex Kill; - - LiveInBlock(MachineDomTreeNode *node) : DomNode(node), Value(0) {} - }; + /// LRCalc - Cache for computing live ranges and SSA update. Each instance + /// can only handle non-overlapping live ranges, so use a separate + /// LiveRangeCalc instance for the complement interval when in spill mode. + LiveRangeCalc LRCalc[2]; - /// LiveInBlocks - List of live-in blocks used by findReachingDefs() and - /// updateSSA(). This list is usually empty, it exists here to avoid frequent - /// reallocations. - SmallVector<LiveInBlock, 16> LiveInBlocks; + /// getLRCalc - Return the LRCalc to use for RegIdx. In spill mode, the + /// complement interval can overlap the other intervals, so it gets its own + /// LRCalc instance. When not in spill mode, all intervals can share one. + LiveRangeCalc &getLRCalc(unsigned RegIdx) { + return LRCalc[SpillMode != SM_Partition && RegIdx != 0]; + } /// defValue - define a value in RegIdx from ParentVNI at Idx. /// Idx does not have to be ParentVNI->def, but it must be contained within @@ -298,9 +297,11 @@ class SplitEditor { /// Return the new LI value. VNInfo *defValue(unsigned RegIdx, const VNInfo *ParentVNI, SlotIndex Idx); - /// markComplexMapped - Mark ParentVNI as complex mapped in RegIdx regardless - /// of the number of defs. - void markComplexMapped(unsigned RegIdx, const VNInfo *ParentVNI); + /// forceRecompute - Force the live range of ParentVNI in RegIdx to be + /// recomputed by LiveRangeCalc::extend regardless of the number of defs. + /// This is used for values whose live range doesn't match RegAssign exactly. + /// They could have rematerialized, or back-copies may have been moved. + void forceRecompute(unsigned RegIdx, const VNInfo *ParentVNI); /// defFromParent - Define Reg from ParentVNI at UseIdx using either /// rematerialization or a COPY from parent. Return the new value. @@ -310,22 +311,18 @@ class SplitEditor { MachineBasicBlock &MBB, MachineBasicBlock::iterator I); - /// extendRange - Extend the live range of Edit.get(RegIdx) so it reaches Idx. - /// Insert PHIDefs as needed to preserve SSA form. - void extendRange(unsigned RegIdx, SlotIndex Idx); + /// removeBackCopies - Remove the copy instructions that defines the values + /// in the vector in the complement interval. + void removeBackCopies(SmallVectorImpl<VNInfo*> &Copies); - /// findReachingDefs - Starting from MBB, add blocks to LiveInBlocks until all - /// reaching defs for LI are found. - /// @param LI Live interval whose value is needed. - /// @param MBB Block where LI should be live-in. - /// @param Kill Kill point in MBB. - /// @return Unique value seen, or NULL. - VNInfo *findReachingDefs(LiveInterval *LI, MachineBasicBlock *MBB, - SlotIndex Kill); + /// getShallowDominator - Returns the least busy dominator of MBB that is + /// also dominated by DefMBB. Busy is measured by loop depth. + MachineBasicBlock *findShallowDominator(MachineBasicBlock *MBB, + MachineBasicBlock *DefMBB); - /// updateSSA - Compute and insert PHIDefs such that all blocks in - // LiveInBlocks get a known live-in value. Add live ranges to the blocks. - void updateSSA(); + /// hoistCopiesForSize - Hoist back-copies to the complement interval in a + /// way that minimizes code size. This implements the SM_Size spill mode. + void hoistCopiesForSize(); /// transferValues - Transfer values to the new ranges. /// Return true if any ranges were skipped. @@ -348,7 +345,7 @@ public: MachineDominatorTree&); /// reset - Prepare for a new split. - void reset(LiveRangeEdit&); + void reset(LiveRangeEdit&, ComplementSpillMode = SM_Partition); /// Create a new virtual register and live interval. /// Return the interval index, starting from 1. Interval index 0 is the @@ -423,10 +420,6 @@ public: /// split, and doesn't call finish(). void splitSingleBlock(const SplitAnalysis::BlockInfo &BI); - /// splitSingleBlocks - Split CurLI into a separate live interval inside each - /// basic block in Blocks. - void splitSingleBlocks(const SplitAnalysis::BlockPtrSet &Blocks); - /// splitLiveThroughBlock - Split CurLI in the given block such that it /// enters the block in IntvIn and leaves it in IntvOut. There may be uses in /// the block, but they will be ignored when placing split points. diff --git a/lib/CodeGen/Splitter.cpp b/lib/CodeGen/Splitter.cpp index ec75df4..77973b7 100644 --- a/lib/CodeGen/Splitter.cpp +++ b/lib/CodeGen/Splitter.cpp @@ -11,7 +11,6 @@ #include "Splitter.h" -#include "RegisterCoalescer.h" #include "llvm/Module.h" #include "llvm/CodeGen/CalcSpillWeights.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" @@ -20,6 +19,7 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/SlotIndexes.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -262,7 +262,7 @@ namespace llvm { au.addPreserved<MachineDominatorTree>(); au.addRequired<MachineLoopInfo>(); au.addPreserved<MachineLoopInfo>(); - au.addPreserved<RegisterCoalescer>(); + au.addPreservedID(RegisterCoalescerPassID); au.addPreserved<CalculateSpillWeights>(); au.addPreserved<LiveStacks>(); au.addRequired<SlotIndexes>(); diff --git a/lib/CodeGen/StackProtector.cpp b/lib/CodeGen/StackProtector.cpp index d3cbd15..1f0e5a2 100644 --- a/lib/CodeGen/StackProtector.cpp +++ b/lib/CodeGen/StackProtector.cpp @@ -123,7 +123,7 @@ bool StackProtector::RequiresStackProtector() const { // protectors. return true; - if (const ArrayType *AT = dyn_cast<ArrayType>(AI->getAllocatedType())) { + if (ArrayType *AT = dyn_cast<ArrayType>(AI->getAllocatedType())) { // We apparently only care about character arrays. if (!AT->getElementType()->isIntegerTy(8)) continue; @@ -165,7 +165,7 @@ bool StackProtector::InsertStackProtectors() { // StackGuard = load __stack_chk_guard // call void @llvm.stackprotect.create(StackGuard, StackGuardSlot) // - const PointerType *PtrTy = Type::getInt8PtrTy(RI->getContext()); + PointerType *PtrTy = Type::getInt8PtrTy(RI->getContext()); unsigned AddressSpace, Offset; if (TLI->getStackCookieLocation(AddressSpace, Offset)) { Constant *OffsetVal = diff --git a/lib/CodeGen/StrongPHIElimination.cpp b/lib/CodeGen/StrongPHIElimination.cpp index 227eb47..260cc0e 100644 --- a/lib/CodeGen/StrongPHIElimination.cpp +++ b/lib/CodeGen/StrongPHIElimination.cpp @@ -47,6 +47,7 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Target/TargetInstrInfo.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/Statistic.h" #include "llvm/Support/Debug.h" using namespace llvm; @@ -673,7 +674,7 @@ void StrongPHIElimination::InsertCopiesForPHI(MachineInstr *PHI, if (PHIColor && SrcColor == PHIColor) { LiveInterval &SrcInterval = LI->getInterval(SrcReg); SlotIndex PredIndex = LI->getMBBEndIdx(PredBB); - VNInfo *SrcVNI = SrcInterval.getVNInfoAt(PredIndex.getPrevIndex()); + VNInfo *SrcVNI = SrcInterval.getVNInfoBefore(PredIndex); assert(SrcVNI); SrcVNI->setHasPHIKill(true); continue; diff --git a/lib/CodeGen/TailDuplication.cpp b/lib/CodeGen/TailDuplication.cpp index 6b801cb..3a6211a 100644 --- a/lib/CodeGen/TailDuplication.cpp +++ b/lib/CodeGen/TailDuplication.cpp @@ -25,6 +25,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/Statistic.h" diff --git a/lib/CodeGen/TargetInstrInfoImpl.cpp b/lib/CodeGen/TargetInstrInfoImpl.cpp index 86e71d8..f32678f 100644 --- a/lib/CodeGen/TargetInstrInfoImpl.cpp +++ b/lib/CodeGen/TargetInstrInfoImpl.cpp @@ -74,23 +74,25 @@ MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI, assert(MI->getOperand(Idx1).isReg() && MI->getOperand(Idx2).isReg() && "This only knows how to commute register operands so far"); + unsigned Reg0 = HasDef ? MI->getOperand(0).getReg() : 0; unsigned Reg1 = MI->getOperand(Idx1).getReg(); unsigned Reg2 = MI->getOperand(Idx2).getReg(); bool Reg1IsKill = MI->getOperand(Idx1).isKill(); bool Reg2IsKill = MI->getOperand(Idx2).isKill(); - bool ChangeReg0 = false; - if (HasDef && MI->getOperand(0).getReg() == Reg1) { - // Must be two address instruction! - assert(MI->getDesc().getOperandConstraint(0, MCOI::TIED_TO) && - "Expecting a two-address instruction!"); + // If destination is tied to either of the commuted source register, then + // it must be updated. + if (HasDef && Reg0 == Reg1 && + MI->getDesc().getOperandConstraint(Idx1, MCOI::TIED_TO) == 0) { Reg2IsKill = false; - ChangeReg0 = true; + Reg0 = Reg2; + } else if (HasDef && Reg0 == Reg2 && + MI->getDesc().getOperandConstraint(Idx2, MCOI::TIED_TO) == 0) { + Reg1IsKill = false; + Reg0 = Reg1; } if (NewMI) { // Create a new instruction. - unsigned Reg0 = HasDef - ? (ChangeReg0 ? Reg2 : MI->getOperand(0).getReg()) : 0; bool Reg0IsDead = HasDef ? MI->getOperand(0).isDead() : false; MachineFunction &MF = *MI->getParent()->getParent(); if (HasDef) @@ -104,8 +106,8 @@ MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI, .addReg(Reg1, getKillRegState(Reg2IsKill)); } - if (ChangeReg0) - MI->getOperand(0).setReg(Reg2); + if (HasDef) + MI->getOperand(0).setReg(Reg0); MI->getOperand(Idx2).setReg(Reg1); MI->getOperand(Idx1).setReg(Reg2); MI->getOperand(Idx2).setIsKill(Reg1IsKill); @@ -160,6 +162,42 @@ bool TargetInstrInfoImpl::PredicateInstruction(MachineInstr *MI, return MadeChange; } +bool TargetInstrInfoImpl::hasLoadFromStackSlot(const MachineInstr *MI, + const MachineMemOperand *&MMO, + int &FrameIndex) const { + for (MachineInstr::mmo_iterator o = MI->memoperands_begin(), + oe = MI->memoperands_end(); + o != oe; + ++o) { + if ((*o)->isLoad() && (*o)->getValue()) + if (const FixedStackPseudoSourceValue *Value = + dyn_cast<const FixedStackPseudoSourceValue>((*o)->getValue())) { + FrameIndex = Value->getFrameIndex(); + MMO = *o; + return true; + } + } + return false; +} + +bool TargetInstrInfoImpl::hasStoreToStackSlot(const MachineInstr *MI, + const MachineMemOperand *&MMO, + int &FrameIndex) const { + for (MachineInstr::mmo_iterator o = MI->memoperands_begin(), + oe = MI->memoperands_end(); + o != oe; + ++o) { + if ((*o)->isStore() && (*o)->getValue()) + if (const FixedStackPseudoSourceValue *Value = + dyn_cast<const FixedStackPseudoSourceValue>((*o)->getValue())) { + FrameIndex = Value->getFrameIndex(); + MMO = *o; + return true; + } + } + return false; +} + void TargetInstrInfoImpl::reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, unsigned DestReg, @@ -324,6 +362,19 @@ isReallyTriviallyReMaterializableGeneric(const MachineInstr *MI, const TargetInstrInfo &TII = *TM.getInstrInfo(); const TargetRegisterInfo &TRI = *TM.getRegisterInfo(); + // Remat clients assume operand 0 is the defined register. + if (!MI->getNumOperands() || !MI->getOperand(0).isReg()) + return false; + unsigned DefReg = MI->getOperand(0).getReg(); + + // A sub-register definition can only be rematerialized if the instruction + // doesn't read the other parts of the register. Otherwise it is really a + // read-modify-write operation on the full virtual register which cannot be + // moved safely. + if (TargetRegisterInfo::isVirtualRegister(DefReg) && + MI->getOperand(0).getSubReg() && MI->readsVirtualRegister(DefReg)) + return false; + // A load from a fixed stack slot can be rematerialized. This may be // redundant with subsequent checks, but it's target-independent, // simple, and a common case. @@ -383,8 +434,9 @@ isReallyTriviallyReMaterializableGeneric(const MachineInstr *MI, continue; } - // Only allow one virtual-register def, and that in the first operand. - if (MO.isDef() != (i == 0)) + // Only allow one virtual-register def. There may be multiple defs of the + // same virtual register, though. + if (MO.isDef() && Reg != DefReg) return false; // Don't allow any virtual-register uses. Rematting an instruction with diff --git a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index a3c5620..fb87154 100644 --- a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -43,153 +43,6 @@ using namespace dwarf; // ELF //===----------------------------------------------------------------------===// -TargetLoweringObjectFileELF::TargetLoweringObjectFileELF() - : TargetLoweringObjectFile(), - TLSDataSection(0), - TLSBSSSection(0), - DataRelSection(0), - DataRelLocalSection(0), - DataRelROSection(0), - DataRelROLocalSection(0), - MergeableConst4Section(0), - MergeableConst8Section(0), - MergeableConst16Section(0) { -} - -void TargetLoweringObjectFileELF::Initialize(MCContext &Ctx, - const TargetMachine &TM) { - TargetLoweringObjectFile::Initialize(Ctx, TM); - - BSSSection = - getContext().getELFSection(".bss", ELF::SHT_NOBITS, - ELF::SHF_WRITE |ELF::SHF_ALLOC, - SectionKind::getBSS()); - - TextSection = - getContext().getELFSection(".text", ELF::SHT_PROGBITS, - ELF::SHF_EXECINSTR | - ELF::SHF_ALLOC, - SectionKind::getText()); - - DataSection = - getContext().getELFSection(".data", ELF::SHT_PROGBITS, - ELF::SHF_WRITE |ELF::SHF_ALLOC, - SectionKind::getDataRel()); - - ReadOnlySection = - getContext().getELFSection(".rodata", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC, - SectionKind::getReadOnly()); - - TLSDataSection = - getContext().getELFSection(".tdata", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC | ELF::SHF_TLS | - ELF::SHF_WRITE, - SectionKind::getThreadData()); - - TLSBSSSection = - getContext().getELFSection(".tbss", ELF::SHT_NOBITS, - ELF::SHF_ALLOC | ELF::SHF_TLS | - ELF::SHF_WRITE, - SectionKind::getThreadBSS()); - - DataRelSection = - getContext().getELFSection(".data.rel", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC |ELF::SHF_WRITE, - SectionKind::getDataRel()); - - DataRelLocalSection = - getContext().getELFSection(".data.rel.local", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC |ELF::SHF_WRITE, - SectionKind::getDataRelLocal()); - - DataRelROSection = - getContext().getELFSection(".data.rel.ro", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC |ELF::SHF_WRITE, - SectionKind::getReadOnlyWithRel()); - - DataRelROLocalSection = - getContext().getELFSection(".data.rel.ro.local", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC |ELF::SHF_WRITE, - SectionKind::getReadOnlyWithRelLocal()); - - MergeableConst4Section = - getContext().getELFSection(".rodata.cst4", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC |ELF::SHF_MERGE, - SectionKind::getMergeableConst4()); - - MergeableConst8Section = - getContext().getELFSection(".rodata.cst8", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC |ELF::SHF_MERGE, - SectionKind::getMergeableConst8()); - - MergeableConst16Section = - getContext().getELFSection(".rodata.cst16", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC |ELF::SHF_MERGE, - SectionKind::getMergeableConst16()); - - StaticCtorSection = - getContext().getELFSection(".ctors", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC |ELF::SHF_WRITE, - SectionKind::getDataRel()); - - StaticDtorSection = - getContext().getELFSection(".dtors", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC |ELF::SHF_WRITE, - SectionKind::getDataRel()); - - // Exception Handling Sections. - - // FIXME: We're emitting LSDA info into a readonly section on ELF, even though - // it contains relocatable pointers. In PIC mode, this is probably a big - // runtime hit for C++ apps. Either the contents of the LSDA need to be - // adjusted or this should be a data section. - LSDASection = - getContext().getELFSection(".gcc_except_table", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC, - SectionKind::getReadOnly()); - // Debug Info Sections. - DwarfAbbrevSection = - getContext().getELFSection(".debug_abbrev", ELF::SHT_PROGBITS, 0, - SectionKind::getMetadata()); - DwarfInfoSection = - getContext().getELFSection(".debug_info", ELF::SHT_PROGBITS, 0, - SectionKind::getMetadata()); - DwarfLineSection = - getContext().getELFSection(".debug_line", ELF::SHT_PROGBITS, 0, - SectionKind::getMetadata()); - DwarfFrameSection = - getContext().getELFSection(".debug_frame", ELF::SHT_PROGBITS, 0, - SectionKind::getMetadata()); - DwarfPubNamesSection = - getContext().getELFSection(".debug_pubnames", ELF::SHT_PROGBITS, 0, - SectionKind::getMetadata()); - DwarfPubTypesSection = - getContext().getELFSection(".debug_pubtypes", ELF::SHT_PROGBITS, 0, - SectionKind::getMetadata()); - DwarfStrSection = - getContext().getELFSection(".debug_str", ELF::SHT_PROGBITS, 0, - SectionKind::getMetadata()); - DwarfLocSection = - getContext().getELFSection(".debug_loc", ELF::SHT_PROGBITS, 0, - SectionKind::getMetadata()); - DwarfARangesSection = - getContext().getELFSection(".debug_aranges", ELF::SHT_PROGBITS, 0, - SectionKind::getMetadata()); - DwarfRangesSection = - getContext().getELFSection(".debug_ranges", ELF::SHT_PROGBITS, 0, - SectionKind::getMetadata()); - DwarfMacroInfoSection = - getContext().getELFSection(".debug_macinfo", ELF::SHT_PROGBITS, 0, - SectionKind::getMetadata()); -} - -const MCSection *TargetLoweringObjectFileELF::getEHFrameSection() const { - return getContext().getELFSection(".eh_frame", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC, - SectionKind::getDataRel()); -} - MCSymbol * TargetLoweringObjectFileELF::getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang, @@ -493,221 +346,6 @@ getExprForDwarfGlobalReference(const GlobalValue *GV, Mangler *Mang, // MachO //===----------------------------------------------------------------------===// -TargetLoweringObjectFileMachO::TargetLoweringObjectFileMachO() - : TargetLoweringObjectFile(), - TLSDataSection(0), - TLSBSSSection(0), - TLSTLVSection(0), - TLSThreadInitSection(0), - CStringSection(0), - UStringSection(0), - TextCoalSection(0), - ConstTextCoalSection(0), - ConstDataSection(0), - DataCoalSection(0), - DataCommonSection(0), - DataBSSSection(0), - FourByteConstantSection(0), - EightByteConstantSection(0), - SixteenByteConstantSection(0), - LazySymbolPointerSection(0), - NonLazySymbolPointerSection(0) { -} - -void TargetLoweringObjectFileMachO::Initialize(MCContext &Ctx, - const TargetMachine &TM) { - IsFunctionEHFrameSymbolPrivate = false; - SupportsWeakOmittedEHFrame = false; - - // .comm doesn't support alignment before Leopard. - Triple T(((LLVMTargetMachine&)TM).getTargetTriple()); - if (T.isMacOSX() && T.isMacOSXVersionLT(10, 5)) - CommDirectiveSupportsAlignment = false; - - TargetLoweringObjectFile::Initialize(Ctx, TM); - - TextSection // .text - = getContext().getMachOSection("__TEXT", "__text", - MCSectionMachO::S_ATTR_PURE_INSTRUCTIONS, - SectionKind::getText()); - DataSection // .data - = getContext().getMachOSection("__DATA", "__data", 0, - SectionKind::getDataRel()); - - TLSDataSection // .tdata - = getContext().getMachOSection("__DATA", "__thread_data", - MCSectionMachO::S_THREAD_LOCAL_REGULAR, - SectionKind::getDataRel()); - TLSBSSSection // .tbss - = getContext().getMachOSection("__DATA", "__thread_bss", - MCSectionMachO::S_THREAD_LOCAL_ZEROFILL, - SectionKind::getThreadBSS()); - - // TODO: Verify datarel below. - TLSTLVSection // .tlv - = getContext().getMachOSection("__DATA", "__thread_vars", - MCSectionMachO::S_THREAD_LOCAL_VARIABLES, - SectionKind::getDataRel()); - - TLSThreadInitSection - = getContext().getMachOSection("__DATA", "__thread_init", - MCSectionMachO::S_THREAD_LOCAL_INIT_FUNCTION_POINTERS, - SectionKind::getDataRel()); - - CStringSection // .cstring - = getContext().getMachOSection("__TEXT", "__cstring", - MCSectionMachO::S_CSTRING_LITERALS, - SectionKind::getMergeable1ByteCString()); - UStringSection - = getContext().getMachOSection("__TEXT","__ustring", 0, - SectionKind::getMergeable2ByteCString()); - FourByteConstantSection // .literal4 - = getContext().getMachOSection("__TEXT", "__literal4", - MCSectionMachO::S_4BYTE_LITERALS, - SectionKind::getMergeableConst4()); - EightByteConstantSection // .literal8 - = getContext().getMachOSection("__TEXT", "__literal8", - MCSectionMachO::S_8BYTE_LITERALS, - SectionKind::getMergeableConst8()); - - // ld_classic doesn't support .literal16 in 32-bit mode, and ld64 falls back - // to using it in -static mode. - SixteenByteConstantSection = 0; - if (TM.getRelocationModel() != Reloc::Static && - TM.getTargetData()->getPointerSize() == 32) - SixteenByteConstantSection = // .literal16 - getContext().getMachOSection("__TEXT", "__literal16", - MCSectionMachO::S_16BYTE_LITERALS, - SectionKind::getMergeableConst16()); - - ReadOnlySection // .const - = getContext().getMachOSection("__TEXT", "__const", 0, - SectionKind::getReadOnly()); - - TextCoalSection - = getContext().getMachOSection("__TEXT", "__textcoal_nt", - MCSectionMachO::S_COALESCED | - MCSectionMachO::S_ATTR_PURE_INSTRUCTIONS, - SectionKind::getText()); - ConstTextCoalSection - = getContext().getMachOSection("__TEXT", "__const_coal", - MCSectionMachO::S_COALESCED, - SectionKind::getReadOnly()); - ConstDataSection // .const_data - = getContext().getMachOSection("__DATA", "__const", 0, - SectionKind::getReadOnlyWithRel()); - DataCoalSection - = getContext().getMachOSection("__DATA","__datacoal_nt", - MCSectionMachO::S_COALESCED, - SectionKind::getDataRel()); - DataCommonSection - = getContext().getMachOSection("__DATA","__common", - MCSectionMachO::S_ZEROFILL, - SectionKind::getBSS()); - DataBSSSection - = getContext().getMachOSection("__DATA","__bss", MCSectionMachO::S_ZEROFILL, - SectionKind::getBSS()); - - - LazySymbolPointerSection - = getContext().getMachOSection("__DATA", "__la_symbol_ptr", - MCSectionMachO::S_LAZY_SYMBOL_POINTERS, - SectionKind::getMetadata()); - NonLazySymbolPointerSection - = getContext().getMachOSection("__DATA", "__nl_symbol_ptr", - MCSectionMachO::S_NON_LAZY_SYMBOL_POINTERS, - SectionKind::getMetadata()); - - if (TM.getRelocationModel() == Reloc::Static) { - StaticCtorSection - = getContext().getMachOSection("__TEXT", "__constructor", 0, - SectionKind::getDataRel()); - StaticDtorSection - = getContext().getMachOSection("__TEXT", "__destructor", 0, - SectionKind::getDataRel()); - } else { - StaticCtorSection - = getContext().getMachOSection("__DATA", "__mod_init_func", - MCSectionMachO::S_MOD_INIT_FUNC_POINTERS, - SectionKind::getDataRel()); - StaticDtorSection - = getContext().getMachOSection("__DATA", "__mod_term_func", - MCSectionMachO::S_MOD_TERM_FUNC_POINTERS, - SectionKind::getDataRel()); - } - - // Exception Handling. - LSDASection = getContext().getMachOSection("__TEXT", "__gcc_except_tab", 0, - SectionKind::getReadOnlyWithRel()); - - if (T.isMacOSX() && !T.isMacOSXVersionLT(10, 6)) - CompactUnwindSection = - getContext().getMachOSection("__LD", "__compact_unwind", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getReadOnly()); - - // Debug Information. - DwarfAbbrevSection = - getContext().getMachOSection("__DWARF", "__debug_abbrev", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfInfoSection = - getContext().getMachOSection("__DWARF", "__debug_info", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfLineSection = - getContext().getMachOSection("__DWARF", "__debug_line", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfFrameSection = - getContext().getMachOSection("__DWARF", "__debug_frame", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfPubNamesSection = - getContext().getMachOSection("__DWARF", "__debug_pubnames", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfPubTypesSection = - getContext().getMachOSection("__DWARF", "__debug_pubtypes", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfStrSection = - getContext().getMachOSection("__DWARF", "__debug_str", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfLocSection = - getContext().getMachOSection("__DWARF", "__debug_loc", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfARangesSection = - getContext().getMachOSection("__DWARF", "__debug_aranges", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfRangesSection = - getContext().getMachOSection("__DWARF", "__debug_ranges", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfMacroInfoSection = - getContext().getMachOSection("__DWARF", "__debug_macinfo", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - DwarfDebugInlineSection = - getContext().getMachOSection("__DWARF", "__debug_inlined", - MCSectionMachO::S_ATTR_DEBUG, - SectionKind::getMetadata()); - - TLSExtraDataSection = TLSTLVSection; -} - -const MCSection *TargetLoweringObjectFileMachO::getEHFrameSection() const { - return getContext().getMachOSection("__TEXT", "__eh_frame", - MCSectionMachO::S_COALESCED | - MCSectionMachO::S_ATTR_NO_TOC | - MCSectionMachO::S_ATTR_STRIP_STATIC_SYMS | - MCSectionMachO::S_ATTR_LIVE_SUPPORT, - SectionKind::getReadOnly()); -} - const MCSection *TargetLoweringObjectFileMachO:: getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind, Mangler *Mang, const TargetMachine &TM) const { @@ -905,183 +543,10 @@ getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang, return SSym; } -unsigned TargetLoweringObjectFileMachO::getPersonalityEncoding() const { - return DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4; -} - -unsigned TargetLoweringObjectFileMachO::getLSDAEncoding() const { - return DW_EH_PE_pcrel; -} - -unsigned TargetLoweringObjectFileMachO::getFDEEncoding(bool CFI) const { - return DW_EH_PE_pcrel; -} - -unsigned TargetLoweringObjectFileMachO::getTTypeEncoding() const { - return DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4; -} - //===----------------------------------------------------------------------===// // COFF //===----------------------------------------------------------------------===// -TargetLoweringObjectFileCOFF::TargetLoweringObjectFileCOFF() - : TargetLoweringObjectFile(), - DrectveSection(0), - PDataSection(0), - XDataSection(0) { -} - -void TargetLoweringObjectFileCOFF::Initialize(MCContext &Ctx, - const TargetMachine &TM) { - TargetLoweringObjectFile::Initialize(Ctx, TM); - TextSection = - getContext().getCOFFSection(".text", - COFF::IMAGE_SCN_CNT_CODE | - COFF::IMAGE_SCN_MEM_EXECUTE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getText()); - DataSection = - getContext().getCOFFSection(".data", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); - ReadOnlySection = - getContext().getCOFFSection(".rdata", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getReadOnly()); - StaticCtorSection = - getContext().getCOFFSection(".ctors", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); - StaticDtorSection = - getContext().getCOFFSection(".dtors", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); - - // FIXME: We're emitting LSDA info into a readonly section on COFF, even - // though it contains relocatable pointers. In PIC mode, this is probably a - // big runtime hit for C++ apps. Either the contents of the LSDA need to be - // adjusted or this should be a data section. - LSDASection = - getContext().getCOFFSection(".gcc_except_table", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getReadOnly()); - // Debug info. - DwarfAbbrevSection = - getContext().getCOFFSection(".debug_abbrev", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfInfoSection = - getContext().getCOFFSection(".debug_info", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfLineSection = - getContext().getCOFFSection(".debug_line", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfFrameSection = - getContext().getCOFFSection(".debug_frame", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfPubNamesSection = - getContext().getCOFFSection(".debug_pubnames", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfPubTypesSection = - getContext().getCOFFSection(".debug_pubtypes", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfStrSection = - getContext().getCOFFSection(".debug_str", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfLocSection = - getContext().getCOFFSection(".debug_loc", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfARangesSection = - getContext().getCOFFSection(".debug_aranges", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfRangesSection = - getContext().getCOFFSection(".debug_ranges", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - DwarfMacroInfoSection = - getContext().getCOFFSection(".debug_macinfo", - COFF::IMAGE_SCN_MEM_DISCARDABLE | - COFF::IMAGE_SCN_MEM_READ, - SectionKind::getMetadata()); - - DrectveSection = - getContext().getCOFFSection(".drectve", - COFF::IMAGE_SCN_LNK_INFO, - SectionKind::getMetadata()); - - PDataSection = - getContext().getCOFFSection(".pdata", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); - - XDataSection = - getContext().getCOFFSection(".xdata", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); -} - -const MCSection *TargetLoweringObjectFileCOFF::getEHFrameSection() const { - return getContext().getCOFFSection(".eh_frame", - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); -} - -const MCSection *TargetLoweringObjectFileCOFF::getWin64EHFuncTableSection( - StringRef suffix) const { - if (suffix == "") - return PDataSection; - return getContext().getCOFFSection((".pdata"+suffix).str(), - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); -} - -const MCSection *TargetLoweringObjectFileCOFF::getWin64EHTableSection( - StringRef suffix) const { - if (suffix == "") - return XDataSection; - return getContext().getCOFFSection((".xdata"+suffix).str(), - COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | - COFF::IMAGE_SCN_MEM_READ | - COFF::IMAGE_SCN_MEM_WRITE, - SectionKind::getDataRel()); -} - - static unsigned getCOFFSectionFlags(SectionKind K) { unsigned Flags = 0; diff --git a/lib/CodeGen/TwoAddressInstructionPass.cpp b/lib/CodeGen/TwoAddressInstructionPass.cpp index 6d6244e..d879378 100644 --- a/lib/CodeGen/TwoAddressInstructionPass.cpp +++ b/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -177,6 +177,10 @@ char &llvm::TwoAddressInstructionPassID = TwoAddressInstructionPass::ID; bool TwoAddressInstructionPass::Sink3AddrInstruction(MachineBasicBlock *MBB, MachineInstr *MI, unsigned SavedReg, MachineBasicBlock::iterator OldPos) { + // FIXME: Shouldn't we be trying to do this before we three-addressify the + // instruction? After this transformation is done, we no longer need + // the instruction to be in three-address form. + // Check if it's safe to move this instruction. bool SeenStore = true; // Be conservative. if (!MI->isSafeToMove(TII, AA, SeenStore)) @@ -217,7 +221,11 @@ bool TwoAddressInstructionPass::Sink3AddrInstruction(MachineBasicBlock *MBB, break; } - if (!KillMI || KillMI->getParent() != MBB || KillMI == MI) + // If we find the instruction that kills SavedReg, and it is in an + // appropriate location, we can try to sink the current instruction + // past it. + if (!KillMI || KillMI->getParent() != MBB || KillMI == MI || + KillMI->getDesc().isTerminator()) return false; // If any of the definitions are used by another instruction between the @@ -1041,6 +1049,9 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &MF) { DEBUG(dbgs() << "********** Function: " << MF.getFunction()->getName() << '\n'); + // This pass takes the function out of SSA form. + MRI->leaveSSA(); + // ReMatRegs - Keep track of the registers whose def's are remat'ed. BitVector ReMatRegs(MRI->getNumVirtRegs()); diff --git a/lib/CodeGen/VirtRegMap.cpp b/lib/CodeGen/VirtRegMap.cpp index 7557979..8a1cdc0 100644 --- a/lib/CodeGen/VirtRegMap.cpp +++ b/lib/CodeGen/VirtRegMap.cpp @@ -41,8 +41,8 @@ #include <algorithm> using namespace llvm; -STATISTIC(NumSpills , "Number of register spills"); -STATISTIC(NumIdCopies, "Number of identity moves eliminated after rewriting"); +STATISTIC(NumSpillSlots, "Number of spill slots allocated"); +STATISTIC(NumIdCopies, "Number of identity moves eliminated after rewriting"); //===----------------------------------------------------------------------===// // VirtRegMap implementation @@ -111,6 +111,7 @@ unsigned VirtRegMap::createSpillSlot(const TargetRegisterClass *RC) { unsigned Idx = SS-LowSpillSlot; while (Idx >= SpillSlotToUsesMap.size()) SpillSlotToUsesMap.resize(SpillSlotToUsesMap.size()*2); + ++NumSpillSlots; return SS; } @@ -130,7 +131,6 @@ int VirtRegMap::assignVirt2StackSlot(unsigned virtReg) { assert(Virt2StackSlotMap[virtReg] == NO_STACK_SLOT && "attempt to assign stack slot to already spilled register"); const TargetRegisterClass* RC = MF->getRegInfo().getRegClass(virtReg); - ++NumSpills; return Virt2StackSlotMap[virtReg] = createSpillSlot(RC); } @@ -285,14 +285,24 @@ void VirtRegMap::rewrite(SlotIndexes *Indexes) { // Preserve semantics of sub-register operands. if (MO.getSubReg()) { // A virtual register kill refers to the whole register, so we may - // have to add <imp-use,kill> operands for the super-register. - if (MO.isUse()) { - if (MO.isKill() && !MO.isUndef()) - SuperKills.push_back(PhysReg); - } else if (MO.isDead()) - SuperDeads.push_back(PhysReg); - else - SuperDefs.push_back(PhysReg); + // have to add <imp-use,kill> operands for the super-register. A + // partial redef always kills and redefines the super-register. + if (MO.readsReg() && (MO.isDef() || MO.isKill())) + SuperKills.push_back(PhysReg); + + if (MO.isDef()) { + // The <def,undef> flag only makes sense for sub-register defs, and + // we are substituting a full physreg. An <imp-use,kill> operand + // from the SuperKills list will represent the partial read of the + // super-register. + MO.setIsUndef(false); + + // Also add implicit defs for the super-register. + if (MO.isDead()) + SuperDeads.push_back(PhysReg); + else + SuperDefs.push_back(PhysReg); + } // PhysReg operands cannot have subregister indexes. PhysReg = TRI->getSubReg(PhysReg, MO.getSubReg()); |
