diff options
Diffstat (limited to 'lib/CodeGen')
61 files changed, 3441 insertions, 2261 deletions
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index ded4b3f..5a0c27b 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -205,16 +205,10 @@ void AsmPrinter::EmitLinkage(unsigned Linkage, MCSymbol *GVSym) const { OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Global); // .weak_definition _foo OutStreamer.EmitSymbolAttribute(GVSym, MCSA_WeakDefinition); - } else if (const char *LinkOnce = MAI->getLinkOnceDirective()) { + } else if (MAI->getLinkOnceDirective() != 0) { // .globl _foo OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Global); - // FIXME: linkonce should be a section attribute, handled by COFF Section - // assignment. - // http://sourceware.org/binutils/docs-2.20/as/Linkonce.html#Linkonce - // .linkonce discard - // FIXME: It would be nice to use .linkonce samesize for non-common - // globals. - OutStreamer.EmitRawText(StringRef(LinkOnce)); + //NOTE: linkonce is handled by the section the symbol was assigned to. } else { // .weak _foo OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Weak); @@ -247,6 +241,12 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { if (EmitSpecialLLVMGlobal(GV)) return; + if (isVerbose()) { + WriteAsOperand(OutStreamer.GetCommentOS(), GV, + /*PrintType=*/false, GV->getParent()); + OutStreamer.GetCommentOS() << '\n'; + } + MCSymbol *GVSym = Mang->getSymbol(GV); EmitVisibility(GVSym, GV->getVisibility()); @@ -316,17 +316,57 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { OutStreamer.EmitZerofill(TheSection, GVSym, Size, 1 << AlignLog); return; } + + // Handle thread local data for mach-o which requires us to output an + // additional structure of data and mangle the original symbol so that we + // can reference it later. + if (GVKind.isThreadLocal() && MAI->hasMachoTBSSDirective()) { + // Emit the .tbss symbol + MCSymbol *MangSym = + OutContext.GetOrCreateSymbol(GVSym->getName() + Twine("$tlv$init")); + + if (GVKind.isThreadBSS()) + OutStreamer.EmitTBSSSymbol(TheSection, MangSym, Size, 1 << AlignLog); + else if (GVKind.isThreadData()) { + OutStreamer.SwitchSection(TheSection); + + EmitLinkage(GV->getLinkage(), MangSym); + EmitAlignment(AlignLog, GV); + OutStreamer.EmitLabel(MangSym); + + EmitGlobalConstant(GV->getInitializer()); + } + + OutStreamer.AddBlankLine(); + + // Emit the variable struct for the runtime. + const MCSection *TLVSect + = getObjFileLowering().getTLSExtraDataSection(); + + OutStreamer.SwitchSection(TLVSect); + // Emit the linkage here. + EmitLinkage(GV->getLinkage(), GVSym); + OutStreamer.EmitLabel(GVSym); + + // Three pointers in size: + // - __tlv_bootstrap - used to make sure support exists + // - spare pointer, used when mapped by the runtime + // - pointer to mangled symbol above with initializer + unsigned PtrSize = TD->getPointerSizeInBits()/8; + OutStreamer.EmitSymbolValue(GetExternalSymbolSymbol("__tlv_bootstrap"), + PtrSize, 0); + OutStreamer.EmitIntValue(0, PtrSize, 0); + OutStreamer.EmitSymbolValue(MangSym, PtrSize, 0); + + OutStreamer.AddBlankLine(); + return; + } OutStreamer.SwitchSection(TheSection); EmitLinkage(GV->getLinkage(), GVSym); EmitAlignment(AlignLog, GV); - if (isVerbose()) { - WriteAsOperand(OutStreamer.GetCommentOS(), GV, - /*PrintType=*/false, GV->getParent()); - OutStreamer.GetCommentOS() << '\n'; - } OutStreamer.EmitLabel(GVSym); EmitGlobalConstant(GV->getInitializer()); @@ -408,7 +448,13 @@ void AsmPrinter::EmitFunctionHeader() { /// EmitFunctionEntryLabel - Emit the label that is the entrypoint for the /// function. This can be overridden by targets as required to do custom stuff. void AsmPrinter::EmitFunctionEntryLabel() { - OutStreamer.EmitLabel(CurrentFnSym); + // The function label could have already been emitted if two symbols end up + // conflicting due to asm renaming. Detect this and emit an error. + if (CurrentFnSym->isUndefined()) + return OutStreamer.EmitLabel(CurrentFnSym); + + report_fatal_error("'" + Twine(CurrentFnSym->getName()) + + "' label emitted multiple times to assembly file"); } @@ -503,7 +549,7 @@ static bool EmitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) { // cast away const; DIetc do not take const operands for some reason. DIVariable V(const_cast<MDNode*>(MI->getOperand(2).getMetadata())); if (V.getContext().isSubprogram()) - OS << DISubprogram(V.getContext().getNode()).getDisplayName() << ":"; + OS << DISubprogram(V.getContext()).getDisplayName() << ":"; OS << V.getName() << " <- "; // Register or immediate value. Register 0 means undef. diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp index 37d10e5..ba6fed2 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp @@ -53,6 +53,17 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, unsigned LocCookie) const { } SourceMgr SrcMgr; + + // Ensure the buffer is newline terminated. + char *TmpString = 0; + if (Str.back() != '\n') { + TmpString = new char[Str.size() + 2]; + memcpy(TmpString, Str.data(), Str.size()); + TmpString[Str.size()] = '\n'; + TmpString[Str.size() + 1] = 0; + isNullTerminated = true; + Str = TmpString; + } // If the current LLVMContext has an inline asm handler, set it in SourceMgr. LLVMContext &LLVMCtx = MMI->getModule()->getContext(); @@ -84,6 +95,9 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, unsigned LocCookie) const { /*NoFinalize*/ true); if (Res && !HasDiagHandler) report_fatal_error("Error parsing inline asm\n"); + + if (TmpString) + delete[] TmpString; } diff --git a/lib/CodeGen/AsmPrinter/DIE.h b/lib/CodeGen/AsmPrinter/DIE.h index 9cb8314..d56c094 100644 --- a/lib/CodeGen/AsmPrinter/DIE.h +++ b/lib/CodeGen/AsmPrinter/DIE.h @@ -315,6 +315,10 @@ namespace llvm { /// virtual void EmitValue(AsmPrinter *AP, unsigned Form) const; + /// getValue - Get MCSymbol. + /// + const MCSymbol *getValue() const { return Label; } + /// SizeOf - Determine size of label value in bytes. /// virtual unsigned SizeOf(AsmPrinter *AP, unsigned Form) const; diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index e9e9ba5..890507c 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -47,6 +47,10 @@ static cl::opt<bool> PrintDbgScope("print-dbgscope", cl::Hidden, static cl::opt<bool> DisableDebugInfoPrinting("disable-debug-info-print", cl::Hidden, cl::desc("Disable debug info printing")); +static cl::opt<bool> UnknownLocations("use-unknown-locations", cl::Hidden, + cl::desc("Make an absense of debug location information explicit."), + cl::init(false)); + namespace { const char *DWARFGroupName = "DWARF Emission"; const char *DbgTimerName = "DWARF Debug Writer"; @@ -78,12 +82,12 @@ class CompileUnit { /// GVToDieMap - Tracks the mapping of unit level debug informaton /// variables to debug information entries. /// FIXME : Rename GVToDieMap -> NodeToDieMap - DenseMap<MDNode *, DIE *> GVToDieMap; + DenseMap<const MDNode *, DIE *> GVToDieMap; /// GVToDIEEntryMap - Tracks the mapping of unit level debug informaton /// descriptors to debug information entries using a DIEEntry proxy. /// FIXME : Rename - DenseMap<MDNode *, DIEEntry *> GVToDIEEntryMap; + DenseMap<const MDNode *, DIEEntry *> GVToDIEEntryMap; /// Globals - A map of globally visible named entities for this unit. /// @@ -119,24 +123,24 @@ public: /// getDIE - Returns the debug information entry map slot for the /// specified debug variable. - DIE *getDIE(MDNode *N) { return GVToDieMap.lookup(N); } + DIE *getDIE(const MDNode *N) { return GVToDieMap.lookup(N); } /// insertDIE - Insert DIE into the map. - void insertDIE(MDNode *N, DIE *D) { + void insertDIE(const MDNode *N, DIE *D) { GVToDieMap.insert(std::make_pair(N, D)); } /// getDIEEntry - Returns the debug information entry for the speciefied /// debug variable. - DIEEntry *getDIEEntry(MDNode *N) { - DenseMap<MDNode *, DIEEntry *>::iterator I = GVToDIEEntryMap.find(N); + DIEEntry *getDIEEntry(const MDNode *N) { + DenseMap<const MDNode *, DIEEntry *>::iterator I = GVToDIEEntryMap.find(N); if (I == GVToDIEEntryMap.end()) return NULL; return I->second; } /// insertDIEEntry - Insert debug information entry into the map. - void insertDIEEntry(MDNode *N, DIEEntry *E) { + void insertDIEEntry(const MDNode *N, DIEEntry *E) { GVToDIEEntryMap.insert(std::make_pair(N, E)); } @@ -164,31 +168,18 @@ public: /// class DbgVariable { DIVariable Var; // Variable Descriptor. - unsigned FrameIndex; // Variable frame index. - const MachineInstr *DbgValueMInsn; // DBG_VALUE - // DbgValueLabel - DBG_VALUE is effective from this label. - MCSymbol *DbgValueLabel; - DbgVariable *const AbstractVar; // Abstract variable for this variable. - DIE *TheDIE; + DIE *TheDIE; // Variable DIE. + unsigned DotDebugLocOffset; // Offset in DotDebugLocEntries. public: // AbsVar may be NULL. - DbgVariable(DIVariable V, unsigned I, DbgVariable *AbsVar) - : Var(V), FrameIndex(I), DbgValueMInsn(0), - DbgValueLabel(0), AbstractVar(AbsVar), TheDIE(0) {} - DbgVariable(DIVariable V, const MachineInstr *MI, DbgVariable *AbsVar) - : Var(V), FrameIndex(0), DbgValueMInsn(MI), DbgValueLabel(0), - AbstractVar(AbsVar), TheDIE(0) - {} + DbgVariable(DIVariable V) : Var(V), TheDIE(0), DotDebugLocOffset(~0U) {} // Accessors. DIVariable getVariable() const { return Var; } - unsigned getFrameIndex() const { return FrameIndex; } - const MachineInstr *getDbgValue() const { return DbgValueMInsn; } - MCSymbol *getDbgValueLabel() const { return DbgValueLabel; } - void setDbgValueLabel(MCSymbol *L) { DbgValueLabel = L; } - DbgVariable *getAbstractVariable() const { return AbstractVar; } void setDIE(DIE *D) { TheDIE = D; } DIE *getDIE() const { return TheDIE; } + void setDotDebugLocOffset(unsigned O) { DotDebugLocOffset = O; } + unsigned getDotDebugLocOffset() const { return DotDebugLocOffset; } }; //===----------------------------------------------------------------------===// @@ -204,7 +195,7 @@ class DbgScope { DbgScope *Parent; // Parent to this scope. DIDescriptor Desc; // Debug info descriptor for scope. // Location at which this scope is inlined. - AssertingVH<MDNode> InlinedAtLocation; + AssertingVH<const MDNode> InlinedAtLocation; bool AbstractScope; // Abstract Scope const MachineInstr *LastInsn; // Last instruction of this scope. const MachineInstr *FirstInsn; // First instruction of this scope. @@ -217,7 +208,7 @@ class DbgScope { // Private state for dump() mutable unsigned IndentLevel; public: - DbgScope(DbgScope *P, DIDescriptor D, MDNode *I = 0) + DbgScope(DbgScope *P, DIDescriptor D, const MDNode *I = 0) : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(false), LastInsn(0), FirstInsn(0), DFSIn(0), DFSOut(0), IndentLevel(0) {} @@ -227,8 +218,8 @@ public: DbgScope *getParent() const { return Parent; } void setParent(DbgScope *P) { Parent = P; } DIDescriptor getDesc() const { return Desc; } - MDNode *getInlinedAt() const { return InlinedAtLocation; } - MDNode *getScopeNode() const { return Desc.getNode(); } + const MDNode *getInlinedAt() const { return InlinedAtLocation; } + const MDNode *getScopeNode() const { return Desc; } const SmallVector<DbgScope *, 4> &getScopes() { return Scopes; } const SmallVector<DbgVariable *, 8> &getVariables() { return Variables; } const SmallVector<DbgRange, 4> &getRanges() { return Ranges; } @@ -300,7 +291,7 @@ public: void DbgScope::dump() const { raw_ostream &err = dbgs(); err.indent(IndentLevel); - MDNode *N = Desc.getNode(); + const MDNode *N = Desc; N->dump(); if (AbstractScope) err << "Abstract Scope\n"; @@ -322,15 +313,15 @@ DbgScope::~DbgScope() { } DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M) - : Asm(A), MMI(Asm->MMI), ModuleCU(0), + : Asm(A), MMI(Asm->MMI), FirstCU(0), AbbreviationsSet(InitAbbreviationsSetSize), CurrentFnDbgScope(0), PrevLabel(NULL) { NextStringPoolNumber = 0; DwarfFrameSectionSym = DwarfInfoSectionSym = DwarfAbbrevSectionSym = 0; DwarfStrSectionSym = TextSectionSym = 0; - DwarfDebugRangeSectionSym = 0; - FunctionBeginSym = 0; + DwarfDebugRangeSectionSym = DwarfDebugLocSectionSym = 0; + FunctionBeginSym = FunctionEndSym = 0; if (TimePassesIsEnabled) { NamedRegionTimer T(DbgTimerName, DWARFGroupName); beginModule(M); @@ -444,8 +435,8 @@ void DwarfDebug::addBlock(DIE *Die, unsigned Attribute, unsigned Form, /// addSourceLine - Add location information to specified debug information /// entry. void DwarfDebug::addSourceLine(DIE *Die, const DIVariable *V) { - // If there is no compile unit specified, don't add a line #. - if (!V->getCompileUnit().Verify()) + // Verify variable. + if (!V->Verify()) return; unsigned Line = V->getLineNumber(); @@ -458,9 +449,9 @@ void DwarfDebug::addSourceLine(DIE *Die, const DIVariable *V) { /// addSourceLine - Add location information to specified debug information /// entry. -void DwarfDebug::addSourceLine(DIE *Die, const DIGlobal *G) { - // If there is no compile unit specified, don't add a line #. - if (!G->getCompileUnit().Verify()) +void DwarfDebug::addSourceLine(DIE *Die, const DIGlobalVariable *G) { + // Verify global variable. + if (!G->Verify()) return; unsigned Line = G->getLineNumber(); @@ -474,8 +465,8 @@ void DwarfDebug::addSourceLine(DIE *Die, const DIGlobal *G) { /// addSourceLine - Add location information to specified debug information /// entry. void DwarfDebug::addSourceLine(DIE *Die, const DISubprogram *SP) { - // If there is no compile unit specified, don't add a line #. - if (!SP->getCompileUnit().Verify()) + // Verify subprogram. + if (!SP->Verify()) return; // If the line number is 0, don't add it. if (SP->getLineNumber() == 0) @@ -494,9 +485,8 @@ void DwarfDebug::addSourceLine(DIE *Die, const DISubprogram *SP) { /// addSourceLine - Add location information to specified debug information /// entry. void DwarfDebug::addSourceLine(DIE *Die, const DIType *Ty) { - // If there is no compile unit specified, don't add a line #. - DICompileUnit CU = Ty->getCompileUnit(); - if (!CU.Verify()) + // Verify type. + if (!Ty->Verify()) return; unsigned Line = Ty->getLineNumber(); @@ -512,8 +502,8 @@ void DwarfDebug::addSourceLine(DIE *Die, const DIType *Ty) { /// addSourceLine - Add location information to specified debug information /// entry. void DwarfDebug::addSourceLine(DIE *Die, const DINameSpace *NS) { - // If there is no compile unit specified, don't add a line #. - if (!NS->getCompileUnit().Verify()) + // Verify namespace. + if (!NS->Verify()) return; unsigned Line = NS->getLineNumber(); @@ -560,16 +550,16 @@ DIType DwarfDebug::getBlockByrefType(DIType Ty, std::string Name) { unsigned tag = Ty.getTag(); if (tag == dwarf::DW_TAG_pointer_type) { - DIDerivedType DTy = DIDerivedType(Ty.getNode()); + DIDerivedType DTy = DIDerivedType(Ty); subType = DTy.getTypeDerivedFrom(); } - DICompositeType blockStruct = DICompositeType(subType.getNode()); + DICompositeType blockStruct = DICompositeType(subType); DIArray Elements = blockStruct.getTypeArray(); for (unsigned i = 0, N = Elements.getNumElements(); i < N; ++i) { DIDescriptor Element = Elements.getElement(i); - DIDerivedType DT = DIDerivedType(Element.getNode()); + DIDerivedType DT = DIDerivedType(Element); if (Name == DT.getName()) return (DT.getTypeDerivedFrom()); } @@ -700,12 +690,12 @@ void DwarfDebug::addBlockByrefAddress(DbgVariable *&DV, DIE *Die, StringRef varName = VD.getName(); if (Tag == dwarf::DW_TAG_pointer_type) { - DIDerivedType DTy = DIDerivedType(Ty.getNode()); + DIDerivedType DTy = DIDerivedType(Ty); TmpTy = DTy.getTypeDerivedFrom(); isPointer = true; } - DICompositeType blockStruct = DICompositeType(TmpTy.getNode()); + DICompositeType blockStruct = DICompositeType(TmpTy); // Find the __forwarding field and the variable field in the __Block_byref // struct. @@ -715,7 +705,7 @@ void DwarfDebug::addBlockByrefAddress(DbgVariable *&DV, DIE *Die, for (unsigned i = 0, N = Fields.getNumElements(); i < N; ++i) { DIDescriptor Element = Fields.getElement(i); - DIDerivedType DT = DIDerivedType(Element.getNode()); + DIDerivedType DT = DIDerivedType(Element); StringRef fieldName = DT.getName(); if (fieldName == "__forwarding") forwardingField = Element; @@ -725,9 +715,9 @@ void DwarfDebug::addBlockByrefAddress(DbgVariable *&DV, DIE *Die, // Get the offsets for the forwarding field and the variable field. unsigned forwardingFieldOffset = - DIDerivedType(forwardingField.getNode()).getOffsetInBits() >> 3; + DIDerivedType(forwardingField).getOffsetInBits() >> 3; unsigned varFieldOffset = - DIDerivedType(varField.getNode()).getOffsetInBits() >> 3; + DIDerivedType(varField).getOffsetInBits() >> 3; // Decode the original location, and use that as the start of the byref // variable's location. @@ -813,7 +803,7 @@ void DwarfDebug::addAddress(DIE *Die, unsigned Attribute, } /// addRegisterAddress - Add register location entry in variable DIE. -bool DwarfDebug::addRegisterAddress(DIE *Die, DbgVariable *DV, +bool DwarfDebug::addRegisterAddress(DIE *Die, const MCSymbol *VS, const MachineOperand &MO) { assert (MO.isReg() && "Invalid machine operand!"); if (!MO.getReg()) @@ -821,26 +811,26 @@ bool DwarfDebug::addRegisterAddress(DIE *Die, DbgVariable *DV, MachineLocation Location; Location.set(MO.getReg()); addAddress(Die, dwarf::DW_AT_location, Location); - if (MCSymbol *VS = DV->getDbgValueLabel()) + if (VS) addLabel(Die, dwarf::DW_AT_start_scope, dwarf::DW_FORM_addr, VS); return true; } /// addConstantValue - Add constant value entry in variable DIE. -bool DwarfDebug::addConstantValue(DIE *Die, DbgVariable *DV, +bool DwarfDebug::addConstantValue(DIE *Die, const MCSymbol *VS, const MachineOperand &MO) { assert (MO.isImm() && "Invalid machine operand!"); DIEBlock *Block = new (DIEValueAllocator) DIEBlock(); unsigned Imm = MO.getImm(); addUInt(Block, 0, dwarf::DW_FORM_udata, Imm); addBlock(Die, dwarf::DW_AT_const_value, 0, Block); - if (MCSymbol *VS = DV->getDbgValueLabel()) + if (VS) addLabel(Die, dwarf::DW_AT_start_scope, dwarf::DW_FORM_addr, VS); return true; } /// addConstantFPValue - Add constant value entry in variable DIE. -bool DwarfDebug::addConstantFPValue(DIE *Die, DbgVariable *DV, +bool DwarfDebug::addConstantFPValue(DIE *Die, const MCSymbol *VS, const MachineOperand &MO) { assert (MO.isFPImm() && "Invalid machine operand!"); DIEBlock *Block = new (DIEValueAllocator) DIEBlock(); @@ -862,10 +852,8 @@ bool DwarfDebug::addConstantFPValue(DIE *Die, DbgVariable *DV, (unsigned char)0xFF & FltPtr[Start]); addBlock(Die, dwarf::DW_AT_const_value, 0, Block); - - if (MCSymbol *VS = DV->getDbgValueLabel()) - addLabel(Die, dwarf::DW_AT_start_scope, dwarf::DW_FORM_addr, - VS); + if (VS) + addLabel(Die, dwarf::DW_AT_start_scope, dwarf::DW_FORM_addr, VS); return true; } @@ -873,34 +861,35 @@ bool DwarfDebug::addConstantFPValue(DIE *Die, DbgVariable *DV, /// addToContextOwner - Add Die into the list of its context owner's children. void DwarfDebug::addToContextOwner(DIE *Die, DIDescriptor Context) { if (Context.isType()) { - DIE *ContextDIE = getOrCreateTypeDIE(DIType(Context.getNode())); + DIE *ContextDIE = getOrCreateTypeDIE(DIType(Context)); ContextDIE->addChild(Die); } else if (Context.isNameSpace()) { - DIE *ContextDIE = getOrCreateNameSpace(DINameSpace(Context.getNode())); + DIE *ContextDIE = getOrCreateNameSpace(DINameSpace(Context)); ContextDIE->addChild(Die); - } else if (DIE *ContextDIE = ModuleCU->getDIE(Context.getNode())) + } else if (DIE *ContextDIE = getCompileUnit(Context)->getDIE(Context)) ContextDIE->addChild(Die); else - ModuleCU->addDie(Die); + getCompileUnit(Context)->addDie(Die); } /// getOrCreateTypeDIE - Find existing DIE or create new DIE for the /// given DIType. DIE *DwarfDebug::getOrCreateTypeDIE(DIType Ty) { - DIE *TyDIE = ModuleCU->getDIE(Ty.getNode()); + CompileUnit *TypeCU = getCompileUnit(Ty); + DIE *TyDIE = TypeCU->getDIE(Ty); if (TyDIE) return TyDIE; // Create new type. TyDIE = new DIE(dwarf::DW_TAG_base_type); - ModuleCU->insertDIE(Ty.getNode(), TyDIE); + TypeCU->insertDIE(Ty, TyDIE); if (Ty.isBasicType()) - constructTypeDIE(*TyDIE, DIBasicType(Ty.getNode())); + constructTypeDIE(*TyDIE, DIBasicType(Ty)); else if (Ty.isCompositeType()) - constructTypeDIE(*TyDIE, DICompositeType(Ty.getNode())); + constructTypeDIE(*TyDIE, DICompositeType(Ty)); else { assert(Ty.isDerivedType() && "Unknown kind of DIType"); - constructTypeDIE(*TyDIE, DIDerivedType(Ty.getNode())); + constructTypeDIE(*TyDIE, DIDerivedType(Ty)); } addToContextOwner(TyDIE, Ty.getContext()); @@ -909,11 +898,12 @@ DIE *DwarfDebug::getOrCreateTypeDIE(DIType Ty) { /// addType - Add a new type attribute to the specified entity. void DwarfDebug::addType(DIE *Entity, DIType Ty) { - if (!Ty.isValid()) + if (!Ty.Verify()) return; // Check for pre-existence. - DIEEntry *Entry = ModuleCU->getDIEEntry(Ty.getNode()); + CompileUnit *TypeCU = getCompileUnit(Ty); + DIEEntry *Entry = TypeCU->getDIEEntry(Ty); // If it exists then use the existing value. if (Entry) { Entity->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, Entry); @@ -925,7 +915,7 @@ void DwarfDebug::addType(DIE *Entity, DIType Ty) { // Set up proxy. Entry = createDIEEntry(Buffer); - ModuleCU->insertDIEEntry(Ty.getNode(), Entry); + TypeCU->insertDIEEntry(Ty, Entry); Entity->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, Entry); } @@ -994,9 +984,9 @@ void DwarfDebug::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { // Add enumerators to enumeration type. for (unsigned i = 0, N = Elements.getNumElements(); i < N; ++i) { DIE *ElemDie = NULL; - DIDescriptor Enum(Elements.getElement(i).getNode()); + DIDescriptor Enum(Elements.getElement(i)); if (Enum.isEnumerator()) { - ElemDie = constructEnumTypeDIE(DIEnumerator(Enum.getNode())); + ElemDie = constructEnumTypeDIE(DIEnumerator(Enum)); Buffer.addChild(ElemDie); } } @@ -1006,7 +996,7 @@ void DwarfDebug::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { // Add return type. DIArray Elements = CTy.getTypeArray(); DIDescriptor RTy = Elements.getElement(0); - addType(&Buffer, DIType(RTy.getNode())); + addType(&Buffer, DIType(RTy)); // Add prototype flag. addUInt(&Buffer, dwarf::DW_AT_prototyped, dwarf::DW_FORM_flag, 1); @@ -1015,7 +1005,7 @@ void DwarfDebug::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { for (unsigned i = 1, N = Elements.getNumElements(); i < N; ++i) { DIE *Arg = new DIE(dwarf::DW_TAG_formal_parameter); DIDescriptor Ty = Elements.getElement(i); - addType(Arg, DIType(Ty.getNode())); + addType(Arg, DIType(Ty)); Buffer.addChild(Arg); } } @@ -1036,9 +1026,9 @@ void DwarfDebug::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { DIDescriptor Element = Elements.getElement(i); DIE *ElemDie = NULL; if (Element.isSubprogram()) - ElemDie = createSubprogramDIE(DISubprogram(Element.getNode())); + ElemDie = createSubprogramDIE(DISubprogram(Element)); else if (Element.isVariable()) { - DIVariable DV(Element.getNode()); + DIVariable DV(Element); ElemDie = new DIE(dwarf::DW_TAG_variable); addString(ElemDie, dwarf::DW_AT_name, dwarf::DW_FORM_string, DV.getName()); @@ -1047,7 +1037,7 @@ void DwarfDebug::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { addUInt(ElemDie, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1); addSourceLine(ElemDie, &DV); } else if (Element.isDerivedType()) - ElemDie = createMemberDIE(DIDerivedType(Element.getNode())); + ElemDie = createMemberDIE(DIDerivedType(Element)); else continue; Buffer.addChild(ElemDie); @@ -1062,9 +1052,9 @@ void DwarfDebug::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { dwarf::DW_FORM_data1, RLang); DICompositeType ContainingType = CTy.getContainingType(); - if (DIDescriptor(ContainingType.getNode()).isCompositeType()) + if (DIDescriptor(ContainingType).isCompositeType()) addDIEEntry(&Buffer, dwarf::DW_AT_containing_type, dwarf::DW_FORM_ref4, - getOrCreateTypeDIE(DIType(ContainingType.getNode()))); + getOrCreateTypeDIE(DIType(ContainingType))); break; } default: @@ -1120,22 +1110,23 @@ void DwarfDebug::constructArrayTypeDIE(DIE &Buffer, DIArray Elements = CTy->getTypeArray(); // Get an anonymous type for index type. - DIE *IdxTy = ModuleCU->getIndexTyDie(); + CompileUnit *TheCU = getCompileUnit(*CTy); + DIE *IdxTy = TheCU->getIndexTyDie(); if (!IdxTy) { // Construct an anonymous type for index type. IdxTy = new DIE(dwarf::DW_TAG_base_type); addUInt(IdxTy, dwarf::DW_AT_byte_size, 0, sizeof(int32_t)); addUInt(IdxTy, dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, dwarf::DW_ATE_signed); - ModuleCU->addDie(IdxTy); - ModuleCU->setIndexTyDie(IdxTy); + TheCU->addDie(IdxTy); + TheCU->setIndexTyDie(IdxTy); } // Add subranges to array type. for (unsigned i = 0, N = Elements.getNumElements(); i < N; ++i) { DIDescriptor Element = Elements.getElement(i); if (Element.getTag() == dwarf::DW_TAG_subrange_type) - constructSubrangeDIE(Buffer, DISubrange(Element.getNode()), IdxTy); + constructSubrangeDIE(Buffer, DISubrange(Element), IdxTy); } } @@ -1262,7 +1253,8 @@ DIE *DwarfDebug::createMemberDIE(const DIDerivedType &DT) { /// createSubprogramDIE - Create new DIE using SP. DIE *DwarfDebug::createSubprogramDIE(const DISubprogram &SP, bool MakeDecl) { - DIE *SPDie = ModuleCU->getDIE(SP.getNode()); + CompileUnit *SPCU = getCompileUnit(SP); + DIE *SPDie = SPCU->getDIE(SP); if (SPDie) return SPDie; @@ -1292,7 +1284,7 @@ DIE *DwarfDebug::createSubprogramDIE(const DISubprogram &SP, bool MakeDecl) { if (Args.getNumElements() == 0 || SPTag != dwarf::DW_TAG_subroutine_type) addType(SPDie, SPTy); else - addType(SPDie, DIType(Args.getElement(0).getNode())); + addType(SPDie, DIType(Args.getElement(0))); unsigned VK = SP.getVirtuality(); if (VK) { @@ -1302,7 +1294,7 @@ DIE *DwarfDebug::createSubprogramDIE(const DISubprogram &SP, bool MakeDecl) { addUInt(Block, 0, dwarf::DW_FORM_data1, SP.getVirtualIndex()); addBlock(SPDie, dwarf::DW_AT_vtable_elem_location, 0, Block); ContainingTypeMap.insert(std::make_pair(SPDie, - SP.getContainingType().getNode())); + SP.getContainingType())); } if (MakeDecl || !SP.isDefinition()) { @@ -1317,7 +1309,7 @@ DIE *DwarfDebug::createSubprogramDIE(const DISubprogram &SP, bool MakeDecl) { if (SPTag == dwarf::DW_TAG_subroutine_type) for (unsigned i = 1, N = Args.getNumElements(); i < N; ++i) { DIE *Arg = new DIE(dwarf::DW_TAG_formal_parameter); - DIType ATy = DIType(DIType(Args.getElement(i).getNode())); + DIType ATy = DIType(DIType(Args.getElement(i))); addType(Arg, ATy); if (ATy.isArtificial()) addUInt(Arg, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1); @@ -1335,12 +1327,12 @@ DIE *DwarfDebug::createSubprogramDIE(const DISubprogram &SP, bool MakeDecl) { addUInt(SPDie, dwarf::DW_AT_APPLE_optimized, dwarf::DW_FORM_flag, 1); // DW_TAG_inlined_subroutine may refer to this DIE. - ModuleCU->insertDIE(SP.getNode(), SPDie); + SPCU->insertDIE(SP, SPDie); return SPDie; } -DbgScope *DwarfDebug::getOrCreateAbstractScope(MDNode *N) { +DbgScope *DwarfDebug::getOrCreateAbstractScope(const MDNode *N) { assert(N && "Invalid Scope encoding!"); DbgScope *AScope = AbstractScopes.lookup(N); @@ -1353,7 +1345,7 @@ DbgScope *DwarfDebug::getOrCreateAbstractScope(MDNode *N) { if (Scope.isLexicalBlock()) { DILexicalBlock DB(N); DIDescriptor ParentDesc = DB.getContext(); - Parent = getOrCreateAbstractScope(ParentDesc.getNode()); + Parent = getOrCreateAbstractScope(ParentDesc); } AScope = new DbgScope(Parent, DIDescriptor(N), NULL); @@ -1369,14 +1361,14 @@ DbgScope *DwarfDebug::getOrCreateAbstractScope(MDNode *N) { /// isSubprogramContext - Return true if Context is either a subprogram /// or another context nested inside a subprogram. -static bool isSubprogramContext(MDNode *Context) { +static bool isSubprogramContext(const MDNode *Context) { if (!Context) return false; DIDescriptor D(Context); if (D.isSubprogram()) return true; if (D.isType()) - return isSubprogramContext(DIType(Context).getContext().getNode()); + return isSubprogramContext(DIType(Context).getContext()); return false; } @@ -1384,8 +1376,9 @@ static bool isSubprogramContext(MDNode *Context) { /// attach appropriate DW_AT_low_pc and DW_AT_high_pc attributes. /// If there are global variables in this scope then create and insert /// DIEs for these variables. -DIE *DwarfDebug::updateSubprogramScopeDIE(MDNode *SPNode) { - DIE *SPDie = ModuleCU->getDIE(SPNode); +DIE *DwarfDebug::updateSubprogramScopeDIE(const MDNode *SPNode) { + CompileUnit *SPCU = getCompileUnit(SPNode); + DIE *SPDie = SPCU->getDIE(SPNode); assert(SPDie && "Unable to find subprogram DIE!"); DISubprogram SP(SPNode); @@ -1396,7 +1389,7 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(MDNode *SPNode) { // specification DIE for a function defined inside a function. if (SP.isDefinition() && !SP.getContext().isCompileUnit() && !SP.getContext().isFile() && - !isSubprogramContext(SP.getContext().getNode())) { + !isSubprogramContext(SP.getContext())) { addUInt(SPDie, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1); // Add arguments. @@ -1406,7 +1399,7 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(MDNode *SPNode) { if (SPTag == dwarf::DW_TAG_subroutine_type) for (unsigned i = 1, N = Args.getNumElements(); i < N; ++i) { DIE *Arg = new DIE(dwarf::DW_TAG_formal_parameter); - DIType ATy = DIType(DIType(Args.getElement(i).getNode())); + DIType ATy = DIType(DIType(Args.getElement(i))); addType(Arg, ATy); if (ATy.isArtificial()) addUInt(Arg, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1); @@ -1416,7 +1409,7 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(MDNode *SPNode) { SPDie = new DIE(dwarf::DW_TAG_subprogram); addDIEEntry(SPDie, dwarf::DW_AT_specification, dwarf::DW_FORM_ref4, SPDeclDie); - ModuleCU->addDie(SPDie); + SPCU->addDie(SPDie); } addLabel(SPDie, dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, @@ -1451,18 +1444,18 @@ DIE *DwarfDebug::constructLexicalScopeDIE(DbgScope *Scope) { DebugRangeSymbols.size() * Asm->getTargetData().getPointerSize()); for (SmallVector<DbgRange, 4>::const_iterator RI = Ranges.begin(), RE = Ranges.end(); RI != RE; ++RI) { - DebugRangeSymbols.push_back(LabelsBeforeInsn.lookup(RI->first)); - DebugRangeSymbols.push_back(LabelsAfterInsn.lookup(RI->second)); + DebugRangeSymbols.push_back(getLabelBeforeInsn(RI->first)); + DebugRangeSymbols.push_back(getLabelAfterInsn(RI->second)); } DebugRangeSymbols.push_back(NULL); DebugRangeSymbols.push_back(NULL); return ScopeDIE; } - MCSymbol *Start = LabelsBeforeInsn.lookup(RI->first); - MCSymbol *End = LabelsAfterInsn.lookup(RI->second); + const MCSymbol *Start = getLabelBeforeInsn(RI->first); + const MCSymbol *End = getLabelAfterInsn(RI->second); - if (Start == 0 || End == 0) return 0; + if (End == 0) return 0; assert(Start->isDefined() && "Invalid starting label for an inlined scope!"); assert(End->isDefined() && "Invalid end label for an inlined scope!"); @@ -1487,10 +1480,10 @@ DIE *DwarfDebug::constructInlinedScopeDIE(DbgScope *Scope) { // For now, use first instruction range and emit low_pc/high_pc pair and // corresponding .debug_inlined section entry for this pair. SmallVector<DbgRange, 4>::const_iterator RI = Ranges.begin(); - MCSymbol *StartLabel = LabelsBeforeInsn.lookup(RI->first); - MCSymbol *EndLabel = LabelsAfterInsn.lookup(RI->second); + const MCSymbol *StartLabel = getLabelBeforeInsn(RI->first); + const MCSymbol *EndLabel = getLabelAfterInsn(RI->second); - if (StartLabel == 0 || EndLabel == 0) { + if (StartLabel == FunctionBeginSym || EndLabel == 0) { assert (0 && "Unexpected Start and End labels for a inlined scope!"); return 0; } @@ -1504,8 +1497,9 @@ DIE *DwarfDebug::constructInlinedScopeDIE(DbgScope *Scope) { DIScope DS(Scope->getScopeNode()); DIE *ScopeDIE = new DIE(dwarf::DW_TAG_inlined_subroutine); - DISubprogram InlinedSP = getDISubprogram(DS.getNode()); - DIE *OriginDIE = ModuleCU->getDIE(InlinedSP.getNode()); + DISubprogram InlinedSP = getDISubprogram(DS); + CompileUnit *TheCU = getCompileUnit(InlinedSP); + DIE *OriginDIE = TheCU->getDIE(InlinedSP); assert(OriginDIE && "Unable to find Origin DIE!"); addDIEEntry(ScopeDIE, dwarf::DW_AT_abstract_origin, dwarf::DW_FORM_ref4, OriginDIE); @@ -1516,18 +1510,18 @@ DIE *DwarfDebug::constructInlinedScopeDIE(DbgScope *Scope) { InlinedSubprogramDIEs.insert(OriginDIE); // Track the start label for this inlined function. - DenseMap<MDNode *, SmallVector<InlineInfoLabels, 4> >::iterator - I = InlineInfo.find(InlinedSP.getNode()); + DenseMap<const MDNode *, SmallVector<InlineInfoLabels, 4> >::iterator + I = InlineInfo.find(InlinedSP); if (I == InlineInfo.end()) { - InlineInfo[InlinedSP.getNode()].push_back(std::make_pair(StartLabel, + InlineInfo[InlinedSP].push_back(std::make_pair(StartLabel, ScopeDIE)); - InlinedSPNodes.push_back(InlinedSP.getNode()); + InlinedSPNodes.push_back(InlinedSP); } else I->second.push_back(std::make_pair(StartLabel, ScopeDIE)); DILocation DL(Scope->getInlinedAt()); - addUInt(ScopeDIE, dwarf::DW_AT_call_file, 0, ModuleCU->getID()); + addUInt(ScopeDIE, dwarf::DW_AT_call_file, 0, TheCU->getID()); addUInt(ScopeDIE, dwarf::DW_AT_call_line, 0, DL.getLineNumber()); return ScopeDIE; @@ -1560,22 +1554,15 @@ DIE *DwarfDebug::constructVariableDIE(DbgVariable *DV, DbgScope *Scope) { // Define variable debug information entry. DIE *VariableDie = new DIE(Tag); - DIE *AbsDIE = NULL; - if (DbgVariable *AV = DV->getAbstractVariable()) - AbsDIE = AV->getDIE(); - - if (AbsDIE) { - DIScope DS(Scope->getScopeNode()); - DISubprogram InlinedSP = getDISubprogram(DS.getNode()); - DIE *OriginSPDIE = ModuleCU->getDIE(InlinedSP.getNode()); - (void) OriginSPDIE; - assert(OriginSPDIE && "Unable to find Origin DIE for the SP!"); - DIE *AbsDIE = DV->getAbstractVariable()->getDIE(); - assert(AbsDIE && "Unable to find Origin DIE for the Variable!"); + DenseMap<const DbgVariable *, const DbgVariable *>::iterator + V2AVI = VarToAbstractVarMap.find(DV); + if (V2AVI != VarToAbstractVarMap.end()) + AbsDIE = V2AVI->second->getDIE(); + + if (AbsDIE) addDIEEntry(VariableDie, dwarf::DW_AT_abstract_origin, dwarf::DW_FORM_ref4, AbsDIE); - } else { addString(VariableDie, dwarf::DW_AT_name, dwarf::DW_FORM_string, Name); addSourceLine(VariableDie, &VD); @@ -1589,55 +1576,76 @@ DIE *DwarfDebug::constructVariableDIE(DbgVariable *DV, DbgScope *Scope) { addType(VariableDie, VD.getType()); } + if (Tag == dwarf::DW_TAG_formal_parameter && VD.getType().isArtificial()) + addUInt(VariableDie, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1); + + if (Scope->isAbstractScope()) { + DV->setDIE(VariableDie); + return VariableDie; + } + // Add variable address. - if (!Scope->isAbstractScope()) { - // Check if variable is described by DBG_VALUE instruction. - if (const MachineInstr *DVInsn = DV->getDbgValue()) { - bool updated = false; - // FIXME : Handle getNumOperands != 3 - if (DVInsn->getNumOperands() == 3) { - if (DVInsn->getOperand(0).isReg()) - updated = addRegisterAddress(VariableDie, DV, DVInsn->getOperand(0)); - else if (DVInsn->getOperand(0).isImm()) - updated = addConstantValue(VariableDie, DV, DVInsn->getOperand(0)); - else if (DVInsn->getOperand(0).isFPImm()) - updated = addConstantFPValue(VariableDie, DV, DVInsn->getOperand(0)); - } else { - MachineLocation Location = Asm->getDebugValueLocation(DVInsn); - if (Location.getReg()) { - addAddress(VariableDie, dwarf::DW_AT_location, Location); - if (MCSymbol *VS = DV->getDbgValueLabel()) - addLabel(VariableDie, dwarf::DW_AT_start_scope, dwarf::DW_FORM_addr, - VS); - updated = true; - } - } - if (!updated) { - // If variableDie is not updated then DBG_VALUE instruction does not - // have valid variable info. - delete VariableDie; - return NULL; - } - } - else { - MachineLocation Location; - unsigned FrameReg; - const TargetRegisterInfo *RI = Asm->TM.getRegisterInfo(); - int Offset = RI->getFrameIndexReference(*Asm->MF, DV->getFrameIndex(), - FrameReg); - Location.set(FrameReg, Offset); - - if (VD.hasComplexAddress()) - addComplexAddress(DV, VariableDie, dwarf::DW_AT_location, Location); - else if (VD.isBlockByrefVariable()) - addBlockByrefAddress(DV, VariableDie, dwarf::DW_AT_location, Location); - else + + unsigned Offset = DV->getDotDebugLocOffset(); + if (Offset != ~0U) { + addLabel(VariableDie, dwarf::DW_AT_location, dwarf::DW_FORM_data4, + Asm->GetTempSymbol("debug_loc", Offset)); + DV->setDIE(VariableDie); + UseDotDebugLocEntry.insert(VariableDie); + return VariableDie; + } + + // Check if variable is described by a DBG_VALUE instruction. + DenseMap<const DbgVariable *, const MachineInstr *>::iterator DVI = + DbgVariableToDbgInstMap.find(DV); + if (DVI != DbgVariableToDbgInstMap.end()) { + const MachineInstr *DVInsn = DVI->second; + const MCSymbol *DVLabel = findVariableLabel(DV); + bool updated = false; + // FIXME : Handle getNumOperands != 3 + if (DVInsn->getNumOperands() == 3) { + if (DVInsn->getOperand(0).isReg()) + updated = addRegisterAddress(VariableDie, DVLabel, DVInsn->getOperand(0)); + else if (DVInsn->getOperand(0).isImm()) + updated = addConstantValue(VariableDie, DVLabel, DVInsn->getOperand(0)); + else if (DVInsn->getOperand(0).isFPImm()) + updated = addConstantFPValue(VariableDie, DVLabel, DVInsn->getOperand(0)); + } else { + MachineLocation Location = Asm->getDebugValueLocation(DVInsn); + if (Location.getReg()) { addAddress(VariableDie, dwarf::DW_AT_location, Location); + if (DVLabel) + addLabel(VariableDie, dwarf::DW_AT_start_scope, dwarf::DW_FORM_addr, + DVLabel); + updated = true; + } } - } + if (!updated) { + // If variableDie is not updated then DBG_VALUE instruction does not + // have valid variable info. + delete VariableDie; + return NULL; + } + DV->setDIE(VariableDie); + return VariableDie; + } - if (Tag == dwarf::DW_TAG_formal_parameter && VD.getType().isArtificial()) - addUInt(VariableDie, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1); + // .. else use frame index, if available. + MachineLocation Location; + unsigned FrameReg; + const TargetRegisterInfo *RI = Asm->TM.getRegisterInfo(); + int FI = 0; + if (findVariableFrameIndex(DV, &FI)) { + int Offset = RI->getFrameIndexReference(*Asm->MF, FI, FrameReg); + Location.set(FrameReg, Offset); + + if (VD.hasComplexAddress()) + addComplexAddress(DV, VariableDie, dwarf::DW_AT_location, Location); + else if (VD.isBlockByrefVariable()) + addBlockByrefAddress(DV, VariableDie, dwarf::DW_AT_location, Location); + else + addAddress(VariableDie, dwarf::DW_AT_location, Location); + } DV->setDIE(VariableDie); return VariableDie; @@ -1651,14 +1659,15 @@ void DwarfDebug::addPubTypes(DISubprogram SP) { DIArray Args = SPTy.getTypeArray(); for (unsigned i = 0, e = Args.getNumElements(); i != e; ++i) { - DIType ATy(Args.getElement(i).getNode()); - if (!ATy.isValid()) + DIType ATy(Args.getElement(i)); + if (!ATy.Verify()) continue; DICompositeType CATy = getDICompositeType(ATy); - if (DIDescriptor(CATy.getNode()).Verify() && !CATy.getName().empty() + if (DIDescriptor(CATy).Verify() && !CATy.getName().empty() && !CATy.isForwardDecl()) { - if (DIEEntry *Entry = ModuleCU->getDIEEntry(CATy.getNode())) - ModuleCU->addGlobalType(CATy.getName(), Entry->getEntry()); + CompileUnit *TheCU = getCompileUnit(CATy); + if (DIEEntry *Entry = TheCU->getDIEEntry(CATy)) + TheCU->addGlobalType(CATy.getName(), Entry->getEntry()); } } } @@ -1674,9 +1683,9 @@ DIE *DwarfDebug::constructScopeDIE(DbgScope *Scope) { ScopeDIE = constructInlinedScopeDIE(Scope); else if (DS.isSubprogram()) { if (Scope->isAbstractScope()) - ScopeDIE = ModuleCU->getDIE(DS.getNode()); + ScopeDIE = getCompileUnit(DS)->getDIE(DS); else - ScopeDIE = updateSubprogramScopeDIE(DS.getNode()); + ScopeDIE = updateSubprogramScopeDIE(DS); } else ScopeDIE = constructLexicalScopeDIE(Scope); @@ -1700,7 +1709,7 @@ DIE *DwarfDebug::constructScopeDIE(DbgScope *Scope) { } if (DS.isSubprogram()) - addPubTypes(DISubprogram(DS.getNode())); + addPubTypes(DISubprogram(DS)); return ScopeDIE; } @@ -1744,11 +1753,12 @@ unsigned DwarfDebug::GetOrCreateSourceID(StringRef DirName, StringRef FileName){ /// getOrCreateNameSpace - Create a DIE for DINameSpace. DIE *DwarfDebug::getOrCreateNameSpace(DINameSpace NS) { - DIE *NDie = ModuleCU->getDIE(NS.getNode()); + CompileUnit *TheCU = getCompileUnit(NS); + DIE *NDie = TheCU->getDIE(NS); if (NDie) return NDie; NDie = new DIE(dwarf::DW_TAG_namespace); - ModuleCU->insertDIE(NS.getNode(), NDie); + TheCU->insertDIE(NS, NDie); if (!NS.getName().empty()) addString(NDie, dwarf::DW_AT_name, dwarf::DW_FORM_string, NS.getName()); addSourceLine(NDie, &NS); @@ -1756,12 +1766,10 @@ DIE *DwarfDebug::getOrCreateNameSpace(DINameSpace NS) { return NDie; } -void DwarfDebug::constructCompileUnit(MDNode *N) { +/// constructCompileUnit - Create new CompileUnit for the given +/// metadata node with tag DW_TAG_compile_unit. +void DwarfDebug::constructCompileUnit(const MDNode *N) { DICompileUnit DIUnit(N); - // Use first compile unit marked as isMain as the compile unit for this - // module. - if (ModuleCU || !DIUnit.isMain()) - return; StringRef FN = DIUnit.getFilename(); StringRef Dir = DIUnit.getDirectory(); unsigned ID = GetOrCreateSourceID(Dir, FN); @@ -1794,12 +1802,44 @@ void DwarfDebug::constructCompileUnit(MDNode *N) { addUInt(Die, dwarf::DW_AT_APPLE_major_runtime_vers, dwarf::DW_FORM_data1, RVer); - assert(!ModuleCU && - "ModuleCU assigned since the top of constructCompileUnit"); - ModuleCU = new CompileUnit(ID, Die); + CompileUnit *NewCU = new CompileUnit(ID, Die); + if (!FirstCU) + FirstCU = NewCU; + CUMap.insert(std::make_pair(N, NewCU)); +} + +/// getCompielUnit - Get CompileUnit DIE. +CompileUnit *DwarfDebug::getCompileUnit(const MDNode *N) const { + assert (N && "Invalid DwarfDebug::getCompileUnit argument!"); + DIDescriptor D(N); + const MDNode *CUNode = NULL; + if (D.isCompileUnit()) + CUNode = N; + else if (D.isSubprogram()) + CUNode = DISubprogram(N).getCompileUnit(); + else if (D.isType()) + CUNode = DIType(N).getCompileUnit(); + else if (D.isGlobalVariable()) + CUNode = DIGlobalVariable(N).getCompileUnit(); + else if (D.isVariable()) + CUNode = DIVariable(N).getCompileUnit(); + else if (D.isNameSpace()) + CUNode = DINameSpace(N).getCompileUnit(); + else if (D.isFile()) + CUNode = DIFile(N).getCompileUnit(); + else + return FirstCU; + + DenseMap<const MDNode *, CompileUnit *>::const_iterator I + = CUMap.find(CUNode); + if (I == CUMap.end()) + return FirstCU; + return I->second; } -void DwarfDebug::constructGlobalVariableDIE(MDNode *N) { + +/// constructGlobalVariableDIE - Construct global variable DIE. +void DwarfDebug::constructGlobalVariableDIE(const MDNode *N) { DIGlobalVariable DI_GV(N); // If debug information is malformed then ignore it. @@ -1807,7 +1847,8 @@ void DwarfDebug::constructGlobalVariableDIE(MDNode *N) { return; // Check for pre-existence. - if (ModuleCU->getDIE(DI_GV.getNode())) + CompileUnit *TheCU = getCompileUnit(N); + if (TheCU->getDIE(DI_GV)) return; DIE *VariableDie = createGlobalVariableDIE(DI_GV); @@ -1815,7 +1856,7 @@ void DwarfDebug::constructGlobalVariableDIE(MDNode *N) { return; // Add to map. - ModuleCU->insertDIE(N, VariableDie); + TheCU->insertDIE(N, VariableDie); // Add to context owner. DIDescriptor GVContext = DI_GV.getContext(); @@ -1823,7 +1864,7 @@ void DwarfDebug::constructGlobalVariableDIE(MDNode *N) { // or a subprogram. if (DI_GV.isDefinition() && !GVContext.isCompileUnit() && !GVContext.isFile() && - !isSubprogramContext(GVContext.getNode())) { + !isSubprogramContext(GVContext)) { // Create specification DIE. DIE *VariableSpecDIE = new DIE(dwarf::DW_TAG_variable); addDIEEntry(VariableSpecDIE, dwarf::DW_AT_specification, @@ -1834,7 +1875,7 @@ void DwarfDebug::constructGlobalVariableDIE(MDNode *N) { Asm->Mang->getSymbol(DI_GV.getGlobal())); addBlock(VariableSpecDIE, dwarf::DW_AT_location, 0, Block); addUInt(VariableDie, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1); - ModuleCU->addDie(VariableSpecDIE); + TheCU->addDie(VariableSpecDIE); } else { DIEBlock *Block = new (DIEValueAllocator) DIEBlock(); addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr); @@ -1845,23 +1886,25 @@ void DwarfDebug::constructGlobalVariableDIE(MDNode *N) { addToContextOwner(VariableDie, GVContext); // Expose as global. FIXME - need to check external flag. - ModuleCU->addGlobal(DI_GV.getName(), VariableDie); + TheCU->addGlobal(DI_GV.getName(), VariableDie); DIType GTy = DI_GV.getType(); if (GTy.isCompositeType() && !GTy.getName().empty() && !GTy.isForwardDecl()) { - DIEEntry *Entry = ModuleCU->getDIEEntry(GTy.getNode()); + DIEEntry *Entry = TheCU->getDIEEntry(GTy); assert(Entry && "Missing global type!"); - ModuleCU->addGlobalType(GTy.getName(), Entry->getEntry()); + TheCU->addGlobalType(GTy.getName(), Entry->getEntry()); } return; } -void DwarfDebug::constructSubprogramDIE(MDNode *N) { +/// construct SubprogramDIE - Construct subprogram DIE. +void DwarfDebug::constructSubprogramDIE(const MDNode *N) { DISubprogram SP(N); // Check for pre-existence. - if (ModuleCU->getDIE(N)) + CompileUnit *TheCU = getCompileUnit(N); + if (TheCU->getDIE(N)) return; if (!SP.isDefinition()) @@ -1872,13 +1915,13 @@ void DwarfDebug::constructSubprogramDIE(MDNode *N) { DIE *SubprogramDie = createSubprogramDIE(SP); // Add to map. - ModuleCU->insertDIE(N, SubprogramDie); + TheCU->insertDIE(N, SubprogramDie); // Add to context owner. addToContextOwner(SubprogramDie, SP.getContext()); // Expose as global. - ModuleCU->addGlobal(SP.getName(), SubprogramDie); + TheCU->addGlobal(SP.getName(), SubprogramDie); return; } @@ -1952,7 +1995,7 @@ void DwarfDebug::beginModule(Module *M) { /// endModule - Emit all Dwarf sections that should come after the content. /// void DwarfDebug::endModule() { - if (!ModuleCU) return; + if (!FirstCU) return; // Attach DW_AT_inline attribute with inlined subprogram DIEs. for (SmallPtrSet<DIE *, 4>::iterator AI = InlinedSubprogramDIEs.begin(), @@ -1961,12 +2004,12 @@ void DwarfDebug::endModule() { addUInt(ISP, dwarf::DW_AT_inline, 0, dwarf::DW_INL_inlined); } - for (DenseMap<DIE *, MDNode *>::iterator CI = ContainingTypeMap.begin(), + for (DenseMap<DIE *, const MDNode *>::iterator CI = ContainingTypeMap.begin(), CE = ContainingTypeMap.end(); CI != CE; ++CI) { DIE *SPDie = CI->first; - MDNode *N = dyn_cast_or_null<MDNode>(CI->second); + const MDNode *N = dyn_cast_or_null<MDNode>(CI->second); if (!N) continue; - DIE *NDie = ModuleCU->getDIE(N); + DIE *NDie = getCompileUnit(N)->getDIE(N); if (!NDie) continue; addDIEEntry(SPDie, dwarf::DW_AT_containing_type, dwarf::DW_FORM_ref4, NDie); } @@ -2027,68 +2070,48 @@ void DwarfDebug::endModule() { // Emit info into a debug str section. emitDebugStr(); - delete ModuleCU; - ModuleCU = NULL; // Reset for the next Module, if any. + for (DenseMap<const MDNode *, CompileUnit *>::iterator I = CUMap.begin(), + E = CUMap.end(); I != E; ++I) + delete I->second; + FirstCU = NULL; // Reset for the next Module, if any. } /// findAbstractVariable - Find abstract variable, if any, associated with Var. -DbgVariable *DwarfDebug::findAbstractVariable(DIVariable &Var, - unsigned FrameIdx, +DbgVariable *DwarfDebug::findAbstractVariable(DIVariable &Var, DebugLoc ScopeLoc) { - DbgVariable *AbsDbgVariable = AbstractVariables.lookup(Var.getNode()); + DbgVariable *AbsDbgVariable = AbstractVariables.lookup(Var); if (AbsDbgVariable) return AbsDbgVariable; - LLVMContext &Ctx = Var.getNode()->getContext(); + LLVMContext &Ctx = Var->getContext(); DbgScope *Scope = AbstractScopes.lookup(ScopeLoc.getScope(Ctx)); if (!Scope) return NULL; - AbsDbgVariable = new DbgVariable(Var, FrameIdx, - NULL /* No more-abstract variable*/); + AbsDbgVariable = new DbgVariable(Var); Scope->addVariable(AbsDbgVariable); - AbstractVariables[Var.getNode()] = AbsDbgVariable; + AbstractVariables[Var] = AbsDbgVariable; return AbsDbgVariable; } -/// findAbstractVariable - Find abstract variable, if any, associated with Var. -/// FIXME : Refactor findAbstractVariable. -DbgVariable *DwarfDebug::findAbstractVariable(DIVariable &Var, - const MachineInstr *MI, - DebugLoc ScopeLoc) { - - DbgVariable *AbsDbgVariable = AbstractVariables.lookup(Var.getNode()); - if (AbsDbgVariable) - return AbsDbgVariable; - - LLVMContext &Ctx = Var.getNode()->getContext(); - DbgScope *Scope = AbstractScopes.lookup(ScopeLoc.getScope(Ctx)); - if (!Scope) - return NULL; - - AbsDbgVariable = new DbgVariable(Var, MI, - NULL /* No more-abstract variable*/); - Scope->addVariable(AbsDbgVariable); - AbstractVariables[Var.getNode()] = AbsDbgVariable; - DbgValueStartMap[MI] = AbsDbgVariable; - return AbsDbgVariable; -} - -/// collectVariableInfo - Populate DbgScope entries with variables' info. -void DwarfDebug::collectVariableInfo() { +/// collectVariableInfoFromMMITable - Collect variable information from +/// side table maintained by MMI. +void +DwarfDebug::collectVariableInfoFromMMITable(const MachineFunction * MF, + SmallPtrSet<const MDNode *, 16> &Processed) { const LLVMContext &Ctx = Asm->MF->getFunction()->getContext(); - MachineModuleInfo::VariableDbgInfoMapTy &VMap = MMI->getVariableDbgInfo(); for (MachineModuleInfo::VariableDbgInfoMapTy::iterator VI = VMap.begin(), VE = VMap.end(); VI != VE; ++VI) { - MDNode *Var = VI->first; + const MDNode *Var = VI->first; if (!Var) continue; + Processed.insert(Var); DIVariable DV(Var); const std::pair<unsigned, DebugLoc> &VP = VI->second; DbgScope *Scope = 0; - if (MDNode *IA = VP.second.getInlinedAt(Ctx)) + if (const MDNode *IA = VP.second.getInlinedAt(Ctx)) Scope = ConcreteScopes.lookup(IA); if (Scope == 0) Scope = DbgScopeMap.lookup(VP.second.getScope(Ctx)); @@ -2097,100 +2120,192 @@ void DwarfDebug::collectVariableInfo() { if (Scope == 0) continue; - DbgVariable *AbsDbgVariable = findAbstractVariable(DV, VP.first, VP.second); - DbgVariable *RegVar = new DbgVariable(DV, VP.first, AbsDbgVariable); + DbgVariable *AbsDbgVariable = findAbstractVariable(DV, VP.second); + DbgVariable *RegVar = new DbgVariable(DV); + recordVariableFrameIndex(RegVar, VP.first); Scope->addVariable(RegVar); + if (AbsDbgVariable) { + recordVariableFrameIndex(AbsDbgVariable, VP.first); + VarToAbstractVarMap[RegVar] = AbsDbgVariable; + } } +} +/// isDbgValueInUndefinedReg - Return true if debug value, encoded by +/// DBG_VALUE instruction, is in undefined reg. +static bool isDbgValueInUndefinedReg(const MachineInstr *MI) { + assert (MI->isDebugValue() && "Invalid DBG_VALUE machine instruction!"); + if (MI->getOperand(0).isReg() && !MI->getOperand(0).getReg()) + return true; + return false; +} + +/// isDbgValueInDefinedReg - Return true if debug value, encoded by +/// DBG_VALUE instruction, is in a defined reg. +static bool isDbgValueInDefinedReg(const MachineInstr *MI) { + assert (MI->isDebugValue() && "Invalid DBG_VALUE machine instruction!"); + if (MI->getOperand(0).isReg() && MI->getOperand(0).getReg()) + return true; + return false; +} + +/// collectVariableInfo - Populate DbgScope entries with variables' info. +void DwarfDebug::collectVariableInfo(const MachineFunction *MF) { + SmallPtrSet<const MDNode *, 16> Processed; + + /// collection info from MMI table. + collectVariableInfoFromMMITable(MF, Processed); + + SmallVector<const MachineInstr *, 8> DbgValues; // Collect variable information from DBG_VALUE machine instructions; for (MachineFunction::const_iterator I = Asm->MF->begin(), E = Asm->MF->end(); - I != E; ++I) { + I != E; ++I) for (MachineBasicBlock::const_iterator II = I->begin(), IE = I->end(); II != IE; ++II) { const MachineInstr *MInsn = II; - if (!MInsn->isDebugValue()) + if (!MInsn->isDebugValue() || isDbgValueInUndefinedReg(MInsn)) continue; + DbgValues.push_back(MInsn); + } - // Ignore Undef values. - if (MInsn->getOperand(0).isReg() && !MInsn->getOperand(0).getReg()) - continue; + // This is a collection of DBV_VALUE instructions describing same variable. + SmallVector<const MachineInstr *, 4> MultipleValues; + for(SmallVector<const MachineInstr *, 8>::iterator I = DbgValues.begin(), + E = DbgValues.end(); I != E; ++I) { + const MachineInstr *MInsn = *I; + MultipleValues.clear(); + if (isDbgValueInDefinedReg(MInsn)) + MultipleValues.push_back(MInsn); + DIVariable DV(MInsn->getOperand(MInsn->getNumOperands() - 1).getMetadata()); + if (Processed.count(DV) != 0) + continue; + + for (SmallVector<const MachineInstr *, 8>::iterator MI = I+1, + ME = DbgValues.end(); MI != ME; ++MI) { + const MDNode *Var = + (*MI)->getOperand((*MI)->getNumOperands()-1).getMetadata(); + if (Var == DV && isDbgValueInDefinedReg(*MI)) + MultipleValues.push_back(*MI); + } + + DbgScope *Scope = findDbgScope(MInsn); + if (!Scope && DV.getTag() == dwarf::DW_TAG_arg_variable) + Scope = CurrentFnDbgScope; + // If variable scope is not found then skip this variable. + if (!Scope) + continue; - DIVariable DV( - const_cast<MDNode *>(MInsn->getOperand(MInsn->getNumOperands() - 1) - .getMetadata())); - if (DV.getTag() == dwarf::DW_TAG_arg_variable) { - // FIXME Handle inlined subroutine arguments. - DbgVariable *ArgVar = new DbgVariable(DV, MInsn, NULL); - CurrentFnDbgScope->addVariable(ArgVar); - DbgValueStartMap[MInsn] = ArgVar; + Processed.insert(DV); + DbgVariable *RegVar = new DbgVariable(DV); + Scope->addVariable(RegVar); + if (DV.getTag() != dwarf::DW_TAG_arg_variable) + DbgVariableLabelsMap[RegVar] = getLabelBeforeInsn(MInsn); + if (DbgVariable *AbsVar = findAbstractVariable(DV, MInsn->getDebugLoc())) { + DbgVariableToDbgInstMap[AbsVar] = MInsn; + VarToAbstractVarMap[RegVar] = AbsVar; + } + if (MultipleValues.size() <= 1) { + DbgVariableToDbgInstMap[RegVar] = MInsn; + continue; + } + + // handle multiple DBG_VALUE instructions describing one variable. + if (DotDebugLocEntries.empty()) + RegVar->setDotDebugLocOffset(0); + else + RegVar->setDotDebugLocOffset(DotDebugLocEntries.size()); + const MachineInstr *Begin = NULL; + const MachineInstr *End = NULL; + for (SmallVector<const MachineInstr *, 4>::iterator + MVI = MultipleValues.begin(), MVE = MultipleValues.end(); MVI != MVE; ++MVI) { + if (!Begin) { + Begin = *MVI; continue; + } + End = *MVI; + MachineLocation MLoc; + MLoc.set(Begin->getOperand(0).getReg(), 0); + const MCSymbol *FLabel = getLabelBeforeInsn(Begin); + const MCSymbol *SLabel = getLabelBeforeInsn(End); + DotDebugLocEntries.push_back(DotDebugLocEntry(FLabel, SLabel, MLoc)); + Begin = End; + if (MVI + 1 == MVE) { + // If End is the last instruction then its value is valid + // until the end of the funtion. + MLoc.set(End->getOperand(0).getReg(), 0); + DotDebugLocEntries. + push_back(DotDebugLocEntry(SLabel, FunctionEndSym, MLoc)); } + } + DotDebugLocEntries.push_back(DotDebugLocEntry()); + } - DebugLoc DL = MInsn->getDebugLoc(); - if (DL.isUnknown()) continue; - DbgScope *Scope = 0; - if (MDNode *IA = DL.getInlinedAt(Ctx)) - Scope = ConcreteScopes.lookup(IA); - if (Scope == 0) - Scope = DbgScopeMap.lookup(DL.getScope(Ctx)); - - // If variable scope is not found then skip this variable. - if (Scope == 0) + // Collect info for variables that were optimized out. + if (NamedMDNode *NMD = + MF->getFunction()->getParent()->getNamedMetadata("llvm.dbg.lv")) { + for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { + DIVariable DV(cast_or_null<MDNode>(NMD->getOperand(i))); + if (!DV || !Processed.insert(DV)) continue; - - DbgVariable *AbsDbgVariable = findAbstractVariable(DV, MInsn, DL); - DbgVariable *RegVar = new DbgVariable(DV, MInsn, AbsDbgVariable); - DbgValueStartMap[MInsn] = RegVar; - Scope->addVariable(RegVar); + DbgScope *Scope = DbgScopeMap.lookup(DV.getContext()); + if (Scope) + Scope->addVariable(new DbgVariable(DV)); } } } +/// getLabelBeforeInsn - Return Label preceding the instruction. +const MCSymbol *DwarfDebug::getLabelBeforeInsn(const MachineInstr *MI) { + DenseMap<const MachineInstr *, MCSymbol *>::iterator I = + LabelsBeforeInsn.find(MI); + if (I == LabelsBeforeInsn.end()) + // FunctionBeginSym always preceeds all the instruction in current function. + return FunctionBeginSym; + return I->second; +} + +/// getLabelAfterInsn - Return Label immediately following the instruction. +const MCSymbol *DwarfDebug::getLabelAfterInsn(const MachineInstr *MI) { + DenseMap<const MachineInstr *, MCSymbol *>::iterator I = + LabelsAfterInsn.find(MI); + if (I == LabelsAfterInsn.end()) + return NULL; + return I->second; +} + /// beginScope - Process beginning of a scope. void DwarfDebug::beginScope(const MachineInstr *MI) { - // Check location. - DebugLoc DL = MI->getDebugLoc(); - if (DL.isUnknown()) + if (InsnNeedsLabel.count(MI) == 0) { + LabelsBeforeInsn[MI] = PrevLabel; return; + } - MDNode *Scope = DL.getScope(Asm->MF->getFunction()->getContext()); - - // FIXME: Should only verify each scope once! - if (!DIScope(Scope).Verify()) + // Check location. + DebugLoc DL = MI->getDebugLoc(); + if (!DL.isUnknown()) { + const MDNode *Scope = DL.getScope(Asm->MF->getFunction()->getContext()); + PrevLabel = recordSourceLine(DL.getLine(), DL.getCol(), Scope); + PrevInstLoc = DL; + LabelsBeforeInsn[MI] = PrevLabel; return; + } - // DBG_VALUE instruction establishes new value. + // If location is unknown then use temp label for this DBG_VALUE + // instruction. if (MI->isDebugValue()) { - DenseMap<const MachineInstr *, DbgVariable *>::iterator DI - = DbgValueStartMap.find(MI); - if (DI != DbgValueStartMap.end()) { - MCSymbol *Label = NULL; - if (DL == PrevInstLoc) - Label = PrevLabel; - else { - Label = recordSourceLine(DL.getLine(), DL.getCol(), Scope); - PrevInstLoc = DL; - PrevLabel = Label; - } - DI->second->setDbgValueLabel(Label); - } + PrevLabel = MMI->getContext().CreateTempSymbol(); + Asm->OutStreamer.EmitLabel(PrevLabel); + LabelsBeforeInsn[MI] = PrevLabel; return; } - // Emit a label to indicate location change. This is used for line - // table even if this instruction does not start a new scope. - MCSymbol *Label = NULL; - if (DL == PrevInstLoc) - Label = PrevLabel; - else { - Label = recordSourceLine(DL.getLine(), DL.getCol(), Scope); - PrevInstLoc = DL; - PrevLabel = Label; + if (UnknownLocations) { + PrevLabel = recordSourceLine(0, 0, 0); + LabelsBeforeInsn[MI] = PrevLabel; + return; } - // If this instruction begins a scope then note down corresponding label. - if (InsnsBeginScopeSet.count(MI) != 0) - LabelsBeforeInsn[MI] = Label; + assert (0 && "Instruction is not processed!"); } /// endScope - Process end of a scope. @@ -2204,7 +2319,7 @@ void DwarfDebug::endScope(const MachineInstr *MI) { } /// getOrCreateDbgScope - Create DbgScope for the scope. -DbgScope *DwarfDebug::getOrCreateDbgScope(MDNode *Scope, MDNode *InlinedAt) { +DbgScope *DwarfDebug::getOrCreateDbgScope(const MDNode *Scope, const MDNode *InlinedAt) { if (!InlinedAt) { DbgScope *WScope = DbgScopeMap.lookup(Scope); if (WScope) @@ -2213,7 +2328,7 @@ DbgScope *DwarfDebug::getOrCreateDbgScope(MDNode *Scope, MDNode *InlinedAt) { DbgScopeMap.insert(std::make_pair(Scope, WScope)); if (DIDescriptor(Scope).isLexicalBlock()) { DbgScope *Parent = - getOrCreateDbgScope(DILexicalBlock(Scope).getContext().getNode(), NULL); + getOrCreateDbgScope(DILexicalBlock(Scope).getContext(), NULL); WScope->setParent(Parent); Parent->addScope(WScope); } @@ -2235,7 +2350,7 @@ DbgScope *DwarfDebug::getOrCreateDbgScope(MDNode *Scope, MDNode *InlinedAt) { DbgScopeMap.insert(std::make_pair(InlinedAt, WScope)); DILocation DL(InlinedAt); DbgScope *Parent = - getOrCreateDbgScope(DL.getScope().getNode(), DL.getOrigLocation().getNode()); + getOrCreateDbgScope(DL.getScope(), DL.getOrigLocation()); WScope->setParent(Parent); Parent->addScope(WScope); @@ -2249,13 +2364,13 @@ DbgScope *DwarfDebug::getOrCreateDbgScope(MDNode *Scope, MDNode *InlinedAt) { /// machine instruction encodes valid location info. static bool hasValidLocation(LLVMContext &Ctx, const MachineInstr *MInsn, - MDNode *&Scope, MDNode *&InlinedAt) { + const MDNode *&Scope, const MDNode *&InlinedAt) { if (MInsn->isDebugValue()) return false; DebugLoc DL = MInsn->getDebugLoc(); if (DL.isUnknown()) return false; - MDNode *S = DL.getScope(Ctx); + const MDNode *S = DL.getScope(Ctx); // There is no need to create another DIE for compile unit. For all // other scopes, create one DbgScope now. This will be translated @@ -2307,8 +2422,8 @@ void printDbgScopeInfo(LLVMContext &Ctx, const MachineFunction *MF, for (MachineBasicBlock::const_iterator II = I->begin(), IE = I->end(); II != IE; ++II) { const MachineInstr *MInsn = II; - MDNode *Scope = NULL; - MDNode *InlinedAt = NULL; + const MDNode *Scope = NULL; + const MDNode *InlinedAt = NULL; // Check if instruction has valid location information. if (hasValidLocation(Ctx, MInsn, Scope, InlinedAt)) { @@ -2344,8 +2459,8 @@ bool DwarfDebug::extractScopeInformation() { LLVMContext &Ctx = Asm->MF->getFunction()->getContext(); SmallVector<DbgRange, 4> MIRanges; DenseMap<const MachineInstr *, DbgScope *> MI2ScopeMap; - MDNode *PrevScope = NULL; - MDNode *PrevInlinedAt = NULL; + const MDNode *PrevScope = NULL; + const MDNode *PrevInlinedAt = NULL; const MachineInstr *RangeBeginMI = NULL; const MachineInstr *PrevMI = NULL; for (MachineFunction::const_iterator I = Asm->MF->begin(), E = Asm->MF->end(); @@ -2353,8 +2468,8 @@ bool DwarfDebug::extractScopeInformation() { for (MachineBasicBlock::const_iterator II = I->begin(), IE = I->end(); II != IE; ++II) { const MachineInstr *MInsn = II; - MDNode *Scope = NULL; - MDNode *InlinedAt = NULL; + const MDNode *Scope = NULL; + const MDNode *InlinedAt = NULL; // Check if instruction has valid location information. if (!hasValidLocation(Ctx, MInsn, Scope, InlinedAt)) { @@ -2476,8 +2591,6 @@ static DebugLoc FindFirstDebugLoc(const MachineFunction *MF) { void DwarfDebug::beginFunction(const MachineFunction *MF) { if (!MMI->hasDebugInfo()) return; if (!extractScopeInformation()) return; - - collectVariableInfo(); FunctionBeginSym = Asm->GetTempSymbol("func_begin", Asm->getFunctionNumber()); @@ -2489,7 +2602,7 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { DebugLoc FDL = FindFirstDebugLoc(MF); if (FDL.isUnknown()) return; - MDNode *Scope = FDL.getScope(MF->getFunction()->getContext()); + const MDNode *Scope = FDL.getScope(MF->getFunction()->getContext()); DISubprogram SP = getDISubprogram(Scope); unsigned Line, Col; @@ -2502,6 +2615,40 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { } recordSourceLine(Line, Col, Scope); + + DebugLoc PrevLoc; + for (MachineFunction::const_iterator I = MF->begin(), E = MF->end(); + I != E; ++I) + for (MachineBasicBlock::const_iterator II = I->begin(), IE = I->end(); + II != IE; ++II) { + const MachineInstr *MI = II; + DebugLoc DL = MI->getDebugLoc(); + if (MI->isDebugValue()) { + // DBG_VALUE needs a label if the variable is local variable or + // an argument whose location is changing. + assert (MI->getNumOperands() > 1 && "Invalid machine instruction!"); + DIVariable DV(MI->getOperand(MI->getNumOperands() - 1).getMetadata()); + if (!DV.Verify()) continue; + if (DV.getTag() != dwarf::DW_TAG_arg_variable) + InsnNeedsLabel.insert(MI); + else if (!ProcessedArgs.insert(DV)) + InsnNeedsLabel.insert(MI); + } else { + // If location is unknown then instruction needs a location only if + // UnknownLocations flag is set. + if (DL.isUnknown()) { + if (UnknownLocations && !PrevLoc.isUnknown()) + InsnNeedsLabel.insert(MI); + } else if (DL != PrevLoc) + // Otherwise, instruction needs a location only if it is new location. + InsnNeedsLabel.insert(MI); + } + + if (!DL.isUnknown() || UnknownLocations) + PrevLoc = DL; + } + + PrevLabel = FunctionBeginSym; } /// endFunction - Gather and emit post-function debug information. @@ -2510,10 +2657,15 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { if (!MMI->hasDebugInfo() || DbgScopeMap.empty()) return; if (CurrentFnDbgScope) { + // Define end label for subprogram. - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("func_end", - Asm->getFunctionNumber())); + FunctionEndSym = Asm->GetTempSymbol("func_end", + Asm->getFunctionNumber()); + // Assumes in correct section after the entry point. + Asm->OutStreamer.EmitLabel(FunctionEndSym); + collectVariableInfo(MF); + // Get function line info. if (!Lines.empty()) { // Get section line info. @@ -2543,10 +2695,15 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { // Clear debug info CurrentFnDbgScope = NULL; + InsnNeedsLabel.clear(); + ProcessedArgs.clear(); + DbgVariableToFrameIndexMap.clear(); + VarToAbstractVarMap.clear(); + DbgVariableToDbgInstMap.clear(); + DbgVariableLabelsMap.clear(); DeleteContainerSeconds(DbgScopeMap); InsnsBeginScopeSet.clear(); InsnsEndScopeSet.clear(); - DbgValueStartMap.clear(); ConcreteScopes.clear(); DeleteContainerSeconds(AbstractScopes); AbstractScopesList.clear(); @@ -2557,30 +2714,82 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { PrevLabel = NULL; } +/// recordVariableFrameIndex - Record a variable's index. +void DwarfDebug::recordVariableFrameIndex(const DbgVariable *V, int Index) { + assert (V && "Invalid DbgVariable!"); + DbgVariableToFrameIndexMap[V] = Index; +} + +/// findVariableFrameIndex - Return true if frame index for the variable +/// is found. Update FI to hold value of the index. +bool DwarfDebug::findVariableFrameIndex(const DbgVariable *V, int *FI) { + assert (V && "Invalid DbgVariable!"); + DenseMap<const DbgVariable *, int>::iterator I = + DbgVariableToFrameIndexMap.find(V); + if (I == DbgVariableToFrameIndexMap.end()) + return false; + *FI = I->second; + return true; +} + +/// findVariableLabel - Find MCSymbol for the variable. +const MCSymbol *DwarfDebug::findVariableLabel(const DbgVariable *V) { + DenseMap<const DbgVariable *, const MCSymbol *>::iterator I + = DbgVariableLabelsMap.find(V); + if (I == DbgVariableLabelsMap.end()) + return NULL; + else return I->second; +} + +/// findDbgScope - Find DbgScope for the debug loc attached with an +/// instruction. +DbgScope *DwarfDebug::findDbgScope(const MachineInstr *MInsn) { + DbgScope *Scope = NULL; + LLVMContext &Ctx = + MInsn->getParent()->getParent()->getFunction()->getContext(); + DebugLoc DL = MInsn->getDebugLoc(); + + if (DL.isUnknown()) + return Scope; + + if (const MDNode *IA = DL.getInlinedAt(Ctx)) + Scope = ConcreteScopes.lookup(IA); + if (Scope == 0) + Scope = DbgScopeMap.lookup(DL.getScope(Ctx)); + + return Scope; +} + + /// recordSourceLine - Register a source line with debug info. Returns the /// unique label that was emitted and which provides correspondence to /// the source line list. -MCSymbol *DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, MDNode *S) { +MCSymbol *DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S) { StringRef Dir; StringRef Fn; - DIDescriptor Scope(S); - if (Scope.isCompileUnit()) { - DICompileUnit CU(S); - Dir = CU.getDirectory(); - Fn = CU.getFilename(); - } else if (Scope.isSubprogram()) { - DISubprogram SP(S); - Dir = SP.getDirectory(); - Fn = SP.getFilename(); - } else if (Scope.isLexicalBlock()) { - DILexicalBlock DB(S); - Dir = DB.getDirectory(); - Fn = DB.getFilename(); - } else - assert(0 && "Unexpected scope info"); + unsigned Src = 1; + if (S) { + DIDescriptor Scope(S); + + if (Scope.isCompileUnit()) { + DICompileUnit CU(S); + Dir = CU.getDirectory(); + Fn = CU.getFilename(); + } else if (Scope.isSubprogram()) { + DISubprogram SP(S); + Dir = SP.getDirectory(); + Fn = SP.getFilename(); + } else if (Scope.isLexicalBlock()) { + DILexicalBlock DB(S); + Dir = DB.getDirectory(); + Fn = DB.getFilename(); + } else + assert(0 && "Unexpected scope info"); + + Src = GetOrCreateSourceID(Dir, Fn); + } - unsigned Src = GetOrCreateSourceID(Dir, Fn); MCSymbol *Label = MMI->getContext().CreateTempSymbol(); Lines.push_back(SrcLineInfo(Line, Col, Src, Label)); @@ -2643,14 +2852,18 @@ DwarfDebug::computeSizeAndOffset(DIE *Die, unsigned Offset, bool Last) { /// computeSizeAndOffsets - Compute the size and offset of all the DIEs. /// void DwarfDebug::computeSizeAndOffsets() { - // Compute size of compile unit header. - static unsigned Offset = - sizeof(int32_t) + // Length of Compilation Unit Info - sizeof(int16_t) + // DWARF version number - sizeof(int32_t) + // Offset Into Abbrev. Section - sizeof(int8_t); // Pointer Size (in bytes) - - computeSizeAndOffset(ModuleCU->getCUDie(), Offset, true); + unsigned PrevOffset = 0; + for (DenseMap<const MDNode *, CompileUnit *>::iterator I = CUMap.begin(), + E = CUMap.end(); I != E; ++I) { + // Compute size of compile unit header. + static unsigned Offset = PrevOffset + + sizeof(int32_t) + // Length of Compilation Unit Info + sizeof(int16_t) + // DWARF version number + sizeof(int32_t) + // Offset Into Abbrev. Section + sizeof(int8_t); // Pointer Size (in bytes) + computeSizeAndOffset(I->second->getCUDie(), Offset, true); + PrevOffset = Offset; + } } /// EmitSectionSym - Switch to the specified MCSection and emit an assembler @@ -2694,6 +2907,9 @@ void DwarfDebug::EmitSectionLabels() { DwarfDebugRangeSectionSym = EmitSectionSym(Asm, TLOF.getDwarfRangesSection(), "debug_range"); + DwarfDebugLocSectionSym = EmitSectionSym(Asm, TLOF.getDwarfLocSection(), + "section_debug_loc"); + TextSectionSym = EmitSectionSym(Asm, TLOF.getTextSection(), "text_begin"); EmitSectionSym(Asm, TLOF.getDataSection()); } @@ -2745,6 +2961,14 @@ void DwarfDebug::emitDIE(DIE *Die) { 4); break; } + case dwarf::DW_AT_location: { + if (UseDotDebugLocEntry.count(Die) != 0) { + DIELabel *L = cast<DIELabel>(Values[i]); + Asm->EmitLabelDifference(L->getValue(), DwarfDebugLocSectionSym, 4); + } else + Values[i]->EmitValue(Asm, Form); + break; + } default: // Emit an attribute using the defined form. Values[i]->EmitValue(Asm, Form); @@ -2771,37 +2995,41 @@ void DwarfDebug::emitDebugInfo() { // Start debug info section. Asm->OutStreamer.SwitchSection( Asm->getObjFileLowering().getDwarfInfoSection()); - DIE *Die = ModuleCU->getCUDie(); - - // Emit the compile units header. - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("info_begin", - ModuleCU->getID())); - - // Emit size of content not including length itself - unsigned ContentSize = Die->getSize() + - sizeof(int16_t) + // DWARF version number - sizeof(int32_t) + // Offset Into Abbrev. Section - sizeof(int8_t) + // Pointer Size (in bytes) - sizeof(int32_t); // FIXME - extra pad for gdb bug. - - Asm->OutStreamer.AddComment("Length of Compilation Unit Info"); - Asm->EmitInt32(ContentSize); - Asm->OutStreamer.AddComment("DWARF version number"); - Asm->EmitInt16(dwarf::DWARF_VERSION); - Asm->OutStreamer.AddComment("Offset Into Abbrev. Section"); - Asm->EmitSectionOffset(Asm->GetTempSymbol("abbrev_begin"), - DwarfAbbrevSectionSym); - Asm->OutStreamer.AddComment("Address Size (in bytes)"); - Asm->EmitInt8(Asm->getTargetData().getPointerSize()); - - emitDIE(Die); - // FIXME - extra padding for gdb bug. - Asm->OutStreamer.AddComment("4 extra padding bytes for GDB"); - Asm->EmitInt8(0); - Asm->EmitInt8(0); - Asm->EmitInt8(0); - Asm->EmitInt8(0); - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("info_end", ModuleCU->getID())); + for (DenseMap<const MDNode *, CompileUnit *>::iterator I = CUMap.begin(), + E = CUMap.end(); I != E; ++I) { + CompileUnit *TheCU = I->second; + DIE *Die = TheCU->getCUDie(); + + // Emit the compile units header. + Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("info_begin", + TheCU->getID())); + + // Emit size of content not including length itself + unsigned ContentSize = Die->getSize() + + sizeof(int16_t) + // DWARF version number + sizeof(int32_t) + // Offset Into Abbrev. Section + sizeof(int8_t) + // Pointer Size (in bytes) + sizeof(int32_t); // FIXME - extra pad for gdb bug. + + Asm->OutStreamer.AddComment("Length of Compilation Unit Info"); + Asm->EmitInt32(ContentSize); + Asm->OutStreamer.AddComment("DWARF version number"); + Asm->EmitInt16(dwarf::DWARF_VERSION); + Asm->OutStreamer.AddComment("Offset Into Abbrev. Section"); + Asm->EmitSectionOffset(Asm->GetTempSymbol("abbrev_begin"), + DwarfAbbrevSectionSym); + Asm->OutStreamer.AddComment("Address Size (in bytes)"); + Asm->EmitInt8(Asm->getTargetData().getPointerSize()); + + emitDIE(Die); + // FIXME - extra padding for gdb bug. + Asm->OutStreamer.AddComment("4 extra padding bytes for GDB"); + Asm->EmitInt8(0); + Asm->EmitInt8(0); + Asm->EmitInt8(0); + Asm->EmitInt8(0); + Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("info_end", TheCU->getID())); + } } /// emitAbbreviations - Emit the abbreviation section. @@ -2967,8 +3195,6 @@ void DwarfDebug::emitDebugLines() { MCSymbol *Label = LineInfo.getLabel(); if (!Label->isDefined()) continue; // Not emitted, in dead code. - if (LineInfo.getLine() == 0) continue; - if (Asm->isVerbose()) { std::pair<unsigned, unsigned> SrcID = getSourceDirectoryAndFileIds(LineInfo.getSourceID()); @@ -3128,91 +3354,99 @@ emitFunctionDebugFrame(const FunctionDebugFrameInfo &DebugFrameInfo) { /// emitDebugPubNames - Emit visible names into a debug pubnames section. /// void DwarfDebug::emitDebugPubNames() { - // Start the dwarf pubnames section. - Asm->OutStreamer.SwitchSection( - Asm->getObjFileLowering().getDwarfPubNamesSection()); - - Asm->OutStreamer.AddComment("Length of Public Names Info"); - Asm->EmitLabelDifference( - Asm->GetTempSymbol("pubnames_end", ModuleCU->getID()), - Asm->GetTempSymbol("pubnames_begin", ModuleCU->getID()), 4); - - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubnames_begin", - ModuleCU->getID())); - - Asm->OutStreamer.AddComment("DWARF Version"); - Asm->EmitInt16(dwarf::DWARF_VERSION); - - Asm->OutStreamer.AddComment("Offset of Compilation Unit Info"); - Asm->EmitSectionOffset(Asm->GetTempSymbol("info_begin", ModuleCU->getID()), - DwarfInfoSectionSym); - - Asm->OutStreamer.AddComment("Compilation Unit Length"); - Asm->EmitLabelDifference(Asm->GetTempSymbol("info_end", ModuleCU->getID()), - Asm->GetTempSymbol("info_begin", ModuleCU->getID()), - 4); - - const StringMap<DIE*> &Globals = ModuleCU->getGlobals(); - for (StringMap<DIE*>::const_iterator - GI = Globals.begin(), GE = Globals.end(); GI != GE; ++GI) { - const char *Name = GI->getKeyData(); - DIE *Entity = GI->second; - - Asm->OutStreamer.AddComment("DIE offset"); - Asm->EmitInt32(Entity->getOffset()); + for (DenseMap<const MDNode *, CompileUnit *>::iterator I = CUMap.begin(), + E = CUMap.end(); I != E; ++I) { + CompileUnit *TheCU = I->second; + // Start the dwarf pubnames section. + Asm->OutStreamer.SwitchSection( + Asm->getObjFileLowering().getDwarfPubNamesSection()); - if (Asm->isVerbose()) - Asm->OutStreamer.AddComment("External Name"); - Asm->OutStreamer.EmitBytes(StringRef(Name, strlen(Name)+1), 0); + Asm->OutStreamer.AddComment("Length of Public Names Info"); + Asm->EmitLabelDifference( + Asm->GetTempSymbol("pubnames_end", TheCU->getID()), + Asm->GetTempSymbol("pubnames_begin", TheCU->getID()), 4); + + Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubnames_begin", + TheCU->getID())); + + Asm->OutStreamer.AddComment("DWARF Version"); + Asm->EmitInt16(dwarf::DWARF_VERSION); + + Asm->OutStreamer.AddComment("Offset of Compilation Unit Info"); + Asm->EmitSectionOffset(Asm->GetTempSymbol("info_begin", TheCU->getID()), + DwarfInfoSectionSym); + + Asm->OutStreamer.AddComment("Compilation Unit Length"); + Asm->EmitLabelDifference(Asm->GetTempSymbol("info_end", TheCU->getID()), + Asm->GetTempSymbol("info_begin", TheCU->getID()), + 4); + + const StringMap<DIE*> &Globals = TheCU->getGlobals(); + for (StringMap<DIE*>::const_iterator + GI = Globals.begin(), GE = Globals.end(); GI != GE; ++GI) { + const char *Name = GI->getKeyData(); + DIE *Entity = GI->second; + + Asm->OutStreamer.AddComment("DIE offset"); + Asm->EmitInt32(Entity->getOffset()); + + if (Asm->isVerbose()) + Asm->OutStreamer.AddComment("External Name"); + Asm->OutStreamer.EmitBytes(StringRef(Name, strlen(Name)+1), 0); + } + + Asm->OutStreamer.AddComment("End Mark"); + Asm->EmitInt32(0); + Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubnames_end", + TheCU->getID())); } - - Asm->OutStreamer.AddComment("End Mark"); - Asm->EmitInt32(0); - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubnames_end", - ModuleCU->getID())); } void DwarfDebug::emitDebugPubTypes() { - // Start the dwarf pubnames section. - Asm->OutStreamer.SwitchSection( - Asm->getObjFileLowering().getDwarfPubTypesSection()); - Asm->OutStreamer.AddComment("Length of Public Types Info"); - Asm->EmitLabelDifference( - Asm->GetTempSymbol("pubtypes_end", ModuleCU->getID()), - Asm->GetTempSymbol("pubtypes_begin", ModuleCU->getID()), 4); - - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubtypes_begin", - ModuleCU->getID())); - - if (Asm->isVerbose()) Asm->OutStreamer.AddComment("DWARF Version"); - Asm->EmitInt16(dwarf::DWARF_VERSION); - - Asm->OutStreamer.AddComment("Offset of Compilation ModuleCU Info"); - Asm->EmitSectionOffset(Asm->GetTempSymbol("info_begin", ModuleCU->getID()), - DwarfInfoSectionSym); - - Asm->OutStreamer.AddComment("Compilation ModuleCU Length"); - Asm->EmitLabelDifference(Asm->GetTempSymbol("info_end", ModuleCU->getID()), - Asm->GetTempSymbol("info_begin", ModuleCU->getID()), - 4); - - const StringMap<DIE*> &Globals = ModuleCU->getGlobalTypes(); - for (StringMap<DIE*>::const_iterator - GI = Globals.begin(), GE = Globals.end(); GI != GE; ++GI) { - const char *Name = GI->getKeyData(); - DIE * Entity = GI->second; - - if (Asm->isVerbose()) Asm->OutStreamer.AddComment("DIE offset"); - Asm->EmitInt32(Entity->getOffset()); + for (DenseMap<const MDNode *, CompileUnit *>::iterator I = CUMap.begin(), + E = CUMap.end(); I != E; ++I) { + CompileUnit *TheCU = I->second; + // Start the dwarf pubnames section. + Asm->OutStreamer.SwitchSection( + Asm->getObjFileLowering().getDwarfPubTypesSection()); + Asm->OutStreamer.AddComment("Length of Public Types Info"); + Asm->EmitLabelDifference( + Asm->GetTempSymbol("pubtypes_end", TheCU->getID()), + Asm->GetTempSymbol("pubtypes_begin", TheCU->getID()), 4); + + Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubtypes_begin", + TheCU->getID())); - if (Asm->isVerbose()) Asm->OutStreamer.AddComment("External Name"); - Asm->OutStreamer.EmitBytes(StringRef(Name, GI->getKeyLength()+1), 0); + if (Asm->isVerbose()) Asm->OutStreamer.AddComment("DWARF Version"); + Asm->EmitInt16(dwarf::DWARF_VERSION); + + Asm->OutStreamer.AddComment("Offset of Compilation Unit Info"); + Asm->EmitSectionOffset(Asm->GetTempSymbol("info_begin", TheCU->getID()), + DwarfInfoSectionSym); + + Asm->OutStreamer.AddComment("Compilation Unit Length"); + Asm->EmitLabelDifference(Asm->GetTempSymbol("info_end", TheCU->getID()), + Asm->GetTempSymbol("info_begin", TheCU->getID()), + 4); + + const StringMap<DIE*> &Globals = TheCU->getGlobalTypes(); + for (StringMap<DIE*>::const_iterator + GI = Globals.begin(), GE = Globals.end(); GI != GE; ++GI) { + const char *Name = GI->getKeyData(); + DIE * Entity = GI->second; + + if (Asm->isVerbose()) Asm->OutStreamer.AddComment("DIE offset"); + Asm->EmitInt32(Entity->getOffset()); + + if (Asm->isVerbose()) Asm->OutStreamer.AddComment("External Name"); + Asm->OutStreamer.EmitBytes(StringRef(Name, GI->getKeyLength()+1), 0); + } + + Asm->OutStreamer.AddComment("End Mark"); + Asm->EmitInt32(0); + Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubtypes_end", + TheCU->getID())); } - - Asm->OutStreamer.AddComment("End Mark"); - Asm->EmitInt32(0); - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubtypes_end", - ModuleCU->getID())); } /// emitDebugStr - Emit visible names into a debug str section. @@ -3248,9 +3482,39 @@ void DwarfDebug::emitDebugStr() { /// emitDebugLoc - Emit visible names into a debug loc section. /// void DwarfDebug::emitDebugLoc() { + if (DotDebugLocEntries.empty()) + return; + // Start the dwarf loc section. Asm->OutStreamer.SwitchSection( - Asm->getObjFileLowering().getDwarfLocSection()); + Asm->getObjFileLowering().getDwarfLocSection()); + unsigned char Size = Asm->getTargetData().getPointerSize(); + Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("debug_loc", 0)); + unsigned index = 1; + for (SmallVector<DotDebugLocEntry, 4>::iterator I = DotDebugLocEntries.begin(), + E = DotDebugLocEntries.end(); I != E; ++I, ++index) { + DotDebugLocEntry Entry = *I; + if (Entry.isEmpty()) { + Asm->OutStreamer.EmitIntValue(0, Size, /*addrspace*/0); + Asm->OutStreamer.EmitIntValue(0, Size, /*addrspace*/0); + Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("debug_loc", index)); + } else { + Asm->OutStreamer.EmitSymbolValue(Entry.Begin, Size, 0); + Asm->OutStreamer.EmitSymbolValue(Entry.End, Size, 0); + const TargetRegisterInfo *RI = Asm->TM.getRegisterInfo(); + unsigned Reg = RI->getDwarfRegNum(Entry.Loc.getReg(), false); + if (Reg < 32) { + Asm->OutStreamer.AddComment("Loc expr size"); + Asm->EmitInt16(1); + Asm->EmitInt8(dwarf::DW_OP_reg0 + Reg); + } else { + Asm->OutStreamer.AddComment("Loc expr size"); + Asm->EmitInt16(1+MCAsmInfo::getULEB128Size(Reg)); + Asm->EmitInt8(dwarf::DW_OP_regx); + Asm->EmitULEB128(Reg); + } + } + } } /// EmitDebugARanges - Emit visible names into a debug aranges section. @@ -3310,7 +3574,7 @@ void DwarfDebug::emitDebugInlineInfo() { if (!Asm->MAI->doesDwarfUsesInlineInfoSection()) return; - if (!ModuleCU) + if (!FirstCU) return; Asm->OutStreamer.SwitchSection( @@ -3327,11 +3591,11 @@ void DwarfDebug::emitDebugInlineInfo() { Asm->OutStreamer.AddComment("Address Size (in bytes)"); Asm->EmitInt8(Asm->getTargetData().getPointerSize()); - for (SmallVector<MDNode *, 4>::iterator I = InlinedSPNodes.begin(), + for (SmallVector<const MDNode *, 4>::iterator I = InlinedSPNodes.begin(), E = InlinedSPNodes.end(); I != E; ++I) { - MDNode *Node = *I; - DenseMap<MDNode *, SmallVector<InlineInfoLabels, 4> >::iterator II + const MDNode *Node = *I; + DenseMap<const MDNode *, SmallVector<InlineInfoLabels, 4> >::iterator II = InlineInfo.find(Node); SmallVector<InlineInfoLabels, 4> &Labels = II->second; DISubprogram SP(Node); diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.h b/lib/CodeGen/AsmPrinter/DwarfDebug.h index b964b23..0d6116f 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.h +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.h @@ -15,6 +15,7 @@ #define CODEGEN_ASMPRINTER_DWARFDEBUG_H__ #include "llvm/CodeGen/AsmPrinter.h" +#include "llvm/CodeGen/MachineLocation.h" #include "DIE.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/FoldingSet.h" @@ -30,7 +31,6 @@ class DbgConcreteScope; class DbgScope; class DbgVariable; class MachineFrameInfo; -class MachineLocation; class MachineModuleInfo; class MachineOperand; class MCAsmInfo; @@ -82,8 +82,8 @@ class DwarfDebug { // Attributes used to construct specific Dwarf sections. // - /// ModuleCU - All DIEs are inserted in ModuleCU. - CompileUnit *ModuleCU; + CompileUnit *FirstCU; + DenseMap <const MDNode *, CompileUnit *> CUMap; /// AbbreviationsSet - Used to uniquely define abbreviations. /// @@ -146,15 +146,15 @@ class DwarfDebug { /// DbgScopeMap - Tracks the scopes in the current function. Owns the /// contained DbgScope*s. /// - DenseMap<MDNode *, DbgScope *> DbgScopeMap; + DenseMap<const MDNode *, DbgScope *> DbgScopeMap; /// ConcreteScopes - Tracks the concrete scopees in the current function. /// These scopes are also included in DbgScopeMap. - DenseMap<MDNode *, DbgScope *> ConcreteScopes; + DenseMap<const MDNode *, DbgScope *> ConcreteScopes; /// AbstractScopes - Tracks the abstract scopes a module. These scopes are /// not included DbgScopeMap. AbstractScopes owns its DbgScope*s. - DenseMap<MDNode *, DbgScope *> AbstractScopes; + DenseMap<const MDNode *, DbgScope *> AbstractScopes; /// AbstractScopesList - Tracks abstract scopes constructed while processing /// a function. This list is cleared during endFunction(). @@ -162,13 +162,43 @@ class DwarfDebug { /// AbstractVariables - Collection on abstract variables. Owned by the /// DbgScopes in AbstractScopes. - DenseMap<MDNode *, DbgVariable *> AbstractVariables; - - /// DbgValueStartMap - Tracks starting scope of variable DIEs. - /// If the scope of an object begins sometime after the low pc value for the - /// scope most closely enclosing the object, the object entry may have a - /// DW_AT_start_scope attribute. - DenseMap<const MachineInstr *, DbgVariable *> DbgValueStartMap; + DenseMap<const MDNode *, DbgVariable *> AbstractVariables; + + /// DbgVariableToFrameIndexMap - Tracks frame index used to find + /// variable's value. + DenseMap<const DbgVariable *, int> DbgVariableToFrameIndexMap; + + /// DbgVariableToDbgInstMap - Maps DbgVariable to corresponding DBG_VALUE + /// machine instruction. + DenseMap<const DbgVariable *, const MachineInstr *> DbgVariableToDbgInstMap; + + /// DbgVariableLabelsMap - Maps DbgVariable to corresponding MCSymbol. + DenseMap<const DbgVariable *, const MCSymbol *> DbgVariableLabelsMap; + + /// DotDebugLocEntry - This struct describes location entries emitted in + /// .debug_loc section. + typedef struct DotDebugLocEntry { + const MCSymbol *Begin; + const MCSymbol *End; + MachineLocation Loc; + DotDebugLocEntry() : Begin(0), End(0) {} + DotDebugLocEntry(const MCSymbol *B, const MCSymbol *E, + MachineLocation &L) : Begin(B), End(E), Loc(L) {} + /// Empty entries are also used as a trigger to emit temp label. Such + /// labels are referenced is used to find debug_loc offset for a given DIE. + bool isEmpty() { return Begin == 0 && End == 0; } + } DotDebugLocEntry; + + /// DotDebugLocEntries - Collection of DotDebugLocEntry. + SmallVector<DotDebugLocEntry, 4> DotDebugLocEntries; + + /// UseDotDebugLocEntry - DW_AT_location attributes for the DIEs in this set + /// idetifies corresponding .debug_loc entry offset. + SmallPtrSet<const DIE *, 4> UseDotDebugLocEntry; + + /// VarToAbstractVarMap - Maps DbgVariable with corresponding Abstract + /// DbgVariable, if any. + DenseMap<const DbgVariable *, const DbgVariable *> VarToAbstractVarMap; /// InliendSubprogramDIEs - Collection of subprgram DIEs that are marked /// (at the end of the module) as DW_AT_inline. @@ -177,7 +207,7 @@ class DwarfDebug { /// ContainingTypeMap - This map is used to keep track of subprogram DIEs that /// need DW_AT_containing_type attribute. This attribute points to a DIE that /// corresponds to the MDNode mapped with the subprogram DIE. - DenseMap<DIE *, MDNode *> ContainingTypeMap; + DenseMap<DIE *, const MDNode *> ContainingTypeMap; typedef SmallVector<DbgScope *, 2> ScopeVector; SmallPtrSet<const MachineInstr *, 8> InsnsBeginScopeSet; @@ -185,9 +215,9 @@ class DwarfDebug { /// InlineInfo - Keep track of inlined functions and their location. This /// information is used to populate debug_inlined section. - typedef std::pair<MCSymbol*, DIE *> InlineInfoLabels; - DenseMap<MDNode*, SmallVector<InlineInfoLabels, 4> > InlineInfo; - SmallVector<MDNode *, 4> InlinedSPNodes; + typedef std::pair<const MCSymbol *, DIE *> InlineInfoLabels; + DenseMap<const MDNode *, SmallVector<InlineInfoLabels, 4> > InlineInfo; + SmallVector<const MDNode *, 4> InlinedSPNodes; /// LabelsBeforeInsn - Maps instruction with label emitted before /// instruction. @@ -197,6 +227,13 @@ class DwarfDebug { /// instruction. DenseMap<const MachineInstr *, MCSymbol *> LabelsAfterInsn; + /// insnNeedsLabel - Collection of instructions that need a label to mark + /// a debuggging information entity. + SmallPtrSet<const MachineInstr *, 8> InsnNeedsLabel; + + /// ProcessedArgs - Collection of arguments already processed. + SmallPtrSet<const MDNode *, 8> ProcessedArgs; + SmallVector<const MCSymbol *, 8> DebugRangeSymbols; /// Previous instruction's location information. This is used to determine @@ -219,8 +256,8 @@ class DwarfDebug { // section offsets and are created by EmitSectionLabels. MCSymbol *DwarfFrameSectionSym, *DwarfInfoSectionSym, *DwarfAbbrevSectionSym; MCSymbol *DwarfStrSectionSym, *TextSectionSym, *DwarfDebugRangeSectionSym; - - MCSymbol *FunctionBeginSym; + MCSymbol *DwarfDebugLocSectionSym; + MCSymbol *FunctionBeginSym, *FunctionEndSym; private: /// getSourceDirectoryAndFileIds - Return the directory and file ids that @@ -295,7 +332,7 @@ private: /// addSourceLine - Add location information to specified debug information /// entry. void addSourceLine(DIE *Die, const DIVariable *V); - void addSourceLine(DIE *Die, const DIGlobal *G); + void addSourceLine(DIE *Die, const DIGlobalVariable *G); void addSourceLine(DIE *Die, const DISubprogram *SP); void addSourceLine(DIE *Die, const DIType *Ty); void addSourceLine(DIE *Die, const DINameSpace *NS); @@ -306,13 +343,13 @@ private: const MachineLocation &Location); /// addRegisterAddress - Add register location entry in variable DIE. - bool addRegisterAddress(DIE *Die, DbgVariable *DV, const MachineOperand &MO); + bool addRegisterAddress(DIE *Die, const MCSymbol *VS, const MachineOperand &MO); /// addConstantValue - Add constant value entry in variable DIE. - bool addConstantValue(DIE *Die, DbgVariable *DV, const MachineOperand &MO); + bool addConstantValue(DIE *Die, const MCSymbol *VS, const MachineOperand &MO); /// addConstantFPValue - Add constant value entry in variable DIE. - bool addConstantFPValue(DIE *Die, DbgVariable *DV, const MachineOperand &MO); + bool addConstantFPValue(DIE *Die, const MCSymbol *VS, const MachineOperand &MO); /// addComplexAddress - Start with the address based on the location provided, /// and generate the DWARF information necessary to find the actual variable @@ -380,21 +417,18 @@ private: DIE *createSubprogramDIE(const DISubprogram &SP, bool MakeDecl = false); /// getOrCreateDbgScope - Create DbgScope for the scope. - DbgScope *getOrCreateDbgScope(MDNode *Scope, MDNode *InlinedAt); + DbgScope *getOrCreateDbgScope(const MDNode *Scope, const MDNode *InlinedAt); - DbgScope *getOrCreateAbstractScope(MDNode *N); + DbgScope *getOrCreateAbstractScope(const MDNode *N); /// findAbstractVariable - Find abstract variable associated with Var. - DbgVariable *findAbstractVariable(DIVariable &Var, unsigned FrameIdx, - DebugLoc Loc); - DbgVariable *findAbstractVariable(DIVariable &Var, const MachineInstr *MI, - DebugLoc Loc); + DbgVariable *findAbstractVariable(DIVariable &Var, DebugLoc Loc); /// updateSubprogramScopeDIE - Find DIE for the given subprogram and /// attach appropriate DW_AT_low_pc and DW_AT_high_pc attributes. /// If there are global variables in this scope then create and insert /// DIEs for these variables. - DIE *updateSubprogramScopeDIE(MDNode *SPNode); + DIE *updateSubprogramScopeDIE(const MDNode *SPNode); /// constructLexicalScope - Construct new DW_TAG_lexical_block /// for this scope and attach DW_AT_low_pc/DW_AT_high_pc labels. @@ -506,11 +540,18 @@ private: /// maps as well. unsigned GetOrCreateSourceID(StringRef DirName, StringRef FileName); - void constructCompileUnit(MDNode *N); + /// constructCompileUnit - Create new CompileUnit for the given + /// metadata node with tag DW_TAG_compile_unit. + void constructCompileUnit(const MDNode *N); + + /// getCompielUnit - Get CompileUnit DIE. + CompileUnit *getCompileUnit(const MDNode *N) const; - void constructGlobalVariableDIE(MDNode *N); + /// constructGlobalVariableDIE - Construct global variable DIE. + void constructGlobalVariableDIE(const MDNode *N); - void constructSubprogramDIE(MDNode *N); + /// construct SubprogramDIE - Construct subprogram DIE. + void constructSubprogramDIE(const MDNode *N); // FIXME: This should go away in favor of complex addresses. /// Find the type the programmer originally declared the variable to be @@ -521,7 +562,7 @@ private: /// recordSourceLine - Register a source line with debug info. Returns the /// unique label that was emitted and which provides correspondence to /// the source line list. - MCSymbol *recordSourceLine(unsigned Line, unsigned Col, MDNode *Scope); + MCSymbol *recordSourceLine(unsigned Line, unsigned Col, const MDNode *Scope); /// getSourceLineCount - Return the number of source lines in the debug /// info. @@ -529,6 +570,20 @@ private: return Lines.size(); } + /// recordVariableFrameIndex - Record a variable's index. + void recordVariableFrameIndex(const DbgVariable *V, int Index); + + /// findVariableFrameIndex - Return true if frame index for the variable + /// is found. Update FI to hold value of the index. + bool findVariableFrameIndex(const DbgVariable *V, int *FI); + + /// findVariableLabel - Find MCSymbol for the variable. + const MCSymbol *findVariableLabel(const DbgVariable *V); + + /// findDbgScope - Find DbgScope for the debug loc attached with an + /// instruction. + DbgScope *findDbgScope(const MachineInstr *MI); + /// identifyScopeMarkers() - Indentify instructions that are marking /// beginning of or end of a scope. void identifyScopeMarkers(); @@ -538,8 +593,12 @@ private: bool extractScopeInformation(); /// collectVariableInfo - Populate DbgScope entries with variables' info. - void collectVariableInfo(); + void collectVariableInfo(const MachineFunction *); + /// collectVariableInfoFromMMITable - Collect variable information from + /// side table maintained by MMI. + void collectVariableInfoFromMMITable(const MachineFunction * MF, + SmallPtrSet<const MDNode *, 16> &P); public: //===--------------------------------------------------------------------===// // Main entry points. @@ -563,6 +622,12 @@ public: /// void endFunction(const MachineFunction *MF); + /// getLabelBeforeInsn - Return Label preceding the instruction. + const MCSymbol *getLabelBeforeInsn(const MachineInstr *MI); + + /// getLabelAfterInsn - Return Label immediately following the instruction. + const MCSymbol *getLabelAfterInsn(const MachineInstr *MI); + /// beginScope - Process beginning of a scope. void beginScope(const MachineInstr *MI); diff --git a/lib/CodeGen/AsmPrinter/DwarfException.cpp b/lib/CodeGen/AsmPrinter/DwarfException.cpp index 0ff1036..c872840 100644 --- a/lib/CodeGen/AsmPrinter/DwarfException.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfException.cpp @@ -189,7 +189,7 @@ void DwarfException::EmitFDE(const FunctionEHFrameInfo &EHFrameInfo) { // EH Frame, but some environments do not handle weak absolute symbols. If // UnwindTablesMandatory is set we cannot do this optimization; the unwind // info is to be available for non-EH uses. - if (!EHFrameInfo.hasCalls && !UnwindTablesMandatory && + if (!EHFrameInfo.adjustsStack && !UnwindTablesMandatory && (!TheFunc->isWeakForLinker() || !Asm->MAI->getWeakDefDirective() || TLOF.getSupportsWeakOmittedEHFrame())) { @@ -949,11 +949,12 @@ void DwarfException::EndFunction() { TLOF.isFunctionEHFrameSymbolPrivate()); // Save EH frame information - EHFrames.push_back(FunctionEHFrameInfo(FunctionEHSym, - Asm->getFunctionNumber(), - MMI->getPersonalityIndex(), - Asm->MF->getFrameInfo()->hasCalls(), - !MMI->getLandingPads().empty(), - MMI->getFrameMoves(), - Asm->MF->getFunction())); + EHFrames. + push_back(FunctionEHFrameInfo(FunctionEHSym, + Asm->getFunctionNumber(), + MMI->getPersonalityIndex(), + Asm->MF->getFrameInfo()->adjustsStack(), + !MMI->getLandingPads().empty(), + MMI->getFrameMoves(), + Asm->MF->getFunction())); } diff --git a/lib/CodeGen/AsmPrinter/DwarfException.h b/lib/CodeGen/AsmPrinter/DwarfException.h index 5839f8c..bc311e6 100644 --- a/lib/CodeGen/AsmPrinter/DwarfException.h +++ b/lib/CodeGen/AsmPrinter/DwarfException.h @@ -45,7 +45,7 @@ class DwarfException { MCSymbol *FunctionEHSym; // L_foo.eh unsigned Number; unsigned PersonalityIndex; - bool hasCalls; + bool adjustsStack; bool hasLandingPads; std::vector<MachineMove> Moves; const Function *function; @@ -55,7 +55,7 @@ class DwarfException { const std::vector<MachineMove> &M, const Function *f): FunctionEHSym(EHSym), Number(Num), PersonalityIndex(P), - hasCalls(hC), hasLandingPads(hL), Moves(M), function (f) { } + adjustsStack(hC), hasLandingPads(hL), Moves(M), function (f) { } }; std::vector<FunctionEHFrameInfo> EHFrames; diff --git a/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp b/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp index a8c3c7b..f92127f 100644 --- a/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp @@ -104,6 +104,21 @@ void OcamlGCMetadataPrinter::finishAssembly(AsmPrinter &AP) { AP.OutStreamer.SwitchSection(AP.getObjFileLowering().getDataSection()); EmitCamlGlobal(getModule(), AP, "frametable"); + int NumDescriptors = 0; + for (iterator I = begin(), IE = end(); I != IE; ++I) { + GCFunctionInfo &FI = **I; + for (GCFunctionInfo::iterator J = FI.begin(), JE = FI.end(); J != JE; ++J) { + NumDescriptors++; + } + } + + if (NumDescriptors >= 1<<16) { + // Very rude! + report_fatal_error(" Too much descriptor for ocaml GC"); + } + AP.EmitInt16(NumDescriptors); + AP.EmitAlignment(IntPtrSize == 4 ? 2 : 3); + for (iterator I = begin(), IE = end(); I != IE; ++I) { GCFunctionInfo &FI = **I; @@ -135,11 +150,13 @@ void OcamlGCMetadataPrinter::finishAssembly(AsmPrinter &AP) { for (GCFunctionInfo::live_iterator K = FI.live_begin(J), KE = FI.live_end(J); K != KE; ++K) { - assert(K->StackOffset < 1<<16 && - "GC root stack offset is outside of fixed stack frame and out " - "of range for ocaml GC!"); - - AP.EmitInt32(K->StackOffset); + if (K->StackOffset >= 1<<16) { + // Very rude! + report_fatal_error( + "GC root stack offset is outside of fixed stack frame and out " + "of range for ocaml GC!"); + } + AP.EmitInt16(K->StackOffset); } AP.EmitAlignment(IntPtrSize == 4 ? 2 : 3); diff --git a/lib/CodeGen/CriticalAntiDepBreaker.cpp b/lib/CodeGen/CriticalAntiDepBreaker.cpp index 759fbaa..fd957b1 100644 --- a/lib/CodeGen/CriticalAntiDepBreaker.cpp +++ b/lib/CodeGen/CriticalAntiDepBreaker.cpp @@ -26,7 +26,7 @@ using namespace llvm; CriticalAntiDepBreaker:: -CriticalAntiDepBreaker(MachineFunction& MFi) : +CriticalAntiDepBreaker(MachineFunction& MFi) : AntiDepBreaker(), MF(MFi), MRI(MF.getRegInfo()), TRI(MF.getTarget().getRegisterInfo()), @@ -172,7 +172,7 @@ void CriticalAntiDepBreaker::PrescanInstruction(MachineInstr *MI) { unsigned Reg = MO.getReg(); if (Reg == 0) continue; const TargetRegisterClass *NewRC = 0; - + if (i < MI->getDesc().getNumOperands()) NewRC = MI->getDesc().OpInfo[i].getRegClass(TRI); @@ -422,7 +422,7 @@ BreakAntiDependencies(const std::vector<SUnit>& SUnits, // breaking anti-dependence edges that aren't going to significantly // impact the overall schedule. There are a limited number of registers // and we want to save them for the important edges. - // + // // TODO: Instructions with multiple defs could have multiple // anti-dependencies. The current code here only knows how to break one // edge per instruction. Note that we'd have to be able to break all of diff --git a/lib/CodeGen/IntrinsicLowering.cpp b/lib/CodeGen/IntrinsicLowering.cpp index e1c52f7..63bb5f2 100644 --- a/lib/CodeGen/IntrinsicLowering.cpp +++ b/lib/CodeGen/IntrinsicLowering.cpp @@ -83,6 +83,12 @@ static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI, return NewCI; } +// VisualStudio defines setjmp as _setjmp +#if defined(_MSC_VER) && defined(setjmp) +#define setjmp_undefined_for_visual_studio +#undef setjmp +#endif + void IntrinsicLowering::AddPrototypes(Module &M) { LLVMContext &Context = M.getContext(); for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp index 331dc7d..b584704 100644 --- a/lib/CodeGen/LLVMTargetMachine.cpp +++ b/lib/CodeGen/LLVMTargetMachine.cpp @@ -65,6 +65,12 @@ static cl::opt<bool> PrintISelInput("print-isel-input", cl::Hidden, cl::desc("Print LLVM IR input to isel pass")); static cl::opt<bool> PrintGCInfo("print-gc", cl::Hidden, cl::desc("Dump garbage collector data")); +static cl::opt<bool> ShowMCEncoding("show-mc-encoding", cl::Hidden, + cl::desc("Show encoding in .s output")); +static cl::opt<bool> ShowMCInst("show-mc-inst", cl::Hidden, + cl::desc("Show instruction structure in .s output")); +static cl::opt<bool> EnableMCLogging("enable-mc-api-logging", cl::Hidden, + cl::desc("Enable MC API logging")); static cl::opt<bool> VerifyMachineCode("verify-machineinstrs", cl::Hidden, cl::desc("Verify generated machine code"), cl::init(getenv("LLVM_VERIFY_MACHINEINSTRS")!=NULL)); @@ -131,21 +137,33 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, case CGFT_AssemblyFile: { MCInstPrinter *InstPrinter = getTarget().createMCInstPrinter(MAI.getAssemblerDialect(), MAI); + + // Create a code emitter if asked to show the encoding. + // + // FIXME: These are currently leaked. + MCCodeEmitter *MCE = 0; + if (ShowMCEncoding) + MCE = getTarget().createCodeEmitter(*this, *Context); + AsmStreamer.reset(createAsmStreamer(*Context, Out, getTargetData()->isLittleEndian(), getVerboseAsm(), InstPrinter, - /*codeemitter*/0)); + MCE, ShowMCInst)); break; } case CGFT_ObjectFile: { // Create the code emitter for the target if it exists. If not, .o file // emission fails. + // + // FIXME: These are currently leaked. MCCodeEmitter *MCE = getTarget().createCodeEmitter(*this, *Context); TargetAsmBackend *TAB = getTarget().createAsmBackend(TargetTriple); if (MCE == 0 || TAB == 0) return true; - - AsmStreamer.reset(createMachOStreamer(*Context, *TAB, Out, MCE)); + + AsmStreamer.reset(getTarget().createObjectStreamer(TargetTriple, *Context, + *TAB, Out, MCE, + hasMCRelaxAll())); break; } case CGFT_Null: @@ -154,7 +172,10 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, AsmStreamer.reset(createNullStreamer(*Context)); break; } - + + if (EnableMCLogging) + AsmStreamer.reset(createLoggingStreamer(AsmStreamer.take(), errs())); + // Create the AsmPrinter, which takes ownership of AsmStreamer if successful. FunctionPass *Printer = getTarget().createAsmPrinter(*this, *AsmStreamer); if (Printer == 0) diff --git a/lib/CodeGen/LatencyPriorityQueue.cpp b/lib/CodeGen/LatencyPriorityQueue.cpp index f1bd573..03b4eab 100644 --- a/lib/CodeGen/LatencyPriorityQueue.cpp +++ b/lib/CodeGen/LatencyPriorityQueue.cpp @@ -68,7 +68,7 @@ SUnit *LatencyPriorityQueue::getSingleUnscheduledPred(SUnit *SU) { return OnlyAvailablePred; } -void LatencyPriorityQueue::push_impl(SUnit *SU) { +void LatencyPriorityQueue::push(SUnit *SU) { // Look at all of the successors of this node. Count the number of nodes that // this node is the sole unscheduled node for. unsigned NumNodesBlocking = 0; @@ -79,7 +79,7 @@ void LatencyPriorityQueue::push_impl(SUnit *SU) { } NumNodesSolelyBlocking[SU->NodeNum] = NumNodesBlocking; - Queue.push(SU); + Queue.push_back(SU); } @@ -114,3 +114,25 @@ void LatencyPriorityQueue::AdjustPriorityOfUnscheduledPreds(SUnit *SU) { // NumNodesSolelyBlocking value. push(OnlyAvailablePred); } + +SUnit *LatencyPriorityQueue::pop() { + if (empty()) return NULL; + std::vector<SUnit *>::iterator Best = Queue.begin(); + for (std::vector<SUnit *>::iterator I = next(Queue.begin()), + E = Queue.end(); I != E; ++I) + if (Picker(*Best, *I)) + Best = I; + SUnit *V = *Best; + if (Best != prior(Queue.end())) + std::swap(*Best, Queue.back()); + Queue.pop_back(); + return V; +} + +void LatencyPriorityQueue::remove(SUnit *SU) { + assert(!Queue.empty() && "Queue is empty!"); + std::vector<SUnit *>::iterator I = std::find(Queue.begin(), Queue.end(), SU); + if (I != prior(Queue.end())) + std::swap(*I, Queue.back()); + Queue.pop_back(); +} diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp index ca9921c..a6d38ad 100644 --- a/lib/CodeGen/LiveIntervalAnalysis.cpp +++ b/lib/CodeGen/LiveIntervalAnalysis.cpp @@ -263,7 +263,7 @@ static void printRegName(unsigned reg, const TargetRegisterInfo* tri_) { #endif static -bool MultipleDefsByMI(const MachineInstr &MI, unsigned MOIdx) { +bool MultipleDefsBySameMI(const MachineInstr &MI, unsigned MOIdx) { unsigned Reg = MI.getOperand(MOIdx).getReg(); for (unsigned i = MOIdx+1, e = MI.getNumOperands(); i < e; ++i) { const MachineOperand &MO = MI.getOperand(i); @@ -279,6 +279,24 @@ bool MultipleDefsByMI(const MachineInstr &MI, unsigned MOIdx) { return false; } +/// isPartialRedef - Return true if the specified def at the specific index is +/// partially re-defining the specified live interval. A common case of this is +/// a definition of the sub-register. +bool LiveIntervals::isPartialRedef(SlotIndex MIIdx, MachineOperand &MO, + LiveInterval &interval) { + if (!MO.getSubReg() || MO.isEarlyClobber()) + return false; + + SlotIndex RedefIndex = MIIdx.getDefIndex(); + const LiveRange *OldLR = + interval.getLiveRangeContaining(RedefIndex.getUseIndex()); + if (OldLR->valno->isDefAccurate()) { + MachineInstr *DefMI = getInstructionFromIndex(OldLR->valno->def); + return DefMI->findRegisterDefOperandIdx(interval.reg) != -1; + } + return false; +} + void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, MachineBasicBlock::iterator mi, SlotIndex MIIdx, @@ -302,15 +320,20 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, // of inputs. if (MO.isEarlyClobber()) defIndex = MIIdx.getUseIndex(); - VNInfo *ValNo; + + // Make sure the first definition is not a partial redefinition. Add an + // <imp-def> of the full register. + if (MO.getSubReg()) + mi->addRegisterDefined(interval.reg); + MachineInstr *CopyMI = NULL; unsigned SrcReg, DstReg, SrcSubReg, DstSubReg; if (mi->isExtractSubreg() || mi->isInsertSubreg() || mi->isSubregToReg() || tii_->isMoveInstr(*mi, SrcReg, DstReg, SrcSubReg, DstSubReg)) CopyMI = mi; - // Earlyclobbers move back one. - ValNo = interval.getNextValue(defIndex, CopyMI, true, VNInfoAllocator); + VNInfo *ValNo = interval.getNextValue(defIndex, CopyMI, true, + VNInfoAllocator); assert(ValNo->id == 0 && "First value in interval is not 0?"); // Loop over all of the blocks that the vreg is defined in. There are @@ -389,9 +412,9 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, } } else { - if (MultipleDefsByMI(*mi, MOIdx)) - // Mutple defs of the same virtual register by the same instruction. e.g. - // %reg1031:5<def>, %reg1031:6<def> = VLD1q16 %reg1024<kill>, ... + if (MultipleDefsBySameMI(*mi, MOIdx)) + // Multiple defs of the same virtual register by the same instruction. + // e.g. %reg1031:5<def>, %reg1031:6<def> = VLD1q16 %reg1024<kill>, ... // This is likely due to elimination of REG_SEQUENCE instructions. Return // here since there is nothing to do. return; @@ -400,13 +423,21 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, // must be due to phi elimination or two addr elimination. If this is // the result of two address elimination, then the vreg is one of the // def-and-use register operand. - if (mi->isRegTiedToUseOperand(MOIdx)) { + + // It may also be partial redef like this: + // 80 %reg1041:6<def> = VSHRNv4i16 %reg1034<kill>, 12, pred:14, pred:%reg0 + // 120 %reg1041:5<def> = VSHRNv4i16 %reg1039<kill>, 12, pred:14, pred:%reg0 + bool PartReDef = isPartialRedef(MIIdx, MO, interval); + if (PartReDef || mi->isRegTiedToUseOperand(MOIdx)) { // If this is a two-address definition, then we have already processed // the live range. The only problem is that we didn't realize there // are actually two values in the live interval. Because of this we // need to take the LiveRegion that defines this register and split it // into two values. - assert(interval.containsOneValue()); + // Two-address vregs should always only be redefined once. This means + // that at this point, there should be exactly one value number in it. + assert((PartReDef || interval.containsOneValue()) && + "Unexpected 2-addr liveint!"); SlotIndex DefIndex = interval.getValNumInfo(0)->def.getDefIndex(); SlotIndex RedefIndex = MIIdx.getDefIndex(); if (MO.isEarlyClobber()) @@ -420,10 +451,6 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, // because the 2-addr copy must be in the same MBB as the redef. interval.removeRange(DefIndex, RedefIndex); - // Two-address vregs should always only be redefined once. This means - // that at this point, there should be exactly one value number in it. - assert(interval.containsOneValue() && "Unexpected 2-addr liveint!"); - // The new value number (#1) is defined by the instruction we claimed // defined value #0. VNInfo *ValNo = interval.getNextValue(OldValNo->def, OldValNo->getCopy(), @@ -434,6 +461,12 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, // Value#0 is now defined by the 2-addr instruction. OldValNo->def = RedefIndex; OldValNo->setCopy(0); + + // A re-def may be a copy. e.g. %reg1030:6<def> = VMOVD %reg1026, ... + unsigned SrcReg, DstReg, SrcSubReg, DstSubReg; + if (PartReDef && + tii_->isMoveInstr(*mi, SrcReg, DstReg, SrcSubReg, DstSubReg)) + OldValNo->setCopy(&*mi); // Add the new live interval which replaces the range for the input copy. LiveRange LR(DefIndex, RedefIndex, ValNo); @@ -451,8 +484,7 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, dbgs() << " RESULT: "; interval.print(dbgs(), tri_); }); - } else { - assert(lv_->isPHIJoin(interval.reg) && "Multiply defined register"); + } else if (lv_->isPHIJoin(interval.reg)) { // In the case of PHI elimination, each variable definition is only // live until the end of the block. We've already taken care of the // rest of the live range. @@ -475,6 +507,8 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, ValNo->addKill(indexes_->getTerminatorGap(mbb)); ValNo->setHasPHIKill(true); DEBUG(dbgs() << " phi-join +" << LR); + } else { + llvm_unreachable("Multiply defined register"); } } @@ -528,7 +562,7 @@ void LiveIntervals::handlePhysicalRegisterDef(MachineBasicBlock *MBB, end = baseIndex.getDefIndex(); goto exit; } else { - int DefIdx = mi->findRegisterDefOperandIdx(interval.reg, false, tri_); + int DefIdx = mi->findRegisterDefOperandIdx(interval.reg,false,false,tri_); if (DefIdx != -1) { if (mi->isRegTiedToUseOperand(DefIdx)) { // Two-address instruction. @@ -590,7 +624,7 @@ void LiveIntervals::handleRegisterDef(MachineBasicBlock *MBB, for (const unsigned* AS = tri_->getSubRegisters(MO.getReg()); *AS; ++AS) // If MI also modifies the sub-register explicitly, avoid processing it // more than once. Do not pass in TRI here so it checks for exact match. - if (!MI->modifiesRegister(*AS)) + if (!MI->definesRegister(*AS)) handlePhysicalRegisterDef(MBB, MI, MIIdx, MO, getOrCreateInterval(*AS), 0); } @@ -631,7 +665,7 @@ void LiveIntervals::handleLiveInRegister(MachineBasicBlock *MBB, end = baseIndex.getDefIndex(); SeenDefUse = true; break; - } else if (mi->modifiesRegister(interval.reg, tri_)) { + } else if (mi->definesRegister(interval.reg, tri_)) { // Another instruction redefines the register before it is ever read. // Then the register is essentially dead at the instruction that defines // it. Hence its interval is: @@ -1343,7 +1377,8 @@ rewriteInstructionsForSpills(const LiveInterval &li, bool TrySplit, MI->eraseFromParent(); continue; } - assert(!O.isImplicit() && "Spilling register that's used as implicit use?"); + assert(!(O.isImplicit() && O.isUse()) && + "Spilling register that's used as implicit use?"); SlotIndex index = getInstructionIndex(MI); if (index < start || index >= end) continue; @@ -1605,7 +1640,7 @@ LiveIntervals::getSpillWeight(bool isDef, bool isUse, unsigned loopDepth) { // overflow a float. This expression behaves like 10^d for small d, but is // more tempered for large d. At d=200 we get 6.7e33 which leaves a bit of // headroom before overflow. - float lc = powf(1 + (100.0f / (loopDepth+10)), (float)loopDepth); + float lc = std::pow(1 + (100.0f / (loopDepth+10)), (float)loopDepth); return (isDef + isUse) * lc; } diff --git a/lib/CodeGen/LowerSubregs.cpp b/lib/CodeGen/LowerSubregs.cpp index b4ef648..b0348a5 100644 --- a/lib/CodeGen/LowerSubregs.cpp +++ b/lib/CodeGen/LowerSubregs.cpp @@ -140,7 +140,8 @@ bool LowerSubregsInstructionPass::LowerExtract(MachineInstr *MI) { // Insert copy const TargetRegisterClass *TRCS = TRI->getPhysicalRegisterRegClass(DstReg); const TargetRegisterClass *TRCD = TRI->getPhysicalRegisterRegClass(SrcReg); - bool Emitted = TII->copyRegToReg(*MBB, MI, DstReg, SrcReg, TRCD, TRCS); + bool Emitted = TII->copyRegToReg(*MBB, MI, DstReg, SrcReg, TRCD, TRCS, + MI->getDebugLoc()); (void)Emitted; assert(Emitted && "Subreg and Dst must be of compatible register class"); // Transfer the kill/dead flags, if needed. @@ -193,7 +194,8 @@ bool LowerSubregsInstructionPass::LowerSubregToReg(MachineInstr *MI) { // Insert sub-register copy const TargetRegisterClass *TRC0= TRI->getPhysicalRegisterRegClass(DstSubReg); const TargetRegisterClass *TRC1= TRI->getPhysicalRegisterRegClass(InsReg); - bool Emitted = TII->copyRegToReg(*MBB, MI, DstSubReg, InsReg, TRC0, TRC1); + bool Emitted = TII->copyRegToReg(*MBB, MI, DstSubReg, InsReg, TRC0, TRC1, + MI->getDebugLoc()); (void)Emitted; assert(Emitted && "Subreg and Dst must be of compatible register class"); // Transfer the kill/dead flags, if needed. @@ -262,7 +264,8 @@ bool LowerSubregsInstructionPass::LowerInsert(MachineInstr *MI) { BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(TargetOpcode::KILL), DstSubReg); else { - bool Emitted = TII->copyRegToReg(*MBB, MI, DstSubReg, InsReg, TRC0, TRC1); + bool Emitted = TII->copyRegToReg(*MBB, MI, DstSubReg, InsReg, TRC0, TRC1, + MI->getDebugLoc()); (void)Emitted; assert(Emitted && "Subreg and Dst must be of compatible register class"); } diff --git a/lib/CodeGen/MachineCSE.cpp b/lib/CodeGen/MachineCSE.cpp index 84c3d71..6f4f7a8 100644 --- a/lib/CodeGen/MachineCSE.cpp +++ b/lib/CodeGen/MachineCSE.cpp @@ -23,6 +23,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/ScopedHashTable.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" using namespace llvm; @@ -30,6 +31,9 @@ using namespace llvm; STATISTIC(NumCoalesces, "Number of copies coalesced"); STATISTIC(NumCSEs, "Number of common subexpression eliminated"); +static cl::opt<bool> CSEPhysDef("machine-cse-phys-defs", + cl::init(false), cl::Hidden); + namespace { class MachineCSE : public MachineFunctionPass { const TargetInstrInfo *TII; @@ -39,7 +43,7 @@ namespace { MachineRegisterInfo *MRI; public: static char ID; // Pass identification - MachineCSE() : MachineFunctionPass(&ID), CurrVN(0) {} + MachineCSE() : MachineFunctionPass(&ID), LookAheadLimit(5), CurrVN(0) {} virtual bool runOnMachineFunction(MachineFunction &MF); @@ -52,6 +56,7 @@ namespace { } private: + const unsigned LookAheadLimit; typedef ScopedHashTableScope<MachineInstr*, unsigned, MachineInstrExpressionTrait> ScopeType; DenseMap<MachineBasicBlock*, ScopeType*> ScopeMap; @@ -62,8 +67,12 @@ namespace { bool PerformTrivialCoalescing(MachineInstr *MI, MachineBasicBlock *MBB); bool isPhysDefTriviallyDead(unsigned Reg, MachineBasicBlock::const_iterator I, - MachineBasicBlock::const_iterator E); - bool hasLivePhysRegDefUse(MachineInstr *MI, MachineBasicBlock *MBB); + MachineBasicBlock::const_iterator E) const ; + bool hasLivePhysRegDefUse(const MachineInstr *MI, + const MachineBasicBlock *MBB, + unsigned &PhysDef) const; + bool PhysRegDefReaches(MachineInstr *CSMI, MachineInstr *MI, + unsigned PhysDef) const; bool isCSECandidate(MachineInstr *MI); bool isProfitableToCSE(unsigned CSReg, unsigned Reg, MachineInstr *CSMI, MachineInstr *MI); @@ -112,6 +121,7 @@ bool MachineCSE::PerformTrivialCoalescing(MachineInstr *MI, DEBUG(dbgs() << "Coalescing: " << *DefMI); DEBUG(dbgs() << "*** to: " << *MI); MO.setReg(SrcReg); + MRI->clearKillFlags(SrcReg); if (NewRC != SRC) MRI->setRegClass(SrcReg, NewRC); DefMI->eraseFromParent(); @@ -123,10 +133,11 @@ bool MachineCSE::PerformTrivialCoalescing(MachineInstr *MI, return Changed; } -bool MachineCSE::isPhysDefTriviallyDead(unsigned Reg, - MachineBasicBlock::const_iterator I, - MachineBasicBlock::const_iterator E) { - unsigned LookAheadLeft = 5; +bool +MachineCSE::isPhysDefTriviallyDead(unsigned Reg, + MachineBasicBlock::const_iterator I, + MachineBasicBlock::const_iterator E) const { + unsigned LookAheadLeft = LookAheadLimit; while (LookAheadLeft) { // Skip over dbg_value's. while (I != E && I->isDebugValue()) @@ -144,6 +155,7 @@ bool MachineCSE::isPhysDefTriviallyDead(unsigned Reg, if (!TRI->regsOverlap(MO.getReg(), Reg)) continue; if (MO.isUse()) + // Found a use! return false; SeenDef = true; } @@ -159,41 +171,73 @@ bool MachineCSE::isPhysDefTriviallyDead(unsigned Reg, } /// hasLivePhysRegDefUse - Return true if the specified instruction read / write -/// physical registers (except for dead defs of physical registers). -bool MachineCSE::hasLivePhysRegDefUse(MachineInstr *MI, MachineBasicBlock *MBB){ - unsigned PhysDef = 0; +/// physical registers (except for dead defs of physical registers). It also +/// returns the physical register def by reference if it's the only one. +bool MachineCSE::hasLivePhysRegDefUse(const MachineInstr *MI, + const MachineBasicBlock *MBB, + unsigned &PhysDef) const { + PhysDef = 0; for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - MachineOperand &MO = MI->getOperand(i); + const MachineOperand &MO = MI->getOperand(i); if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); if (!Reg) continue; - if (TargetRegisterInfo::isPhysicalRegister(Reg)) { - if (MO.isUse()) - // Can't touch anything to read a physical register. - return true; - if (MO.isDead()) - // If the def is dead, it's ok. - continue; - // Ok, this is a physical register def that's not marked "dead". That's - // common since this pass is run before livevariables. We can scan - // forward a few instructions and check if it is obviously dead. - if (PhysDef) - // Multiple physical register defs. These are rare, forget about it. - return true; - PhysDef = Reg; + if (TargetRegisterInfo::isVirtualRegister(Reg)) + continue; + if (MO.isUse()) + // Can't touch anything to read a physical register. + return true; + if (MO.isDead()) + // If the def is dead, it's ok. + continue; + // Ok, this is a physical register def that's not marked "dead". That's + // common since this pass is run before livevariables. We can scan + // forward a few instructions and check if it is obviously dead. + if (PhysDef) { + // Multiple physical register defs. These are rare, forget about it. + PhysDef = 0; + return true; } + PhysDef = Reg; } if (PhysDef) { - MachineBasicBlock::iterator I = MI; I = llvm::next(I); + MachineBasicBlock::const_iterator I = MI; I = llvm::next(I); if (!isPhysDefTriviallyDead(PhysDef, I, MBB->end())) return true; } return false; } +bool MachineCSE::PhysRegDefReaches(MachineInstr *CSMI, MachineInstr *MI, + unsigned PhysDef) const { + // For now conservatively returns false if the common subexpression is + // not in the same basic block as the given instruction. + MachineBasicBlock *MBB = MI->getParent(); + if (CSMI->getParent() != MBB) + return false; + MachineBasicBlock::const_iterator I = CSMI; I = llvm::next(I); + MachineBasicBlock::const_iterator E = MI; + unsigned LookAheadLeft = LookAheadLimit; + while (LookAheadLeft) { + // Skip over dbg_value's. + while (I != E && I->isDebugValue()) + ++I; + + if (I == E) + return true; + if (I->modifiesRegister(PhysDef, TRI)) + return false; + + --LookAheadLeft; + ++I; + } + + return false; +} + static bool isCopy(const MachineInstr *MI, const TargetInstrInfo *TII) { unsigned SrcReg, DstReg, SrcSubIdx, DstSubIdx; return TII->isMoveInstr(*MI, SrcReg, DstReg, SrcSubIdx, DstSubIdx) || @@ -326,9 +370,20 @@ bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) { // If the instruction defines a physical register and the value *may* be // used, then it's not safe to replace it with a common subexpression. - if (FoundCSE && hasLivePhysRegDefUse(MI, MBB)) + unsigned PhysDef = 0; + if (FoundCSE && hasLivePhysRegDefUse(MI, MBB, PhysDef)) { FoundCSE = false; + // ... Unless the CS is local and it also defines the physical register + // which is not clobbered in between. + if (PhysDef && CSEPhysDef) { + unsigned CSVN = VNT.lookup(MI); + MachineInstr *CSMI = Exps[CSVN]; + if (PhysRegDefReaches(CSMI, MI, PhysDef)) + FoundCSE = true; + } + } + if (!FoundCSE) { VNT.insert(MI, CurrVN++); Exps.push_back(MI); @@ -365,8 +420,10 @@ bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) { // Actually perform the elimination. if (DoCSE) { - for (unsigned i = 0, e = CSEPairs.size(); i != e; ++i) + for (unsigned i = 0, e = CSEPairs.size(); i != e; ++i) { MRI->replaceRegWith(CSEPairs[i].first, CSEPairs[i].second); + MRI->clearKillFlags(CSEPairs[i].second); + } MI->eraseFromParent(); ++NumCSEs; } else { diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp index 3cf10b3..a38c881 100644 --- a/lib/CodeGen/MachineFunction.cpp +++ b/lib/CodeGen/MachineFunction.cpp @@ -398,8 +398,14 @@ void MachineFunction::viewCFGOnly() const unsigned MachineFunction::addLiveIn(unsigned PReg, const TargetRegisterClass *RC) { assert(RC->contains(PReg) && "Not the correct regclass!"); - unsigned VReg = getRegInfo().createVirtualRegister(RC); - getRegInfo().addLiveIn(PReg, VReg); + MachineRegisterInfo &MRI = getRegInfo(); + unsigned VReg = MRI.getLiveInVirtReg(PReg); + if (VReg) { + assert(MRI.getRegClass(VReg) == RC && "Register class mismatch!"); + return VReg; + } + VReg = MRI.createVirtualRegister(RC); + MRI.addLiveIn(PReg, VReg); return VReg; } diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp index 99b5beb..e54cd5c 100644 --- a/lib/CodeGen/MachineInstr.cpp +++ b/lib/CodeGen/MachineInstr.cpp @@ -219,8 +219,12 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const { OS << "%physreg" << getReg(); } - if (getSubReg() != 0) - OS << ':' << getSubReg(); + if (getSubReg() != 0) { + if (TM) + OS << ':' << TM->getRegisterInfo()->getSubRegIndexName(getSubReg()); + else + OS << ':' << getSubReg(); + } if (isDef() || isKill() || isDead() || isImplicit() || isUndef() || isEarlyClobber()) { @@ -781,25 +785,57 @@ int MachineInstr::findRegisterUseOperandIdx(unsigned Reg, bool isKill, } return -1; } - + +/// readsWritesVirtualRegister - Return a pair of bools (reads, writes) +/// indicating if this instruction reads or writes Reg. This also considers +/// partial defines. +std::pair<bool,bool> +MachineInstr::readsWritesVirtualRegister(unsigned Reg, + SmallVectorImpl<unsigned> *Ops) const { + bool PartDef = false; // Partial redefine. + bool FullDef = false; // Full define. + bool Use = false; + + for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { + const MachineOperand &MO = getOperand(i); + if (!MO.isReg() || MO.getReg() != Reg) + continue; + if (Ops) + Ops->push_back(i); + if (MO.isUse()) + Use |= !MO.isUndef(); + else if (MO.getSubReg()) + PartDef = true; + else + FullDef = true; + } + // A partial redefine uses Reg unless there is also a full define. + return std::make_pair(Use || (PartDef && !FullDef), PartDef || FullDef); +} + /// findRegisterDefOperandIdx() - Returns the operand index that is a def of /// the specified register or -1 if it is not found. If isDead is true, defs /// that are not dead are skipped. If TargetRegisterInfo is non-null, then it /// also checks if there is a def of a super-register. -int MachineInstr::findRegisterDefOperandIdx(unsigned Reg, bool isDead, - const TargetRegisterInfo *TRI) const { +int +MachineInstr::findRegisterDefOperandIdx(unsigned Reg, bool isDead, bool Overlap, + const TargetRegisterInfo *TRI) const { + bool isPhys = TargetRegisterInfo::isPhysicalRegister(Reg); for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { const MachineOperand &MO = getOperand(i); if (!MO.isReg() || !MO.isDef()) continue; unsigned MOReg = MO.getReg(); - if (MOReg == Reg || - (TRI && - TargetRegisterInfo::isPhysicalRegister(MOReg) && - TargetRegisterInfo::isPhysicalRegister(Reg) && - TRI->isSubRegister(MOReg, Reg))) - if (!isDead || MO.isDead()) - return i; + bool Found = (MOReg == Reg); + if (!Found && TRI && isPhys && + TargetRegisterInfo::isPhysicalRegister(MOReg)) { + if (Overlap) + Found = TRI->regsOverlap(MOReg, Reg); + else + Found = TRI->isSubRegister(MOReg, Reg); + } + if (Found && (!isDead || MO.isDead())) + return i; } return -1; } @@ -938,6 +974,16 @@ isRegTiedToDefOperand(unsigned UseOpIdx, unsigned *DefOpIdx) const { return true; } +/// clearKillInfo - Clears kill flags on all operands. +/// +void MachineInstr::clearKillInfo() { + for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { + MachineOperand &MO = getOperand(i); + if (MO.isReg() && MO.isUse()) + MO.setIsKill(false); + } +} + /// copyKillDeadInfo - Copies kill / dead operand properties from MI. /// void MachineInstr::copyKillDeadInfo(const MachineInstr *MI) { @@ -1355,11 +1401,21 @@ bool MachineInstr::addRegisterDead(unsigned IncomingReg, void MachineInstr::addRegisterDefined(unsigned IncomingReg, const TargetRegisterInfo *RegInfo) { - MachineOperand *MO = findRegisterDefOperand(IncomingReg, false, RegInfo); - if (!MO || MO->getSubReg()) - addOperand(MachineOperand::CreateReg(IncomingReg, - true /*IsDef*/, - true /*IsImp*/)); + if (TargetRegisterInfo::isPhysicalRegister(IncomingReg)) { + MachineOperand *MO = findRegisterDefOperand(IncomingReg, false, RegInfo); + if (MO) + return; + } else { + for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { + const MachineOperand &MO = getOperand(i); + if (MO.isReg() && MO.getReg() == IncomingReg && MO.isDef() && + MO.getSubReg() == 0) + return; + } + } + addOperand(MachineOperand::CreateReg(IncomingReg, + true /*IsDef*/, + true /*IsImp*/)); } unsigned diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp index b2e757d..6120617 100644 --- a/lib/CodeGen/MachineLICM.cpp +++ b/lib/CodeGen/MachineLICM.cpp @@ -738,8 +738,10 @@ bool MachineLICM::EliminateCSE(MachineInstr *MI, "Instructions with different phys regs are not identical!"); if (MO.isReg() && MO.isDef() && - !TargetRegisterInfo::isPhysicalRegister(MO.getReg())) + !TargetRegisterInfo::isPhysicalRegister(MO.getReg())) { RegInfo->replaceRegWith(MO.getReg(), Dup->getOperand(i).getReg()); + RegInfo->clearKillFlags(Dup->getOperand(i).getReg()); + } } MI->eraseFromParent(); ++NumCSEed; @@ -784,6 +786,15 @@ void MachineLICM::Hoist(MachineInstr *MI) { // Otherwise, splice the instruction to the preheader. CurPreheader->splice(CurPreheader->getFirstTerminator(),MI->getParent(),MI); + // Clear the kill flags of any register this instruction defines, + // since they may need to be live throughout the entire loop + // rather than just live for part of it. + for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { + MachineOperand &MO = MI->getOperand(i); + if (MO.isReg() && MO.isDef() && !MO.isDead()) + RegInfo->clearKillFlags(MO.getReg()); + } + // Add to the CSE map. if (CI != CSEMap.end()) CI->second.push_back(MI); diff --git a/lib/CodeGen/MachineRegisterInfo.cpp b/lib/CodeGen/MachineRegisterInfo.cpp index ea5ca0c..70bf7e5 100644 --- a/lib/CodeGen/MachineRegisterInfo.cpp +++ b/lib/CodeGen/MachineRegisterInfo.cpp @@ -133,6 +133,15 @@ bool MachineRegisterInfo::hasOneNonDBGUse(unsigned RegNo) const { return ++UI == use_nodbg_end(); } +/// clearKillFlags - Iterate over all the uses of the given register and +/// clear the kill flag from the MachineOperand. This function is used by +/// optimization passes which extend register lifetimes and need only +/// preserve conservative kill flag information. +void MachineRegisterInfo::clearKillFlags(unsigned Reg) const { + for (use_iterator UI = use_begin(Reg), UE = use_end(); UI != UE; ++UI) + UI.getOperand().setIsKill(false); +} + bool MachineRegisterInfo::isLiveIn(unsigned Reg) const { for (livein_iterator I = livein_begin(), E = livein_end(); I != E; ++I) if (I->first == Reg || I->second == Reg) @@ -156,6 +165,15 @@ unsigned MachineRegisterInfo::getLiveInPhysReg(unsigned VReg) const { return 0; } +/// getLiveInVirtReg - If PReg is a live-in physical register, return the +/// corresponding live-in physical register. +unsigned MachineRegisterInfo::getLiveInVirtReg(unsigned PReg) const { + for (livein_iterator I = livein_begin(), E = livein_end(); I != E; ++I) + if (I->first == PReg) + return I->second; + return 0; +} + static cl::opt<bool> SchedLiveInCopies("schedule-livein-copies", cl::Hidden, cl::desc("Schedule copies of livein registers"), @@ -218,7 +236,8 @@ static void EmitLiveInCopy(MachineBasicBlock *MBB, --Pos; } - bool Emitted = TII.copyRegToReg(*MBB, Pos, VirtReg, PhysReg, RC, RC); + bool Emitted = TII.copyRegToReg(*MBB, Pos, VirtReg, PhysReg, RC, RC, + DebugLoc()); assert(Emitted && "Unable to issue a live-in copy instruction!\n"); (void) Emitted; @@ -253,7 +272,8 @@ MachineRegisterInfo::EmitLiveInCopies(MachineBasicBlock *EntryMBB, if (LI->second) { const TargetRegisterClass *RC = getRegClass(LI->second); bool Emitted = TII.copyRegToReg(*EntryMBB, EntryMBB->begin(), - LI->second, LI->first, RC, RC); + LI->second, LI->first, RC, RC, + DebugLoc()); assert(Emitted && "Unable to issue a live-in copy instruction!\n"); (void) Emitted; } @@ -265,6 +285,15 @@ MachineRegisterInfo::EmitLiveInCopies(MachineBasicBlock *EntryMBB, EntryMBB->addLiveIn(I->first); } +void MachineRegisterInfo::closePhysRegsUsed(const TargetRegisterInfo &TRI) { + for (int i = UsedPhysRegs.find_first(); i >= 0; + i = UsedPhysRegs.find_next(i)) + for (const unsigned *SS = TRI.getSubRegisters(i); + unsigned SubReg = *SS; ++SS) + if (SubReg > unsigned(i)) + UsedPhysRegs.set(SubReg); +} + #ifndef NDEBUG void MachineRegisterInfo::dumpUses(unsigned Reg) const { for (use_iterator I = use_begin(Reg), E = use_end(); I != E; ++I) diff --git a/lib/CodeGen/MachineSSAUpdater.cpp b/lib/CodeGen/MachineSSAUpdater.cpp index b8996d4..84d6df2 100644 --- a/lib/CodeGen/MachineSSAUpdater.cpp +++ b/lib/CodeGen/MachineSSAUpdater.cpp @@ -26,39 +26,17 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Utils/SSAUpdaterImpl.h" using namespace llvm; -/// BBInfo - Per-basic block information used internally by MachineSSAUpdater. -class MachineSSAUpdater::BBInfo { -public: - MachineBasicBlock *BB; // Back-pointer to the corresponding block. - unsigned AvailableVal; // Value to use in this block. - BBInfo *DefBB; // Block that defines the available value. - int BlkNum; // Postorder number. - BBInfo *IDom; // Immediate dominator. - unsigned NumPreds; // Number of predecessor blocks. - BBInfo **Preds; // Array[NumPreds] of predecessor blocks. - MachineInstr *PHITag; // Marker for existing PHIs that match. - - BBInfo(MachineBasicBlock *ThisBB, unsigned V) - : BB(ThisBB), AvailableVal(V), DefBB(V ? this : 0), BlkNum(0), IDom(0), - NumPreds(0), Preds(0), PHITag(0) { } -}; - -typedef DenseMap<MachineBasicBlock*, MachineSSAUpdater::BBInfo*> BBMapTy; - typedef DenseMap<MachineBasicBlock*, unsigned> AvailableValsTy; static AvailableValsTy &getAvailableVals(void *AV) { return *static_cast<AvailableValsTy*>(AV); } -static BBMapTy *getBBMap(void *BM) { - return static_cast<BBMapTy*>(BM); -} - MachineSSAUpdater::MachineSSAUpdater(MachineFunction &MF, SmallVectorImpl<MachineInstr*> *NewPHI) - : AV(0), BM(0), InsertedPHIs(NewPHI) { + : AV(0), InsertedPHIs(NewPHI) { TII = MF.getTarget().getInstrInfo(); MRI = &MF.getRegInfo(); } @@ -134,7 +112,8 @@ static MachineInstr *InsertNewDef(unsigned Opcode, MachineBasicBlock *BB, MachineBasicBlock::iterator I, const TargetRegisterClass *RC, - MachineRegisterInfo *MRI, const TargetInstrInfo *TII) { + MachineRegisterInfo *MRI, + const TargetInstrInfo *TII) { unsigned NewVR = MRI->createVirtualRegister(RC); return BuildMI(*BB, I, DebugLoc(), TII->get(Opcode), NewVR); } @@ -263,438 +242,131 @@ void MachineSSAUpdater::ReplaceRegWith(unsigned OldReg, unsigned NewReg) { I->second = NewReg; } -/// GetValueAtEndOfBlockInternal - Check to see if AvailableVals has an entry -/// for the specified BB and if so, return it. If not, construct SSA form by -/// first calculating the required placement of PHIs and then inserting new -/// PHIs where needed. -unsigned MachineSSAUpdater::GetValueAtEndOfBlockInternal(MachineBasicBlock *BB){ - AvailableValsTy &AvailableVals = getAvailableVals(AV); - if (unsigned V = AvailableVals[BB]) - return V; +/// MachinePHIiter - Iterator for PHI operands. This is used for the +/// PHI_iterator in the SSAUpdaterImpl template. +namespace { + class MachinePHIiter { + private: + MachineInstr *PHI; + unsigned idx; + + public: + explicit MachinePHIiter(MachineInstr *P) // begin iterator + : PHI(P), idx(1) {} + MachinePHIiter(MachineInstr *P, bool) // end iterator + : PHI(P), idx(PHI->getNumOperands()) {} + + MachinePHIiter &operator++() { idx += 2; return *this; } + bool operator==(const MachinePHIiter& x) const { return idx == x.idx; } + bool operator!=(const MachinePHIiter& x) const { return !operator==(x); } + unsigned getIncomingValue() { return PHI->getOperand(idx).getReg(); } + MachineBasicBlock *getIncomingBlock() { + return PHI->getOperand(idx+1).getMBB(); + } + }; +} - // Pool allocation used internally by GetValueAtEndOfBlock. - BumpPtrAllocator Allocator; - BBMapTy BBMapObj; - BM = &BBMapObj; +/// SSAUpdaterTraits<MachineSSAUpdater> - Traits for the SSAUpdaterImpl +/// template, specialized for MachineSSAUpdater. +namespace llvm { +template<> +class SSAUpdaterTraits<MachineSSAUpdater> { +public: + typedef MachineBasicBlock BlkT; + typedef unsigned ValT; + typedef MachineInstr PhiT; + + typedef MachineBasicBlock::succ_iterator BlkSucc_iterator; + static BlkSucc_iterator BlkSucc_begin(BlkT *BB) { return BB->succ_begin(); } + static BlkSucc_iterator BlkSucc_end(BlkT *BB) { return BB->succ_end(); } + + typedef MachinePHIiter PHI_iterator; + static inline PHI_iterator PHI_begin(PhiT *PHI) { return PHI_iterator(PHI); } + static inline PHI_iterator PHI_end(PhiT *PHI) { + return PHI_iterator(PHI, true); + } - SmallVector<BBInfo*, 100> BlockList; - BuildBlockList(BB, &BlockList, &Allocator); + /// FindPredecessorBlocks - Put the predecessors of BB into the Preds + /// vector. + static void FindPredecessorBlocks(MachineBasicBlock *BB, + SmallVectorImpl<MachineBasicBlock*> *Preds){ + for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(), + E = BB->pred_end(); PI != E; ++PI) + Preds->push_back(*PI); + } - // Special case: bail out if BB is unreachable. - if (BlockList.size() == 0) { - BM = 0; + /// GetUndefVal - Create an IMPLICIT_DEF instruction with a new register. + /// Add it into the specified block and return the register. + static unsigned GetUndefVal(MachineBasicBlock *BB, + MachineSSAUpdater *Updater) { // Insert an implicit_def to represent an undef value. MachineInstr *NewDef = InsertNewDef(TargetOpcode::IMPLICIT_DEF, BB, BB->getFirstTerminator(), - VRC, MRI, TII); - unsigned V = NewDef->getOperand(0).getReg(); - AvailableVals[BB] = V; - return V; - } - - FindDominators(&BlockList); - FindPHIPlacement(&BlockList); - FindAvailableVals(&BlockList); - - BM = 0; - return BBMapObj[BB]->DefBB->AvailableVal; -} - -/// FindPredecessorBlocks - Put the predecessors of Info->BB into the Preds -/// vector, set Info->NumPreds, and allocate space in Info->Preds. -static void FindPredecessorBlocks(MachineSSAUpdater::BBInfo *Info, - SmallVectorImpl<MachineBasicBlock*> *Preds, - BumpPtrAllocator *Allocator) { - MachineBasicBlock *BB = Info->BB; - for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(), - E = BB->pred_end(); PI != E; ++PI) - Preds->push_back(*PI); - - Info->NumPreds = Preds->size(); - Info->Preds = static_cast<MachineSSAUpdater::BBInfo**> - (Allocator->Allocate(Info->NumPreds * sizeof(MachineSSAUpdater::BBInfo*), - AlignOf<MachineSSAUpdater::BBInfo*>::Alignment)); -} - -/// BuildBlockList - Starting from the specified basic block, traverse back -/// through its predecessors until reaching blocks with known values. Create -/// BBInfo structures for the blocks and append them to the block list. -void MachineSSAUpdater::BuildBlockList(MachineBasicBlock *BB, - BlockListTy *BlockList, - BumpPtrAllocator *Allocator) { - AvailableValsTy &AvailableVals = getAvailableVals(AV); - BBMapTy *BBMap = getBBMap(BM); - SmallVector<BBInfo*, 10> RootList; - SmallVector<BBInfo*, 64> WorkList; - - BBInfo *Info = new (*Allocator) BBInfo(BB, 0); - (*BBMap)[BB] = Info; - WorkList.push_back(Info); - - // Search backward from BB, creating BBInfos along the way and stopping when - // reaching blocks that define the value. Record those defining blocks on - // the RootList. - SmallVector<MachineBasicBlock*, 10> Preds; - while (!WorkList.empty()) { - Info = WorkList.pop_back_val(); - Preds.clear(); - FindPredecessorBlocks(Info, &Preds, Allocator); - - // Treat an unreachable predecessor as a definition with 'undef'. - if (Info->NumPreds == 0) { - // Insert an implicit_def to represent an undef value. - MachineInstr *NewDef = InsertNewDef(TargetOpcode::IMPLICIT_DEF, - Info->BB, - Info->BB->getFirstTerminator(), - VRC, MRI, TII); - Info->AvailableVal = NewDef->getOperand(0).getReg(); - Info->DefBB = Info; - RootList.push_back(Info); - continue; - } - - for (unsigned p = 0; p != Info->NumPreds; ++p) { - MachineBasicBlock *Pred = Preds[p]; - // Check if BBMap already has a BBInfo for the predecessor block. - BBMapTy::value_type &BBMapBucket = BBMap->FindAndConstruct(Pred); - if (BBMapBucket.second) { - Info->Preds[p] = BBMapBucket.second; - continue; - } - - // Create a new BBInfo for the predecessor. - unsigned PredVal = AvailableVals.lookup(Pred); - BBInfo *PredInfo = new (*Allocator) BBInfo(Pred, PredVal); - BBMapBucket.second = PredInfo; - Info->Preds[p] = PredInfo; - - if (PredInfo->AvailableVal) { - RootList.push_back(PredInfo); - continue; - } - WorkList.push_back(PredInfo); - } + Updater->VRC, Updater->MRI, + Updater->TII); + return NewDef->getOperand(0).getReg(); } - // Now that we know what blocks are backwards-reachable from the starting - // block, do a forward depth-first traversal to assign postorder numbers - // to those blocks. - BBInfo *PseudoEntry = new (*Allocator) BBInfo(0, 0); - unsigned BlkNum = 1; - - // Initialize the worklist with the roots from the backward traversal. - while (!RootList.empty()) { - Info = RootList.pop_back_val(); - Info->IDom = PseudoEntry; - Info->BlkNum = -1; - WorkList.push_back(Info); + /// CreateEmptyPHI - Create a PHI instruction that defines a new register. + /// Add it into the specified block and return the register. + static unsigned CreateEmptyPHI(MachineBasicBlock *BB, unsigned NumPreds, + MachineSSAUpdater *Updater) { + MachineBasicBlock::iterator Loc = BB->empty() ? BB->end() : BB->front(); + MachineInstr *PHI = InsertNewDef(TargetOpcode::PHI, BB, Loc, + Updater->VRC, Updater->MRI, + Updater->TII); + return PHI->getOperand(0).getReg(); } - while (!WorkList.empty()) { - Info = WorkList.back(); - - if (Info->BlkNum == -2) { - // All the successors have been handled; assign the postorder number. - Info->BlkNum = BlkNum++; - // If not a root, put it on the BlockList. - if (!Info->AvailableVal) - BlockList->push_back(Info); - WorkList.pop_back(); - continue; - } - - // Leave this entry on the worklist, but set its BlkNum to mark that its - // successors have been put on the worklist. When it returns to the top - // the list, after handling its successors, it will be assigned a number. - Info->BlkNum = -2; - - // Add unvisited successors to the work list. - for (MachineBasicBlock::succ_iterator SI = Info->BB->succ_begin(), - E = Info->BB->succ_end(); SI != E; ++SI) { - BBInfo *SuccInfo = (*BBMap)[*SI]; - if (!SuccInfo || SuccInfo->BlkNum) - continue; - SuccInfo->BlkNum = -1; - WorkList.push_back(SuccInfo); - } + /// AddPHIOperand - Add the specified value as an operand of the PHI for + /// the specified predecessor block. + static void AddPHIOperand(MachineInstr *PHI, unsigned Val, + MachineBasicBlock *Pred) { + PHI->addOperand(MachineOperand::CreateReg(Val, false)); + PHI->addOperand(MachineOperand::CreateMBB(Pred)); } - PseudoEntry->BlkNum = BlkNum; -} -/// IntersectDominators - This is the dataflow lattice "meet" operation for -/// finding dominators. Given two basic blocks, it walks up the dominator -/// tree until it finds a common dominator of both. It uses the postorder -/// number of the blocks to determine how to do that. -static MachineSSAUpdater::BBInfo * -IntersectDominators(MachineSSAUpdater::BBInfo *Blk1, - MachineSSAUpdater::BBInfo *Blk2) { - while (Blk1 != Blk2) { - while (Blk1->BlkNum < Blk2->BlkNum) { - Blk1 = Blk1->IDom; - if (!Blk1) - return Blk2; - } - while (Blk2->BlkNum < Blk1->BlkNum) { - Blk2 = Blk2->IDom; - if (!Blk2) - return Blk1; - } - } - return Blk1; -} - -/// FindDominators - Calculate the dominator tree for the subset of the CFG -/// corresponding to the basic blocks on the BlockList. This uses the -/// algorithm from: "A Simple, Fast Dominance Algorithm" by Cooper, Harvey and -/// Kennedy, published in Software--Practice and Experience, 2001, 4:1-10. -/// Because the CFG subset does not include any edges leading into blocks that -/// define the value, the results are not the usual dominator tree. The CFG -/// subset has a single pseudo-entry node with edges to a set of root nodes -/// for blocks that define the value. The dominators for this subset CFG are -/// not the standard dominators but they are adequate for placing PHIs within -/// the subset CFG. -void MachineSSAUpdater::FindDominators(BlockListTy *BlockList) { - bool Changed; - do { - Changed = false; - // Iterate over the list in reverse order, i.e., forward on CFG edges. - for (BlockListTy::reverse_iterator I = BlockList->rbegin(), - E = BlockList->rend(); I != E; ++I) { - BBInfo *Info = *I; - - // Start with the first predecessor. - assert(Info->NumPreds > 0 && "unreachable block"); - BBInfo *NewIDom = Info->Preds[0]; - - // Iterate through the block's other predecessors. - for (unsigned p = 1; p != Info->NumPreds; ++p) { - BBInfo *Pred = Info->Preds[p]; - NewIDom = IntersectDominators(NewIDom, Pred); - } - - // Check if the IDom value has changed. - if (NewIDom != Info->IDom) { - Info->IDom = NewIDom; - Changed = true; - } - } - } while (Changed); -} - -/// IsDefInDomFrontier - Search up the dominator tree from Pred to IDom for -/// any blocks containing definitions of the value. If one is found, then the -/// successor of Pred is in the dominance frontier for the definition, and -/// this function returns true. -static bool IsDefInDomFrontier(const MachineSSAUpdater::BBInfo *Pred, - const MachineSSAUpdater::BBInfo *IDom) { - for (; Pred != IDom; Pred = Pred->IDom) { - if (Pred->DefBB == Pred) - return true; + /// InstrIsPHI - Check if an instruction is a PHI. + /// + static MachineInstr *InstrIsPHI(MachineInstr *I) { + if (I && I->isPHI()) + return I; + return 0; } - return false; -} - -/// FindPHIPlacement - PHIs are needed in the iterated dominance frontiers of -/// the known definitions. Iteratively add PHIs in the dom frontiers until -/// nothing changes. Along the way, keep track of the nearest dominating -/// definitions for non-PHI blocks. -void MachineSSAUpdater::FindPHIPlacement(BlockListTy *BlockList) { - bool Changed; - do { - Changed = false; - // Iterate over the list in reverse order, i.e., forward on CFG edges. - for (BlockListTy::reverse_iterator I = BlockList->rbegin(), - E = BlockList->rend(); I != E; ++I) { - BBInfo *Info = *I; - - // If this block already needs a PHI, there is nothing to do here. - if (Info->DefBB == Info) - continue; - - // Default to use the same def as the immediate dominator. - BBInfo *NewDefBB = Info->IDom->DefBB; - for (unsigned p = 0; p != Info->NumPreds; ++p) { - if (IsDefInDomFrontier(Info->Preds[p], Info->IDom)) { - // Need a PHI here. - NewDefBB = Info; - break; - } - } - // Check if anything changed. - if (NewDefBB != Info->DefBB) { - Info->DefBB = NewDefBB; - Changed = true; - } - } - } while (Changed); -} - -/// FindAvailableVal - If this block requires a PHI, first check if an existing -/// PHI matches the PHI placement and reaching definitions computed earlier, -/// and if not, create a new PHI. Visit all the block's predecessors to -/// calculate the available value for each one and fill in the incoming values -/// for a new PHI. -void MachineSSAUpdater::FindAvailableVals(BlockListTy *BlockList) { - AvailableValsTy &AvailableVals = getAvailableVals(AV); - - // Go through the worklist in forward order (i.e., backward through the CFG) - // and check if existing PHIs can be used. If not, create empty PHIs where - // they are needed. - for (BlockListTy::iterator I = BlockList->begin(), E = BlockList->end(); - I != E; ++I) { - BBInfo *Info = *I; - // Check if there needs to be a PHI in BB. - if (Info->DefBB != Info) - continue; - - // Look for an existing PHI. - FindExistingPHI(Info->BB, BlockList); - if (Info->AvailableVal) - continue; - - MachineBasicBlock::iterator Loc = - Info->BB->empty() ? Info->BB->end() : Info->BB->front(); - MachineInstr *InsertedPHI = InsertNewDef(TargetOpcode::PHI, Info->BB, Loc, - VRC, MRI, TII); - unsigned PHI = InsertedPHI->getOperand(0).getReg(); - Info->AvailableVal = PHI; - AvailableVals[Info->BB] = PHI; + /// ValueIsPHI - Check if the instruction that defines the specified register + /// is a PHI instruction. + static MachineInstr *ValueIsPHI(unsigned Val, MachineSSAUpdater *Updater) { + return InstrIsPHI(Updater->MRI->getVRegDef(Val)); } - // Now go back through the worklist in reverse order to fill in the arguments - // for any new PHIs added in the forward traversal. - for (BlockListTy::reverse_iterator I = BlockList->rbegin(), - E = BlockList->rend(); I != E; ++I) { - BBInfo *Info = *I; - - if (Info->DefBB != Info) { - // Record the available value at join nodes to speed up subsequent - // uses of this SSAUpdater for the same value. - if (Info->NumPreds > 1) - AvailableVals[Info->BB] = Info->DefBB->AvailableVal; - continue; - } - - // Check if this block contains a newly added PHI. - unsigned PHI = Info->AvailableVal; - MachineInstr *InsertedPHI = MRI->getVRegDef(PHI); - if (!InsertedPHI->isPHI() || InsertedPHI->getNumOperands() > 1) - continue; - - // Iterate through the block's predecessors. - MachineInstrBuilder MIB(InsertedPHI); - for (unsigned p = 0; p != Info->NumPreds; ++p) { - BBInfo *PredInfo = Info->Preds[p]; - MachineBasicBlock *Pred = PredInfo->BB; - // Skip to the nearest preceding definition. - if (PredInfo->DefBB != PredInfo) - PredInfo = PredInfo->DefBB; - MIB.addReg(PredInfo->AvailableVal).addMBB(Pred); - } - - DEBUG(dbgs() << " Inserted PHI: " << *InsertedPHI << "\n"); - - // If the client wants to know about all new instructions, tell it. - if (InsertedPHIs) InsertedPHIs->push_back(InsertedPHI); + /// ValueIsNewPHI - Like ValueIsPHI but also check if the PHI has no source + /// operands, i.e., it was just added. + static MachineInstr *ValueIsNewPHI(unsigned Val, MachineSSAUpdater *Updater) { + MachineInstr *PHI = ValueIsPHI(Val, Updater); + if (PHI && PHI->getNumOperands() <= 1) + return PHI; + return 0; } -} -/// FindExistingPHI - Look through the PHI nodes in a block to see if any of -/// them match what is needed. -void MachineSSAUpdater::FindExistingPHI(MachineBasicBlock *BB, - BlockListTy *BlockList) { - for (MachineBasicBlock::iterator BBI = BB->begin(), BBE = BB->end(); - BBI != BBE && BBI->isPHI(); ++BBI) { - if (CheckIfPHIMatches(BBI)) { - RecordMatchingPHI(BBI); - break; - } - // Match failed: clear all the PHITag values. - for (BlockListTy::iterator I = BlockList->begin(), E = BlockList->end(); - I != E; ++I) - (*I)->PHITag = 0; + /// GetPHIValue - For the specified PHI instruction, return the register + /// that it defines. + static unsigned GetPHIValue(MachineInstr *PHI) { + return PHI->getOperand(0).getReg(); } -} - -/// CheckIfPHIMatches - Check if a PHI node matches the placement and values -/// in the BBMap. -bool MachineSSAUpdater::CheckIfPHIMatches(MachineInstr *PHI) { - BBMapTy *BBMap = getBBMap(BM); - SmallVector<MachineInstr*, 20> WorkList; - WorkList.push_back(PHI); - - // Mark that the block containing this PHI has been visited. - (*BBMap)[PHI->getParent()]->PHITag = PHI; - - while (!WorkList.empty()) { - PHI = WorkList.pop_back_val(); - - // Iterate through the PHI's incoming values. - for (unsigned i = 1, e = PHI->getNumOperands(); i != e; i += 2) { - unsigned IncomingVal = PHI->getOperand(i).getReg(); - BBInfo *PredInfo = (*BBMap)[PHI->getOperand(i+1).getMBB()]; - // Skip to the nearest preceding definition. - if (PredInfo->DefBB != PredInfo) - PredInfo = PredInfo->DefBB; - - // Check if it matches the expected value. - if (PredInfo->AvailableVal) { - if (IncomingVal == PredInfo->AvailableVal) - continue; - return false; - } - - // Check if the value is a PHI in the correct block. - MachineInstr *IncomingPHIVal = MRI->getVRegDef(IncomingVal); - if (!IncomingPHIVal->isPHI() || - IncomingPHIVal->getParent() != PredInfo->BB) - return false; - - // If this block has already been visited, check if this PHI matches. - if (PredInfo->PHITag) { - if (IncomingPHIVal == PredInfo->PHITag) - continue; - return false; - } - PredInfo->PHITag = IncomingPHIVal; +}; - WorkList.push_back(IncomingPHIVal); - } - } - return true; -} +} // End llvm namespace -/// RecordMatchingPHI - For a PHI node that matches, record it and its input -/// PHIs in both the BBMap and the AvailableVals mapping. -void MachineSSAUpdater::RecordMatchingPHI(MachineInstr *PHI) { - BBMapTy *BBMap = getBBMap(BM); +/// GetValueAtEndOfBlockInternal - Check to see if AvailableVals has an entry +/// for the specified BB and if so, return it. If not, construct SSA form by +/// first calculating the required placement of PHIs and then inserting new +/// PHIs where needed. +unsigned MachineSSAUpdater::GetValueAtEndOfBlockInternal(MachineBasicBlock *BB){ AvailableValsTy &AvailableVals = getAvailableVals(AV); - SmallVector<MachineInstr*, 20> WorkList; - WorkList.push_back(PHI); - - // Record this PHI. - MachineBasicBlock *BB = PHI->getParent(); - AvailableVals[BB] = PHI->getOperand(0).getReg(); - (*BBMap)[BB]->AvailableVal = PHI->getOperand(0).getReg(); - - while (!WorkList.empty()) { - PHI = WorkList.pop_back_val(); - - // Iterate through the PHI's incoming values. - for (unsigned i = 1, e = PHI->getNumOperands(); i != e; i += 2) { - unsigned IncomingVal = PHI->getOperand(i).getReg(); - MachineInstr *IncomingPHIVal = MRI->getVRegDef(IncomingVal); - if (!IncomingPHIVal->isPHI()) continue; - BB = IncomingPHIVal->getParent(); - BBInfo *Info = (*BBMap)[BB]; - if (!Info || Info->AvailableVal) - continue; - - // Record the PHI and add it to the worklist. - AvailableVals[BB] = IncomingVal; - Info->AvailableVal = IncomingVal; - WorkList.push_back(IncomingPHIVal); - } - } + if (unsigned V = AvailableVals[BB]) + return V; + + SSAUpdaterImpl<MachineSSAUpdater> Impl(this, &AvailableVals, InsertedPHIs); + return Impl.GetValue(BB); } diff --git a/lib/CodeGen/MachineSink.cpp b/lib/CodeGen/MachineSink.cpp index ef489dc..1610e6c 100644 --- a/lib/CodeGen/MachineSink.cpp +++ b/lib/CodeGen/MachineSink.cpp @@ -314,5 +314,10 @@ bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) { // Move the instruction. SuccToSinkTo->splice(InsertPos, ParentBlock, MI, ++MachineBasicBlock::iterator(MI)); + + // Conservatively, clear any kill flags, since it's possible that + // they are no longer correct. + MI->clearKillInfo(); + return true; } diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp index 0b75c55..8baf01c 100644 --- a/lib/CodeGen/MachineVerifier.cpp +++ b/lib/CodeGen/MachineVerifier.cpp @@ -47,7 +47,7 @@ namespace { MachineVerifier(Pass *pass, bool allowDoubleDefs) : PASS(pass), allowVirtDoubleDefs(allowDoubleDefs), - allowPhysDoubleDefs(allowDoubleDefs), + allowPhysDoubleDefs(true), OutFileName(getenv("LLVM_VERIFY_MACHINEINSTRS")) {} @@ -552,19 +552,23 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { regsLiveInButUnused.erase(Reg); bool isKill = false; - if (MO->isKill()) { - isKill = true; - // Tied operands on two-address instuctions MUST NOT have a <kill> flag. - if (MI->isRegTiedToDefOperand(MONum)) + unsigned defIdx; + if (MI->isRegTiedToDefOperand(MONum, &defIdx)) { + // A two-addr use counts as a kill if use and def are the same. + unsigned DefReg = MI->getOperand(defIdx).getReg(); + if (Reg == DefReg) { + isKill = true; + // ANd in that case an explicit kill flag is not allowed. + if (MO->isKill()) report("Illegal kill flag on two-address instruction operand", MO, MONum); - } else { - // TwoAddress instr modifying a reg is treated as kill+def. - unsigned defIdx; - if (MI->isRegTiedToDefOperand(MONum, &defIdx) && - MI->getOperand(defIdx).getReg() == Reg) - isKill = true; - } + } else if (TargetRegisterInfo::isPhysicalRegister(Reg)) { + report("Two-address instruction operands must be identical", + MO, MONum); + } + } else + isKill = MO->isKill(); + if (isKill) { addRegWithSubRegs(regsKilled, Reg); @@ -631,11 +635,14 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { // Virtual register. const TargetRegisterClass *RC = MRI->getRegClass(Reg); if (SubIdx) { - if (RC->subregclasses_begin()+SubIdx >= RC->subregclasses_end()) { + const TargetRegisterClass *SRC = RC->getSubRegisterRegClass(SubIdx); + if (!SRC) { report("Invalid subregister index for virtual register", MO, MONum); + *OS << "Register class " << RC->getName() + << " does not support subreg index " << SubIdx << "\n"; return; } - RC = *(RC->subregclasses_begin()+SubIdx); + RC = SRC; } if (const TargetRegisterClass *DRC = TOI.getRegClass(TRI)) { if (RC != DRC && !RC->hasSuperClass(DRC)) { diff --git a/lib/CodeGen/PHIElimination.cpp b/lib/CodeGen/PHIElimination.cpp index 1651719..edbc13f 100644 --- a/lib/CodeGen/PHIElimination.cpp +++ b/lib/CodeGen/PHIElimination.cpp @@ -27,10 +27,8 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" -#include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" #include <algorithm> #include <map> using namespace llvm; @@ -88,10 +86,6 @@ bool llvm::PHIElimination::runOnMachineFunction(MachineFunction &MF) { ImpDefs.clear(); VRegPHIUseCount.clear(); - // Eliminate REG_SEQUENCE instructions. Their whole purpose was to preseve - // SSA form. - Changed |= EliminateRegSequences(MF); - return Changed; } @@ -216,7 +210,8 @@ void llvm::PHIElimination::LowerAtomicPHINode( } else { entry = IncomingReg = MF.getRegInfo().createVirtualRegister(RC); } - TII->copyRegToReg(MBB, AfterPHIsIt, DestReg, IncomingReg, RC, RC); + TII->copyRegToReg(MBB, AfterPHIsIt, DestReg, IncomingReg, RC, RC, + MPhi->getDebugLoc()); } // Update live variable information if there is any. @@ -298,7 +293,8 @@ void llvm::PHIElimination::LowerAtomicPHINode( // Insert the copy. if (!reusedIncoming && IncomingReg) - TII->copyRegToReg(opBlock, InsertPos, IncomingReg, SrcReg, RC, RC); + TII->copyRegToReg(opBlock, InsertPos, IncomingReg, SrcReg, RC, RC, + MPhi->getDebugLoc()); // Now update live variable information if we have it. Otherwise we're done if (!LV) continue; @@ -449,58 +445,3 @@ MachineBasicBlock *PHIElimination::SplitCriticalEdge(MachineBasicBlock *A, return NMBB; } - -static void UpdateRegSequenceSrcs(unsigned SrcReg, - unsigned DstReg, unsigned SrcIdx, - MachineRegisterInfo *MRI) { - for (MachineRegisterInfo::reg_iterator RI = MRI->reg_begin(SrcReg), - UE = MRI->reg_end(); RI != UE; ) { - MachineOperand &MO = RI.getOperand(); - ++RI; - MO.setReg(DstReg); - MO.setSubReg(SrcIdx); - } -} - -/// EliminateRegSequences - Eliminate REG_SEQUENCE instructions as second part -/// of de-ssa process. This replaces sources of REG_SEQUENCE as sub-register -/// references of the register defined by REG_SEQUENCE. e.g. -/// -/// %reg1029<def>, %reg1030<def> = VLD1q16 %reg1024<kill>, ... -/// %reg1031<def> = REG_SEQUENCE %reg1029<kill>, 5, %reg1030<kill>, 6 -/// => -/// %reg1031:5<def>, %reg1031:6<def> = VLD1q16 %reg1024<kill>, ... -bool PHIElimination::EliminateRegSequences(MachineFunction &MF) { - bool Changed = false; - - for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) - for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end(); - BBI != BBE; ) { - MachineInstr &MI = *BBI; - ++BBI; - if (MI.getOpcode() != TargetOpcode::REG_SEQUENCE) - continue; - unsigned DstReg = MI.getOperand(0).getReg(); - if (MI.getOperand(0).getSubReg() || - TargetRegisterInfo::isPhysicalRegister(DstReg) || - !(MI.getNumOperands() & 1)) { - DEBUG(dbgs() << "Illegal REG_SEQUENCE instruction:" << MI); - llvm_unreachable(0); - } - for (unsigned i = 1, e = MI.getNumOperands(); i < e; i += 2) { - unsigned SrcReg = MI.getOperand(i).getReg(); - if (MI.getOperand(i).getSubReg() || - TargetRegisterInfo::isPhysicalRegister(SrcReg)) { - DEBUG(dbgs() << "Illegal REG_SEQUENCE instruction:" << MI); - llvm_unreachable(0); - } - unsigned SrcIdx = MI.getOperand(i+1).getImm(); - UpdateRegSequenceSrcs(SrcReg, DstReg, SrcIdx, MRI); - } - - MI.eraseFromParent(); - Changed = true; - } - - return Changed; -} diff --git a/lib/CodeGen/PHIElimination.h b/lib/CodeGen/PHIElimination.h index 3292aa2..7dedf03 100644 --- a/lib/CodeGen/PHIElimination.h +++ b/lib/CodeGen/PHIElimination.h @@ -94,8 +94,6 @@ namespace llvm { return I; } - bool EliminateRegSequences(MachineFunction &MF); - typedef std::pair<unsigned, unsigned> BBVRegPair; typedef DenseMap<BBVRegPair, unsigned> VRegPHIUse; diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp index d3e1295..9714ea6 100644 --- a/lib/CodeGen/PostRASchedulerList.cpp +++ b/lib/CodeGen/PostRASchedulerList.cpp @@ -114,7 +114,7 @@ namespace { /// AvailableQueue - The priority queue to use for the available SUnits. /// LatencyPriorityQueue AvailableQueue; - + /// PendingQueue - This contains all of the instructions whose operands have /// been issued, but their results are not ready yet (due to the latency of /// the operation). Once the operands becomes available, the instruction is @@ -158,7 +158,7 @@ namespace { /// Schedule - Schedule the instruction range using list scheduling. /// void Schedule(); - + /// Observe - Update liveness information to account for the current /// instruction, which will not be scheduled. /// @@ -179,7 +179,7 @@ namespace { void ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle); void ListScheduleTopDown(); void StartBlockForKills(MachineBasicBlock *BB); - + // ToggleKillFlag - Toggle a register operand kill flag. Other // adjustments may be made to the instruction if necessary. Return // true if the operand has been deleted, false if not. @@ -197,13 +197,13 @@ static bool isSchedulingBoundary(const MachineInstr *MI, if (MI->getDesc().isTerminator() || MI->isLabel()) return true; - // Don't attempt to schedule around any instruction that modifies + // Don't attempt to schedule around any instruction that defines // a stack-oriented pointer, as it's unlikely to be profitable. This // saves compile time, because it doesn't require every single // stack slot reference to depend on the instruction that does the // modification. const TargetLowering &TLI = *MF.getTarget().getTargetLowering(); - if (MI->modifiesRegister(TLI.getStackPointerRegisterToSaveRestore())) + if (MI->definesRegister(TLI.getStackPointerRegisterToSaveRestore())) return true; return false; @@ -227,9 +227,10 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) { // Check for antidep breaking override... if (EnableAntiDepBreaking.getPosition() > 0) { - AntiDepMode = (EnableAntiDepBreaking == "all") ? TargetSubtarget::ANTIDEP_ALL : - (EnableAntiDepBreaking == "critical") ? TargetSubtarget::ANTIDEP_CRITICAL : - TargetSubtarget::ANTIDEP_NONE; + AntiDepMode = (EnableAntiDepBreaking == "all") ? + TargetSubtarget::ANTIDEP_ALL : + (EnableAntiDepBreaking == "critical") + ? TargetSubtarget::ANTIDEP_CRITICAL : TargetSubtarget::ANTIDEP_NONE; } DEBUG(dbgs() << "PostRAScheduler\n"); @@ -240,10 +241,10 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) { ScheduleHazardRecognizer *HR = EnablePostRAHazardAvoidance ? (ScheduleHazardRecognizer *)new ExactHazardRecognizer(InstrItins) : (ScheduleHazardRecognizer *)new SimpleHazardRecognizer(); - AntiDepBreaker *ADB = + AntiDepBreaker *ADB = ((AntiDepMode == TargetSubtarget::ANTIDEP_ALL) ? (AntiDepBreaker *)new AggressiveAntiDepBreaker(Fn, CriticalPathRCs) : - ((AntiDepMode == TargetSubtarget::ANTIDEP_CRITICAL) ? + ((AntiDepMode == TargetSubtarget::ANTIDEP_CRITICAL) ? (AntiDepBreaker *)new CriticalAntiDepBreaker(Fn) : NULL)); SchedulePostRATDList Scheduler(Fn, MLI, MDT, HR, ADB, AA); @@ -265,17 +266,6 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) { // Initialize register live-range state for scheduling in this block. Scheduler.StartBlock(MBB); - // FIXME: Temporary workaround for <rdar://problem/7759363>: The post-RA - // scheduler has some sort of problem with DebugValue instructions that - // causes an assertion in LeaksContext.h to fail occasionally. Just - // remove all those instructions for now. - for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); - I != E; ) { - MachineInstr *MI = &*I++; - if (MI->isDebugValue()) - MI->eraseFromParent(); - } - // Schedule each sequence of instructions not interrupted by a label // or anything else that effectively needs to shut down scheduling. MachineBasicBlock::iterator Current = MBB->end(); @@ -310,7 +300,7 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) { return true; } - + /// StartBlock - Initialize register live-range state for scheduling in /// this block. /// @@ -331,10 +321,10 @@ void SchedulePostRATDList::Schedule() { BuildSchedGraph(AA); if (AntiDepBreak != NULL) { - unsigned Broken = + unsigned Broken = AntiDepBreak->BreakAntiDependencies(SUnits, Begin, InsertPos, InsertPosIndex); - + if (Broken != 0) { // We made changes. Update the dependency graph. // Theoretically we could update the graph in place: @@ -347,7 +337,7 @@ void SchedulePostRATDList::Schedule() { EntrySU = SUnit(); ExitSU = SUnit(); BuildSchedGraph(AA); - + NumFixedAnti += Broken; } } @@ -425,7 +415,7 @@ bool SchedulePostRATDList::ToggleKillFlag(MachineInstr *MI, MO.setIsKill(true); return false; } - + // If MO itself is live, clear the kill flag... if (KillIndices[MO.getReg()] != ~0u) { MO.setIsKill(false); @@ -464,7 +454,7 @@ void SchedulePostRATDList::FixupKills(MachineBasicBlock *MBB) { BitVector ReservedRegs = TRI->getReservedRegs(MF); StartBlockForKills(MBB); - + // Examine block from end to start... unsigned Count = MBB->size(); for (MachineBasicBlock::iterator I = MBB->end(), E = MBB->begin(); @@ -484,9 +474,9 @@ void SchedulePostRATDList::FixupKills(MachineBasicBlock *MBB) { if (!MO.isDef()) continue; // Ignore two-addr defs. if (MI->isRegTiedToUseOperand(i)) continue; - + KillIndices[Reg] = ~0u; - + // Repeat for all subregs. for (const unsigned *Subreg = TRI->getSubRegisters(Reg); *Subreg; ++Subreg) { @@ -521,17 +511,17 @@ void SchedulePostRATDList::FixupKills(MachineBasicBlock *MBB) { if (kill) kill = (KillIndices[Reg] == ~0u); } - + if (MO.isKill() != kill) { DEBUG(dbgs() << "Fixing " << MO << " in "); // Warning: ToggleKillFlag may invalidate MO. ToggleKillFlag(MI, MO); DEBUG(MI->dump()); } - + killedRegs.insert(Reg); } - + // Mark any used register (that is not using undef) and subregs as // now live... for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { @@ -541,7 +531,7 @@ void SchedulePostRATDList::FixupKills(MachineBasicBlock *MBB) { if ((Reg == 0) || ReservedRegs.test(Reg)) continue; KillIndices[Reg] = Count; - + for (const unsigned *Subreg = TRI->getSubRegisters(Reg); *Subreg; ++Subreg) { KillIndices[*Subreg] = Count; @@ -573,7 +563,7 @@ void SchedulePostRATDList::ReleaseSucc(SUnit *SU, SDep *SuccEdge) { // available. This is the max of the start time of all predecessors plus // their latencies. SuccSU->setDepthToAtLeast(SU->getDepth() + SuccEdge->getLatency()); - + // If all the node's predecessors are scheduled, this node is ready // to be scheduled. Ignore the special ExitSU node. if (SuccSU->NumPredsLeft == 0 && SuccSU != &ExitSU) @@ -594,9 +584,9 @@ void SchedulePostRATDList::ReleaseSuccessors(SUnit *SU) { void SchedulePostRATDList::ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle) { DEBUG(dbgs() << "*** Scheduling [" << CurCycle << "]: "); DEBUG(SU->dump(this)); - + Sequence.push_back(SU); - assert(CurCycle >= SU->getDepth() && + assert(CurCycle >= SU->getDepth() && "Node scheduled above its depth!"); SU->setDepthToAtLeast(CurCycle); @@ -609,7 +599,7 @@ void SchedulePostRATDList::ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle) { /// schedulers. void SchedulePostRATDList::ListScheduleTopDown() { unsigned CurCycle = 0; - + // We're scheduling top-down but we're visiting the regions in // bottom-up order, so we don't know the hazards at the start of a // region. So assume no hazards (this should usually be ok as most diff --git a/lib/CodeGen/PreAllocSplitting.cpp b/lib/CodeGen/PreAllocSplitting.cpp index 2d49beb..96e7327 100644 --- a/lib/CodeGen/PreAllocSplitting.cpp +++ b/lib/CodeGen/PreAllocSplitting.cpp @@ -882,7 +882,7 @@ MachineInstr* PreAllocSplitting::FoldSpill(unsigned vreg, !RefsInMBB.count(FoldPt)) --FoldPt; - int OpIdx = FoldPt->findRegisterDefOperandIdx(vreg, false); + int OpIdx = FoldPt->findRegisterDefOperandIdx(vreg); if (OpIdx == -1) return 0; @@ -1061,7 +1061,8 @@ bool PreAllocSplitting::SplitRegLiveInterval(LiveInterval *LI) { // Add spill. SS = CreateSpillStackSlot(CurrLI->reg, RC); - TII->storeRegToStackSlot(*BarrierMBB, SpillPt, CurrLI->reg, true, SS, RC); + TII->storeRegToStackSlot(*BarrierMBB, SpillPt, CurrLI->reg, true, SS, RC, + TRI); SpillMI = prior(SpillPt); SpillIndex = LIs->InsertMachineInstrInMaps(SpillMI); } @@ -1097,7 +1098,8 @@ bool PreAllocSplitting::SplitRegLiveInterval(LiveInterval *LI) { } // Add spill. SS = CreateSpillStackSlot(CurrLI->reg, RC); - TII->storeRegToStackSlot(*DefMBB, SpillPt, CurrLI->reg, false, SS, RC); + TII->storeRegToStackSlot(*DefMBB, SpillPt, CurrLI->reg, false, SS, RC, + TRI); SpillMI = prior(SpillPt); SpillIndex = LIs->InsertMachineInstrInMaps(SpillMI); } @@ -1116,7 +1118,7 @@ bool PreAllocSplitting::SplitRegLiveInterval(LiveInterval *LI) { RestoreIndex = LIs->getInstructionIndex(RestorePt); FoldedRestore = true; } else { - TII->loadRegFromStackSlot(*BarrierMBB, RestorePt, CurrLI->reg, SS, RC); + TII->loadRegFromStackSlot(*BarrierMBB, RestorePt, CurrLI->reg, SS, RC, TRI); MachineInstr *LoadMI = prior(RestorePt); RestoreIndex = LIs->InsertMachineInstrInMaps(LoadMI); } @@ -1152,7 +1154,7 @@ PreAllocSplitting::SplitRegLiveIntervals(const TargetRegisterClass **RCs, // codegen is not modelling. Ignore these barriers for now. if (!TII->isSafeToMoveRegClassDefs(*RC)) continue; - std::vector<unsigned> &VRs = MRI->getRegClassVirtRegs(*RC); + const std::vector<unsigned> &VRs = MRI->getRegClassVirtRegs(*RC); for (unsigned i = 0, e = VRs.size(); i != e; ++i) { unsigned Reg = VRs[i]; if (!LIs->hasInterval(Reg)) diff --git a/lib/CodeGen/ProcessImplicitDefs.cpp b/lib/CodeGen/ProcessImplicitDefs.cpp index d7179b3..62f525f 100644 --- a/lib/CodeGen/ProcessImplicitDefs.cpp +++ b/lib/CodeGen/ProcessImplicitDefs.cpp @@ -46,7 +46,7 @@ bool ProcessImplicitDefs::CanTurnIntoImplicitDef(MachineInstr *MI, const TargetInstrInfo *tii_) { unsigned SrcReg, DstReg, SrcSubReg, DstSubReg; if (tii_->isMoveInstr(*MI, SrcReg, DstReg, SrcSubReg, DstSubReg) && - Reg == SrcReg) + Reg == SrcReg && SrcSubReg == 0 && DstSubReg == 0) return true; if (OpIdx == 2 && MI->isSubregToReg()) @@ -89,6 +89,8 @@ bool ProcessImplicitDefs::runOnMachineFunction(MachineFunction &fn) { MachineInstr *MI = &*I; ++I; if (MI->isImplicitDef()) { + if (MI->getOperand(0).getSubReg()) + continue; unsigned Reg = MI->getOperand(0).getReg(); ImpDefRegs.insert(Reg); if (TargetRegisterInfo::isPhysicalRegister(Reg)) { @@ -218,7 +220,7 @@ bool ProcessImplicitDefs::runOnMachineFunction(MachineFunction &fn) { // Turn a copy use into an implicit_def. unsigned SrcReg, DstReg, SrcSubReg, DstSubReg; if (tii_->isMoveInstr(*RMI, SrcReg, DstReg, SrcSubReg, DstSubReg) && - Reg == SrcReg) { + Reg == SrcReg && SrcSubReg == 0 && DstSubReg == 0) { RMI->setDesc(tii_->get(TargetOpcode::IMPLICIT_DEF)); bool isKill = false; diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp index a454b62..e778024 100644 --- a/lib/CodeGen/PrologEpilogInserter.cpp +++ b/lib/CodeGen/PrologEpilogInserter.cpp @@ -58,8 +58,9 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) { FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(Fn); FrameConstantRegMap.clear(); - // Calculate the MaxCallFrameSize and HasCalls variables for the function's - // frame information. Also eliminates call frame pseudo instructions. + // Calculate the MaxCallFrameSize and AdjustsStack variables for the + // function's frame information. Also eliminates call frame pseudo + // instructions. calculateCallsInformation(Fn); // Allow the target machine to make some adjustments to the function @@ -91,8 +92,8 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) { // Add prolog and epilog code to the function. This function is required // to align the stack frame as necessary for any stack variables or - // called functions. Because of this, calculateCalleeSavedRegisters - // must be called before this function in order to set the HasCalls + // called functions. Because of this, calculateCalleeSavedRegisters() + // must be called before this function in order to set the AdjustsStack // and MaxCallFrameSize variables. if (!F->hasFnAttr(Attribute::Naked)) insertPrologEpilogCode(Fn); @@ -126,7 +127,7 @@ void PEI::getAnalysisUsage(AnalysisUsage &AU) const { } #endif -/// calculateCallsInformation - Calculate the MaxCallFrameSize and HasCalls +/// calculateCallsInformation - Calculate the MaxCallFrameSize and AdjustsStack /// variables for the function's frame information and eliminate call frame /// pseudo instructions. void PEI::calculateCallsInformation(MachineFunction &Fn) { @@ -134,7 +135,7 @@ void PEI::calculateCallsInformation(MachineFunction &Fn) { MachineFrameInfo *MFI = Fn.getFrameInfo(); unsigned MaxCallFrameSize = 0; - bool HasCalls = MFI->hasCalls(); + bool AdjustsStack = MFI->adjustsStack(); // Get the function call frame set-up and tear-down instruction opcode int FrameSetupOpcode = RegInfo->getCallFrameSetupOpcode(); @@ -154,15 +155,15 @@ void PEI::calculateCallsInformation(MachineFunction &Fn) { " instructions should have a single immediate argument!"); unsigned Size = I->getOperand(0).getImm(); if (Size > MaxCallFrameSize) MaxCallFrameSize = Size; - HasCalls = true; + AdjustsStack = true; FrameSDOps.push_back(I); } else if (I->isInlineAsm()) { // An InlineAsm might be a call; assume it is to get the stack frame // aligned correctly for calls. - HasCalls = true; + AdjustsStack = true; } - MFI->setHasCalls(HasCalls); + MFI->setAdjustsStack(AdjustsStack); MFI->setMaxCallFrameSize(MaxCallFrameSize); for (std::vector<MachineBasicBlock::iterator>::iterator @@ -289,12 +290,13 @@ void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { return; const TargetInstrInfo &TII = *Fn.getTarget().getInstrInfo(); + const TargetRegisterInfo *TRI = Fn.getTarget().getRegisterInfo(); MachineBasicBlock::iterator I; if (! ShrinkWrapThisFunction) { // Spill using target interface. I = EntryBlock->begin(); - if (!TII.spillCalleeSavedRegisters(*EntryBlock, I, CSI)) { + if (!TII.spillCalleeSavedRegisters(*EntryBlock, I, CSI, TRI)) { for (unsigned i = 0, e = CSI.size(); i != e; ++i) { // Add the callee-saved register as live-in. // It's killed at the spill. @@ -302,7 +304,7 @@ void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { // Insert the spill to the stack frame. TII.storeRegToStackSlot(*EntryBlock, I, CSI[i].getReg(), true, - CSI[i].getFrameIdx(), CSI[i].getRegClass()); + CSI[i].getFrameIdx(), CSI[i].getRegClass(),TRI); } } @@ -324,11 +326,11 @@ void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { // Restore all registers immediately before the return and any // terminators that preceed it. - if (!TII.restoreCalleeSavedRegisters(*MBB, I, CSI)) { + if (!TII.restoreCalleeSavedRegisters(*MBB, I, CSI, TRI)) { for (unsigned i = 0, e = CSI.size(); i != e; ++i) { TII.loadRegFromStackSlot(*MBB, I, CSI[i].getReg(), CSI[i].getFrameIdx(), - CSI[i].getRegClass()); + CSI[i].getRegClass(), TRI); assert(I != MBB->begin() && "loadRegFromStackSlot didn't insert any code!"); // Insert in reverse order. loadRegFromStackSlot can insert @@ -375,7 +377,7 @@ void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { TII.storeRegToStackSlot(*MBB, I, blockCSI[i].getReg(), true, blockCSI[i].getFrameIdx(), - blockCSI[i].getRegClass()); + blockCSI[i].getRegClass(), TRI); } } @@ -423,7 +425,7 @@ void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { for (unsigned i = 0, e = blockCSI.size(); i != e; ++i) { TII.loadRegFromStackSlot(*MBB, I, blockCSI[i].getReg(), blockCSI[i].getFrameIdx(), - blockCSI[i].getRegClass()); + blockCSI[i].getRegClass(), TRI); assert(I != MBB->begin() && "loadRegFromStackSlot didn't insert any code!"); // Insert in reverse order. loadRegFromStackSlot can insert @@ -576,7 +578,7 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { // If we have reserved argument space for call sites in the function // immediately on entry to the current function, count it as part of the // overall stack size. - if (MFI->hasCalls() && RegInfo->hasReservedCallFrame(Fn)) + if (MFI->adjustsStack() && RegInfo->hasReservedCallFrame(Fn)) Offset += MFI->getMaxCallFrameSize(); // Round up the size to a multiple of the alignment. If the function has @@ -585,13 +587,14 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { // otherwise, for leaf functions, align to the TransientStackAlignment // value. unsigned StackAlign; - if (MFI->hasCalls() || MFI->hasVarSizedObjects() || + if (MFI->adjustsStack() || MFI->hasVarSizedObjects() || (RegInfo->needsStackRealignment(Fn) && MFI->getObjectIndexEnd() != 0)) StackAlign = TFI.getStackAlignment(); else StackAlign = TFI.getTransientStackAlignment(); - // If the frame pointer is eliminated, all frame offsets will be relative - // to SP not FP; align to MaxAlign so this works. + + // If the frame pointer is eliminated, all frame offsets will be relative to + // SP not FP. Align to MaxAlign so this works. StackAlign = std::max(StackAlign, MaxAlign); unsigned AlignMask = StackAlign - 1; Offset = (Offset + AlignMask) & ~uint64_t(AlignMask); @@ -601,7 +604,6 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { MFI->setStackSize(Offset - LocalAreaOffset); } - /// insertPrologEpilogCode - Scan the function for modified callee saved /// registers, insert spill code for these callee saved registers, then add /// prolog and epilog code to the function. @@ -620,7 +622,6 @@ void PEI::insertPrologEpilogCode(MachineFunction &Fn) { } } - /// replaceFrameIndices - Replace all MO_FrameIndex operands with physical /// register references and actual offsets. /// diff --git a/lib/CodeGen/RegAllocFast.cpp b/lib/CodeGen/RegAllocFast.cpp index 2caf1df..b3b5760 100644 --- a/lib/CodeGen/RegAllocFast.cpp +++ b/lib/CodeGen/RegAllocFast.cpp @@ -18,7 +18,6 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/Target/TargetInstrInfo.h" @@ -38,6 +37,7 @@ using namespace llvm; STATISTIC(NumStores, "Number of stores added"); STATISTIC(NumLoads , "Number of loads added"); +STATISTIC(NumCopies, "Number of copies coalesced"); static RegisterRegAlloc fastRegAlloc("fast", "fast register allocator", createFastRegisterAllocator); @@ -46,77 +46,80 @@ namespace { class RAFast : public MachineFunctionPass { public: static char ID; - RAFast() : MachineFunctionPass(&ID), StackSlotForVirtReg(-1) {} + RAFast() : MachineFunctionPass(&ID), StackSlotForVirtReg(-1), + isBulkSpilling(false) {} private: const TargetMachine *TM; MachineFunction *MF; + MachineRegisterInfo *MRI; const TargetRegisterInfo *TRI; const TargetInstrInfo *TII; + // Basic block currently being allocated. + MachineBasicBlock *MBB; + // StackSlotForVirtReg - Maps virtual regs to the frame index where these // values are spilled. IndexedMap<int, VirtReg2IndexFunctor> StackSlotForVirtReg; - // Virt2PhysRegMap - This map contains entries for each virtual register + // Everything we know about a live virtual register. + struct LiveReg { + MachineInstr *LastUse; // Last instr to use reg. + unsigned PhysReg; // Currently held here. + unsigned short LastOpNum; // OpNum on LastUse. + bool Dirty; // Register needs spill. + + LiveReg(unsigned p=0) : LastUse(0), PhysReg(p), LastOpNum(0), + Dirty(false) {} + }; + + typedef DenseMap<unsigned, LiveReg> LiveRegMap; + typedef LiveRegMap::value_type LiveRegEntry; + + // LiveVirtRegs - This map contains entries for each virtual register // that is currently available in a physical register. - IndexedMap<unsigned, VirtReg2IndexFunctor> Virt2PhysRegMap; + LiveRegMap LiveVirtRegs; - unsigned &getVirt2PhysRegMapSlot(unsigned VirtReg) { - return Virt2PhysRegMap[VirtReg]; - } + // RegState - Track the state of a physical register. + enum RegState { + // A disabled register is not available for allocation, but an alias may + // be in use. A register can only be moved out of the disabled state if + // all aliases are disabled. + regDisabled, - // PhysRegsUsed - This array is effectively a map, containing entries for - // each physical register that currently has a value (ie, it is in - // Virt2PhysRegMap). The value mapped to is the virtual register - // corresponding to the physical register (the inverse of the - // Virt2PhysRegMap), or 0. The value is set to 0 if this register is pinned - // because it is used by a future instruction, and to -2 if it is not - // allocatable. If the entry for a physical register is -1, then the - // physical register is "not in the map". - // - std::vector<int> PhysRegsUsed; + // A free register is not currently in use and can be allocated + // immediately without checking aliases. + regFree, + + // A reserved register has been assigned expolicitly (e.g., setting up a + // call parameter), and it remains reserved until it is used. + regReserved + + // A register state may also be a virtual register number, indication that + // the physical register is currently allocated to a virtual register. In + // that case, LiveVirtRegs contains the inverse mapping. + }; + + // PhysRegState - One of the RegState enums, or a virtreg. + std::vector<unsigned> PhysRegState; // UsedInInstr - BitVector of physregs that are used in the current // instruction, and so cannot be allocated. BitVector UsedInInstr; - // Virt2LastUseMap - This maps each virtual register to its last use - // (MachineInstr*, operand index pair). - IndexedMap<std::pair<MachineInstr*, unsigned>, VirtReg2IndexFunctor> - Virt2LastUseMap; - - std::pair<MachineInstr*,unsigned>& getVirtRegLastUse(unsigned Reg) { - assert(TargetRegisterInfo::isVirtualRegister(Reg) && "Illegal VirtReg!"); - return Virt2LastUseMap[Reg]; - } - - // VirtRegModified - This bitset contains information about which virtual - // registers need to be spilled back to memory when their registers are - // scavenged. If a virtual register has simply been rematerialized, there - // is no reason to spill it to memory when we need the register back. - // - BitVector VirtRegModified; - - // UsedInMultipleBlocks - Tracks whether a particular register is used in - // more than one block. - BitVector UsedInMultipleBlocks; - - void markVirtRegModified(unsigned Reg, bool Val = true) { - assert(TargetRegisterInfo::isVirtualRegister(Reg) && "Illegal VirtReg!"); - Reg -= TargetRegisterInfo::FirstVirtualRegister; - if (Val) - VirtRegModified.set(Reg); - else - VirtRegModified.reset(Reg); - } + // Allocatable - vector of allocatable physical registers. + BitVector Allocatable; - bool isVirtRegModified(unsigned Reg) const { - assert(TargetRegisterInfo::isVirtualRegister(Reg) && "Illegal VirtReg!"); - assert(Reg - TargetRegisterInfo::FirstVirtualRegister < - VirtRegModified.size() && "Illegal virtual register!"); - return VirtRegModified[Reg - TargetRegisterInfo::FirstVirtualRegister]; - } + // isBulkSpilling - This flag is set when LiveRegMap will be cleared + // completely after spilling all live registers. LiveRegMap entries should + // not be erased. + bool isBulkSpilling; + enum { + spillClean = 1, + spillDirty = 100, + spillImpossible = ~0u + }; public: virtual const char *getPassName() const { return "Fast Register Allocator"; @@ -124,104 +127,34 @@ namespace { virtual void getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesCFG(); - AU.addRequired<LiveVariables>(); AU.addRequiredID(PHIEliminationID); AU.addRequiredID(TwoAddressInstructionPassID); MachineFunctionPass::getAnalysisUsage(AU); } private: - /// runOnMachineFunction - Register allocate the whole function bool runOnMachineFunction(MachineFunction &Fn); - - /// AllocateBasicBlock - Register allocate the specified basic block. - void AllocateBasicBlock(MachineBasicBlock &MBB); - - - /// areRegsEqual - This method returns true if the specified registers are - /// related to each other. To do this, it checks to see if they are equal - /// or if the first register is in the alias set of the second register. - /// - bool areRegsEqual(unsigned R1, unsigned R2) const { - if (R1 == R2) return true; - for (const unsigned *AliasSet = TRI->getAliasSet(R2); - *AliasSet; ++AliasSet) { - if (*AliasSet == R1) return true; - } - return false; - } - - /// getStackSpaceFor - This returns the frame index of the specified virtual - /// register on the stack, allocating space if necessary. + void AllocateBasicBlock(); int getStackSpaceFor(unsigned VirtReg, const TargetRegisterClass *RC); - - /// removePhysReg - This method marks the specified physical register as no - /// longer being in use. - /// - void removePhysReg(unsigned PhysReg); - - /// spillVirtReg - This method spills the value specified by PhysReg into - /// the virtual register slot specified by VirtReg. It then updates the RA - /// data structures to indicate the fact that PhysReg is now available. - /// - void spillVirtReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, - unsigned VirtReg, unsigned PhysReg); - - /// spillPhysReg - This method spills the specified physical register into - /// the virtual register slot associated with it. If OnlyVirtRegs is set to - /// true, then the request is ignored if the physical register does not - /// contain a virtual register. - /// - void spillPhysReg(MachineBasicBlock &MBB, MachineInstr *I, - unsigned PhysReg, bool OnlyVirtRegs = false); - - /// assignVirtToPhysReg - This method updates local state so that we know - /// that PhysReg is the proper container for VirtReg now. The physical - /// register must not be used for anything else when this is called. - /// - void assignVirtToPhysReg(unsigned VirtReg, unsigned PhysReg); - - /// isPhysRegAvailable - Return true if the specified physical register is - /// free and available for use. This also includes checking to see if - /// aliased registers are all free... - /// - bool isPhysRegAvailable(unsigned PhysReg) const; - - /// isPhysRegSpillable - Can PhysReg be freed by spilling? - bool isPhysRegSpillable(unsigned PhysReg) const; - - /// getFreeReg - Look to see if there is a free register available in the - /// specified register class. If not, return 0. - /// - unsigned getFreeReg(const TargetRegisterClass *RC); - - /// getReg - Find a physical register to hold the specified virtual - /// register. If all compatible physical registers are used, this method - /// spills the last used virtual register to the stack, and uses that - /// register. If NoFree is true, that means the caller knows there isn't - /// a free register, do not call getFreeReg(). - unsigned getReg(MachineBasicBlock &MBB, MachineInstr *MI, - unsigned VirtReg, bool NoFree = false); - - /// reloadVirtReg - This method transforms the specified virtual - /// register use to refer to a physical register. This method may do this - /// in one of several ways: if the register is available in a physical - /// register already, it uses that physical register. If the value is not - /// in a physical register, and if there are physical registers available, - /// it loads it into a register: PhysReg if that is an available physical - /// register, otherwise any physical register of the right class. - /// If register pressure is high, and it is possible, it tries to fold the - /// load of the virtual register into the instruction itself. It avoids - /// doing this if register pressure is low to improve the chance that - /// subsequent instructions can use the reloaded value. This method - /// returns the modified instruction. - /// - MachineInstr *reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI, - unsigned OpNum, SmallSet<unsigned, 4> &RRegs, - unsigned PhysReg); - - void reloadPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I, - unsigned PhysReg); + bool isLastUseOfLocalReg(MachineOperand&); + + void addKillFlag(const LiveReg&); + void killVirtReg(LiveRegMap::iterator); + void killVirtReg(unsigned VirtReg); + void spillVirtReg(MachineBasicBlock::iterator MI, LiveRegMap::iterator); + void spillVirtReg(MachineBasicBlock::iterator MI, unsigned VirtReg); + + void usePhysReg(MachineOperand&); + void definePhysReg(MachineInstr *MI, unsigned PhysReg, RegState NewState); + unsigned calcSpillCost(unsigned PhysReg) const; + void assignVirtToPhysReg(LiveRegEntry &LRE, unsigned PhysReg); + void allocVirtReg(MachineInstr *MI, LiveRegEntry &LRE, unsigned Hint); + LiveRegMap::iterator defineVirtReg(MachineInstr *MI, unsigned OpNum, + unsigned VirtReg, unsigned Hint); + LiveRegMap::iterator reloadVirtReg(MachineInstr *MI, unsigned OpNum, + unsigned VirtReg, unsigned Hint); + void spillAll(MachineInstr *MI); + bool setPhysReg(MachineInstr *MI, unsigned OpNum, unsigned PhysReg); }; char RAFast::ID = 0; } @@ -243,687 +176,668 @@ int RAFast::getStackSpaceFor(unsigned VirtReg, const TargetRegisterClass *RC) { return FrameIdx; } - -/// removePhysReg - This method marks the specified physical register as no -/// longer being in use. -/// -void RAFast::removePhysReg(unsigned PhysReg) { - PhysRegsUsed[PhysReg] = -1; // PhyReg no longer used -} - - -/// spillVirtReg - This method spills the value specified by PhysReg into the -/// virtual register slot specified by VirtReg. It then updates the RA data -/// structures to indicate the fact that PhysReg is now available. +/// isLastUseOfLocalReg - Return true if MO is the only remaining reference to +/// its virtual register, and it is guaranteed to be a block-local register. /// -void RAFast::spillVirtReg(MachineBasicBlock &MBB, - MachineBasicBlock::iterator I, - unsigned VirtReg, unsigned PhysReg) { - assert(VirtReg && "Spilling a physical register is illegal!" - " Must not have appropriate kill for the register or use exists beyond" - " the intended one."); - DEBUG(dbgs() << " Spilling register " << TRI->getName(PhysReg) - << " containing %reg" << VirtReg); - - if (!isVirtRegModified(VirtReg)) { - DEBUG(dbgs() << " which has not been modified, so no store necessary!"); - std::pair<MachineInstr*, unsigned> &LastUse = getVirtRegLastUse(VirtReg); - if (LastUse.first) - LastUse.first->getOperand(LastUse.second).setIsKill(); - } else { - // Otherwise, there is a virtual register corresponding to this physical - // register. We only need to spill it into its stack slot if it has been - // modified. - const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(VirtReg); - int FrameIndex = getStackSpaceFor(VirtReg, RC); - DEBUG(dbgs() << " to stack slot #" << FrameIndex); - // If the instruction reads the register that's spilled, (e.g. this can - // happen if it is a move to a physical register), then the spill - // instruction is not a kill. - bool isKill = !(I != MBB.end() && I->readsRegister(PhysReg)); - TII->storeRegToStackSlot(MBB, I, PhysReg, isKill, FrameIndex, RC); - ++NumStores; // Update statistics - } +bool RAFast::isLastUseOfLocalReg(MachineOperand &MO) { + // Check for non-debug uses or defs following MO. + // This is the most likely way to fail - fast path it. + MachineOperand *Next = &MO; + while ((Next = Next->getNextOperandForReg())) + if (!Next->isDebug()) + return false; - getVirt2PhysRegMapSlot(VirtReg) = 0; // VirtReg no longer available + // If the register has ever been spilled or reloaded, we conservatively assume + // it is a global register used in multiple blocks. + if (StackSlotForVirtReg[MO.getReg()] != -1) + return false; - DEBUG(dbgs() << '\n'); - removePhysReg(PhysReg); + // Check that the use/def chain has exactly one operand - MO. + return &MRI->reg_nodbg_begin(MO.getReg()).getOperand() == &MO; } - -/// spillPhysReg - This method spills the specified physical register into the -/// virtual register slot associated with it. If OnlyVirtRegs is set to true, -/// then the request is ignored if the physical register does not contain a -/// virtual register. -/// -void RAFast::spillPhysReg(MachineBasicBlock &MBB, MachineInstr *I, - unsigned PhysReg, bool OnlyVirtRegs) { - if (PhysRegsUsed[PhysReg] != -1) { // Only spill it if it's used! - assert(PhysRegsUsed[PhysReg] != -2 && "Non allocable reg used!"); - if (PhysRegsUsed[PhysReg] || !OnlyVirtRegs) - spillVirtReg(MBB, I, PhysRegsUsed[PhysReg], PhysReg); - return; +/// addKillFlag - Set kill flags on last use of a virtual register. +void RAFast::addKillFlag(const LiveReg &LR) { + if (!LR.LastUse) return; + MachineOperand &MO = LR.LastUse->getOperand(LR.LastOpNum); + if (MO.isUse() && !LR.LastUse->isRegTiedToDefOperand(LR.LastOpNum)) { + if (MO.getReg() == LR.PhysReg) + MO.setIsKill(); + else + LR.LastUse->addRegisterKilled(LR.PhysReg, TRI, true); } +} - // If the selected register aliases any other registers, we must make - // sure that one of the aliases isn't alive. - for (const unsigned *AliasSet = TRI->getAliasSet(PhysReg); - *AliasSet; ++AliasSet) { - if (PhysRegsUsed[*AliasSet] == -1 || // Spill aliased register. - PhysRegsUsed[*AliasSet] == -2) // If allocatable. - continue; - - if (PhysRegsUsed[*AliasSet]) - spillVirtReg(MBB, I, PhysRegsUsed[*AliasSet], *AliasSet); - } +/// killVirtReg - Mark virtreg as no longer available. +void RAFast::killVirtReg(LiveRegMap::iterator LRI) { + addKillFlag(LRI->second); + const LiveReg &LR = LRI->second; + assert(PhysRegState[LR.PhysReg] == LRI->first && "Broken RegState mapping"); + PhysRegState[LR.PhysReg] = regFree; + // Erase from LiveVirtRegs unless we're spilling in bulk. + if (!isBulkSpilling) + LiveVirtRegs.erase(LRI); } +/// killVirtReg - Mark virtreg as no longer available. +void RAFast::killVirtReg(unsigned VirtReg) { + assert(TargetRegisterInfo::isVirtualRegister(VirtReg) && + "killVirtReg needs a virtual register"); + LiveRegMap::iterator LRI = LiveVirtRegs.find(VirtReg); + if (LRI != LiveVirtRegs.end()) + killVirtReg(LRI); +} -/// assignVirtToPhysReg - This method updates local state so that we know -/// that PhysReg is the proper container for VirtReg now. The physical -/// register must not be used for anything else when this is called. -/// -void RAFast::assignVirtToPhysReg(unsigned VirtReg, unsigned PhysReg) { - assert(PhysRegsUsed[PhysReg] == -1 && "Phys reg already assigned!"); - // Update information to note the fact that this register was just used, and - // it holds VirtReg. - PhysRegsUsed[PhysReg] = VirtReg; - getVirt2PhysRegMapSlot(VirtReg) = PhysReg; - UsedInInstr.set(PhysReg); +/// spillVirtReg - This method spills the value specified by VirtReg into the +/// corresponding stack slot if needed. If isKill is set, the register is also +/// killed. +void RAFast::spillVirtReg(MachineBasicBlock::iterator MI, unsigned VirtReg) { + assert(TargetRegisterInfo::isVirtualRegister(VirtReg) && + "Spilling a physical register is illegal!"); + LiveRegMap::iterator LRI = LiveVirtRegs.find(VirtReg); + assert(LRI != LiveVirtRegs.end() && "Spilling unmapped virtual register"); + spillVirtReg(MI, LRI); } +/// spillVirtReg - Do the actual work of spilling. +void RAFast::spillVirtReg(MachineBasicBlock::iterator MI, + LiveRegMap::iterator LRI) { + LiveReg &LR = LRI->second; + assert(PhysRegState[LR.PhysReg] == LRI->first && "Broken RegState mapping"); + + if (LR.Dirty) { + // If this physreg is used by the instruction, we want to kill it on the + // instruction, not on the spill. + bool SpillKill = LR.LastUse != MI; + LR.Dirty = false; + DEBUG(dbgs() << "Spilling %reg" << LRI->first + << " in " << TRI->getName(LR.PhysReg)); + const TargetRegisterClass *RC = MRI->getRegClass(LRI->first); + int FI = getStackSpaceFor(LRI->first, RC); + DEBUG(dbgs() << " to stack slot #" << FI << "\n"); + TII->storeRegToStackSlot(*MBB, MI, LR.PhysReg, SpillKill, FI, RC, TRI); + ++NumStores; // Update statistics -/// isPhysRegAvailable - Return true if the specified physical register is free -/// and available for use. This also includes checking to see if aliased -/// registers are all free... -/// -bool RAFast::isPhysRegAvailable(unsigned PhysReg) const { - if (PhysRegsUsed[PhysReg] != -1) return false; - - // If the selected register aliases any other allocated registers, it is - // not free! - for (const unsigned *AliasSet = TRI->getAliasSet(PhysReg); - *AliasSet; ++AliasSet) - if (PhysRegsUsed[*AliasSet] >= 0) // Aliased register in use? - return false; // Can't use this reg then. - return true; + if (SpillKill) + LR.LastUse = 0; // Don't kill register again + } + killVirtReg(LRI); } -/// isPhysRegSpillable - Return true if the specified physical register can be -/// spilled for use in the current instruction. -/// -bool RAFast::isPhysRegSpillable(unsigned PhysReg) const { - // Test that PhysReg and all aliases are either free or assigned to a VirtReg - // that is not used in the instruction. - if (PhysRegsUsed[PhysReg] != -1 && - (PhysRegsUsed[PhysReg] <= 0 || UsedInInstr.test(PhysReg))) - return false; - - for (const unsigned *AliasSet = TRI->getAliasSet(PhysReg); - *AliasSet; ++AliasSet) - if (PhysRegsUsed[*AliasSet] != -1 && - (PhysRegsUsed[*AliasSet] <= 0 || UsedInInstr.test(*AliasSet))) - return false; - return true; +/// spillAll - Spill all dirty virtregs without killing them. +void RAFast::spillAll(MachineInstr *MI) { + if (LiveVirtRegs.empty()) return; + isBulkSpilling = true; + // The LiveRegMap is keyed by an unsigned (the virtreg number), so the order + // of spilling here is deterministic, if arbitrary. + for (LiveRegMap::iterator i = LiveVirtRegs.begin(), e = LiveVirtRegs.end(); + i != e; ++i) + spillVirtReg(MI, i); + LiveVirtRegs.clear(); + isBulkSpilling = false; } +/// usePhysReg - Handle the direct use of a physical register. +/// Check that the register is not used by a virtreg. +/// Kill the physreg, marking it free. +/// This may add implicit kills to MO->getParent() and invalidate MO. +void RAFast::usePhysReg(MachineOperand &MO) { + unsigned PhysReg = MO.getReg(); + assert(TargetRegisterInfo::isPhysicalRegister(PhysReg) && + "Bad usePhysReg operand"); + + switch (PhysRegState[PhysReg]) { + case regDisabled: + break; + case regReserved: + PhysRegState[PhysReg] = regFree; + // Fall through + case regFree: + UsedInInstr.set(PhysReg); + MO.setIsKill(); + return; + default: + // The physreg was allocated to a virtual register. That means to value we + // wanted has been clobbered. + llvm_unreachable("Instruction uses an allocated register"); + } -/// getFreeReg - Look to see if there is a free register available in the -/// specified register class. If not, return 0. -/// -unsigned RAFast::getFreeReg(const TargetRegisterClass *RC) { - // Get iterators defining the range of registers that are valid to allocate in - // this class, which also specifies the preferred allocation order. - TargetRegisterClass::iterator RI = RC->allocation_order_begin(*MF); - TargetRegisterClass::iterator RE = RC->allocation_order_end(*MF); - - for (; RI != RE; ++RI) - if (isPhysRegAvailable(*RI)) { // Is reg unused? - assert(*RI != 0 && "Cannot use register!"); - return *RI; // Found an unused register! + // Maybe a superregister is reserved? + for (const unsigned *AS = TRI->getAliasSet(PhysReg); + unsigned Alias = *AS; ++AS) { + switch (PhysRegState[Alias]) { + case regDisabled: + break; + case regReserved: + assert(TRI->isSuperRegister(PhysReg, Alias) && + "Instruction is not using a subregister of a reserved register"); + // Leave the superregister in the working set. + PhysRegState[Alias] = regFree; + UsedInInstr.set(Alias); + MO.getParent()->addRegisterKilled(Alias, TRI, true); + return; + case regFree: + if (TRI->isSuperRegister(PhysReg, Alias)) { + // Leave the superregister in the working set. + UsedInInstr.set(Alias); + MO.getParent()->addRegisterKilled(Alias, TRI, true); + return; + } + // Some other alias was in the working set - clear it. + PhysRegState[Alias] = regDisabled; + break; + default: + llvm_unreachable("Instruction uses an alias of an allocated register"); } - return 0; + } + + // All aliases are disabled, bring register into working set. + PhysRegState[PhysReg] = regFree; + UsedInInstr.set(PhysReg); + MO.setIsKill(); } +/// definePhysReg - Mark PhysReg as reserved or free after spilling any +/// virtregs. This is very similar to defineVirtReg except the physreg is +/// reserved instead of allocated. +void RAFast::definePhysReg(MachineInstr *MI, unsigned PhysReg, + RegState NewState) { + UsedInInstr.set(PhysReg); + switch (unsigned VirtReg = PhysRegState[PhysReg]) { + case regDisabled: + break; + default: + spillVirtReg(MI, VirtReg); + // Fall through. + case regFree: + case regReserved: + PhysRegState[PhysReg] = NewState; + return; + } -/// getReg - Find a physical register to hold the specified virtual -/// register. If all compatible physical registers are used, this method spills -/// the last used virtual register to the stack, and uses that register. -/// -unsigned RAFast::getReg(MachineBasicBlock &MBB, MachineInstr *I, - unsigned VirtReg, bool NoFree) { - const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(VirtReg); + // This is a disabled register, disable all aliases. + PhysRegState[PhysReg] = NewState; + for (const unsigned *AS = TRI->getAliasSet(PhysReg); + unsigned Alias = *AS; ++AS) { + UsedInInstr.set(Alias); + switch (unsigned VirtReg = PhysRegState[Alias]) { + case regDisabled: + break; + default: + spillVirtReg(MI, VirtReg); + // Fall through. + case regFree: + case regReserved: + PhysRegState[Alias] = regDisabled; + if (TRI->isSuperRegister(PhysReg, Alias)) + return; + break; + } + } +} - // First check to see if we have a free register of the requested type... - unsigned PhysReg = NoFree ? 0 : getFreeReg(RC); - if (PhysReg != 0) { - // Assign the register. - assignVirtToPhysReg(VirtReg, PhysReg); - return PhysReg; +// calcSpillCost - Return the cost of spilling clearing out PhysReg and +// aliases so it is free for allocation. +// Returns 0 when PhysReg is free or disabled with all aliases disabled - it +// can be allocated directly. +// Returns spillImpossible when PhysReg or an alias can't be spilled. +unsigned RAFast::calcSpillCost(unsigned PhysReg) const { + if (UsedInInstr.test(PhysReg)) + return spillImpossible; + switch (unsigned VirtReg = PhysRegState[PhysReg]) { + case regDisabled: + break; + case regFree: + return 0; + case regReserved: + return spillImpossible; + default: + return LiveVirtRegs.lookup(VirtReg).Dirty ? spillDirty : spillClean; } - // If we didn't find an unused register, scavenge one now! Don't be fancy, - // just grab the first possible register. - TargetRegisterClass::iterator RI = RC->allocation_order_begin(*MF); - TargetRegisterClass::iterator RE = RC->allocation_order_end(*MF); - - for (; RI != RE; ++RI) - if (isPhysRegSpillable(*RI)) { - PhysReg = *RI; + // This is a disabled register, add up const of aliases. + unsigned Cost = 0; + for (const unsigned *AS = TRI->getAliasSet(PhysReg); + unsigned Alias = *AS; ++AS) { + if (UsedInInstr.test(Alias)) + return spillImpossible; + switch (unsigned VirtReg = PhysRegState[Alias]) { + case regDisabled: + break; + case regFree: + ++Cost; + break; + case regReserved: + return spillImpossible; + default: + Cost += LiveVirtRegs.lookup(VirtReg).Dirty ? spillDirty : spillClean; break; } - - assert(PhysReg && "Physical register not assigned!?!?"); - spillPhysReg(MBB, I, PhysReg); - assignVirtToPhysReg(VirtReg, PhysReg); - return PhysReg; + } + return Cost; } -/// reloadVirtReg - This method transforms the specified virtual -/// register use to refer to a physical register. This method may do this in -/// one of several ways: if the register is available in a physical register -/// already, it uses that physical register. If the value is not in a physical -/// register, and if there are physical registers available, it loads it into a -/// register: PhysReg if that is an available physical register, otherwise any -/// register. If register pressure is high, and it is possible, it tries to -/// fold the load of the virtual register into the instruction itself. It -/// avoids doing this if register pressure is low to improve the chance that -/// subsequent instructions can use the reloaded value. This method returns -/// the modified instruction. +/// assignVirtToPhysReg - This method updates local state so that we know +/// that PhysReg is the proper container for VirtReg now. The physical +/// register must not be used for anything else when this is called. /// -MachineInstr *RAFast::reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI, - unsigned OpNum, - SmallSet<unsigned, 4> &ReloadedRegs, - unsigned PhysReg) { - unsigned VirtReg = MI->getOperand(OpNum).getReg(); - - // If the virtual register is already available, just update the instruction - // and return. - if (unsigned PR = getVirt2PhysRegMapSlot(VirtReg)) { - MI->getOperand(OpNum).setReg(PR); // Assign the input register - if (!MI->isDebugValue()) { - // Do not do these for DBG_VALUE as they can affect codegen. - UsedInInstr.set(PR); - getVirtRegLastUse(VirtReg) = std::make_pair(MI, OpNum); +void RAFast::assignVirtToPhysReg(LiveRegEntry &LRE, unsigned PhysReg) { + DEBUG(dbgs() << "Assigning %reg" << LRE.first << " to " + << TRI->getName(PhysReg) << "\n"); + PhysRegState[PhysReg] = LRE.first; + assert(!LRE.second.PhysReg && "Already assigned a physreg"); + LRE.second.PhysReg = PhysReg; +} + +/// allocVirtReg - Allocate a physical register for VirtReg. +void RAFast::allocVirtReg(MachineInstr *MI, LiveRegEntry &LRE, unsigned Hint) { + const unsigned VirtReg = LRE.first; + + assert(TargetRegisterInfo::isVirtualRegister(VirtReg) && + "Can only allocate virtual registers"); + + const TargetRegisterClass *RC = MRI->getRegClass(VirtReg); + + // Ignore invalid hints. + if (Hint && (!TargetRegisterInfo::isPhysicalRegister(Hint) || + !RC->contains(Hint) || !Allocatable.test(Hint))) + Hint = 0; + + // Take hint when possible. + if (Hint) { + switch(calcSpillCost(Hint)) { + default: + definePhysReg(MI, Hint, regFree); + // Fall through. + case 0: + return assignVirtToPhysReg(LRE, Hint); + case spillImpossible: + break; } - return MI; } - // Otherwise, we need to fold it into the current instruction, or reload it. - // If we have registers available to hold the value, use them. - const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(VirtReg); - // If we already have a PhysReg (this happens when the instruction is a - // reg-to-reg copy with a PhysReg destination) use that. - if (!PhysReg || !TargetRegisterInfo::isPhysicalRegister(PhysReg) || - !isPhysRegAvailable(PhysReg)) - PhysReg = getFreeReg(RC); - int FrameIndex = getStackSpaceFor(VirtReg, RC); - - if (PhysReg) { // Register is available, allocate it! - assignVirtToPhysReg(VirtReg, PhysReg); - } else { // No registers available. - // Force some poor hapless value out of the register file to - // make room for the new register, and reload it. - PhysReg = getReg(MBB, MI, VirtReg, true); + TargetRegisterClass::iterator AOB = RC->allocation_order_begin(*MF); + TargetRegisterClass::iterator AOE = RC->allocation_order_end(*MF); + + // First try to find a completely free register. + for (TargetRegisterClass::iterator I = AOB; I != AOE; ++I) { + unsigned PhysReg = *I; + if (PhysRegState[PhysReg] == regFree && !UsedInInstr.test(PhysReg)) + return assignVirtToPhysReg(LRE, PhysReg); } - markVirtRegModified(VirtReg, false); // Note that this reg was just reloaded + DEBUG(dbgs() << "Allocating %reg" << VirtReg << " from " << RC->getName() + << "\n"); + + unsigned BestReg = 0, BestCost = spillImpossible; + for (TargetRegisterClass::iterator I = AOB; I != AOE; ++I) { + unsigned Cost = calcSpillCost(*I); + // Cost is 0 when all aliases are already disabled. + if (Cost == 0) + return assignVirtToPhysReg(LRE, *I); + if (Cost < BestCost) + BestReg = *I, BestCost = Cost; + } - DEBUG(dbgs() << " Reloading %reg" << VirtReg << " into " - << TRI->getName(PhysReg) << "\n"); + if (BestReg) { + definePhysReg(MI, BestReg, regFree); + return assignVirtToPhysReg(LRE, BestReg); + } - // Add move instruction(s) - TII->loadRegFromStackSlot(MBB, MI, PhysReg, FrameIndex, RC); - ++NumLoads; // Update statistics - - MF->getRegInfo().setPhysRegUsed(PhysReg); - MI->getOperand(OpNum).setReg(PhysReg); // Assign the input register - getVirtRegLastUse(VirtReg) = std::make_pair(MI, OpNum); - - if (!ReloadedRegs.insert(PhysReg)) { - std::string msg; - raw_string_ostream Msg(msg); - Msg << "Ran out of registers during register allocation!"; - if (MI->isInlineAsm()) { - Msg << "\nPlease check your inline asm statement for invalid " - << "constraints:\n"; - MI->print(Msg, TM); - } - report_fatal_error(Msg.str()); + // Nothing we can do. + std::string msg; + raw_string_ostream Msg(msg); + Msg << "Ran out of registers during register allocation!"; + if (MI->isInlineAsm()) { + Msg << "\nPlease check your inline asm statement for " + << "invalid constraints:\n"; + MI->print(Msg, TM); } - for (const unsigned *SubRegs = TRI->getSubRegisters(PhysReg); - *SubRegs; ++SubRegs) { - if (ReloadedRegs.insert(*SubRegs)) continue; - - std::string msg; - raw_string_ostream Msg(msg); - Msg << "Ran out of registers during register allocation!"; - if (MI->isInlineAsm()) { - Msg << "\nPlease check your inline asm statement for invalid " - << "constraints:\n"; - MI->print(Msg, TM); + report_fatal_error(Msg.str()); +} + +/// defineVirtReg - Allocate a register for VirtReg and mark it as dirty. +RAFast::LiveRegMap::iterator +RAFast::defineVirtReg(MachineInstr *MI, unsigned OpNum, + unsigned VirtReg, unsigned Hint) { + assert(TargetRegisterInfo::isVirtualRegister(VirtReg) && + "Not a virtual register"); + LiveRegMap::iterator LRI; + bool New; + tie(LRI, New) = LiveVirtRegs.insert(std::make_pair(VirtReg, LiveReg())); + LiveReg &LR = LRI->second; + bool PartialRedef = MI->getOperand(OpNum).getSubReg(); + if (New) { + // If there is no hint, peek at the only use of this register. + if ((!Hint || !TargetRegisterInfo::isPhysicalRegister(Hint)) && + MRI->hasOneNonDBGUse(VirtReg)) { + unsigned SrcReg, DstReg, SrcSubReg, DstSubReg; + // It's a copy, use the destination register as a hint. + if (TII->isMoveInstr(*MRI->use_nodbg_begin(VirtReg), + SrcReg, DstReg, SrcSubReg, DstSubReg)) + Hint = DstReg; + } + allocVirtReg(MI, *LRI, Hint); + // If this is only a partial redefinition, we must reload the other parts. + if (PartialRedef && MI->readsVirtualRegister(VirtReg)) { + const TargetRegisterClass *RC = MRI->getRegClass(VirtReg); + int FI = getStackSpaceFor(VirtReg, RC); + DEBUG(dbgs() << "Reloading for partial redef: %reg" << VirtReg << "\n"); + TII->loadRegFromStackSlot(*MBB, MI, LR.PhysReg, FI, RC, TRI); + ++NumLoads; } - report_fatal_error(Msg.str()); + } else if (LR.LastUse && !PartialRedef) { + // Redefining a live register - kill at the last use, unless it is this + // instruction defining VirtReg multiple times. + if (LR.LastUse != MI || LR.LastUse->getOperand(LR.LastOpNum).isUse()) + addKillFlag(LR); } - - return MI; + assert(LR.PhysReg && "Register not assigned"); + LR.LastUse = MI; + LR.LastOpNum = OpNum; + LR.Dirty = true; + UsedInInstr.set(LR.PhysReg); + return LRI; } -/// isReadModWriteImplicitKill - True if this is an implicit kill for a -/// read/mod/write register, i.e. update partial register. -static bool isReadModWriteImplicitKill(MachineInstr *MI, unsigned Reg) { - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - MachineOperand &MO = MI->getOperand(i); - if (MO.isReg() && MO.getReg() == Reg && MO.isImplicit() && - MO.isDef() && !MO.isDead()) - return true; +/// reloadVirtReg - Make sure VirtReg is available in a physreg and return it. +RAFast::LiveRegMap::iterator +RAFast::reloadVirtReg(MachineInstr *MI, unsigned OpNum, + unsigned VirtReg, unsigned Hint) { + assert(TargetRegisterInfo::isVirtualRegister(VirtReg) && + "Not a virtual register"); + LiveRegMap::iterator LRI; + bool New; + tie(LRI, New) = LiveVirtRegs.insert(std::make_pair(VirtReg, LiveReg())); + LiveReg &LR = LRI->second; + MachineOperand &MO = MI->getOperand(OpNum); + if (New) { + allocVirtReg(MI, *LRI, Hint); + const TargetRegisterClass *RC = MRI->getRegClass(VirtReg); + int FrameIndex = getStackSpaceFor(VirtReg, RC); + DEBUG(dbgs() << "Reloading %reg" << VirtReg << " into " + << TRI->getName(LR.PhysReg) << "\n"); + TII->loadRegFromStackSlot(*MBB, MI, LR.PhysReg, FrameIndex, RC, TRI); + ++NumLoads; + } else if (LR.Dirty) { + if (isLastUseOfLocalReg(MO)) { + DEBUG(dbgs() << "Killing last use: " << MO << "\n"); + MO.setIsKill(); + } else if (MO.isKill()) { + DEBUG(dbgs() << "Clearing dubious kill: " << MO << "\n"); + MO.setIsKill(false); + } + } else if (MO.isKill()) { + // We must remove kill flags from uses of reloaded registers because the + // register would be killed immediately, and there might be a second use: + // %foo = OR %x<kill>, %x + // This would cause a second reload of %x into a different register. + DEBUG(dbgs() << "Clearing clean kill: " << MO << "\n"); + MO.setIsKill(false); } - return false; + assert(LR.PhysReg && "Register not assigned"); + LR.LastUse = MI; + LR.LastOpNum = OpNum; + UsedInInstr.set(LR.PhysReg); + return LRI; } -/// isReadModWriteImplicitDef - True if this is an implicit def for a -/// read/mod/write register, i.e. update partial register. -static bool isReadModWriteImplicitDef(MachineInstr *MI, unsigned Reg) { - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - MachineOperand &MO = MI->getOperand(i); - if (MO.isReg() && MO.getReg() == Reg && MO.isImplicit() && - !MO.isDef() && MO.isKill()) - return true; +// setPhysReg - Change operand OpNum in MI the refer the PhysReg, considering +// subregs. This may invalidate any operand pointers. +// Return true if the operand kills its register. +bool RAFast::setPhysReg(MachineInstr *MI, unsigned OpNum, unsigned PhysReg) { + MachineOperand &MO = MI->getOperand(OpNum); + if (!MO.getSubReg()) { + MO.setReg(PhysReg); + return MO.isKill() || MO.isDead(); } - return false; -} -void RAFast::AllocateBasicBlock(MachineBasicBlock &MBB) { - // loop over each instruction - MachineBasicBlock::iterator MII = MBB.begin(); - - DEBUG({ - const BasicBlock *LBB = MBB.getBasicBlock(); - if (LBB) - dbgs() << "\nStarting RegAlloc of BB: " << LBB->getName(); - }); - - // Add live-in registers as active. - for (MachineBasicBlock::livein_iterator I = MBB.livein_begin(), - E = MBB.livein_end(); I != E; ++I) { - unsigned Reg = *I; - MF->getRegInfo().setPhysRegUsed(Reg); - PhysRegsUsed[Reg] = 0; // It is free and reserved now - for (const unsigned *SubRegs = TRI->getSubRegisters(Reg); - *SubRegs; ++SubRegs) { - if (PhysRegsUsed[*SubRegs] == -2) continue; - PhysRegsUsed[*SubRegs] = 0; // It is free and reserved now - MF->getRegInfo().setPhysRegUsed(*SubRegs); - } + // Handle subregister index. + MO.setReg(PhysReg ? TRI->getSubReg(PhysReg, MO.getSubReg()) : 0); + MO.setSubReg(0); + + // A kill flag implies killing the full register. Add corresponding super + // register kill. + if (MO.isKill()) { + MI->addRegisterKilled(PhysReg, TRI, true); + return true; } + return MO.isDead(); +} + +void RAFast::AllocateBasicBlock() { + DEBUG(dbgs() << "\nAllocating " << *MBB); + + PhysRegState.assign(TRI->getNumRegs(), regDisabled); + assert(LiveVirtRegs.empty() && "Mapping not cleared form last block?"); + + MachineBasicBlock::iterator MII = MBB->begin(); + + // Add live-in registers as live. + for (MachineBasicBlock::livein_iterator I = MBB->livein_begin(), + E = MBB->livein_end(); I != E; ++I) + definePhysReg(MII, *I, regReserved); + + SmallVector<unsigned, 8> PhysECs, VirtDead; + SmallVector<MachineInstr*, 32> Coalesced; // Otherwise, sequentially allocate each instruction in the MBB. - while (MII != MBB.end()) { + while (MII != MBB->end()) { MachineInstr *MI = MII++; const TargetInstrDesc &TID = MI->getDesc(); DEBUG({ - dbgs() << "\nStarting RegAlloc of: " << *MI; - dbgs() << " Regs have values: "; - for (unsigned i = 0; i != TRI->getNumRegs(); ++i) - if (PhysRegsUsed[i] != -1 && PhysRegsUsed[i] != -2) - dbgs() << "[" << TRI->getName(i) - << ",%reg" << PhysRegsUsed[i] << "] "; + dbgs() << "\n>> " << *MI << "Regs:"; + for (unsigned Reg = 1, E = TRI->getNumRegs(); Reg != E; ++Reg) { + if (PhysRegState[Reg] == regDisabled) continue; + dbgs() << " " << TRI->getName(Reg); + switch(PhysRegState[Reg]) { + case regFree: + break; + case regReserved: + dbgs() << "*"; + break; + default: + dbgs() << "=%reg" << PhysRegState[Reg]; + if (LiveVirtRegs[PhysRegState[Reg]].Dirty) + dbgs() << "*"; + assert(LiveVirtRegs[PhysRegState[Reg]].PhysReg == Reg && + "Bad inverse map"); + break; + } + } dbgs() << '\n'; + // Check that LiveVirtRegs is the inverse. + for (LiveRegMap::iterator i = LiveVirtRegs.begin(), + e = LiveVirtRegs.end(); i != e; ++i) { + assert(TargetRegisterInfo::isVirtualRegister(i->first) && + "Bad map key"); + assert(TargetRegisterInfo::isPhysicalRegister(i->second.PhysReg) && + "Bad map value"); + assert(PhysRegState[i->second.PhysReg] == i->first && + "Bad inverse map"); + } }); - // Track registers used by instruction. - UsedInInstr.reset(); - - // Determine whether this is a copy instruction. The cases where the - // source or destination are phys regs are handled specially. - unsigned SrcCopyReg, DstCopyReg, SrcCopySubReg, DstCopySubReg; - unsigned SrcCopyPhysReg = 0U; - bool isCopy = TII->isMoveInstr(*MI, SrcCopyReg, DstCopyReg, - SrcCopySubReg, DstCopySubReg); - if (isCopy && TargetRegisterInfo::isVirtualRegister(SrcCopyReg)) - SrcCopyPhysReg = getVirt2PhysRegMapSlot(SrcCopyReg); - - // Loop over the implicit uses, making sure they don't get reallocated. - if (TID.ImplicitUses) { - for (const unsigned *ImplicitUses = TID.ImplicitUses; - *ImplicitUses; ++ImplicitUses) - UsedInInstr.set(*ImplicitUses); - } - - SmallVector<unsigned, 8> Kills; - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || !MO.isKill()) continue; - - if (!MO.isImplicit()) - Kills.push_back(MO.getReg()); - else if (!isReadModWriteImplicitKill(MI, MO.getReg())) - // These are extra physical register kills when a sub-register - // is defined (def of a sub-register is a read/mod/write of the - // larger registers). Ignore. - Kills.push_back(MO.getReg()); - } - - // If any physical regs are earlyclobber, spill any value they might - // have in them, then mark them unallocatable. - // If any virtual regs are earlyclobber, allocate them now (before - // freeing inputs that are killed). - if (MI->isInlineAsm()) { + // Debug values are not allowed to change codegen in any way. + if (MI->isDebugValue()) { for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || !MO.isDef() || !MO.isEarlyClobber() || - !MO.getReg()) - continue; - - if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) { - unsigned DestVirtReg = MO.getReg(); - unsigned DestPhysReg; - - // If DestVirtReg already has a value, use it. - if (!(DestPhysReg = getVirt2PhysRegMapSlot(DestVirtReg))) - DestPhysReg = getReg(MBB, MI, DestVirtReg); - MF->getRegInfo().setPhysRegUsed(DestPhysReg); - markVirtRegModified(DestVirtReg); - getVirtRegLastUse(DestVirtReg) = - std::make_pair((MachineInstr*)0, 0); - DEBUG(dbgs() << " Assigning " << TRI->getName(DestPhysReg) - << " to %reg" << DestVirtReg << "\n"); - MO.setReg(DestPhysReg); // Assign the earlyclobber register - } else { - unsigned Reg = MO.getReg(); - if (PhysRegsUsed[Reg] == -2) continue; // Something like ESP. - // These are extra physical register defs when a sub-register - // is defined (def of a sub-register is a read/mod/write of the - // larger registers). Ignore. - if (isReadModWriteImplicitDef(MI, MO.getReg())) continue; - - MF->getRegInfo().setPhysRegUsed(Reg); - spillPhysReg(MBB, MI, Reg, true); // Spill any existing value in reg - PhysRegsUsed[Reg] = 0; // It is free and reserved now - - for (const unsigned *SubRegs = TRI->getSubRegisters(Reg); - *SubRegs; ++SubRegs) { - if (PhysRegsUsed[*SubRegs] == -2) continue; - MF->getRegInfo().setPhysRegUsed(*SubRegs); - PhysRegsUsed[*SubRegs] = 0; // It is free and reserved now - } - } + if (!MO.isReg()) continue; + unsigned Reg = MO.getReg(); + if (!Reg || TargetRegisterInfo::isPhysicalRegister(Reg)) continue; + LiveRegMap::iterator LRI = LiveVirtRegs.find(Reg); + if (LRI != LiveVirtRegs.end()) + setPhysReg(MI, i, LRI->second.PhysReg); + else + MO.setReg(0); // We can't allocate a physreg for a DebugValue, sorry! } + // Next instruction. + continue; } - // If a DBG_VALUE says something is located in a spilled register, - // change the DBG_VALUE to be undef, which prevents the register - // from being reloaded here. Doing that would change the generated - // code, unless another use immediately follows this instruction. - if (MI->isDebugValue() && - MI->getNumOperands()==3 && MI->getOperand(0).isReg()) { - unsigned VirtReg = MI->getOperand(0).getReg(); - if (VirtReg && TargetRegisterInfo::isVirtualRegister(VirtReg) && - !getVirt2PhysRegMapSlot(VirtReg)) - MI->getOperand(0).setReg(0U); - } + // If this is a copy, we may be able to coalesce. + unsigned CopySrc, CopyDst, CopySrcSub, CopyDstSub; + if (!TII->isMoveInstr(*MI, CopySrc, CopyDst, CopySrcSub, CopyDstSub)) + CopySrc = CopyDst = 0; - // Get the used operands into registers. This has the potential to spill - // incoming values if we are out of registers. Note that we completely - // ignore physical register uses here. We assume that if an explicit - // physical register is referenced by the instruction, that it is guaranteed - // to be live-in, or the input is badly hosed. - // - SmallSet<unsigned, 4> ReloadedRegs; - for (unsigned i = 0; i != MI->getNumOperands(); ++i) { - MachineOperand &MO = MI->getOperand(i); - // here we are looking for only used operands (never def&use) - if (MO.isReg() && !MO.isDef() && MO.getReg() && !MO.isImplicit() && - TargetRegisterInfo::isVirtualRegister(MO.getReg())) - MI = reloadVirtReg(MBB, MI, i, ReloadedRegs, - isCopy ? DstCopyReg : 0); - } + // Track registers used by instruction. + UsedInInstr.reset(); + PhysECs.clear(); - // If this instruction is the last user of this register, kill the - // value, freeing the register being used, so it doesn't need to be - // spilled to memory. - // - for (unsigned i = 0, e = Kills.size(); i != e; ++i) { - unsigned VirtReg = Kills[i]; - unsigned PhysReg = VirtReg; - if (TargetRegisterInfo::isVirtualRegister(VirtReg)) { - // If the virtual register was never materialized into a register, it - // might not be in the map, but it won't hurt to zero it out anyway. - unsigned &PhysRegSlot = getVirt2PhysRegMapSlot(VirtReg); - PhysReg = PhysRegSlot; - PhysRegSlot = 0; - } else if (PhysRegsUsed[PhysReg] == -2) { - // Unallocatable register dead, ignore. + // First scan. + // Mark physreg uses and early clobbers as used. + // Find the end of the virtreg operands + unsigned VirtOpEnd = 0; + for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { + MachineOperand &MO = MI->getOperand(i); + if (!MO.isReg()) continue; + unsigned Reg = MO.getReg(); + if (!Reg) continue; + if (TargetRegisterInfo::isVirtualRegister(Reg)) { + VirtOpEnd = i+1; continue; - } else { - assert((!PhysRegsUsed[PhysReg] || PhysRegsUsed[PhysReg] == -1) && - "Silently clearing a virtual register?"); } - - if (!PhysReg) continue; - - DEBUG(dbgs() << " Last use of " << TRI->getName(PhysReg) - << "[%reg" << VirtReg <<"], removing it from live set\n"); - removePhysReg(PhysReg); - for (const unsigned *SubRegs = TRI->getSubRegisters(PhysReg); - *SubRegs; ++SubRegs) { - if (PhysRegsUsed[*SubRegs] != -2) { - DEBUG(dbgs() << " Last use of " - << TRI->getName(*SubRegs) << "[%reg" << VirtReg - <<"], removing it from live set\n"); - removePhysReg(*SubRegs); - } + if (!Allocatable.test(Reg)) continue; + if (MO.isUse()) { + usePhysReg(MO); + } else if (MO.isEarlyClobber()) { + definePhysReg(MI, Reg, MO.isDead() ? regFree : regReserved); + PhysECs.push_back(Reg); } } - // Track registers defined by instruction. - UsedInInstr.reset(); - - // Loop over all of the operands of the instruction, spilling registers that - // are defined, and marking explicit destinations in the PhysRegsUsed map. - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { + // Second scan. + // Allocate virtreg uses and early clobbers. + // Collect VirtKills + for (unsigned i = 0; i != VirtOpEnd; ++i) { MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || !MO.isDef() || MO.isImplicit() || !MO.getReg() || - MO.isEarlyClobber() || - !TargetRegisterInfo::isPhysicalRegister(MO.getReg())) - continue; - + if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); - if (PhysRegsUsed[Reg] == -2) continue; // Something like ESP. - // These are extra physical register defs when a sub-register - // is defined (def of a sub-register is a read/mod/write of the - // larger registers). Ignore. - if (isReadModWriteImplicitDef(MI, MO.getReg())) continue; - - MF->getRegInfo().setPhysRegUsed(Reg); - spillPhysReg(MBB, MI, Reg, true); // Spill any existing value in reg - PhysRegsUsed[Reg] = 0; // It is free and reserved now - - for (const unsigned *SubRegs = TRI->getSubRegisters(Reg); - *SubRegs; ++SubRegs) { - if (PhysRegsUsed[*SubRegs] == -2) continue; - - MF->getRegInfo().setPhysRegUsed(*SubRegs); - PhysRegsUsed[*SubRegs] = 0; // It is free and reserved now + if (!Reg || TargetRegisterInfo::isPhysicalRegister(Reg)) continue; + if (MO.isUse()) { + LiveRegMap::iterator LRI = reloadVirtReg(MI, i, Reg, CopyDst); + unsigned PhysReg = LRI->second.PhysReg; + CopySrc = (CopySrc == Reg || CopySrc == PhysReg) ? PhysReg : 0; + if (setPhysReg(MI, i, PhysReg)) + killVirtReg(LRI); + } else if (MO.isEarlyClobber()) { + // Note: defineVirtReg may invalidate MO. + LiveRegMap::iterator LRI = defineVirtReg(MI, i, Reg, 0); + unsigned PhysReg = LRI->second.PhysReg; + setPhysReg(MI, i, PhysReg); + PhysECs.push_back(PhysReg); } } - // Loop over the implicit defs, spilling them as well. - if (TID.ImplicitDefs) { - for (const unsigned *ImplicitDefs = TID.ImplicitDefs; - *ImplicitDefs; ++ImplicitDefs) { - unsigned Reg = *ImplicitDefs; - if (PhysRegsUsed[Reg] != -2) { - spillPhysReg(MBB, MI, Reg, true); - PhysRegsUsed[Reg] = 0; // It is free and reserved now - } - MF->getRegInfo().setPhysRegUsed(Reg); - for (const unsigned *SubRegs = TRI->getSubRegisters(Reg); - *SubRegs; ++SubRegs) { - if (PhysRegsUsed[*SubRegs] == -2) continue; + MRI->addPhysRegsUsed(UsedInInstr); - PhysRegsUsed[*SubRegs] = 0; // It is free and reserved now - MF->getRegInfo().setPhysRegUsed(*SubRegs); - } - } + // Track registers defined by instruction - early clobbers at this point. + UsedInInstr.reset(); + for (unsigned i = 0, e = PhysECs.size(); i != e; ++i) { + unsigned PhysReg = PhysECs[i]; + UsedInInstr.set(PhysReg); + for (const unsigned *AS = TRI->getAliasSet(PhysReg); + unsigned Alias = *AS; ++AS) + UsedInInstr.set(Alias); } - SmallVector<unsigned, 8> DeadDefs; - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - MachineOperand &MO = MI->getOperand(i); - if (MO.isReg() && MO.isDead()) - DeadDefs.push_back(MO.getReg()); + unsigned DefOpEnd = MI->getNumOperands(); + if (TID.isCall()) { + // Spill all virtregs before a call. This serves two purposes: 1. If an + // exception is thrown, the landing pad is going to expect to find registers + // in their spill slots, and 2. we don't have to wade through all the + // <imp-def> operands on the call instruction. + DefOpEnd = VirtOpEnd; + DEBUG(dbgs() << " Spilling remaining registers before call.\n"); + spillAll(MI); } - // Okay, we have allocated all of the source operands and spilled any values - // that would be destroyed by defs of this instruction. Loop over the - // explicit defs and assign them to a register, spilling incoming values if - // we need to scavenge a register. - // - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { + // Third scan. + // Allocate defs and collect dead defs. + for (unsigned i = 0; i != DefOpEnd; ++i) { MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || !MO.isDef() || !MO.getReg() || - MO.isEarlyClobber() || - !TargetRegisterInfo::isVirtualRegister(MO.getReg())) - continue; - - unsigned DestVirtReg = MO.getReg(); - unsigned DestPhysReg; - - // If DestVirtReg already has a value, use it. - if (!(DestPhysReg = getVirt2PhysRegMapSlot(DestVirtReg))) { - // If this is a copy try to reuse the input as the output; - // that will make the copy go away. - // If this is a copy, the source reg is a phys reg, and - // that reg is available, use that phys reg for DestPhysReg. - // If this is a copy, the source reg is a virtual reg, and - // the phys reg that was assigned to that virtual reg is now - // available, use that phys reg for DestPhysReg. (If it's now - // available that means this was the last use of the source.) - if (isCopy && - TargetRegisterInfo::isPhysicalRegister(SrcCopyReg) && - isPhysRegAvailable(SrcCopyReg)) { - DestPhysReg = SrcCopyReg; - assignVirtToPhysReg(DestVirtReg, DestPhysReg); - } else if (isCopy && - TargetRegisterInfo::isVirtualRegister(SrcCopyReg) && - SrcCopyPhysReg && isPhysRegAvailable(SrcCopyPhysReg) && - MF->getRegInfo().getRegClass(DestVirtReg)-> - contains(SrcCopyPhysReg)) { - DestPhysReg = SrcCopyPhysReg; - assignVirtToPhysReg(DestVirtReg, DestPhysReg); - } else - DestPhysReg = getReg(MBB, MI, DestVirtReg); - } - MF->getRegInfo().setPhysRegUsed(DestPhysReg); - markVirtRegModified(DestVirtReg); - getVirtRegLastUse(DestVirtReg) = std::make_pair((MachineInstr*)0, 0); - DEBUG(dbgs() << " Assigning " << TRI->getName(DestPhysReg) - << " to %reg" << DestVirtReg << "\n"); - MO.setReg(DestPhysReg); // Assign the output register - UsedInInstr.set(DestPhysReg); - } + if (!MO.isReg() || !MO.isDef() || !MO.getReg()) continue; + unsigned Reg = MO.getReg(); - // If this instruction defines any registers that are immediately dead, - // kill them now. - // - for (unsigned i = 0, e = DeadDefs.size(); i != e; ++i) { - unsigned VirtReg = DeadDefs[i]; - unsigned PhysReg = VirtReg; - if (TargetRegisterInfo::isVirtualRegister(VirtReg)) { - unsigned &PhysRegSlot = getVirt2PhysRegMapSlot(VirtReg); - PhysReg = PhysRegSlot; - assert(PhysReg != 0); - PhysRegSlot = 0; - } else if (PhysRegsUsed[PhysReg] == -2) { - // Unallocatable register dead, ignore. - continue; - } else if (!PhysReg) + if (TargetRegisterInfo::isPhysicalRegister(Reg)) { + if (!Allocatable.test(Reg)) continue; + definePhysReg(MI, Reg, (MO.isImplicit() || MO.isDead()) ? + regFree : regReserved); continue; - - DEBUG(dbgs() << " Register " << TRI->getName(PhysReg) - << " [%reg" << VirtReg - << "] is never used, removing it from live set\n"); - removePhysReg(PhysReg); - for (const unsigned *AliasSet = TRI->getAliasSet(PhysReg); - *AliasSet; ++AliasSet) { - if (PhysRegsUsed[*AliasSet] != -2) { - DEBUG(dbgs() << " Register " << TRI->getName(*AliasSet) - << " [%reg" << *AliasSet - << "] is never used, removing it from live set\n"); - removePhysReg(*AliasSet); - } } + LiveRegMap::iterator LRI = defineVirtReg(MI, i, Reg, CopySrc); + unsigned PhysReg = LRI->second.PhysReg; + if (setPhysReg(MI, i, PhysReg)) { + VirtDead.push_back(Reg); + CopyDst = 0; // cancel coalescing; + } else + CopyDst = (CopyDst == Reg || CopyDst == PhysReg) ? PhysReg : 0; } - // Finally, if this is a noop copy instruction, zap it. (Except that if - // the copy is dead, it must be kept to avoid messing up liveness info for - // the register scavenger. See pr4100.) - if (TII->isMoveInstr(*MI, SrcCopyReg, DstCopyReg, - SrcCopySubReg, DstCopySubReg) && - SrcCopyReg == DstCopyReg && DeadDefs.empty()) - MBB.erase(MI); + // Kill dead defs after the scan to ensure that multiple defs of the same + // register are allocated identically. We didn't need to do this for uses + // because we are crerating our own kill flags, and they are always at the + // last use. + for (unsigned i = 0, e = VirtDead.size(); i != e; ++i) + killVirtReg(VirtDead[i]); + VirtDead.clear(); + + MRI->addPhysRegsUsed(UsedInInstr); + + if (CopyDst && CopyDst == CopySrc && CopyDstSub == CopySrcSub) { + DEBUG(dbgs() << "-- coalescing: " << *MI); + Coalesced.push_back(MI); + } else { + DEBUG(dbgs() << "<< " << *MI); + } } - MachineBasicBlock::iterator MI = MBB.getFirstTerminator(); - // Spill all physical registers holding virtual registers now. - for (unsigned i = 0, e = TRI->getNumRegs(); i != e; ++i) - if (PhysRegsUsed[i] != -1 && PhysRegsUsed[i] != -2) { - if (unsigned VirtReg = PhysRegsUsed[i]) - spillVirtReg(MBB, MI, VirtReg, i); - else - removePhysReg(i); - } + DEBUG(dbgs() << "Spilling live registers at end of block.\n"); + spillAll(MBB->getFirstTerminator()); + + // Erase all the coalesced copies. We are delaying it until now because + // LiveVirtRegs might refer to the instrs. + for (unsigned i = 0, e = Coalesced.size(); i != e; ++i) + MBB->erase(Coalesced[i]); + NumCopies += Coalesced.size(); + + DEBUG(MBB->dump()); } /// runOnMachineFunction - Register allocate the whole function /// bool RAFast::runOnMachineFunction(MachineFunction &Fn) { - DEBUG(dbgs() << "Machine Function\n"); + DEBUG(dbgs() << "********** FAST REGISTER ALLOCATION **********\n" + << "********** Function: " + << ((Value*)Fn.getFunction())->getName() << '\n'); MF = &Fn; + MRI = &MF->getRegInfo(); TM = &Fn.getTarget(); TRI = TM->getRegisterInfo(); TII = TM->getInstrInfo(); - PhysRegsUsed.assign(TRI->getNumRegs(), -1); UsedInInstr.resize(TRI->getNumRegs()); - - // At various places we want to efficiently check to see whether a register - // is allocatable. To handle this, we mark all unallocatable registers as - // being pinned down, permanently. - { - BitVector Allocable = TRI->getAllocatableSet(Fn); - for (unsigned i = 0, e = Allocable.size(); i != e; ++i) - if (!Allocable[i]) - PhysRegsUsed[i] = -2; // Mark the reg unallocable. - } + Allocatable = TRI->getAllocatableSet(*MF); // initialize the virtual->physical register map to have a 'null' // mapping for all virtual registers - unsigned LastVirtReg = MF->getRegInfo().getLastVirtReg(); + unsigned LastVirtReg = MRI->getLastVirtReg(); StackSlotForVirtReg.grow(LastVirtReg); - Virt2PhysRegMap.grow(LastVirtReg); - Virt2LastUseMap.grow(LastVirtReg); - VirtRegModified.resize(LastVirtReg+1 - - TargetRegisterInfo::FirstVirtualRegister); - UsedInMultipleBlocks.resize(LastVirtReg+1 - - TargetRegisterInfo::FirstVirtualRegister); // Loop over all of the basic blocks, eliminating virtual register references - for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end(); - MBB != MBBe; ++MBB) - AllocateBasicBlock(*MBB); + for (MachineFunction::iterator MBBi = Fn.begin(), MBBe = Fn.end(); + MBBi != MBBe; ++MBBi) { + MBB = &*MBBi; + AllocateBasicBlock(); + } + + // Make sure the set of used physregs is closed under subreg operations. + MRI->closePhysRegsUsed(*TRI); StackSlotForVirtReg.clear(); - PhysRegsUsed.clear(); - VirtRegModified.clear(); - UsedInMultipleBlocks.clear(); - Virt2PhysRegMap.clear(); - Virt2LastUseMap.clear(); return true; } diff --git a/lib/CodeGen/RegAllocLinearScan.cpp b/lib/CodeGen/RegAllocLinearScan.cpp index 6c8fc0c..bc331f0 100644 --- a/lib/CodeGen/RegAllocLinearScan.cpp +++ b/lib/CodeGen/RegAllocLinearScan.cpp @@ -809,7 +809,7 @@ float getConflictWeight(LiveInterval *cur, unsigned Reg, LiveIntervals *li_, MachineInstr *MI = &*I; if (cur->liveAt(li_->getInstructionIndex(MI))) { unsigned loopDepth = loopInfo->getLoopDepth(MI->getParent()); - Conflicts += powf(10.0f, (float)loopDepth); + Conflicts += std::pow(10.0f, (float)loopDepth); } } return Conflicts; diff --git a/lib/CodeGen/RegAllocLocal.cpp b/lib/CodeGen/RegAllocLocal.cpp index 94456d1..321ae12 100644 --- a/lib/CodeGen/RegAllocLocal.cpp +++ b/lib/CodeGen/RegAllocLocal.cpp @@ -37,6 +37,7 @@ using namespace llvm; STATISTIC(NumStores, "Number of stores added"); STATISTIC(NumLoads , "Number of loads added"); +STATISTIC(NumCopies, "Number of copies coalesced"); static RegisterRegAlloc localRegAlloc("local", "local register allocator", @@ -50,6 +51,7 @@ namespace { private: const TargetMachine *TM; MachineFunction *MF; + MachineRegisterInfo *MRI; const TargetRegisterInfo *TRI; const TargetInstrInfo *TII; @@ -297,8 +299,18 @@ void RALocal::storeVirtReg(MachineBasicBlock &MBB, const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(VirtReg); int FrameIndex = getStackSpaceFor(VirtReg, RC); DEBUG(dbgs() << " to stack slot #" << FrameIndex); - TII->storeRegToStackSlot(MBB, I, PhysReg, isKill, FrameIndex, RC); + TII->storeRegToStackSlot(MBB, I, PhysReg, isKill, FrameIndex, RC, TRI); ++NumStores; // Update statistics + + // Mark the spill instruction as last use if we're not killing the register. + if (!isKill) { + MachineInstr *Spill = llvm::prior(I); + int OpNum = Spill->findRegisterUseOperandIdx(PhysReg); + if (OpNum < 0) + getVirtRegLastUse(VirtReg) = std::make_pair((MachineInstr*)0, 0); + else + getVirtRegLastUse(VirtReg) = std::make_pair(Spill, OpNum); + } } /// spillVirtReg - This method spills the value specified by PhysReg into the @@ -506,10 +518,15 @@ MachineInstr *RALocal::reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI, SmallSet<unsigned, 4> &ReloadedRegs, unsigned PhysReg) { unsigned VirtReg = MI->getOperand(OpNum).getReg(); + unsigned SubIdx = MI->getOperand(OpNum).getSubReg(); // If the virtual register is already available, just update the instruction // and return. if (unsigned PR = getVirt2PhysRegMapSlot(VirtReg)) { + if (SubIdx) { + PR = TRI->getSubReg(PR, SubIdx); + MI->getOperand(OpNum).setSubReg(0); + } MI->getOperand(OpNum).setReg(PR); // Assign the input register if (!MI->isDebugValue()) { // Do not do these for DBG_VALUE as they can affect codegen. @@ -543,11 +560,16 @@ MachineInstr *RALocal::reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI, << TRI->getName(PhysReg) << "\n"); // Add move instruction(s) - TII->loadRegFromStackSlot(MBB, MI, PhysReg, FrameIndex, RC); + TII->loadRegFromStackSlot(MBB, MI, PhysReg, FrameIndex, RC, TRI); ++NumLoads; // Update statistics MF->getRegInfo().setPhysRegUsed(PhysReg); - MI->getOperand(OpNum).setReg(PhysReg); // Assign the input register + // Assign the input register. + if (SubIdx) { + MI->getOperand(OpNum).setSubReg(0); + MI->getOperand(OpNum).setReg(TRI->getSubReg(PhysReg, SubIdx)); + } else + MI->getOperand(OpNum).setReg(PhysReg); // Assign the input register getVirtRegLastUse(VirtReg) = std::make_pair(MI, OpNum); if (!ReloadedRegs.insert(PhysReg)) { @@ -626,7 +648,6 @@ static bool precedes(MachineBasicBlock::iterator A, /// ComputeLocalLiveness - Computes liveness of registers within a basic /// block, setting the killed/dead flags as appropriate. void RALocal::ComputeLocalLiveness(MachineBasicBlock& MBB) { - MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); // Keep track of the most recently seen previous use or def of each reg, // so that we can update them with dead/kill markers. DenseMap<unsigned, std::pair<MachineInstr*, unsigned> > LastUseDef; @@ -672,18 +693,26 @@ void RALocal::ComputeLocalLiveness(MachineBasicBlock& MBB) { // - A def followed by a def is dead // - A use followed by a def is a kill if (!MO.isReg() || !MO.getReg() || !MO.isDef()) continue; - + + unsigned SubIdx = MO.getSubReg(); DenseMap<unsigned, std::pair<MachineInstr*, unsigned> >::iterator last = LastUseDef.find(MO.getReg()); if (last != LastUseDef.end()) { // Check if this is a two address instruction. If so, then // the def does not kill the use. - if (last->second.first == I && - I->isRegTiedToUseOperand(i)) + if (last->second.first == I && I->isRegTiedToUseOperand(i)) continue; MachineOperand &lastUD = last->second.first->getOperand(last->second.second); + if (SubIdx && lastUD.getSubReg() != SubIdx) + // Partial re-def, the last def is not dead. + // %reg1024:5<def> = + // %reg1024:6<def> = + // or + // %reg1024:5<def> = op %reg1024, 5 + continue; + if (lastUD.isDef()) lastUD.setIsDead(true); else @@ -732,8 +761,8 @@ void RALocal::ComputeLocalLiveness(MachineBasicBlock& MBB) { // it wouldn't have been otherwise. Nullify the DBG_VALUEs when that // happens. bool UsedByDebugValueOnly = false; - for (MachineRegisterInfo::reg_iterator UI = MRI.reg_begin(MO.getReg()), - UE = MRI.reg_end(); UI != UE; ++UI) { + for (MachineRegisterInfo::reg_iterator UI = MRI->reg_begin(MO.getReg()), + UE = MRI->reg_end(); UI != UE; ++UI) { // Two cases: // - used in another block // - used in the same block before it is defined (loop) @@ -755,8 +784,8 @@ void RALocal::ComputeLocalLiveness(MachineBasicBlock& MBB) { } if (UsedByDebugValueOnly) - for (MachineRegisterInfo::reg_iterator UI = MRI.reg_begin(MO.getReg()), - UE = MRI.reg_end(); UI != UE; ++UI) + for (MachineRegisterInfo::reg_iterator UI = MRI->reg_begin(MO.getReg()), + UE = MRI->reg_end(); UI != UE; ++UI) if (UI->isDebugValue() && (UI->getParent() != &MBB || (MO.isDef() && precedes(&*UI, MI)))) @@ -828,7 +857,8 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) { unsigned SrcCopyReg, DstCopyReg, SrcCopySubReg, DstCopySubReg; unsigned SrcCopyPhysReg = 0U; bool isCopy = TII->isMoveInstr(*MI, SrcCopyReg, DstCopyReg, - SrcCopySubReg, DstCopySubReg); + SrcCopySubReg, DstCopySubReg) && + SrcCopySubReg == DstCopySubReg; if (isCopy && TargetRegisterInfo::isVirtualRegister(SrcCopyReg)) SrcCopyPhysReg = getVirt2PhysRegMapSlot(SrcCopyReg); @@ -878,6 +908,10 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) { std::make_pair((MachineInstr*)0, 0); DEBUG(dbgs() << " Assigning " << TRI->getName(DestPhysReg) << " to %reg" << DestVirtReg << "\n"); + if (unsigned DestSubIdx = MO.getSubReg()) { + MO.setSubReg(0); + DestPhysReg = TRI->getSubReg(DestPhysReg, DestSubIdx); + } MO.setReg(DestPhysReg); // Assign the earlyclobber register } else { unsigned Reg = MO.getReg(); @@ -1073,6 +1107,11 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) { getVirtRegLastUse(DestVirtReg) = std::make_pair((MachineInstr*)0, 0); DEBUG(dbgs() << " Assigning " << TRI->getName(DestPhysReg) << " to %reg" << DestVirtReg << "\n"); + + if (unsigned DestSubIdx = MO.getSubReg()) { + MO.setSubReg(0); + DestPhysReg = TRI->getSubReg(DestPhysReg, DestSubIdx); + } MO.setReg(DestPhysReg); // Assign the output register } @@ -1127,8 +1166,11 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) { // the register scavenger. See pr4100.) if (TII->isMoveInstr(*MI, SrcCopyReg, DstCopyReg, SrcCopySubReg, DstCopySubReg) && - SrcCopyReg == DstCopyReg && DeadDefs.empty()) + SrcCopyReg == DstCopyReg && SrcCopySubReg == DstCopySubReg && + DeadDefs.empty()) { + ++NumCopies; MBB.erase(MI); + } } MachineBasicBlock::iterator MI = MBB.getFirstTerminator(); @@ -1165,6 +1207,7 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) { bool RALocal::runOnMachineFunction(MachineFunction &Fn) { DEBUG(dbgs() << "Machine Function\n"); MF = &Fn; + MRI = &Fn.getRegInfo(); TM = &Fn.getTarget(); TRI = TM->getRegisterInfo(); TII = TM->getInstrInfo(); diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp index 81cfd8f..4fafd28 100644 --- a/lib/CodeGen/RegAllocPBQP.cpp +++ b/lib/CodeGen/RegAllocPBQP.cpp @@ -489,7 +489,7 @@ PBQPRegAlloc::CoalesceMap PBQPRegAlloc::findCoalesces() { // did, but none of their definitions would prevent us from coalescing. // We're good to go with the coalesce. - float cBenefit = powf(10.0f, loopInfo->getLoopDepth(mbb)) / 5.0; + float cBenefit = std::pow(10.0f, (float)loopInfo->getLoopDepth(mbb)) / 5.0; coalescesFound[RegPair(srcReg, dstReg)] = cBenefit; coalescesFound[RegPair(dstReg, srcReg)] = cBenefit; diff --git a/lib/CodeGen/RegisterScavenging.cpp b/lib/CodeGen/RegisterScavenging.cpp index 179984f..690e59f 100644 --- a/lib/CodeGen/RegisterScavenging.cpp +++ b/lib/CodeGen/RegisterScavenging.cpp @@ -343,12 +343,12 @@ unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC, // Spill the scavenged register before I. assert(ScavengingFrameIndex >= 0 && "Cannot scavenge register without an emergency spill slot!"); - TII->storeRegToStackSlot(*MBB, I, SReg, true, ScavengingFrameIndex, RC); + TII->storeRegToStackSlot(*MBB, I, SReg, true, ScavengingFrameIndex, RC,TRI); MachineBasicBlock::iterator II = prior(I); TRI->eliminateFrameIndex(II, SPAdj, NULL, this); // Restore the scavenged register before its use (or first terminator). - TII->loadRegFromStackSlot(*MBB, UseMI, SReg, ScavengingFrameIndex, RC); + TII->loadRegFromStackSlot(*MBB, UseMI, SReg, ScavengingFrameIndex, RC, TRI); II = prior(UseMI); TRI->eliminateFrameIndex(II, SPAdj, NULL, this); } diff --git a/lib/CodeGen/ScheduleDAG.cpp b/lib/CodeGen/ScheduleDAG.cpp index 587f001..da20c12 100644 --- a/lib/CodeGen/ScheduleDAG.cpp +++ b/lib/CodeGen/ScheduleDAG.cpp @@ -27,7 +27,6 @@ ScheduleDAG::ScheduleDAG(MachineFunction &mf) : TM(mf.getTarget()), TII(TM.getInstrInfo()), TRI(TM.getRegisterInfo()), - TLI(TM.getTargetLowering()), MF(mf), MRI(mf.getRegInfo()), EntrySU(), ExitSU() { } diff --git a/lib/CodeGen/ScheduleDAGEmit.cpp b/lib/CodeGen/ScheduleDAGEmit.cpp index 8e03420..ee08e1d 100644 --- a/lib/CodeGen/ScheduleDAGEmit.cpp +++ b/lib/CodeGen/ScheduleDAGEmit.cpp @@ -51,7 +51,8 @@ void ScheduleDAG::EmitPhysRegCopy(SUnit *SU, } } bool Success = TII->copyRegToReg(*BB, InsertPos, Reg, VRI->second, - SU->CopyDstRC, SU->CopySrcRC); + SU->CopyDstRC, SU->CopySrcRC, + DebugLoc()); (void)Success; assert(Success && "copyRegToReg failed!"); } else { @@ -62,7 +63,8 @@ void ScheduleDAG::EmitPhysRegCopy(SUnit *SU, isNew = isNew; // Silence compiler warning. assert(isNew && "Node emitted out of order - early"); bool Success = TII->copyRegToReg(*BB, InsertPos, VRBase, I->getReg(), - SU->CopyDstRC, SU->CopySrcRC); + SU->CopyDstRC, SU->CopySrcRC, + DebugLoc()); (void)Success; assert(Success && "copyRegToReg failed!"); } diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index ca235c3..09202f8 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -210,7 +210,7 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { assert(TRI->isPhysicalRegister(Reg) && "Virtual register encountered!"); if (MO.isDef() && DanglingDebugValue[Reg].first!=0) { - SU->setDbgInstr(DanglingDebugValue[Reg].first); + SU->DbgInstrList.push_back(DanglingDebugValue[Reg].first); DbgValueVec[DanglingDebugValue[Reg].second] = 0; DanglingDebugValue[Reg] = std::make_pair((MachineInstr*)0, 0); } @@ -599,8 +599,8 @@ MachineBasicBlock *ScheduleDAGInstrs::EmitSchedule() { } BB->insert(InsertPos, SU->getInstr()); - if (SU->getDbgInstr()) - BB->insert(InsertPos, SU->getDbgInstr()); + for (unsigned i = 0, e = SU->DbgInstrList.size() ; i < e ; ++i) + BB->insert(InsertPos, SU->DbgInstrList[i]); } // Update the Begin iterator, as the first instruction in the block diff --git a/lib/CodeGen/ScheduleDAGInstrs.h b/lib/CodeGen/ScheduleDAGInstrs.h index d70608f..ad82db2 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.h +++ b/lib/CodeGen/ScheduleDAGInstrs.h @@ -32,7 +32,7 @@ namespace llvm { /// For example, loop induction variable increments should be /// scheduled as soon as possible after the variable's last use. /// - class VISIBILITY_HIDDEN LoopDependencies { + class LLVM_LIBRARY_VISIBILITY LoopDependencies { const MachineLoopInfo &MLI; const MachineDominatorTree &MDT; @@ -94,7 +94,7 @@ namespace llvm { /// ScheduleDAGInstrs - A ScheduleDAG subclass for scheduling lists of /// MachineInstrs. - class VISIBILITY_HIDDEN ScheduleDAGInstrs : public ScheduleDAG { + class LLVM_LIBRARY_VISIBILITY ScheduleDAGInstrs : public ScheduleDAG { const MachineLoopInfo &MLI; const MachineDominatorTree &MDT; const MachineFrameInfo *MFI; diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 3639f80..6bddd78 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -760,12 +760,18 @@ SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) { bool Replace1 = false; SDValue N1 = Op.getOperand(1); - SDValue NN1 = PromoteOperand(N1, PVT, Replace1); - if (NN1.getNode() == 0) - return SDValue(); + SDValue NN1; + if (N0 == N1) + NN1 = NN0; + else { + NN1 = PromoteOperand(N1, PVT, Replace1); + if (NN1.getNode() == 0) + return SDValue(); + } AddToWorkList(NN0.getNode()); - AddToWorkList(NN1.getNode()); + if (NN1.getNode()) + AddToWorkList(NN1.getNode()); if (Replace0) ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode()); @@ -3425,8 +3431,12 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) { // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n))) SDValue NarrowLoad = ReduceLoadWidth(N0.getNode()); if (NarrowLoad.getNode()) { - if (NarrowLoad.getNode() != N0.getNode()) + SDNode* oye = N0.getNode()->getOperand(0).getNode(); + if (NarrowLoad.getNode() != N0.getNode()) { CombineTo(N0.getNode(), NarrowLoad); + // CombineTo deleted the truncate, if needed, but not what's under it. + AddToWorkList(oye); + } return SDValue(N, 0); // Return N so it doesn't get rechecked! } @@ -3564,7 +3574,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) { DAG.getVSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0), N0.getOperand(1), cast<CondCodeSDNode>(N0.getOperand(2))->get()); - return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT); + return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT); } } @@ -3585,9 +3595,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) { N0.getOperand(0), N0.getOperand(1), cast<CondCodeSDNode>(N0.getOperand(2))->get()), NegOne, DAG.getConstant(0, VT)); - } - - + } // fold (sext x) -> (zext x) if the sign bit is known zero. if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) && @@ -3615,8 +3623,12 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) { if (N0.getOpcode() == ISD::TRUNCATE) { SDValue NarrowLoad = ReduceLoadWidth(N0.getNode()); if (NarrowLoad.getNode()) { - if (NarrowLoad.getNode() != N0.getNode()) + SDNode* oye = N0.getNode()->getOperand(0).getNode(); + if (NarrowLoad.getNode() != N0.getNode()) { CombineTo(N0.getNode(), NarrowLoad); + // CombineTo deleted the truncate, if needed, but not what's under it. + AddToWorkList(oye); + } return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, NarrowLoad); } } @@ -3726,8 +3738,48 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) { } } - // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc if (N0.getOpcode() == ISD::SETCC) { + if (!LegalOperations && VT.isVector()) { + // zext(setcc) -> (and (vsetcc), (1, 1, ...) for vectors. + // Only do this before legalize for now. + EVT N0VT = N0.getOperand(0).getValueType(); + EVT EltVT = VT.getVectorElementType(); + SmallVector<SDValue,8> OneOps(VT.getVectorNumElements(), + DAG.getConstant(1, EltVT)); + if (VT.getSizeInBits() == N0VT.getSizeInBits()) { + // We know that the # elements of the results is the same as the + // # elements of the compare (and the # elements of the compare result + // for that matter). Check to see that they are the same size. If so, + // we know that the element size of the sext'd result matches the + // element size of the compare operands. + return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, + DAG.getVSetCC(N->getDebugLoc(), VT, N0.getOperand(0), + N0.getOperand(1), + cast<CondCodeSDNode>(N0.getOperand(2))->get()), + DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT, + &OneOps[0], OneOps.size())); + } else { + // If the desired elements are smaller or larger than the source + // elements we can use a matching integer vector type and then + // truncate/sign extend + EVT MatchingElementType = + EVT::getIntegerVT(*DAG.getContext(), + N0VT.getScalarType().getSizeInBits()); + EVT MatchingVectorType = + EVT::getVectorVT(*DAG.getContext(), MatchingElementType, + N0VT.getVectorNumElements()); + SDValue VsetCC = + DAG.getVSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0), + N0.getOperand(1), + cast<CondCodeSDNode>(N0.getOperand(2))->get()); + return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, + DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT), + DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT, + &OneOps[0], OneOps.size())); + } + } + + // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc SDValue SCC = SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, VT), DAG.getConstant(0, VT), @@ -3780,8 +3832,12 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) { if (N0.getOpcode() == ISD::TRUNCATE) { SDValue NarrowLoad = ReduceLoadWidth(N0.getNode()); if (NarrowLoad.getNode()) { - if (NarrowLoad.getNode() != N0.getNode()) + SDNode* oye = N0.getNode()->getOperand(0).getNode(); + if (NarrowLoad.getNode() != N0.getNode()) { CombineTo(N0.getNode(), NarrowLoad); + // CombineTo deleted the truncate, if needed, but not what's under it. + AddToWorkList(oye); + } return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, NarrowLoad); } } @@ -3883,8 +3939,39 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) { return SDValue(N, 0); // Return N so it doesn't get rechecked! } - // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc if (N0.getOpcode() == ISD::SETCC) { + // aext(setcc) -> sext_in_reg(vsetcc) for vectors. + // Only do this before legalize for now. + if (VT.isVector() && !LegalOperations) { + EVT N0VT = N0.getOperand(0).getValueType(); + // We know that the # elements of the results is the same as the + // # elements of the compare (and the # elements of the compare result + // for that matter). Check to see that they are the same size. If so, + // we know that the element size of the sext'd result matches the + // element size of the compare operands. + if (VT.getSizeInBits() == N0VT.getSizeInBits()) + return DAG.getVSetCC(N->getDebugLoc(), VT, N0.getOperand(0), + N0.getOperand(1), + cast<CondCodeSDNode>(N0.getOperand(2))->get()); + // If the desired elements are smaller or larger than the source + // elements we can use a matching integer vector type and then + // truncate/sign extend + else { + EVT MatchingElementType = + EVT::getIntegerVT(*DAG.getContext(), + N0VT.getScalarType().getSizeInBits()); + EVT MatchingVectorType = + EVT::getVectorVT(*DAG.getContext(), MatchingElementType, + N0VT.getVectorNumElements()); + SDValue VsetCC = + DAG.getVSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0), + N0.getOperand(1), + cast<CondCodeSDNode>(N0.getOperand(2))->get()); + return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT); + } + } + + // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc SDValue SCC = SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, VT), DAG.getConstant(0, VT), @@ -5278,10 +5365,6 @@ bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) { SDValue Offset; ISD::MemIndexedMode AM = ISD::UNINDEXED; if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) { - if (Ptr == Offset && Op->getOpcode() == ISD::ADD) - std::swap(BasePtr, Offset); - if (Ptr != BasePtr) - continue; // Don't create a indexed load / store with zero offset. if (isa<ConstantSDNode>(Offset) && cast<ConstantSDNode>(Offset)->isNullValue()) @@ -5953,6 +6036,10 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) { SDValue InVal = N->getOperand(1); SDValue EltNo = N->getOperand(2); + // If the inserted element is an UNDEF, just use the input vector. + if (InVal.getOpcode() == ISD::UNDEF) + return InVec; + // If the invec is a BUILD_VECTOR and if EltNo is a constant, build a new // vector with the inserted element. if (InVec.getOpcode() == ISD::BUILD_VECTOR && isa<ConstantSDNode>(EltNo)) { @@ -6206,7 +6293,6 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { // all scalar elements the same. if (cast<ShuffleVectorSDNode>(N)->isSplat()) { SDNode *V = N0.getNode(); - // If this is a bit convert that changes the element type of the vector but // not the number of vector elements, look through it. Be careful not to @@ -6338,13 +6424,21 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) { break; } - Ops.push_back(DAG.getNode(N->getOpcode(), LHS.getDebugLoc(), - EltType, LHSOp, RHSOp)); - AddToWorkList(Ops.back().getNode()); - assert((Ops.back().getOpcode() == ISD::UNDEF || - Ops.back().getOpcode() == ISD::Constant || - Ops.back().getOpcode() == ISD::ConstantFP) && - "Scalar binop didn't fold!"); + // If the vector element type is not legal, the BUILD_VECTOR operands + // are promoted and implicitly truncated. Make that explicit here. + if (LHSOp.getValueType() != EltType) + LHSOp = DAG.getNode(ISD::TRUNCATE, LHS.getDebugLoc(), EltType, LHSOp); + if (RHSOp.getValueType() != EltType) + RHSOp = DAG.getNode(ISD::TRUNCATE, RHS.getDebugLoc(), EltType, RHSOp); + + SDValue FoldOp = DAG.getNode(N->getOpcode(), LHS.getDebugLoc(), EltType, + LHSOp, RHSOp); + if (FoldOp.getOpcode() != ISD::UNDEF && + FoldOp.getOpcode() != ISD::Constant && + FoldOp.getOpcode() != ISD::ConstantFP) + break; + Ops.push_back(FoldOp); + AddToWorkList(FoldOp.getNode()); } if (Ops.size() == LHS.getNumOperands()) { diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp index b4c3833..95f4d07 100644 --- a/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -56,6 +56,27 @@ #include "FunctionLoweringInfo.h" using namespace llvm; +bool FastISel::hasTrivialKill(const Value *V) const { + // Don't consider constants or arguments to have trivial kills. + const Instruction *I = dyn_cast<Instruction>(V); + if (!I) + return false; + + // No-op casts are trivially coalesced by fast-isel. + if (const CastInst *Cast = dyn_cast<CastInst>(I)) + if (Cast->isNoopCast(TD.getIntPtrType(Cast->getContext())) && + !hasTrivialKill(Cast->getOperand(0))) + return false; + + // Only instructions with a single use in the same basic block are considered + // to have trivial kills. + return I->hasOneUse() && + !(I->getOpcode() == Instruction::BitCast || + I->getOpcode() == Instruction::PtrToInt || + I->getOpcode() == Instruction::IntToPtr) && + cast<Instruction>(I->use_begin())->getParent() == I->getParent(); +} + unsigned FastISel::getRegForValue(const Value *V) { EVT RealVT = TLI.getValueType(V->getType(), /*AllowUnknown=*/true); // Don't handle non-simple values in FastISel. @@ -78,12 +99,24 @@ unsigned FastISel::getRegForValue(const Value *V) { // cache values defined by Instructions across blocks, and other values // only locally. This is because Instructions already have the SSA // def-dominates-use requirement enforced. - if (ValueMap.count(V)) - return ValueMap[V]; + DenseMap<const Value *, unsigned>::iterator I = ValueMap.find(V); + if (I != ValueMap.end()) + return I->second; unsigned Reg = LocalValueMap[V]; if (Reg != 0) return Reg; + // In bottom-up mode, just create the virtual register which will be used + // to hold the value. It will be materialized later. + if (IsBottomUp) { + Reg = createResultReg(TLI.getRegClassFor(VT)); + if (isa<Instruction>(V)) + ValueMap[V] = Reg; + else + LocalValueMap[V] = Reg; + return Reg; + } + return materializeRegForValue(V, VT); } @@ -123,7 +156,8 @@ unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) { unsigned IntegerReg = getRegForValue(ConstantInt::get(V->getContext(), IntVal)); if (IntegerReg != 0) - Reg = FastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg); + Reg = FastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, + IntegerReg, /*Kill=*/false); } } } else if (const Operator *Op = dyn_cast<Operator>(V)) { @@ -174,25 +208,33 @@ unsigned FastISel::UpdateValueMap(const Value *I, unsigned Reg) { else if (Reg != AssignedReg) { const TargetRegisterClass *RegClass = MRI.getRegClass(Reg); TII.copyRegToReg(*MBB, MBB->end(), AssignedReg, - Reg, RegClass, RegClass); + Reg, RegClass, RegClass, DL); } return AssignedReg; } -unsigned FastISel::getRegForGEPIndex(const Value *Idx) { +std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) { unsigned IdxN = getRegForValue(Idx); if (IdxN == 0) // Unhandled operand. Halt "fast" selection and bail. - return 0; + return std::pair<unsigned, bool>(0, false); + + bool IdxNIsKill = hasTrivialKill(Idx); // If the index is smaller or larger than intptr_t, truncate or extend it. MVT PtrVT = TLI.getPointerTy(); EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false); - if (IdxVT.bitsLT(PtrVT)) - IdxN = FastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN); - else if (IdxVT.bitsGT(PtrVT)) - IdxN = FastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN); - return IdxN; + if (IdxVT.bitsLT(PtrVT)) { + IdxN = FastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, + IdxN, IdxNIsKill); + IdxNIsKill = true; + } + else if (IdxVT.bitsGT(PtrVT)) { + IdxN = FastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, + IdxN, IdxNIsKill); + IdxNIsKill = true; + } + return std::pair<unsigned, bool>(IdxN, IdxNIsKill); } /// SelectBinaryOp - Select and emit code for a binary operator instruction, @@ -224,10 +266,13 @@ bool FastISel::SelectBinaryOp(const User *I, unsigned ISDOpcode) { // Unhandled operand. Halt "fast" selection and bail. return false; + bool Op0IsKill = hasTrivialKill(I->getOperand(0)); + // Check if the second operand is a constant and handle it appropriately. if (ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1))) { unsigned ResultReg = FastEmit_ri(VT.getSimpleVT(), VT.getSimpleVT(), - ISDOpcode, Op0, CI->getZExtValue()); + ISDOpcode, Op0, Op0IsKill, + CI->getZExtValue()); if (ResultReg != 0) { // We successfully emitted code for the given LLVM Instruction. UpdateValueMap(I, ResultReg); @@ -238,7 +283,7 @@ bool FastISel::SelectBinaryOp(const User *I, unsigned ISDOpcode) { // Check if the second operand is a constant float. if (ConstantFP *CF = dyn_cast<ConstantFP>(I->getOperand(1))) { unsigned ResultReg = FastEmit_rf(VT.getSimpleVT(), VT.getSimpleVT(), - ISDOpcode, Op0, CF); + ISDOpcode, Op0, Op0IsKill, CF); if (ResultReg != 0) { // We successfully emitted code for the given LLVM Instruction. UpdateValueMap(I, ResultReg); @@ -251,9 +296,13 @@ bool FastISel::SelectBinaryOp(const User *I, unsigned ISDOpcode) { // Unhandled operand. Halt "fast" selection and bail. return false; + bool Op1IsKill = hasTrivialKill(I->getOperand(1)); + // Now we have both operands in registers. Emit the instruction. unsigned ResultReg = FastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(), - ISDOpcode, Op0, Op1); + ISDOpcode, + Op0, Op0IsKill, + Op1, Op1IsKill); if (ResultReg == 0) // Target-specific code wasn't able to find a machine opcode for // the given ISD opcode and type. Halt "fast" selection and bail. @@ -270,6 +319,8 @@ bool FastISel::SelectGetElementPtr(const User *I) { // Unhandled operand. Halt "fast" selection and bail. return false; + bool NIsKill = hasTrivialKill(I->getOperand(0)); + const Type *Ty = I->getOperand(0)->getType(); MVT VT = TLI.getPointerTy(); for (GetElementPtrInst::const_op_iterator OI = I->op_begin()+1, @@ -282,10 +333,11 @@ bool FastISel::SelectGetElementPtr(const User *I) { uint64_t Offs = TD.getStructLayout(StTy)->getElementOffset(Field); // FIXME: This can be optimized by combining the add with a // subsequent one. - N = FastEmit_ri_(VT, ISD::ADD, N, Offs, VT); + N = FastEmit_ri_(VT, ISD::ADD, N, NIsKill, Offs, VT); if (N == 0) // Unhandled operand. Halt "fast" selection and bail. return false; + NIsKill = true; } Ty = StTy->getElementType(Field); } else { @@ -296,27 +348,31 @@ bool FastISel::SelectGetElementPtr(const User *I) { if (CI->getZExtValue() == 0) continue; uint64_t Offs = TD.getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue(); - N = FastEmit_ri_(VT, ISD::ADD, N, Offs, VT); + N = FastEmit_ri_(VT, ISD::ADD, N, NIsKill, Offs, VT); if (N == 0) // Unhandled operand. Halt "fast" selection and bail. return false; + NIsKill = true; continue; } // N = N + Idx * ElementSize; uint64_t ElementSize = TD.getTypeAllocSize(Ty); - unsigned IdxN = getRegForGEPIndex(Idx); + std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx); + unsigned IdxN = Pair.first; + bool IdxNIsKill = Pair.second; if (IdxN == 0) // Unhandled operand. Halt "fast" selection and bail. return false; if (ElementSize != 1) { - IdxN = FastEmit_ri_(VT, ISD::MUL, IdxN, ElementSize, VT); + IdxN = FastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT); if (IdxN == 0) // Unhandled operand. Halt "fast" selection and bail. return false; + IdxNIsKill = true; } - N = FastEmit_rr(VT, VT, ISD::ADD, N, IdxN); + N = FastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill); if (N == 0) // Unhandled operand. Halt "fast" selection and bail. return false; @@ -338,7 +394,7 @@ bool FastISel::SelectCall(const User *I) { default: break; case Intrinsic::dbg_declare: { const DbgDeclareInst *DI = cast<DbgDeclareInst>(I); - if (!DIDescriptor::ValidDebugInfo(DI->getVariable(), CodeGenOpt::None) || + if (!DIVariable(DI->getVariable()).Verify() || !MF.getMMI().hasDebugInfo()) return true; @@ -402,7 +458,7 @@ bool FastISel::SelectCall(const User *I) { const TargetRegisterClass *RC = TLI.getRegClassFor(VT); unsigned ResultReg = createResultReg(RC); bool InsertedCopy = TII.copyRegToReg(*MBB, MBB->end(), ResultReg, - Reg, RC, RC); + Reg, RC, RC, DL); assert(InsertedCopy && "Can't copy address registers!"); InsertedCopy = InsertedCopy; UpdateValueMap(I, ResultReg); @@ -432,17 +488,19 @@ bool FastISel::SelectCall(const User *I) { const TargetRegisterClass *RC = TLI.getRegClassFor(SrcVT); unsigned ResultReg = createResultReg(RC); bool InsertedCopy = TII.copyRegToReg(*MBB, MBB->end(), ResultReg, Reg, - RC, RC); + RC, RC, DL); assert(InsertedCopy && "Can't copy address registers!"); InsertedCopy = InsertedCopy; + bool ResultRegIsKill = hasTrivialKill(I); + // Cast the register to the type of the selector. if (SrcVT.bitsGT(MVT::i32)) ResultReg = FastEmit_r(SrcVT.getSimpleVT(), MVT::i32, ISD::TRUNCATE, - ResultReg); + ResultReg, ResultRegIsKill); else if (SrcVT.bitsLT(MVT::i32)) ResultReg = FastEmit_r(SrcVT.getSimpleVT(), MVT::i32, - ISD::SIGN_EXTEND, ResultReg); + ISD::SIGN_EXTEND, ResultReg, ResultRegIsKill); if (ResultReg == 0) // Unhandled operand. Halt "fast" selection and bail. return false; @@ -490,12 +548,15 @@ bool FastISel::SelectCast(const User *I, unsigned Opcode) { // Unhandled operand. Halt "fast" selection and bail. return false; + bool InputRegIsKill = hasTrivialKill(I->getOperand(0)); + // If the operand is i1, arrange for the high bits in the register to be zero. if (SrcVT == MVT::i1) { SrcVT = TLI.getTypeToTransformTo(I->getContext(), SrcVT); - InputReg = FastEmitZExtFromI1(SrcVT.getSimpleVT(), InputReg); + InputReg = FastEmitZExtFromI1(SrcVT.getSimpleVT(), InputReg, InputRegIsKill); if (!InputReg) return false; + InputRegIsKill = true; } // If the result is i1, truncate to the target's type for i1 first. if (DstVT == MVT::i1) @@ -504,7 +565,7 @@ bool FastISel::SelectCast(const User *I, unsigned Opcode) { unsigned ResultReg = FastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), Opcode, - InputReg); + InputReg, InputRegIsKill); if (!ResultReg) return false; @@ -536,6 +597,8 @@ bool FastISel::SelectBitCast(const User *I) { if (Op0 == 0) // Unhandled operand. Halt "fast" selection and bail. return false; + + bool Op0IsKill = hasTrivialKill(I->getOperand(0)); // First, try to perform the bitcast by inserting a reg-reg copy. unsigned ResultReg = 0; @@ -545,7 +608,7 @@ bool FastISel::SelectBitCast(const User *I) { ResultReg = createResultReg(DstClass); bool InsertedCopy = TII.copyRegToReg(*MBB, MBB->end(), ResultReg, - Op0, DstClass, SrcClass); + Op0, DstClass, SrcClass, DL); if (!InsertedCopy) ResultReg = 0; } @@ -553,7 +616,7 @@ bool FastISel::SelectBitCast(const User *I) { // If the reg-reg copy failed, select a BIT_CONVERT opcode. if (!ResultReg) ResultReg = FastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), - ISD::BIT_CONVERT, Op0); + ISD::BIT_CONVERT, Op0, Op0IsKill); if (!ResultReg) return false; @@ -609,10 +672,12 @@ FastISel::SelectFNeg(const User *I) { unsigned OpReg = getRegForValue(BinaryOperator::getFNegArgument(I)); if (OpReg == 0) return false; + bool OpRegIsKill = hasTrivialKill(I); + // If the target has ISD::FNEG, use it. EVT VT = TLI.getValueType(I->getType()); unsigned ResultReg = FastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), - ISD::FNEG, OpReg); + ISD::FNEG, OpReg, OpRegIsKill); if (ResultReg != 0) { UpdateValueMap(I, ResultReg); return true; @@ -626,18 +691,19 @@ FastISel::SelectFNeg(const User *I) { return false; unsigned IntReg = FastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(), - ISD::BIT_CONVERT, OpReg); + ISD::BIT_CONVERT, OpReg, OpRegIsKill); if (IntReg == 0) return false; - unsigned IntResultReg = FastEmit_ri_(IntVT.getSimpleVT(), ISD::XOR, IntReg, + unsigned IntResultReg = FastEmit_ri_(IntVT.getSimpleVT(), ISD::XOR, + IntReg, /*Kill=*/true, UINT64_C(1) << (VT.getSizeInBits()-1), IntVT.getSimpleVT()); if (IntResultReg == 0) return false; ResultReg = FastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), - ISD::BIT_CONVERT, IntResultReg); + ISD::BIT_CONVERT, IntResultReg, /*Kill=*/true); if (ResultReg == 0) return false; @@ -782,7 +848,8 @@ FastISel::FastISel(MachineFunction &mf, TM(MF.getTarget()), TD(*TM.getTargetData()), TII(*TM.getInstrInfo()), - TLI(*TM.getTargetLowering()) { + TLI(*TM.getTargetLowering()), + IsBottomUp(false) { } FastISel::~FastISel() {} @@ -793,13 +860,15 @@ unsigned FastISel::FastEmit_(MVT, MVT, } unsigned FastISel::FastEmit_r(MVT, MVT, - unsigned, unsigned /*Op0*/) { + unsigned, + unsigned /*Op0*/, bool /*Op0IsKill*/) { return 0; } unsigned FastISel::FastEmit_rr(MVT, MVT, - unsigned, unsigned /*Op0*/, - unsigned /*Op0*/) { + unsigned, + unsigned /*Op0*/, bool /*Op0IsKill*/, + unsigned /*Op1*/, bool /*Op1IsKill*/) { return 0; } @@ -813,20 +882,23 @@ unsigned FastISel::FastEmit_f(MVT, MVT, } unsigned FastISel::FastEmit_ri(MVT, MVT, - unsigned, unsigned /*Op0*/, + unsigned, + unsigned /*Op0*/, bool /*Op0IsKill*/, uint64_t /*Imm*/) { return 0; } unsigned FastISel::FastEmit_rf(MVT, MVT, - unsigned, unsigned /*Op0*/, + unsigned, + unsigned /*Op0*/, bool /*Op0IsKill*/, const ConstantFP * /*FPImm*/) { return 0; } unsigned FastISel::FastEmit_rri(MVT, MVT, unsigned, - unsigned /*Op0*/, unsigned /*Op1*/, + unsigned /*Op0*/, bool /*Op0IsKill*/, + unsigned /*Op1*/, bool /*Op1IsKill*/, uint64_t /*Imm*/) { return 0; } @@ -836,16 +908,18 @@ unsigned FastISel::FastEmit_rri(MVT, MVT, /// If that fails, it materializes the immediate into a register and try /// FastEmit_rr instead. unsigned FastISel::FastEmit_ri_(MVT VT, unsigned Opcode, - unsigned Op0, uint64_t Imm, - MVT ImmType) { + unsigned Op0, bool Op0IsKill, + uint64_t Imm, MVT ImmType) { // First check if immediate type is legal. If not, we can't use the ri form. - unsigned ResultReg = FastEmit_ri(VT, VT, Opcode, Op0, Imm); + unsigned ResultReg = FastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm); if (ResultReg != 0) return ResultReg; unsigned MaterialReg = FastEmit_i(ImmType, ImmType, ISD::Constant, Imm); if (MaterialReg == 0) return 0; - return FastEmit_rr(VT, VT, Opcode, Op0, MaterialReg); + return FastEmit_rr(VT, VT, Opcode, + Op0, Op0IsKill, + MaterialReg, /*Kill=*/true); } /// FastEmit_rf_ - This method is a wrapper of FastEmit_ri. It first tries @@ -853,10 +927,10 @@ unsigned FastISel::FastEmit_ri_(MVT VT, unsigned Opcode, /// FastEmit_rf. If that fails, it materializes the immediate into a register /// and try FastEmit_rr instead. unsigned FastISel::FastEmit_rf_(MVT VT, unsigned Opcode, - unsigned Op0, const ConstantFP *FPImm, - MVT ImmType) { + unsigned Op0, bool Op0IsKill, + const ConstantFP *FPImm, MVT ImmType) { // First check if immediate type is legal. If not, we can't use the rf form. - unsigned ResultReg = FastEmit_rf(VT, VT, Opcode, Op0, FPImm); + unsigned ResultReg = FastEmit_rf(VT, VT, Opcode, Op0, Op0IsKill, FPImm); if (ResultReg != 0) return ResultReg; @@ -886,11 +960,13 @@ unsigned FastISel::FastEmit_rf_(MVT VT, unsigned Opcode, if (IntegerReg == 0) return 0; MaterialReg = FastEmit_r(IntVT.getSimpleVT(), VT, - ISD::SINT_TO_FP, IntegerReg); + ISD::SINT_TO_FP, IntegerReg, /*Kill=*/true); if (MaterialReg == 0) return 0; } - return FastEmit_rr(VT, VT, Opcode, Op0, MaterialReg); + return FastEmit_rr(VT, VT, Opcode, + Op0, Op0IsKill, + MaterialReg, /*Kill=*/true); } unsigned FastISel::createResultReg(const TargetRegisterClass* RC) { @@ -908,16 +984,16 @@ unsigned FastISel::FastEmitInst_(unsigned MachineInstOpcode, unsigned FastISel::FastEmitInst_r(unsigned MachineInstOpcode, const TargetRegisterClass *RC, - unsigned Op0) { + unsigned Op0, bool Op0IsKill) { unsigned ResultReg = createResultReg(RC); const TargetInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) - BuildMI(MBB, DL, II, ResultReg).addReg(Op0); + BuildMI(MBB, DL, II, ResultReg).addReg(Op0, Op0IsKill * RegState::Kill); else { - BuildMI(MBB, DL, II).addReg(Op0); + BuildMI(MBB, DL, II).addReg(Op0, Op0IsKill * RegState::Kill); bool InsertedCopy = TII.copyRegToReg(*MBB, MBB->end(), ResultReg, - II.ImplicitDefs[0], RC, RC); + II.ImplicitDefs[0], RC, RC, DL); if (!InsertedCopy) ResultReg = 0; } @@ -927,16 +1003,21 @@ unsigned FastISel::FastEmitInst_r(unsigned MachineInstOpcode, unsigned FastISel::FastEmitInst_rr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, - unsigned Op0, unsigned Op1) { + unsigned Op0, bool Op0IsKill, + unsigned Op1, bool Op1IsKill) { unsigned ResultReg = createResultReg(RC); const TargetInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) - BuildMI(MBB, DL, II, ResultReg).addReg(Op0).addReg(Op1); + BuildMI(MBB, DL, II, ResultReg) + .addReg(Op0, Op0IsKill * RegState::Kill) + .addReg(Op1, Op1IsKill * RegState::Kill); else { - BuildMI(MBB, DL, II).addReg(Op0).addReg(Op1); + BuildMI(MBB, DL, II) + .addReg(Op0, Op0IsKill * RegState::Kill) + .addReg(Op1, Op1IsKill * RegState::Kill); bool InsertedCopy = TII.copyRegToReg(*MBB, MBB->end(), ResultReg, - II.ImplicitDefs[0], RC, RC); + II.ImplicitDefs[0], RC, RC, DL); if (!InsertedCopy) ResultReg = 0; } @@ -945,16 +1026,21 @@ unsigned FastISel::FastEmitInst_rr(unsigned MachineInstOpcode, unsigned FastISel::FastEmitInst_ri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, - unsigned Op0, uint64_t Imm) { + unsigned Op0, bool Op0IsKill, + uint64_t Imm) { unsigned ResultReg = createResultReg(RC); const TargetInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) - BuildMI(MBB, DL, II, ResultReg).addReg(Op0).addImm(Imm); + BuildMI(MBB, DL, II, ResultReg) + .addReg(Op0, Op0IsKill * RegState::Kill) + .addImm(Imm); else { - BuildMI(MBB, DL, II).addReg(Op0).addImm(Imm); + BuildMI(MBB, DL, II) + .addReg(Op0, Op0IsKill * RegState::Kill) + .addImm(Imm); bool InsertedCopy = TII.copyRegToReg(*MBB, MBB->end(), ResultReg, - II.ImplicitDefs[0], RC, RC); + II.ImplicitDefs[0], RC, RC, DL); if (!InsertedCopy) ResultReg = 0; } @@ -963,16 +1049,21 @@ unsigned FastISel::FastEmitInst_ri(unsigned MachineInstOpcode, unsigned FastISel::FastEmitInst_rf(unsigned MachineInstOpcode, const TargetRegisterClass *RC, - unsigned Op0, const ConstantFP *FPImm) { + unsigned Op0, bool Op0IsKill, + const ConstantFP *FPImm) { unsigned ResultReg = createResultReg(RC); const TargetInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) - BuildMI(MBB, DL, II, ResultReg).addReg(Op0).addFPImm(FPImm); + BuildMI(MBB, DL, II, ResultReg) + .addReg(Op0, Op0IsKill * RegState::Kill) + .addFPImm(FPImm); else { - BuildMI(MBB, DL, II).addReg(Op0).addFPImm(FPImm); + BuildMI(MBB, DL, II) + .addReg(Op0, Op0IsKill * RegState::Kill) + .addFPImm(FPImm); bool InsertedCopy = TII.copyRegToReg(*MBB, MBB->end(), ResultReg, - II.ImplicitDefs[0], RC, RC); + II.ImplicitDefs[0], RC, RC, DL); if (!InsertedCopy) ResultReg = 0; } @@ -981,16 +1072,24 @@ unsigned FastISel::FastEmitInst_rf(unsigned MachineInstOpcode, unsigned FastISel::FastEmitInst_rri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, - unsigned Op0, unsigned Op1, uint64_t Imm) { + unsigned Op0, bool Op0IsKill, + unsigned Op1, bool Op1IsKill, + uint64_t Imm) { unsigned ResultReg = createResultReg(RC); const TargetInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) - BuildMI(MBB, DL, II, ResultReg).addReg(Op0).addReg(Op1).addImm(Imm); + BuildMI(MBB, DL, II, ResultReg) + .addReg(Op0, Op0IsKill * RegState::Kill) + .addReg(Op1, Op1IsKill * RegState::Kill) + .addImm(Imm); else { - BuildMI(MBB, DL, II).addReg(Op0).addReg(Op1).addImm(Imm); + BuildMI(MBB, DL, II) + .addReg(Op0, Op0IsKill * RegState::Kill) + .addReg(Op1, Op1IsKill * RegState::Kill) + .addImm(Imm); bool InsertedCopy = TII.copyRegToReg(*MBB, MBB->end(), ResultReg, - II.ImplicitDefs[0], RC, RC); + II.ImplicitDefs[0], RC, RC, DL); if (!InsertedCopy) ResultReg = 0; } @@ -1008,7 +1107,7 @@ unsigned FastISel::FastEmitInst_i(unsigned MachineInstOpcode, else { BuildMI(MBB, DL, II).addImm(Imm); bool InsertedCopy = TII.copyRegToReg(*MBB, MBB->end(), ResultReg, - II.ImplicitDefs[0], RC, RC); + II.ImplicitDefs[0], RC, RC, DL); if (!InsertedCopy) ResultReg = 0; } @@ -1016,18 +1115,23 @@ unsigned FastISel::FastEmitInst_i(unsigned MachineInstOpcode, } unsigned FastISel::FastEmitInst_extractsubreg(MVT RetVT, - unsigned Op0, uint32_t Idx) { + unsigned Op0, bool Op0IsKill, + uint32_t Idx) { const TargetRegisterClass* RC = MRI.getRegClass(Op0); unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT)); const TargetInstrDesc &II = TII.get(TargetOpcode::EXTRACT_SUBREG); if (II.getNumDefs() >= 1) - BuildMI(MBB, DL, II, ResultReg).addReg(Op0).addImm(Idx); + BuildMI(MBB, DL, II, ResultReg) + .addReg(Op0, Op0IsKill * RegState::Kill) + .addImm(Idx); else { - BuildMI(MBB, DL, II).addReg(Op0).addImm(Idx); + BuildMI(MBB, DL, II) + .addReg(Op0, Op0IsKill * RegState::Kill) + .addImm(Idx); bool InsertedCopy = TII.copyRegToReg(*MBB, MBB->end(), ResultReg, - II.ImplicitDefs[0], RC, RC); + II.ImplicitDefs[0], RC, RC, DL); if (!InsertedCopy) ResultReg = 0; } @@ -1036,8 +1140,8 @@ unsigned FastISel::FastEmitInst_extractsubreg(MVT RetVT, /// FastEmitZExtFromI1 - Emit MachineInstrs to compute the value of Op /// with all but the least significant bit set to zero. -unsigned FastISel::FastEmitZExtFromI1(MVT VT, unsigned Op) { - return FastEmit_ri(VT, VT, ISD::AND, Op, 1); +unsigned FastISel::FastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) { + return FastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1); } /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks. @@ -1070,6 +1174,7 @@ bool FastISel::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { // emitted yet. for (BasicBlock::const_iterator I = SuccBB->begin(); const PHINode *PN = dyn_cast<PHINode>(I); ++I) { + // Ignore dead phi's. if (PN->use_empty()) continue; @@ -1092,12 +1197,19 @@ bool FastISel::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { const Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB); + // Set the DebugLoc for the copy. Prefer the location of the operand + // if there is one; use the location of the PHI otherwise. + DL = PN->getDebugLoc(); + if (const Instruction *Inst = dyn_cast<Instruction>(PHIOp)) + DL = Inst->getDebugLoc(); + unsigned Reg = getRegForValue(PHIOp); if (Reg == 0) { PHINodesToUpdate.resize(OrigNumPHINodesToUpdate); return false; } PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg)); + DL = DebugLoc(); } } diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index c5dae82..16eb8a7 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -143,7 +143,7 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned, // Create the reg, emit the copy. VRBase = MRI->createVirtualRegister(DstRC); bool Emitted = TII->copyRegToReg(*MBB, InsertPos, VRBase, SrcReg, - DstRC, SrcRC); + DstRC, SrcRC, Node->getDebugLoc()); assert(Emitted && "Unable to issue a copy instruction!\n"); (void) Emitted; @@ -265,7 +265,7 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, unsigned IIOpNum, const TargetInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, - bool IsDebug) { + bool IsDebug, bool IsClone, bool IsCloned) { assert(Op.getValueType() != MVT::Other && Op.getValueType() != MVT::Flag && "Chain and flag operands should occur at end of operand list!"); @@ -289,7 +289,7 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, if (DstRC && SrcRC != DstRC && !SrcRC->hasSuperClass(DstRC)) { unsigned NewVReg = MRI->createVirtualRegister(DstRC); bool Emitted = TII->copyRegToReg(*MBB, InsertPos, NewVReg, VReg, - DstRC, SrcRC); + DstRC, SrcRC, Op.getNode()->getDebugLoc()); assert(Emitted && "Unable to issue a copy instruction!\n"); (void) Emitted; VReg = NewVReg; @@ -297,15 +297,25 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, } // If this value has only one use, that use is a kill. This is a - // conservative approximation. Tied operands are never killed, so we need - // to check that. And that means we need to determine the index of the - // operand. - unsigned Idx = MI->getNumOperands(); - while (Idx > 0 && - MI->getOperand(Idx-1).isReg() && MI->getOperand(Idx-1).isImplicit()) - --Idx; - bool isTied = MI->getDesc().getOperandConstraint(Idx, TOI::TIED_TO) != -1; - bool isKill = Op.hasOneUse() && !isTied && !IsDebug; + // conservative approximation. InstrEmitter does trivial coalescing + // with CopyFromReg nodes, so don't emit kill flags for them. + // Avoid kill flags on Schedule cloned nodes, since there will be + // multiple uses. + // Tied operands are never killed, so we need to check that. And that + // means we need to determine the index of the operand. + bool isKill = Op.hasOneUse() && + Op.getNode()->getOpcode() != ISD::CopyFromReg && + !IsDebug && + !(IsClone || IsCloned); + if (isKill) { + unsigned Idx = MI->getNumOperands(); + while (Idx > 0 && + MI->getOperand(Idx-1).isReg() && MI->getOperand(Idx-1).isImplicit()) + --Idx; + bool isTied = MI->getDesc().getOperandConstraint(Idx, TOI::TIED_TO) != -1; + if (isTied) + isKill = false; + } MI->addOperand(MachineOperand::CreateReg(VReg, isOptDef, false/*isImp*/, isKill, @@ -322,9 +332,10 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op, unsigned IIOpNum, const TargetInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, - bool IsDebug) { + bool IsDebug, bool IsClone, bool IsCloned) { if (Op.isMachineOpcode()) { - AddRegisterOperand(MI, Op, IIOpNum, II, VRBaseMap, IsDebug); + AddRegisterOperand(MI, Op, IIOpNum, II, VRBaseMap, + IsDebug, IsClone, IsCloned); } else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { MI->addOperand(MachineOperand::CreateImm(C->getSExtValue())); } else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Op)) { @@ -373,7 +384,8 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op, assert(Op.getValueType() != MVT::Other && Op.getValueType() != MVT::Flag && "Chain and flag operands should occur at end of operand list!"); - AddRegisterOperand(MI, Op, IIOpNum, II, VRBaseMap, IsDebug); + AddRegisterOperand(MI, Op, IIOpNum, II, VRBaseMap, + IsDebug, IsClone, IsCloned); } } @@ -395,7 +407,8 @@ getSuperRegisterRegClass(const TargetRegisterClass *TRC, /// EmitSubregNode - Generate machine code for subreg nodes. /// void InstrEmitter::EmitSubregNode(SDNode *Node, - DenseMap<SDValue, unsigned> &VRBaseMap){ + DenseMap<SDValue, unsigned> &VRBaseMap, + bool IsClone, bool IsCloned) { unsigned VRBase = 0; unsigned Opc = Node->getMachineOpcode(); @@ -439,7 +452,8 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, // Add def, source, and subreg index MI->addOperand(MachineOperand::CreateReg(VRBase, true)); - AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap); + AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap, /*IsDebug=*/false, + IsClone, IsCloned); MI->addOperand(MachineOperand::CreateImm(SubIdx)); MBB->insert(InsertPos, MI); } else if (Opc == TargetOpcode::INSERT_SUBREG || @@ -473,9 +487,11 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, const ConstantSDNode *SD = cast<ConstantSDNode>(N0); MI->addOperand(MachineOperand::CreateImm(SD->getZExtValue())); } else - AddOperand(MI, N0, 0, 0, VRBaseMap); + AddOperand(MI, N0, 0, 0, VRBaseMap, /*IsDebug=*/false, + IsClone, IsCloned); // Add the subregster being inserted - AddOperand(MI, N1, 0, 0, VRBaseMap); + AddOperand(MI, N1, 0, 0, VRBaseMap, /*IsDebug=*/false, + IsClone, IsCloned); MI->addOperand(MachineOperand::CreateImm(SubIdx)); MBB->insert(InsertPos, MI); } else @@ -503,7 +519,7 @@ InstrEmitter::EmitCopyToRegClassNode(SDNode *Node, // Create the new VReg in the destination class and emit a copy. unsigned NewVReg = MRI->createVirtualRegister(DstRC); bool Emitted = TII->copyRegToReg(*MBB, InsertPos, NewVReg, VReg, - DstRC, SrcRC); + DstRC, SrcRC, Node->getDebugLoc()); assert(Emitted && "Unable to issue a copy instruction for a COPY_TO_REGCLASS node!\n"); (void) Emitted; @@ -517,7 +533,8 @@ InstrEmitter::EmitCopyToRegClassNode(SDNode *Node, /// EmitRegSequence - Generate machine code for REG_SEQUENCE nodes. /// void InstrEmitter::EmitRegSequence(SDNode *Node, - DenseMap<SDValue, unsigned> &VRBaseMap) { + DenseMap<SDValue, unsigned> &VRBaseMap, + bool IsClone, bool IsCloned) { const TargetRegisterClass *RC = TLI->getRegClassFor(Node->getValueType(0)); unsigned NewVReg = MRI->createVirtualRegister(RC); MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), @@ -528,17 +545,21 @@ void InstrEmitter::EmitRegSequence(SDNode *Node, const TargetInstrDesc &II = TII->get(TargetOpcode::REG_SEQUENCE); for (unsigned i = 0; i != NumOps; ++i) { SDValue Op = Node->getOperand(i); -#ifndef NDEBUG if (i & 1) { unsigned SubIdx = cast<ConstantSDNode>(Op)->getZExtValue(); unsigned SubReg = getVR(Node->getOperand(i-1), VRBaseMap); - const TargetRegisterClass *TRC = MRI->getRegClass(SubReg); - const TargetRegisterClass *SRC = - getSuperRegisterRegClass(TRC, SubIdx, Node->getValueType(0)); - assert(SRC == RC && "Invalid subregister index in REG_SEQUENCE"); + const TargetRegisterClass *TRC = MRI->getRegClass(SubReg); + const TargetRegisterClass *SRC = + TRI->getMatchingSuperRegClass(RC, TRC, SubIdx); + if (!SRC) + llvm_unreachable("Invalid subregister index in REG_SEQUENCE"); + if (SRC != RC) { + MRI->setRegClass(NewVReg, SRC); + RC = SRC; + } } -#endif - AddOperand(MI, Op, i+1, &II, VRBaseMap); + AddOperand(MI, Op, i+1, &II, VRBaseMap, /*IsDebug=*/false, + IsClone, IsCloned); } MBB->insert(InsertPos, MI); @@ -579,11 +600,17 @@ InstrEmitter::EmitDbgValue(SDDbgValue *SD, MIB.addReg(0U); // undef else AddOperand(&*MIB, Op, (*MIB).getNumOperands(), &II, VRBaseMap, - true /*IsDebug*/); + /*IsDebug=*/true, /*IsClone=*/false, /*IsCloned=*/false); } else if (SD->getKind() == SDDbgValue::CONST) { const Value *V = SD->getConst(); if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) { - MIB.addImm(CI->getSExtValue()); + // FIXME: SDDbgValues aren't updated with legalization, so it's possible + // to have i128 values in them at this point. As a crude workaround, just + // drop the debug info if this happens. + if (!CI->getValue().isSignedIntN(64)) + MIB.addReg(0U); + else + MIB.addImm(CI->getSExtValue()); } else if (const ConstantFP *CF = dyn_cast<ConstantFP>(V)) { MIB.addFPImm(CF); } else { @@ -612,7 +639,7 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, if (Opc == TargetOpcode::EXTRACT_SUBREG || Opc == TargetOpcode::INSERT_SUBREG || Opc == TargetOpcode::SUBREG_TO_REG) { - EmitSubregNode(Node, VRBaseMap); + EmitSubregNode(Node, VRBaseMap, IsClone, IsCloned); return; } @@ -624,7 +651,7 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, // Handle REG_SEQUENCE specially. if (Opc == TargetOpcode::REG_SEQUENCE) { - EmitRegSequence(Node, VRBaseMap); + EmitRegSequence(Node, VRBaseMap, IsClone, IsCloned); return; } @@ -663,7 +690,7 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, unsigned NumSkip = HasOptPRefs ? II.getNumDefs() - NumResults : 0; for (unsigned i = NumSkip; i != NodeOperands; ++i) AddOperand(MI, Node->getOperand(i), i-NumSkip+II.getNumDefs(), &II, - VRBaseMap); + VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned); // Transfer all of the memory reference descriptions of this instruction. MI->setMemRefs(cast<MachineSDNode>(Node)->memoperands_begin(), @@ -749,7 +776,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, Node->getOperand(1).getValueType()); bool Emitted = TII->copyRegToReg(*MBB, InsertPos, DestReg, SrcReg, - DstTRC, SrcTRC); + DstTRC, SrcTRC, Node->getDebugLoc()); assert(Emitted && "Unable to issue a copy instruction!\n"); (void) Emitted; break; @@ -810,7 +837,8 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, // The addressing mode has been selected, just add all of the // operands to the machine instruction. for (; NumVals; --NumVals, ++i) - AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap); + AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap, + /*IsDebug=*/false, IsClone, IsCloned); break; } } diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.h b/lib/CodeGen/SelectionDAG/InstrEmitter.h index c7e7c71..02c044c 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.h +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.h @@ -65,7 +65,7 @@ class InstrEmitter { unsigned IIOpNum, const TargetInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, - bool IsDebug = false); + bool IsDebug, bool IsClone, bool IsCloned); /// AddOperand - Add the specified operand to the specified machine instr. II /// specifies the instruction information for the node, and IIOpNum is the @@ -75,11 +75,12 @@ class InstrEmitter { unsigned IIOpNum, const TargetInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, - bool IsDebug = false); + bool IsDebug, bool IsClone, bool IsCloned); /// EmitSubregNode - Generate machine code for subreg nodes. /// - void EmitSubregNode(SDNode *Node, DenseMap<SDValue, unsigned> &VRBaseMap); + void EmitSubregNode(SDNode *Node, DenseMap<SDValue, unsigned> &VRBaseMap, + bool IsClone, bool IsCloned); /// EmitCopyToRegClassNode - Generate machine code for COPY_TO_REGCLASS nodes. /// COPY_TO_REGCLASS is just a normal copy, except that the destination @@ -90,7 +91,8 @@ class InstrEmitter { /// EmitRegSequence - Generate machine code for REG_SEQUENCE nodes. /// - void EmitRegSequence(SDNode *Node, DenseMap<SDValue, unsigned> &VRBaseMap); + void EmitRegSequence(SDNode *Node, DenseMap<SDValue, unsigned> &VRBaseMap, + bool IsClone, bool IsCloned); public: /// CountResults - The results of target nodes have register or immediate /// operands first, then an optional chain, and optional flag operands diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index bedfa57..62a37a5 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -23,7 +23,6 @@ #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" -#include "llvm/Target/TargetSubtarget.h" #include "llvm/CallingConv.h" #include "llvm/Constants.h" #include "llvm/DerivedTypes.h" @@ -2027,6 +2026,7 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, return Result; } assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); + // Code below here assumes !isSigned without checking again. // Implementation of unsigned i64 to f64 following the algorithm in // __floatundidf in compiler_rt. This implementation has the advantage @@ -2052,6 +2052,41 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub); } + // Implementation of unsigned i64 to f32. This implementation has the + // advantage of performing rounding correctly. + // TODO: Generalize this for use with other types. + if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) { + EVT SHVT = TLI.getShiftAmountTy(); + + SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, + DAG.getConstant(UINT64_C(0xfffffffffffff800), MVT::i64)); + SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, + DAG.getConstant(UINT64_C(0x800), MVT::i64)); + SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, + DAG.getConstant(UINT64_C(0x7ff), MVT::i64)); + SDValue Ne = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64), + And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE); + SDValue Sel = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ne, Or, Op0); + SDValue Ge = DAG.getSetCC(dl, TLI.getSetCCResultType(MVT::i64), + Op0, DAG.getConstant(UINT64_C(0x0020000000000000), MVT::i64), + ISD::SETUGE); + SDValue Sel2 = DAG.getNode(ISD::SELECT, dl, MVT::i64, Ge, Sel, Op0); + + SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2, + DAG.getConstant(32, SHVT)); + SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh); + SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc); + SDValue TwoP32 = + DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), MVT::f64); + SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt); + SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2); + SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo); + SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2); + return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd, + DAG.getIntPtrConstant(0)); + + } + SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); SDValue SignSet = DAG.getSetCC(dl, TLI.getSetCCResultType(Op0.getValueType()), @@ -2488,6 +2523,8 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, EVT VT = Node->getValueType(0); EVT EltVT = VT.getVectorElementType(); + if (getTypeAction(EltVT) == Promote) + EltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); unsigned NumElems = VT.getVectorNumElements(); SmallVector<SDValue, 8> Ops; for (unsigned i = 0; i != NumElems; ++i) { diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 548454c..8b382bc 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -2314,13 +2314,29 @@ SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), InL); } +static const fltSemantics *EVTToAPFloatSemantics(EVT VT) { + switch (VT.getSimpleVT().SimpleTy) { + default: llvm_unreachable("Unknown FP format"); + case MVT::f32: return &APFloat::IEEEsingle; + case MVT::f64: return &APFloat::IEEEdouble; + case MVT::f80: return &APFloat::x87DoubleExtended; + case MVT::f128: return &APFloat::IEEEquad; + case MVT::ppcf128: return &APFloat::PPCDoubleDouble; + } +} + SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { SDValue Op = N->getOperand(0); EVT SrcVT = Op.getValueType(); EVT DstVT = N->getValueType(0); DebugLoc dl = N->getDebugLoc(); - if (TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ + // The following optimization is valid only if every value in SrcVT (when + // treated as signed) is representable in DstVT. Check that the mantissa + // size of DstVT is >= than the number of bits in SrcVT -1. + const fltSemantics *sem = EVTToAPFloatSemantics(DstVT); + if (APFloat::semanticsPrecision(*sem) >= SrcVT.getSizeInBits()-1 && + TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ // Do a signed conversion then adjust the result. SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op); SignedConv = TLI.LowerOperation(SignedConv, DAG); diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h index d60ad60..c665963 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -32,7 +32,7 @@ namespace llvm { /// involves promoting small sizes to large sizes or splitting up large values /// into small values. /// -class VISIBILITY_HIDDEN DAGTypeLegalizer { +class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer { const TargetLowering &TLI; SelectionDAG &DAG; public: diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp index b92a672..56f5ded 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGList.cpp @@ -30,7 +30,6 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/PriorityQueue.h" #include "llvm/ADT/Statistic.h" #include <climits> using namespace llvm; diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index da02850..820ba66 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -24,7 +24,6 @@ #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/ADT/PriorityQueue.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/STLExtras.h" @@ -53,6 +52,12 @@ static RegisterScheduler "order when possible", createSourceListDAGScheduler); +static RegisterScheduler + hybridListDAGScheduler("list-hybrid", + "Bottom-up rr list scheduling which avoid stalls for " + "long latency instructions", + createHybridListDAGScheduler); + namespace { //===----------------------------------------------------------------------===// /// ScheduleDAGRRList - The actual register reduction list scheduler @@ -64,6 +69,10 @@ private: /// it is top-down. bool isBottomUp; + /// NeedLatency - True if the scheduler will make use of latency information. + /// + bool NeedLatency; + /// AvailableQueue - The priority queue to use for the available SUnits. SchedulingPriorityQueue *AvailableQueue; @@ -80,9 +89,9 @@ private: public: ScheduleDAGRRList(MachineFunction &mf, - bool isbottomup, + bool isbottomup, bool needlatency, SchedulingPriorityQueue *availqueue) - : ScheduleDAGSDNodes(mf), isBottomUp(isbottomup), + : ScheduleDAGSDNodes(mf), isBottomUp(isbottomup), NeedLatency(needlatency), AvailableQueue(availqueue), Topo(SUnits) { } @@ -161,9 +170,11 @@ private: return NewNode; } - /// ForceUnitLatencies - Return true, since register-pressure-reducing - /// scheduling doesn't need actual latency information. - bool ForceUnitLatencies() const { return true; } + /// ForceUnitLatencies - Register-pressure-reducing scheduling doesn't + /// need actual latency information but the hybrid scheduler does. + bool ForceUnitLatencies() const { + return !NeedLatency; + } }; } // end anonymous namespace @@ -213,6 +224,12 @@ void ScheduleDAGRRList::ReleasePred(SUnit *SU, const SDep *PredEdge) { #endif --PredSU->NumSuccsLeft; + if (!ForceUnitLatencies()) { + // Updating predecessor's height. This is now the cycle when the + // predecessor can be scheduled without causing a pipeline stall. + PredSU->setHeightToAtLeast(SU->getHeight() + PredEdge->getLatency()); + } + // If all the node's successors are scheduled, this node is ready // to be scheduled. Ignore the special EntrySU node. if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) { @@ -244,10 +261,15 @@ void ScheduleDAGRRList::ReleasePredecessors(SUnit *SU, unsigned CurCycle) { /// count of its predecessors. If a predecessor pending count is zero, add it to /// the Available queue. void ScheduleDAGRRList::ScheduleNodeBottomUp(SUnit *SU, unsigned CurCycle) { - DEBUG(dbgs() << "*** Scheduling [" << CurCycle << "]: "); + DEBUG(dbgs() << "\n*** Scheduling [" << CurCycle << "]: "); DEBUG(SU->dump(this)); - assert(CurCycle >= SU->getHeight() && "Node scheduled below its height!"); +#ifndef NDEBUG + if (CurCycle < SU->getHeight()) + DEBUG(dbgs() << " Height [" << SU->getHeight() << "] pipeline stall!\n"); +#endif + + // FIXME: Handle noop hazard. SU->setHeightToAtLeast(CurCycle); Sequence.push_back(SU); @@ -339,6 +361,7 @@ void ScheduleDAGRRList::BacktrackBottomUp(SUnit *SU, unsigned BtCycle, SU->isAvailable = false; UnscheduleNodeBottomUp(OldSU); --CurCycle; + AvailableQueue->setCurCycle(CurCycle); } assert(!SU->isSucc(OldSU) && "Something is wrong!"); @@ -386,7 +409,7 @@ SUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) { if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes)) return NULL; - DEBUG(dbgs() << "Unfolding SU # " << SU->NodeNum << "\n"); + DEBUG(dbgs() << "Unfolding SU #" << SU->NodeNum << "\n"); assert(NewNodes.size() == 2 && "Expected a load folding node!"); N = NewNodes[1]; @@ -504,7 +527,7 @@ SUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) { SU = NewSU; } - DEBUG(dbgs() << "Duplicating SU # " << SU->NodeNum << "\n"); + DEBUG(dbgs() << " Duplicating SU #" << SU->NodeNum << "\n"); NewSU = CreateClone(SU); // New SUnit has the exact same predecessors. @@ -786,7 +809,7 @@ void ScheduleDAGRRList::ListScheduleBottomUp() { // Issue copies, these can be expensive cross register class copies. SmallVector<SUnit*, 2> Copies; InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies); - DEBUG(dbgs() << "Adding an edge from SU #" << TrySU->NodeNum + DEBUG(dbgs() << " Adding an edge from SU #" << TrySU->NodeNum << " to SU #" << Copies.front()->NodeNum << "\n"); AddPred(TrySU, SDep(Copies.front(), SDep::Order, /*Latency=*/1, /*Reg=*/0, /*isNormalMemory=*/false, @@ -795,7 +818,7 @@ void ScheduleDAGRRList::ListScheduleBottomUp() { NewDef = Copies.back(); } - DEBUG(dbgs() << "Adding an edge from SU #" << NewDef->NodeNum + DEBUG(dbgs() << " Adding an edge from SU #" << NewDef->NodeNum << " to SU #" << TrySU->NodeNum << "\n"); LiveRegDefs[Reg] = NewDef; AddPred(NewDef, SDep(TrySU, SDep::Order, /*Latency=*/1, @@ -821,6 +844,7 @@ void ScheduleDAGRRList::ListScheduleBottomUp() { if (CurSU) ScheduleNodeBottomUp(CurSU, CurCycle); ++CurCycle; + AvailableQueue->setCurCycle(CurCycle); } // Reverse the order if it is bottom up. @@ -889,6 +913,7 @@ void ScheduleDAGRRList::ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle) { /// schedulers. void ScheduleDAGRRList::ListScheduleTopDown() { unsigned CurCycle = 0; + AvailableQueue->setCurCycle(CurCycle); // Release any successors of the special Entry node. ReleaseSuccessors(&EntrySU); @@ -911,6 +936,7 @@ void ScheduleDAGRRList::ListScheduleTopDown() { if (CurSU) ScheduleNodeTopDown(CurSU, CurCycle); ++CurCycle; + AvailableQueue->setCurCycle(CurCycle); } #ifndef NDEBUG @@ -956,6 +982,16 @@ namespace { bool operator()(const SUnit* left, const SUnit* right) const; }; + + struct hybrid_ls_rr_sort : public std::binary_function<SUnit*, SUnit*, bool> { + RegReductionPriorityQueue<hybrid_ls_rr_sort> *SPQ; + hybrid_ls_rr_sort(RegReductionPriorityQueue<hybrid_ls_rr_sort> *spq) + : SPQ(spq) {} + hybrid_ls_rr_sort(const hybrid_ls_rr_sort &RHS) + : SPQ(RHS.SPQ) {} + + bool operator()(const SUnit* left, const SUnit* right) const; + }; } // end anonymous namespace /// CalcNodeSethiUllmanNumber - Compute Sethi Ullman number. @@ -990,8 +1026,9 @@ CalcNodeSethiUllmanNumber(const SUnit *SU, std::vector<unsigned> &SUNumbers) { namespace { template<class SF> class RegReductionPriorityQueue : public SchedulingPriorityQueue { - PriorityQueue<SUnit*, std::vector<SUnit*>, SF> Queue; - unsigned currentQueueId; + std::vector<SUnit*> Queue; + SF Picker; + unsigned CurQueueId; protected: // SUnits - The SUnits for the current graph. @@ -1007,7 +1044,7 @@ namespace { public: RegReductionPriorityQueue(const TargetInstrInfo *tii, const TargetRegisterInfo *tri) - : Queue(SF(this)), currentQueueId(0), + : Picker(this), CurQueueId(0), TII(tii), TRI(tri), scheduleDAG(NULL) {} void initNodes(std::vector<SUnit> &sunits) { @@ -1067,26 +1104,26 @@ namespace { unsigned getNodeOrdering(const SUnit *SU) const { return scheduleDAG->DAG->GetOrdering(SU->getNode()); } - - unsigned size() const { return Queue.size(); } bool empty() const { return Queue.empty(); } void push(SUnit *U) { assert(!U->NodeQueueId && "Node in the queue already"); - U->NodeQueueId = ++currentQueueId; - Queue.push(U); + U->NodeQueueId = ++CurQueueId; + Queue.push_back(U); } - void push_all(const std::vector<SUnit *> &Nodes) { - for (unsigned i = 0, e = Nodes.size(); i != e; ++i) - push(Nodes[i]); - } - SUnit *pop() { if (empty()) return NULL; - SUnit *V = Queue.top(); - Queue.pop(); + std::vector<SUnit *>::iterator Best = Queue.begin(); + for (std::vector<SUnit *>::iterator I = next(Queue.begin()), + E = Queue.end(); I != E; ++I) + if (Picker(*Best, *I)) + Best = I; + SUnit *V = *Best; + if (Best != prior(Queue.end())) + std::swap(*Best, Queue.back()); + Queue.pop_back(); V->NodeQueueId = 0; return V; } @@ -1094,7 +1131,11 @@ namespace { void remove(SUnit *SU) { assert(!Queue.empty() && "Queue is empty!"); assert(SU->NodeQueueId != 0 && "Not in queue!"); - Queue.erase_one(SU); + std::vector<SUnit *>::iterator I = std::find(Queue.begin(), Queue.end(), + SU); + if (I != prior(Queue.end())) + std::swap(*I, Queue.back()); + Queue.pop_back(); SU->NodeQueueId = 0; } @@ -1117,6 +1158,9 @@ namespace { typedef RegReductionPriorityQueue<src_ls_rr_sort> SrcRegReductionPriorityQueue; + + typedef RegReductionPriorityQueue<hybrid_ls_rr_sort> + HybridBURRPriorityQueue; } /// closestSucc - Returns the scheduled cycle of the successor which is @@ -1203,7 +1247,7 @@ bool bu_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const { } // Source order, otherwise bottom up. -bool src_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const{ +bool src_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const { unsigned LOrder = SPQ->getNodeOrdering(left); unsigned ROrder = SPQ->getNodeOrdering(right); @@ -1215,6 +1259,25 @@ bool src_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const{ return BURRSort(left, right, SPQ); } +bool hybrid_ls_rr_sort::operator()(const SUnit *left, const SUnit *right) const{ + bool LStall = left->SchedulingPref == Sched::Latency && + SPQ->getCurCycle() < left->getHeight(); + bool RStall = right->SchedulingPref == Sched::Latency && + SPQ->getCurCycle() < right->getHeight(); + // If scheduling one of the node will cause a pipeline stall, delay it. + // If scheduling either one of the node will cause a pipeline stall, sort them + // according to their height. + // If neither will cause a pipeline stall, try to reduce register pressure. + if (LStall) { + if (!RStall) + return true; + if (left->getHeight() != right->getHeight()) + return left->getHeight() > right->getHeight(); + } else if (RStall) + return false; + return BURRSort(left, right, SPQ); +} + template<class SF> bool RegReductionPriorityQueue<SF>::canClobber(const SUnit *SU, const SUnit *Op) { @@ -1379,8 +1442,8 @@ void RegReductionPriorityQueue<SF>::PrescheduleNodesWithMultipleUses() { // Ok, the transformation is safe and the heuristics suggest it is // profitable. Update the graph. - DEBUG(dbgs() << "Prescheduling SU # " << SU->NodeNum - << " next to PredSU # " << PredSU->NodeNum + DEBUG(dbgs() << " Prescheduling SU #" << SU->NodeNum + << " next to PredSU #" << PredSU->NodeNum << " to guide scheduling in the presence of multiple uses\n"); for (unsigned i = 0; i != PredSU->Succs.size(); ++i) { SDep Edge = PredSU->Succs[i]; @@ -1469,7 +1532,7 @@ void RegReductionPriorityQueue<SF>::AddPseudoTwoAddrDeps() { (hasCopyToRegUse(SU) && !hasCopyToRegUse(SuccSU)) || (!SU->isCommutable && SuccSU->isCommutable)) && !scheduleDAG->IsReachable(SuccSU, SU)) { - DEBUG(dbgs() << "Adding a pseudo-two-addr edge from SU # " + DEBUG(dbgs() << " Adding a pseudo-two-addr edge from SU #" << SU->NodeNum << " to SU #" << SuccSU->NodeNum << "\n"); scheduleDAG->AddPred(SU, SDep(SuccSU, SDep::Order, /*Latency=*/0, /*Reg=*/0, /*isNormalMemory=*/false, @@ -1563,8 +1626,7 @@ llvm::createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) { BURegReductionPriorityQueue *PQ = new BURegReductionPriorityQueue(TII, TRI); - ScheduleDAGRRList *SD = - new ScheduleDAGRRList(*IS->MF, true, PQ); + ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, false, PQ); PQ->setScheduleDAG(SD); return SD; } @@ -1577,8 +1639,7 @@ llvm::createTDRRListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) { TDRegReductionPriorityQueue *PQ = new TDRegReductionPriorityQueue(TII, TRI); - ScheduleDAGRRList *SD = - new ScheduleDAGRRList(*IS->MF, false, PQ); + ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, false, PQ); PQ->setScheduleDAG(SD); return SD; } @@ -1591,8 +1652,20 @@ llvm::createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) { SrcRegReductionPriorityQueue *PQ = new SrcRegReductionPriorityQueue(TII, TRI); - ScheduleDAGRRList *SD = - new ScheduleDAGRRList(*IS->MF, true, PQ); + ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, false, PQ); + PQ->setScheduleDAG(SD); + return SD; +} + +llvm::ScheduleDAGSDNodes * +llvm::createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) { + const TargetMachine &TM = IS->TM; + const TargetInstrInfo *TII = TM.getInstrInfo(); + const TargetRegisterInfo *TRI = TM.getRegisterInfo(); + + HybridBURRPriorityQueue *PQ = new HybridBURRPriorityQueue(TII, TRI); + + ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, true, PQ); PQ->setScheduleDAG(SD); return SD; } diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index 76e4771..3185c88 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -19,6 +19,7 @@ #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetSubtarget.h" #include "llvm/ADT/DenseMap.h" @@ -44,6 +45,24 @@ void ScheduleDAGSDNodes::Run(SelectionDAG *dag, MachineBasicBlock *bb, ScheduleDAG::Run(bb, insertPos); } +/// NewSUnit - Creates a new SUnit and return a ptr to it. +/// +SUnit *ScheduleDAGSDNodes::NewSUnit(SDNode *N) { +#ifndef NDEBUG + const SUnit *Addr = 0; + if (!SUnits.empty()) + Addr = &SUnits[0]; +#endif + SUnits.push_back(SUnit(N, (unsigned)SUnits.size())); + assert((Addr == 0 || Addr == &SUnits[0]) && + "SUnits std::vector reallocated on the fly!"); + SUnits.back().OrigNode = &SUnits.back(); + SUnit *SU = &SUnits.back(); + const TargetLowering &TLI = DAG->getTargetLoweringInfo(); + SU->SchedulingPref = TLI.getSchedulingPreference(N); + return SU; +} + SUnit *ScheduleDAGSDNodes::Clone(SUnit *Old) { SUnit *SU = NewSUnit(Old->getNode()); SU->OrigNode = Old->OrigNode; @@ -52,6 +71,7 @@ SUnit *ScheduleDAGSDNodes::Clone(SUnit *Old) { SU->isCommutable = Old->isCommutable; SU->hasPhysRegDefs = Old->hasPhysRegDefs; SU->hasPhysRegClobbers = Old->hasPhysRegClobbers; + SU->SchedulingPref = Old->SchedulingPref; Old->isCloned = true; return SU; } @@ -217,9 +237,6 @@ void ScheduleDAGSDNodes::BuildSchedUnits() { // This is a temporary workaround. SUnits.reserve(NumNodes * 2); - // Check to see if the scheduler cares about latencies. - bool UnitLatencies = ForceUnitLatencies(); - // Add all nodes in depth first order. SmallVector<SDNode*, 64> Worklist; SmallPtrSet<SDNode*, 64> Visited; @@ -282,10 +299,7 @@ void ScheduleDAGSDNodes::BuildSchedUnits() { N->setNodeId(NodeSUnit->NodeNum); // Assign the Latency field of NodeSUnit using target-provided information. - if (UnitLatencies) - NodeSUnit->Latency = 1; - else - ComputeLatency(NodeSUnit); + ComputeLatency(NodeSUnit); } } @@ -353,7 +367,7 @@ void ScheduleDAGSDNodes::AddSchedEdges() { const SDep& dep = SDep(OpSU, isChain ? SDep::Order : SDep::Data, OpSU->Latency, PhysReg); if (!isChain && !UnitLatencies) { - ComputeOperandLatency(OpSU, SU, const_cast<SDep &>(dep)); + ComputeOperandLatency(OpN, N, i, const_cast<SDep &>(dep)); ST.adjustSchedDependency(OpSU, SU, const_cast<SDep &>(dep)); } @@ -377,7 +391,17 @@ void ScheduleDAGSDNodes::BuildSchedGraph(AliasAnalysis *AA) { } void ScheduleDAGSDNodes::ComputeLatency(SUnit *SU) { + // Check to see if the scheduler cares about latencies. + if (ForceUnitLatencies()) { + SU->Latency = 1; + return; + } + const InstrItineraryData &InstrItins = TM.getInstrItineraryData(); + if (InstrItins.isEmpty()) { + SU->Latency = 1; + return; + } // Compute the latency for the node. We use the sum of the latencies for // all nodes flagged together into this SUnit. @@ -389,6 +413,37 @@ void ScheduleDAGSDNodes::ComputeLatency(SUnit *SU) { } } +void ScheduleDAGSDNodes::ComputeOperandLatency(SDNode *Def, SDNode *Use, + unsigned OpIdx, SDep& dep) const{ + // Check to see if the scheduler cares about latencies. + if (ForceUnitLatencies()) + return; + + const InstrItineraryData &InstrItins = TM.getInstrItineraryData(); + if (InstrItins.isEmpty()) + return; + + if (dep.getKind() != SDep::Data) + return; + + unsigned DefIdx = Use->getOperand(OpIdx).getResNo(); + if (Def->isMachineOpcode() && Use->isMachineOpcode()) { + const TargetInstrDesc &II = TII->get(Def->getMachineOpcode()); + if (DefIdx >= II.getNumDefs()) + return; + int DefCycle = InstrItins.getOperandCycle(II.getSchedClass(), DefIdx); + if (DefCycle < 0) + return; + const unsigned UseClass = TII->get(Use->getMachineOpcode()).getSchedClass(); + int UseCycle = InstrItins.getOperandCycle(UseClass, OpIdx); + if (UseCycle >= 0) { + int Latency = DefCycle - UseCycle + 1; + if (Latency >= 0) + dep.setLatency(Latency); + } + } +} + void ScheduleDAGSDNodes::dumpNode(const SUnit *SU) const { if (!SU->getNode()) { dbgs() << "PHYS REG COPY\n"; diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h index 7ae8ec2..e8714ba 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h @@ -66,18 +66,7 @@ namespace llvm { /// NewSUnit - Creates a new SUnit and return a ptr to it. /// - SUnit *NewSUnit(SDNode *N) { -#ifndef NDEBUG - const SUnit *Addr = 0; - if (!SUnits.empty()) - Addr = &SUnits[0]; -#endif - SUnits.push_back(SUnit(N, (unsigned)SUnits.size())); - assert((Addr == 0 || Addr == &SUnits[0]) && - "SUnits std::vector reallocated on the fly!"); - SUnits.back().OrigNode = &SUnits.back(); - return &SUnits.back(); - } + SUnit *NewSUnit(SDNode *N); /// Clone - Creates a clone of the specified SUnit. It does not copy the /// predecessors / successors info nor the temporary scheduling states. @@ -94,6 +83,15 @@ namespace llvm { /// virtual void ComputeLatency(SUnit *SU); + /// ComputeOperandLatency - Override dependence edge latency using + /// operand use/def information + /// + virtual void ComputeOperandLatency(SUnit *Def, SUnit *Use, + SDep& dep) const { } + + virtual void ComputeOperandLatency(SDNode *Def, SDNode *Use, + unsigned OpIdx, SDep& dep) const; + virtual MachineBasicBlock *EmitSchedule(); /// Schedule - Order nodes according to selected style, filling diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index e6df742..38bf68b 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -15,6 +15,7 @@ #include "SDNodeOrdering.h" #include "SDNodeDbgValue.h" #include "llvm/Constants.h" +#include "llvm/Analysis/DebugInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Function.h" #include "llvm/GlobalAlias.h" @@ -32,6 +33,7 @@ #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetFrameInfo.h" #include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetSelectionDAGInfo.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetIntrinsicInfo.h" @@ -789,7 +791,8 @@ unsigned SelectionDAG::getEVTAlignment(EVT VT) const { // EntryNode could meaningfully have debug info if we can find it... SelectionDAG::SelectionDAG(const TargetMachine &tm, FunctionLoweringInfo &fli) - : TM(tm), TLI(*tm.getTargetLowering()), FLI(fli), + : TM(tm), TLI(*tm.getTargetLowering()), TSI(*tm.getSelectionDAGInfo()), + FLI(fli), EntryNode(ISD::EntryToken, DebugLoc(), getVTList(MVT::Other)), Root(getEntryNode()), Ordering(0) { AllNodes.push_back(&EntryNode); @@ -963,8 +966,18 @@ SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { EVT EltVT = VT.getScalarType(); if (EltVT==MVT::f32) return getConstantFP(APFloat((float)Val), VT, isTarget); - else + else if (EltVT==MVT::f64) return getConstantFP(APFloat(Val), VT, isTarget); + else if (EltVT==MVT::f80 || EltVT==MVT::f128) { + bool ignored; + APFloat apf = APFloat(Val); + apf.convert(*EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, + &ignored); + return getConstantFP(apf, VT, isTarget); + } else { + assert(0 && "Unsupported type in getConstantFP"); + return SDValue(); + } } SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, @@ -2614,7 +2627,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, } break; case ISD::AND: - assert(VT.isInteger() && N1.getValueType() == N2.getValueType() && + assert(VT.isInteger() && "This operator does not apply to FP types!"); + assert(N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && "Binary operator types must match!"); // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's // worth handling here. @@ -2627,7 +2641,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, case ISD::XOR: case ISD::ADD: case ISD::SUB: - assert(VT.isInteger() && N1.getValueType() == N2.getValueType() && + assert(VT.isInteger() && "This operator does not apply to FP types!"); + assert(N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && "Binary operator types must match!"); // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so // it's worth handling here. @@ -2642,7 +2657,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, case ISD::SDIV: case ISD::SREM: assert(VT.isInteger() && "This operator does not apply to FP types!"); - // fall through + assert(N1.getValueType() == N2.getValueType() && + N1.getValueType() == VT && "Binary operator types must match!"); + break; case ISD::FADD: case ISD::FSUB: case ISD::FMUL: @@ -2665,6 +2682,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, return N1; } } + assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); assert(N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && "Binary operator types must match!"); break; @@ -3525,7 +3543,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, // Then check to see if we should lower the memcpy with target-specific // code. If the target chooses to do this, this is the next best. SDValue Result = - TLI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, + TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline, DstSV, DstSVOff, SrcSV, SrcSVOff); if (Result.getNode()) @@ -3590,7 +3608,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, // Then check to see if we should lower the memmove with target-specific // code. If the target chooses to do this, this is the next best. SDValue Result = - TLI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, + TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, DstSV, DstSVOff, SrcSV, SrcSVOff); if (Result.getNode()) return Result; @@ -3641,7 +3659,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, // Then check to see if we should lower the memset with target-specific // code. If the target chooses to do this, this is the next best. SDValue Result = - TLI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, + TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, DstSV, DstSVOff); if (Result.getNode()) return Result; @@ -5417,6 +5435,8 @@ const EVT *SDNode::getValueTypeList(EVT VT) { sys::SmartScopedLock<true> Lock(*VTMutex); return &(*EVTs->insert(VT).first); } else { + assert(VT.getSimpleVT().SimpleTy < MVT::LAST_VALUETYPE && + "Value type out of range!"); return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; } } @@ -5607,6 +5627,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::LSDAADDR: return "LSDAADDR"; case ISD::EHSELECTION: return "EHSELECTION"; case ISD::EH_RETURN: return "EH_RETURN"; + case ISD::EH_SJLJ_SETJMP: return "EH_SJLJ_SETJMP"; + case ISD::EH_SJLJ_LONGJMP: return "EH_SJLJ_LONGJMP"; case ISD::ConstantPool: return "ConstantPool"; case ISD::ExternalSymbol: return "ExternalSymbol"; case ISD::BlockAddress: return "BlockAddress"; @@ -6008,6 +6030,21 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { if (getNodeId() != -1) OS << " [ID=" << getNodeId() << ']'; + + DebugLoc dl = getDebugLoc(); + if (G && !dl.isUnknown()) { + DIScope + Scope(dl.getScope(G->getMachineFunction().getFunction()->getContext())); + OS << " dbg:"; + // Omit the directory, since it's usually long and uninteresting. + if (Scope.Verify()) + OS << Scope.getFilename(); + else + OS << "<unknown>"; + OS << ':' << dl.getLine(); + if (dl.getCol() != 0) + OS << ':' << dl.getCol(); + } } void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const { diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index a38b204..fbe601f 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3726,6 +3726,12 @@ SelectionDAGBuilder::EmitFuncArgumentDbgValue(const DbgValueInst &DI, return true; } +// VisualStudio defines setjmp as _setjmp +#if defined(_MSC_VER) && defined(setjmp) +#define setjmp_undefined_for_visual_studio +#undef setjmp +#endif + /// visitIntrinsicCall - Lower the call to the specified intrinsic function. If /// we want to emit this as a call to a named external function, return the name /// otherwise lower it and return null. @@ -3818,7 +3824,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { } case Intrinsic::dbg_declare: { const DbgDeclareInst &DI = cast<DbgDeclareInst>(I); - if (!DIDescriptor::ValidDebugInfo(DI.getVariable(), CodeGenOpt::None)) + if (!DIVariable(DI.getVariable()).Verify()) return 0; MDNode *Variable = DI.getVariable(); @@ -3881,7 +3887,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { } case Intrinsic::dbg_value: { const DbgValueInst &DI = cast<DbgValueInst>(I); - if (!DIDescriptor::ValidDebugInfo(DI.getVariable(), CodeGenOpt::None)) + if (!DIVariable(DI.getVariable()).Verify()) return 0; MDNode *Variable = DI.getVariable(); @@ -3900,6 +3906,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDV = DAG.getDbgValue(Variable, V, Offset, dl, SDNodeOrder); DAG.AddDbgValue(SDV, 0, false); } else { + bool createUndef = false; + // FIXME : Why not use getValue() directly ? SDValue &N = NodeMap[V]; if (N.getNode()) { if (!EmitFuncArgumentDbgValue(DI, V, Variable, Offset, N)) { @@ -3907,7 +3915,19 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { N.getResNo(), Offset, dl, SDNodeOrder); DAG.AddDbgValue(SDV, N.getNode(), false); } - } else { + } else if (isa<PHINode>(V) && !V->use_empty()) { + SDValue N = getValue(V); + if (N.getNode()) { + if (!EmitFuncArgumentDbgValue(DI, V, Variable, Offset, N)) { + SDV = DAG.getDbgValue(Variable, N.getNode(), + N.getResNo(), Offset, dl, SDNodeOrder); + DAG.AddDbgValue(SDV, N.getNode(), false); + } + } else + createUndef = true; + } else + createUndef = true; + if (createUndef) { // We may expand this to cover more cases. One case where we have no // data available is an unreferenced parameter; we need this fallback. SDV = DAG.getDbgValue(Variable, UndefValue::get(V->getType()), @@ -4018,6 +4038,17 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { MMI.setCurrentCallSite(CI->getZExtValue()); return 0; } + case Intrinsic::eh_sjlj_setjmp: { + setValue(&I, DAG.getNode(ISD::EH_SJLJ_SETJMP, dl, MVT::i32, getRoot(), + getValue(I.getOperand(1)))); + return 0; + } + case Intrinsic::eh_sjlj_longjmp: { + DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, dl, MVT::Other, + getRoot(), + getValue(I.getOperand(1)))); + return 0; + } case Intrinsic::convertff: case Intrinsic::convertfsi: @@ -4924,7 +4955,7 @@ isAllocatableRegister(unsigned Reg, MachineFunction &MF, namespace llvm { /// AsmOperandInfo - This contains information for each constraint that we are /// lowering. -class VISIBILITY_HIDDEN SDISelAsmOperandInfo : +class LLVM_LIBRARY_VISIBILITY SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { public: /// CallOperand - If this is the result output operand or a clobber diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 422cb7a..65b8d4f 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -25,9 +25,11 @@ #include "llvm/Intrinsics.h" #include "llvm/IntrinsicInst.h" #include "llvm/LLVMContext.h" +#include "llvm/Module.h" #include "llvm/CodeGen/FastISel.h" #include "llvm/CodeGen/GCStrategy.h" #include "llvm/CodeGen/GCMetadata.h" +#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfo.h" @@ -131,11 +133,13 @@ namespace llvm { if (OptLevel == CodeGenOpt::None) return createFastDAGScheduler(IS, OptLevel); - if (TLI.getSchedulingPreference() == TargetLowering::SchedulingForLatency) + if (TLI.getSchedulingPreference() == Sched::Latency) return createTDListDAGScheduler(IS, OptLevel); - assert(TLI.getSchedulingPreference() == - TargetLowering::SchedulingForRegPressure && "Unknown sched type!"); - return createBURRListDAGScheduler(IS, OptLevel); + if (TLI.getSchedulingPreference() == Sched::RegPressure) + return createBURRListDAGScheduler(IS, OptLevel); + assert(TLI.getSchedulingPreference() == Sched::Hybrid && + "Unknown sched type!"); + return createHybridListDAGScheduler(IS, OptLevel); } } @@ -188,6 +192,39 @@ void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { MachineFunctionPass::getAnalysisUsage(AU); } +/// FunctionCallsSetJmp - Return true if the function has a call to setjmp or +/// other function that gcc recognizes as "returning twice". This is used to +/// limit code-gen optimizations on the machine function. +/// +/// FIXME: Remove after <rdar://problem/8031714> is fixed. +static bool FunctionCallsSetJmp(const Function *F) { + const Module *M = F->getParent(); + static const char *ReturnsTwiceFns[] = { + "setjmp", + "sigsetjmp", + "setjmp_syscall", + "savectx", + "qsetjmp", + "vfork", + "getcontext" + }; +#define NUM_RETURNS_TWICE_FNS sizeof(ReturnsTwiceFns) / sizeof(const char *) + + for (unsigned I = 0; I < NUM_RETURNS_TWICE_FNS; ++I) + if (const Function *Callee = M->getFunction(ReturnsTwiceFns[I])) { + if (!Callee->use_empty()) + for (Value::const_use_iterator + I = Callee->use_begin(), E = Callee->use_end(); + I != E; ++I) + if (const CallInst *CI = dyn_cast<CallInst>(I)) + if (CI->getParent()->getParent() == F) + return true; + } + + return false; +#undef NUM_RETURNS_TWICE_FNS +} + bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // Do some sanity-checking on the command-line options. assert((!EnableFastISelVerbose || EnableFastISel) && @@ -218,6 +255,13 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { MachineBasicBlock *EntryMBB = MF->begin(); RegInfo->EmitLiveInCopies(EntryMBB, TRI, TII); + DenseMap<unsigned, unsigned> LiveInMap; + if (!FuncInfo->ArgDbgValues.empty()) + for (MachineRegisterInfo::livein_iterator LI = RegInfo->livein_begin(), + E = RegInfo->livein_end(); LI != E; ++LI) + if (LI->second) + LiveInMap.insert(std::make_pair(LI->first, LI->second)); + // Insert DBG_VALUE instructions for function arguments to the entry block. for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) { MachineInstr *MI = FuncInfo->ArgDbgValues[e-i-1]; @@ -230,8 +274,44 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // FIXME: VR def may not be in entry block. Def->getParent()->insert(llvm::next(InsertPos), MI); } + + // If Reg is live-in then update debug info to track its copy in a vreg. + DenseMap<unsigned, unsigned>::iterator LDI = LiveInMap.find(Reg); + if (LDI != LiveInMap.end()) { + MachineInstr *Def = RegInfo->getVRegDef(LDI->second); + MachineBasicBlock::iterator InsertPos = Def; + const MDNode *Variable = + MI->getOperand(MI->getNumOperands()-1).getMetadata(); + unsigned Offset = MI->getOperand(1).getImm(); + // Def is never a terminator here, so it is ok to increment InsertPos. + BuildMI(*EntryMBB, ++InsertPos, MI->getDebugLoc(), + TII.get(TargetOpcode::DBG_VALUE)) + .addReg(LDI->second, RegState::Debug) + .addImm(Offset).addMetadata(Variable); + } + } + + // Determine if there are any calls in this machine function. + MachineFrameInfo *MFI = MF->getFrameInfo(); + if (!MFI->hasCalls()) { + for (MachineFunction::const_iterator + I = MF->begin(), E = MF->end(); I != E; ++I) { + const MachineBasicBlock *MBB = I; + for (MachineBasicBlock::const_iterator + II = MBB->begin(), IE = MBB->end(); II != IE; ++II) { + const TargetInstrDesc &TID = TM.getInstrInfo()->get(II->getOpcode()); + if (II->isInlineAsm() || (TID.isCall() && !TID.isReturn())) { + MFI->setHasCalls(true); + goto done; + } + } + } + done:; } + // Determine if there is a call to setjmp in the machine function. + MF->setCallsSetJmp(FunctionCallsSetJmp(&Fn)); + // Release function-specific state. SDB and CurDAG are already cleared // at this point. FuncInfo->clear(); @@ -662,6 +742,7 @@ void SelectionDAGISel::DoInstructionSelection() { CurDAG->setRoot(Dummy.getValue()); } + DEBUG(errs() << "===== Instruction selection ends:\n"); PostprocessISelDAG(); diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 8a4a1b1..44a80d3 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -18,7 +18,6 @@ #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetSubtarget.h" #include "llvm/GlobalVariable.h" #include "llvm/DerivedTypes.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -544,7 +543,7 @@ TargetLowering::TargetLowering(const TargetMachine &tm, ExceptionPointerRegister = 0; ExceptionSelectorRegister = 0; BooleanContents = UndefinedBooleanContent; - SchedPreferenceInfo = SchedulingForLatency; + SchedPreferenceInfo = Sched::Latency; JumpBufSize = 0; JumpBufAlignment = 0; IfCvtBlockSizeLimit = 2; @@ -2417,7 +2416,7 @@ std::pair<unsigned, const TargetRegisterClass*> TargetLowering:: getRegForInlineAsmConstraint(const std::string &Constraint, EVT VT) const { if (Constraint[0] != '{') - return std::pair<unsigned, const TargetRegisterClass*>(0, 0); + return std::make_pair(0u, static_cast<TargetRegisterClass*>(0)); assert(*(Constraint.end()-1) == '}' && "Not a brace enclosed constraint?"); // Remove the braces from around the name. @@ -2449,7 +2448,7 @@ getRegForInlineAsmConstraint(const std::string &Constraint, } } - return std::pair<unsigned, const TargetRegisterClass*>(0, 0); + return std::make_pair(0u, static_cast<const TargetRegisterClass*>(0)); } //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp b/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp index d20477f..a081e3c 100644 --- a/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp +++ b/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp @@ -12,9 +12,11 @@ //===----------------------------------------------------------------------===// #include "llvm/Target/TargetSelectionDAGInfo.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; -TargetSelectionDAGInfo::TargetSelectionDAGInfo() { +TargetSelectionDAGInfo::TargetSelectionDAGInfo(const TargetMachine &TM) + : TD(TM.getTargetData()) { } TargetSelectionDAGInfo::~TargetSelectionDAGInfo() { diff --git a/lib/CodeGen/SimpleRegisterCoalescing.cpp b/lib/CodeGen/SimpleRegisterCoalescing.cpp index 1f68a6f..ed3c243 100644 --- a/lib/CodeGen/SimpleRegisterCoalescing.cpp +++ b/lib/CodeGen/SimpleRegisterCoalescing.cpp @@ -460,7 +460,7 @@ bool SimpleRegisterCoalescing::RemoveCopyByCommutingDef(LiveInterval &IntA, unsigned SrcReg, DstReg, SrcSubIdx, DstSubIdx; if (!tii_->isMoveInstr(*UseMI, SrcReg, DstReg, SrcSubIdx, DstSubIdx)) continue; - if (DstReg == IntB.reg) { + if (DstReg == IntB.reg && DstSubIdx == 0) { // This copy will become a noop. If it's defining a new val#, // remove that val# as well. However this live range is being // extended to the end of the existing live range defined by the copy. @@ -624,9 +624,10 @@ SimpleRegisterCoalescing::TrimLiveIntervalToLastUse(SlotIndex CopyIdx, LR->valno->addKill(LastUseIdx.getDefIndex()); unsigned SrcReg, DstReg, SrcSubIdx, DstSubIdx; if (tii_->isMoveInstr(*LastUseMI, SrcReg, DstReg, SrcSubIdx, DstSubIdx) && - DstReg == li.reg) { + DstReg == li.reg && DstSubIdx == 0) { // Last use is itself an identity code. - int DeadIdx = LastUseMI->findRegisterDefOperandIdx(li.reg, false, tri_); + int DeadIdx = LastUseMI->findRegisterDefOperandIdx(li.reg, + false, false, tri_); LastUseMI->getOperand(DeadIdx).setIsDead(); } return true; @@ -810,6 +811,8 @@ SimpleRegisterCoalescing::UpdateRegDefsUses(unsigned SrcReg, unsigned DstReg, unsigned CopySrcReg, CopyDstReg, CopySrcSubIdx, CopyDstSubIdx; if (tii_->isMoveInstr(*UseMI, CopySrcReg, CopyDstReg, CopySrcSubIdx, CopyDstSubIdx) && + CopySrcSubIdx == 0 && + CopyDstSubIdx == 0 && CopySrcReg != CopyDstReg && CopySrcReg == SrcReg && CopyDstReg != UseDstReg) { // If the use is a copy and it won't be coalesced away, and its source @@ -835,8 +838,13 @@ SimpleRegisterCoalescing::UpdateRegDefsUses(unsigned SrcReg, unsigned DstReg, UseMI->isRegTiedToDefOperand(&O-&UseMI->getOperand(0)))) UseMI->addRegisterKilled(DstReg, tri_, true); } - DEBUG(dbgs() << "\t\tupdated: " << li_->getInstructionIndex(UseMI) - << "\t" << *UseMI); + + DEBUG({ + dbgs() << "\t\tupdated: "; + if (!UseMI->isDebugValue()) + dbgs() << li_->getInstructionIndex(UseMI) << "\t"; + dbgs() << *UseMI; + }); continue; } @@ -845,14 +853,21 @@ SimpleRegisterCoalescing::UpdateRegDefsUses(unsigned SrcReg, unsigned DstReg, // EAX: 1 -> AL, 2 -> AX // So RAX's sub-register 2 is AX, RAX's sub-regsiter 3 is EAX, whose // sub-register 2 is also AX. + // + // FIXME: Properly compose subreg indices for all targets. + // if (SubIdx && OldSubIdx && SubIdx != OldSubIdx) - assert(OldSubIdx < SubIdx && "Conflicting sub-register index!"); + ; else if (SubIdx) O.setSubReg(SubIdx); O.setReg(DstReg); - DEBUG(dbgs() << "\t\tupdated: " << li_->getInstructionIndex(UseMI) - << "\t" << *UseMI); + DEBUG({ + dbgs() << "\t\tupdated: "; + if (!UseMI->isDebugValue()) + dbgs() << li_->getInstructionIndex(UseMI) << "\t"; + dbgs() << *UseMI; + }); // After updating the operand, check if the machine instruction has // become a copy. If so, update its val# information. @@ -938,7 +953,7 @@ static void PropagateDeadness(LiveInterval &li, MachineInstr *CopyMI, MachineInstr *DefMI = li_->getInstructionFromIndex(LRStart.getDefIndex()); if (DefMI && DefMI != CopyMI) { - int DeadIdx = DefMI->findRegisterDefOperandIdx(li.reg, false); + int DeadIdx = DefMI->findRegisterDefOperandIdx(li.reg); if (DeadIdx != -1) DefMI->getOperand(DeadIdx).setIsDead(); else @@ -1255,7 +1270,12 @@ SimpleRegisterCoalescing::CanJoinExtractSubRegToPhysReg(unsigned DstReg, unsigned &RealDstReg) { const TargetRegisterClass *RC = mri_->getRegClass(SrcReg); RealDstReg = tri_->getMatchingSuperReg(DstReg, SubIdx, RC); - assert(RealDstReg && "Invalid extract_subreg instruction!"); + if (!RealDstReg) { + DEBUG(dbgs() << "\tIncompatible source regclass: " + << "none of the super-registers of " << tri_->getName(DstReg) + << " are in " << RC->getName() << ".\n"); + return false; + } LiveInterval &RHS = li_->getInterval(SrcReg); // For this type of EXTRACT_SUBREG, conservatively @@ -1293,7 +1313,12 @@ SimpleRegisterCoalescing::CanJoinInsertSubRegToPhysReg(unsigned DstReg, unsigned &RealSrcReg) { const TargetRegisterClass *RC = mri_->getRegClass(DstReg); RealSrcReg = tri_->getMatchingSuperReg(SrcReg, SubIdx, RC); - assert(RealSrcReg && "Invalid extract_subreg instruction!"); + if (!RealSrcReg) { + DEBUG(dbgs() << "\tIncompatible destination regclass: " + << "none of the super-registers of " << tri_->getName(SrcReg) + << " are in " << RC->getName() << ".\n"); + return false; + } LiveInterval &LHS = li_->getInterval(DstReg); if (li_->hasInterval(RealSrcReg) && @@ -1419,7 +1444,8 @@ bool SimpleRegisterCoalescing::JoinCopy(CopyRec &TheCopy, bool &Again) { assert(DstSubRC && "Illegal subregister index"); if (!DstSubRC->contains(SrcSubReg)) { DEBUG(dbgs() << "\tIncompatible destination regclass: " - << tri_->getName(SrcSubReg) << " not in " + << "none of the super-registers of " + << tri_->getName(SrcSubReg) << " are in " << DstSubRC->getName() << ".\n"); return false; // Not coalescable. } @@ -1436,7 +1462,8 @@ bool SimpleRegisterCoalescing::JoinCopy(CopyRec &TheCopy, bool &Again) { assert(SrcSubRC && "Illegal subregister index"); if (!SrcSubRC->contains(DstSubReg)) { DEBUG(dbgs() << "\tIncompatible source regclass: " - << tri_->getName(DstSubReg) << " not in " + << "none of the super-registers of " + << tri_->getName(DstSubReg) << " are in " << SrcSubRC->getName() << ".\n"); (void)DstSubReg; return false; // Not coalescable. @@ -2625,7 +2652,7 @@ SimpleRegisterCoalescing::lastRegisterUse(SlotIndex Start, MachineInstr *UseMI = Use.getParent(); unsigned SrcReg, DstReg, SrcSubIdx, DstSubIdx; if (tii_->isMoveInstr(*UseMI, SrcReg, DstReg, SrcSubIdx, DstSubIdx) && - SrcReg == DstReg) + SrcReg == DstReg && SrcSubIdx == DstSubIdx) // Ignore identity copies. continue; SlotIndex Idx = li_->getInstructionIndex(UseMI); @@ -2654,7 +2681,7 @@ SimpleRegisterCoalescing::lastRegisterUse(SlotIndex Start, // Ignore identity copies. unsigned SrcReg, DstReg, SrcSubIdx, DstSubIdx; if (!(tii_->isMoveInstr(*MI, SrcReg, DstReg, SrcSubIdx, DstSubIdx) && - SrcReg == DstReg)) + SrcReg == DstReg && SrcSubIdx == DstSubIdx)) for (unsigned i = 0, NumOps = MI->getNumOperands(); i != NumOps; ++i) { MachineOperand &Use = MI->getOperand(i); if (Use.isReg() && Use.isUse() && Use.getReg() && @@ -2785,7 +2812,7 @@ bool SimpleRegisterCoalescing::runOnMachineFunction(MachineFunction &fn) { // If the move will be an identity move delete it bool isMove= tii_->isMoveInstr(*MI, SrcReg, DstReg, SrcSubIdx, DstSubIdx); - if (isMove && SrcReg == DstReg) { + if (isMove && SrcReg == DstReg && SrcSubIdx == DstSubIdx) { if (li_->hasInterval(SrcReg)) { LiveInterval &RegInt = li_->getInterval(SrcReg); // If def of this move instruction is dead, remove its live range diff --git a/lib/CodeGen/Spiller.cpp b/lib/CodeGen/Spiller.cpp index 63c5554..a7b2efe 100644 --- a/lib/CodeGen/Spiller.cpp +++ b/lib/CodeGen/Spiller.cpp @@ -51,6 +51,7 @@ protected: MachineFrameInfo *mfi; MachineRegisterInfo *mri; const TargetInstrInfo *tii; + const TargetRegisterInfo *tri; VirtRegMap *vrm; /// Construct a spiller base. @@ -60,6 +61,7 @@ protected: mfi = mf->getFrameInfo(); mri = &mf->getRegInfo(); tii = mf->getTarget().getInstrInfo(); + tri = mf->getTarget().getRegisterInfo(); } /// Add spill ranges for every use/def of the live interval, inserting loads @@ -129,7 +131,8 @@ protected: // Insert reload if necessary. MachineBasicBlock::iterator miItr(mi); if (hasUse) { - tii->loadRegFromStackSlot(*mi->getParent(), miItr, newVReg, ss, trc); + tii->loadRegFromStackSlot(*mi->getParent(), miItr, newVReg, ss, trc, + tri); MachineInstr *loadInstr(prior(miItr)); SlotIndex loadIndex = lis->InsertMachineInstrInMaps(loadInstr).getDefIndex(); @@ -142,8 +145,8 @@ protected: // Insert store if necessary. if (hasDef) { - tii->storeRegToStackSlot(*mi->getParent(), llvm::next(miItr), newVReg, true, - ss, trc); + tii->storeRegToStackSlot(*mi->getParent(), llvm::next(miItr), newVReg, + true, ss, trc, tri); MachineInstr *storeInstr(llvm::next(miItr)); SlotIndex storeIndex = lis->InsertMachineInstrInMaps(storeInstr).getDefIndex(); @@ -333,7 +336,8 @@ private: // Insert a copy at the start of the MBB. The range proceeding the // copy will be attached to the original LiveInterval. MachineBasicBlock *defMBB = lis->getMBBFromIndex(newVNI->def); - tii->copyRegToReg(*defMBB, defMBB->begin(), newVReg, li->reg, trc, trc); + tii->copyRegToReg(*defMBB, defMBB->begin(), newVReg, li->reg, trc, trc, + DebugLoc()); MachineInstr *copyMI = defMBB->begin(); copyMI->addRegisterKilled(li->reg, tri); SlotIndex copyIdx = lis->InsertMachineInstrInMaps(copyMI); @@ -386,7 +390,8 @@ private: if (isTwoAddr && !twoAddrUseIsUndef) { MachineBasicBlock *defMBB = defInst->getParent(); - tii->copyRegToReg(*defMBB, defInst, newVReg, li->reg, trc, trc); + tii->copyRegToReg(*defMBB, defInst, newVReg, li->reg, trc, trc, + DebugLoc()); MachineInstr *copyMI = prior(MachineBasicBlock::iterator(defInst)); SlotIndex copyIdx = lis->InsertMachineInstrInMaps(copyMI); copyMI->addRegisterKilled(li->reg, tri); @@ -446,8 +451,9 @@ private: // reg. MachineBasicBlock *useMBB = useInst->getParent(); MachineBasicBlock::iterator useItr(useInst); - tii->copyRegToReg(*useMBB, next(useItr), li->reg, newVReg, trc, trc); - MachineInstr *copyMI = next(useItr); + tii->copyRegToReg(*useMBB, llvm::next(useItr), li->reg, newVReg, trc, trc, + DebugLoc()); + MachineInstr *copyMI = llvm::next(useItr); copyMI->addRegisterKilled(newVReg, tri); SlotIndex copyIdx = lis->InsertMachineInstrInMaps(copyMI); @@ -483,7 +489,8 @@ private: assert(oldKillRange != 0 && "No kill range?"); tii->copyRegToReg(*killMBB, killMBB->getFirstTerminator(), - li->reg, newVReg, trc, trc); + li->reg, newVReg, trc, trc, + DebugLoc()); MachineInstr *copyMI = prior(killMBB->getFirstTerminator()); copyMI->addRegisterKilled(newVReg, tri); SlotIndex copyIdx = lis->InsertMachineInstrInMaps(copyMI); diff --git a/lib/CodeGen/StackSlotColoring.cpp b/lib/CodeGen/StackSlotColoring.cpp index 42dfd7f..7f3b452 100644 --- a/lib/CodeGen/StackSlotColoring.cpp +++ b/lib/CodeGen/StackSlotColoring.cpp @@ -13,6 +13,8 @@ #define DEBUG_TYPE "stackcoloring" #include "VirtRegMap.h" +#include "llvm/Function.h" +#include "llvm/Module.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveStackAnalysis.h" @@ -116,6 +118,7 @@ namespace { private: void InitializeSlots(); + bool CheckForSetJmpCall(const MachineFunction &MF) const; void ScanForSpillSlotRefs(MachineFunction &MF); bool OverlapWithAssignments(LiveInterval *li, int Color) const; int ColorSlot(LiveInterval *li); @@ -607,7 +610,8 @@ StackSlotColoring::UnfoldAndRewriteInstruction(MachineInstr *MI, int OldFI, DEBUG(MI->dump()); ++NumLoadElim; } else { - TII->copyRegToReg(*MBB, MI, DstReg, Reg, RC, RC); + TII->copyRegToReg(*MBB, MI, DstReg, Reg, RC, RC, + MI->getDebugLoc()); ++NumRegRepl; } @@ -623,7 +627,8 @@ StackSlotColoring::UnfoldAndRewriteInstruction(MachineInstr *MI, int OldFI, DEBUG(MI->dump()); ++NumStoreElim; } else { - TII->copyRegToReg(*MBB, MI, Reg, SrcReg, RC, RC); + TII->copyRegToReg(*MBB, MI, Reg, SrcReg, RC, RC, + MI->getDebugLoc()); ++NumRegRepl; } @@ -697,7 +702,11 @@ bool StackSlotColoring::RemoveDeadStores(MachineBasicBlock* MBB) { bool StackSlotColoring::runOnMachineFunction(MachineFunction &MF) { - DEBUG(dbgs() << "********** Stack Slot Coloring **********\n"); + DEBUG({ + dbgs() << "********** Stack Slot Coloring **********\n" + << "********** Function: " + << MF.getFunction()->getName() << '\n'; + }); MFI = MF.getFrameInfo(); MRI = &MF.getRegInfo(); @@ -716,6 +725,13 @@ bool StackSlotColoring::runOnMachineFunction(MachineFunction &MF) { return false; } + // If there are calls to setjmp or sigsetjmp, don't perform stack slot + // coloring. The stack could be modified before the longjmp is executed, + // resulting in the wrong value being used afterwards. (See + // <rdar://problem/8007500>.) + if (MF.callsSetJmp()) + return false; + // Gather spill slot references ScanForSpillSlotRefs(MF); InitializeSlots(); diff --git a/lib/CodeGen/StrongPHIElimination.cpp b/lib/CodeGen/StrongPHIElimination.cpp index f8f6a55..142398c 100644 --- a/lib/CodeGen/StrongPHIElimination.cpp +++ b/lib/CodeGen/StrongPHIElimination.cpp @@ -696,7 +696,7 @@ void StrongPHIElimination::ScheduleCopies(MachineBasicBlock* MBB, // the Phi defining curr.second MachineBasicBlock::iterator PI = MRI.getVRegDef(curr.second); TII->copyRegToReg(*PI->getParent(), PI, t, - curr.second, RC, RC); + curr.second, RC, RC, DebugLoc()); DEBUG(dbgs() << "Inserted copy from " << curr.second << " to " << t << "\n"); @@ -713,7 +713,7 @@ void StrongPHIElimination::ScheduleCopies(MachineBasicBlock* MBB, // Insert copy from map[curr.first] to curr.second TII->copyRegToReg(*MBB, MBB->getFirstTerminator(), curr.second, - map[curr.first], RC, RC); + map[curr.first], RC, RC, DebugLoc()); map[curr.first] = curr.second; DEBUG(dbgs() << "Inserted copy from " << curr.first << " to " << curr.second << "\n"); @@ -762,7 +762,7 @@ void StrongPHIElimination::ScheduleCopies(MachineBasicBlock* MBB, // Insert a copy from dest to a new temporary t at the end of b unsigned t = MF->getRegInfo().createVirtualRegister(RC); TII->copyRegToReg(*MBB, MBB->getFirstTerminator(), t, - curr.second, RC, RC); + curr.second, RC, RC, DebugLoc()); map[curr.second] = t; MachineBasicBlock::iterator TI = MBB->getFirstTerminator(); @@ -961,7 +961,7 @@ bool StrongPHIElimination::runOnMachineFunction(MachineFunction &Fn) { const TargetInstrInfo *TII = Fn.getTarget().getInstrInfo(); const TargetRegisterClass *RC = Fn.getRegInfo().getRegClass(I->first); TII->copyRegToReg(*SI->second, SI->second->getFirstTerminator(), - I->first, SI->first, RC, RC); + I->first, SI->first, RC, RC, DebugLoc()); LI.renumber(); diff --git a/lib/CodeGen/TailDuplication.cpp b/lib/CodeGen/TailDuplication.cpp index aa6e2b4..f2e2a76 100644 --- a/lib/CodeGen/TailDuplication.cpp +++ b/lib/CodeGen/TailDuplication.cpp @@ -561,7 +561,7 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF, for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) { const TargetRegisterClass *RC = MRI->getRegClass(CopyInfos[i].first); TII->copyRegToReg(*PredBB, Loc, CopyInfos[i].first, - CopyInfos[i].second, RC,RC); + CopyInfos[i].second, RC,RC, DebugLoc()); MachineInstr *CopyMI = prior(Loc); Copies.push_back(CopyMI); } @@ -620,7 +620,7 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF, for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) { const TargetRegisterClass *RC = MRI->getRegClass(CopyInfos[i].first); TII->copyRegToReg(*PrevBB, Loc, CopyInfos[i].first, - CopyInfos[i].second, RC, RC); + CopyInfos[i].second, RC, RC, DebugLoc()); MachineInstr *CopyMI = prior(Loc); Copies.push_back(CopyMI); } diff --git a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 9f95993..71ad3fb 100644 --- a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -22,6 +22,7 @@ #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCSectionELF.h" +#include "llvm/MC/MCSectionCOFF.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Target/Mangler.h" #include "llvm/Target/TargetData.h" @@ -460,6 +461,26 @@ void TargetLoweringObjectFileMachO::Initialize(MCContext &Ctx, = getContext().getMachOSection("__DATA", "__data", 0, SectionKind::getDataRel()); + TLSDataSection // .tdata + = getContext().getMachOSection("__DATA", "__thread_data", + MCSectionMachO::S_THREAD_LOCAL_REGULAR, + SectionKind::getDataRel()); + TLSBSSSection // .tbss + = getContext().getMachOSection("__DATA", "__thread_bss", + MCSectionMachO::S_THREAD_LOCAL_ZEROFILL, + SectionKind::getThreadBSS()); + + // TODO: Verify datarel below. + TLSTLVSection // .tlv + = getContext().getMachOSection("__DATA", "__thread_vars", + MCSectionMachO::S_THREAD_LOCAL_VARIABLES, + SectionKind::getDataRel()); + + TLSThreadInitSection + = getContext().getMachOSection("__DATA", "__thread_init", + MCSectionMachO::S_THREAD_LOCAL_INIT_FUNCTION_POINTERS, + SectionKind::getDataRel()); + CStringSection // .cstring = getContext().getMachOSection("__TEXT", "__cstring", MCSectionMachO::S_CSTRING_LITERALS, @@ -606,6 +627,8 @@ void TargetLoweringObjectFileMachO::Initialize(MCContext &Ctx, getContext().getMachOSection("__DWARF", "__debug_inlined", MCSectionMachO::S_ATTR_DEBUG, SectionKind::getMetadata()); + + TLSExtraDataSection = TLSTLVSection; } const MCSection *TargetLoweringObjectFileMachO:: @@ -646,7 +669,10 @@ getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind, const MCSection *TargetLoweringObjectFileMachO:: SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, Mangler *Mang, const TargetMachine &TM) const { - assert(!Kind.isThreadLocal() && "Darwin doesn't support TLS"); + + // Handle thread local data. + if (Kind.isThreadBSS()) return TLSBSSSection; + if (Kind.isThreadData()) return TLSDataSection; if (Kind.isText()) return GV->isWeakForLinker() ? TextCoalSection : TextSection; @@ -794,94 +820,160 @@ unsigned TargetLoweringObjectFileMachO::getTTypeEncoding() const { // COFF //===----------------------------------------------------------------------===// -typedef StringMap<const MCSectionCOFF*> COFFUniqueMapTy; - -TargetLoweringObjectFileCOFF::~TargetLoweringObjectFileCOFF() { - delete (COFFUniqueMapTy*)UniquingMap; -} - - -const MCSection *TargetLoweringObjectFileCOFF:: -getCOFFSection(StringRef Name, bool isDirective, SectionKind Kind) const { - // Create the map if it doesn't already exist. - if (UniquingMap == 0) - UniquingMap = new COFFUniqueMapTy(); - COFFUniqueMapTy &Map = *(COFFUniqueMapTy*)UniquingMap; - - // Do the lookup, if we have a hit, return it. - const MCSectionCOFF *&Entry = Map[Name]; - if (Entry) return Entry; - - return Entry = MCSectionCOFF::Create(Name, isDirective, Kind, getContext()); -} - void TargetLoweringObjectFileCOFF::Initialize(MCContext &Ctx, const TargetMachine &TM) { - if (UniquingMap != 0) - ((COFFUniqueMapTy*)UniquingMap)->clear(); TargetLoweringObjectFile::Initialize(Ctx, TM); - TextSection = getCOFFSection("\t.text", true, SectionKind::getText()); - DataSection = getCOFFSection("\t.data", true, SectionKind::getDataRel()); + TextSection = + getContext().getCOFFSection(".text", + MCSectionCOFF::IMAGE_SCN_CNT_CODE | + MCSectionCOFF::IMAGE_SCN_MEM_EXECUTE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getText()); + DataSection = + getContext().getCOFFSection(".data", + MCSectionCOFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + MCSectionCOFF::IMAGE_SCN_MEM_READ | + MCSectionCOFF::IMAGE_SCN_MEM_WRITE, + SectionKind::getDataRel()); + ReadOnlySection = + getContext().getCOFFSection(".rdata", + MCSectionCOFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getReadOnly()); StaticCtorSection = - getCOFFSection(".ctors", false, SectionKind::getDataRel()); + getContext().getCOFFSection(".ctors", + MCSectionCOFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + MCSectionCOFF::IMAGE_SCN_MEM_READ | + MCSectionCOFF::IMAGE_SCN_MEM_WRITE, + SectionKind::getDataRel()); StaticDtorSection = - getCOFFSection(".dtors", false, SectionKind::getDataRel()); + getContext().getCOFFSection(".dtors", + MCSectionCOFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + MCSectionCOFF::IMAGE_SCN_MEM_READ | + MCSectionCOFF::IMAGE_SCN_MEM_WRITE, + SectionKind::getDataRel()); // FIXME: We're emitting LSDA info into a readonly section on COFF, even // though it contains relocatable pointers. In PIC mode, this is probably a // big runtime hit for C++ apps. Either the contents of the LSDA need to be // adjusted or this should be a data section. LSDASection = - getCOFFSection(".gcc_except_table", false, SectionKind::getReadOnly()); + getContext().getCOFFSection(".gcc_except_table", + MCSectionCOFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getReadOnly()); EHFrameSection = - getCOFFSection(".eh_frame", false, SectionKind::getDataRel()); + getContext().getCOFFSection(".eh_frame", + MCSectionCOFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + MCSectionCOFF::IMAGE_SCN_MEM_READ | + MCSectionCOFF::IMAGE_SCN_MEM_WRITE, + SectionKind::getDataRel()); // Debug info. - // FIXME: Don't use 'directive' mode here. DwarfAbbrevSection = - getCOFFSection("\t.section\t.debug_abbrev,\"dr\"", - true, SectionKind::getMetadata()); + getContext().getCOFFSection(".debug_abbrev", + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); DwarfInfoSection = - getCOFFSection("\t.section\t.debug_info,\"dr\"", - true, SectionKind::getMetadata()); + getContext().getCOFFSection(".debug_info", + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); DwarfLineSection = - getCOFFSection("\t.section\t.debug_line,\"dr\"", - true, SectionKind::getMetadata()); + getContext().getCOFFSection(".debug_line", + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); DwarfFrameSection = - getCOFFSection("\t.section\t.debug_frame,\"dr\"", - true, SectionKind::getMetadata()); + getContext().getCOFFSection(".debug_frame", + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); DwarfPubNamesSection = - getCOFFSection("\t.section\t.debug_pubnames,\"dr\"", - true, SectionKind::getMetadata()); + getContext().getCOFFSection(".debug_pubnames", + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); DwarfPubTypesSection = - getCOFFSection("\t.section\t.debug_pubtypes,\"dr\"", - true, SectionKind::getMetadata()); + getContext().getCOFFSection(".debug_pubtypes", + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); DwarfStrSection = - getCOFFSection("\t.section\t.debug_str,\"dr\"", - true, SectionKind::getMetadata()); + getContext().getCOFFSection(".debug_str", + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); DwarfLocSection = - getCOFFSection("\t.section\t.debug_loc,\"dr\"", - true, SectionKind::getMetadata()); + getContext().getCOFFSection(".debug_loc", + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); DwarfARangesSection = - getCOFFSection("\t.section\t.debug_aranges,\"dr\"", - true, SectionKind::getMetadata()); + getContext().getCOFFSection(".debug_aranges", + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); DwarfRangesSection = - getCOFFSection("\t.section\t.debug_ranges,\"dr\"", - true, SectionKind::getMetadata()); + getContext().getCOFFSection(".debug_ranges", + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); DwarfMacroInfoSection = - getCOFFSection("\t.section\t.debug_macinfo,\"dr\"", - true, SectionKind::getMetadata()); + getContext().getCOFFSection(".debug_macinfo", + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE | + MCSectionCOFF::IMAGE_SCN_MEM_READ, + SectionKind::getMetadata()); + + DrectveSection = + getContext().getCOFFSection(".drectve", + MCSectionCOFF::IMAGE_SCN_LNK_INFO, + SectionKind::getMetadata()); +} + +static unsigned +getCOFFSectionFlags(SectionKind K) { + unsigned Flags = 0; + + if (!K.isMetadata()) + Flags |= + MCSectionCOFF::IMAGE_SCN_MEM_DISCARDABLE; + else if (K.isText()) + Flags |= + MCSectionCOFF::IMAGE_SCN_MEM_EXECUTE | + MCSectionCOFF::IMAGE_SCN_CNT_CODE; + else if (K.isBSS ()) + Flags |= + MCSectionCOFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA | + MCSectionCOFF::IMAGE_SCN_MEM_READ | + MCSectionCOFF::IMAGE_SCN_MEM_WRITE; + else if (K.isReadOnly()) + Flags |= + MCSectionCOFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + MCSectionCOFF::IMAGE_SCN_MEM_READ; + else if (K.isWriteable()) + Flags |= + MCSectionCOFF::IMAGE_SCN_CNT_INITIALIZED_DATA | + MCSectionCOFF::IMAGE_SCN_MEM_READ | + MCSectionCOFF::IMAGE_SCN_MEM_WRITE; + + return Flags; } const MCSection *TargetLoweringObjectFileCOFF:: getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind, Mangler *Mang, const TargetMachine &TM) const { - return getCOFFSection(GV->getSection(), false, Kind); + return getContext().getCOFFSection(GV->getSection(), + getCOFFSectionFlags(Kind), + Kind); } static const char *getCOFFSectionPrefixForUniqueGlobal(SectionKind Kind) { if (Kind.isText()) return ".text$linkonce"; + if (Kind.isBSS ()) + return ".bss$linkonce"; if (Kind.isWriteable()) return ".data$linkonce"; return ".rdata$linkonce"; @@ -900,7 +992,13 @@ SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind, SmallString<128> Name(Prefix, Prefix+strlen(Prefix)); MCSymbol *Sym = Mang->getSymbol(GV); Name.append(Sym->getName().begin(), Sym->getName().end()); - return getCOFFSection(Name.str(), false, Kind); + + unsigned Characteristics = getCOFFSectionFlags(Kind); + + Characteristics |= MCSectionCOFF::IMAGE_SCN_LNK_COMDAT; + + return getContext().getCOFFSection(Name.str(), Characteristics, + MCSectionCOFF::IMAGE_COMDAT_SELECT_EXACT_MATCH, Kind); } if (Kind.isText()) diff --git a/lib/CodeGen/TwoAddressInstructionPass.cpp b/lib/CodeGen/TwoAddressInstructionPass.cpp index c288ae0..3d10dc1 100644 --- a/lib/CodeGen/TwoAddressInstructionPass.cpp +++ b/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -40,6 +40,7 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallSet.h" @@ -77,6 +78,10 @@ namespace { // registers from virtual registers. e.g. r1 = move v1024. DenseMap<unsigned, unsigned> DstRegMap; + /// RegSequences - Keep track the list of REG_SEQUENCE instructions seen + /// during the initial walk of the machine function. + SmallVector<MachineInstr*, 16> RegSequences; + bool Sink3AddrInstruction(MachineBasicBlock *MBB, MachineInstr *MI, unsigned Reg, MachineBasicBlock::iterator OldPos); @@ -123,6 +128,13 @@ namespace { void ProcessCopy(MachineInstr *MI, MachineBasicBlock *MBB, SmallPtrSet<MachineInstr*, 8> &Processed); + void CoalesceExtSubRegs(SmallVector<unsigned,4> &Srcs, unsigned DstReg); + + /// EliminateRegSequences - Eliminate REG_SEQUENCE instructions as part + /// of the de-ssa process. This replaces sources of REG_SEQUENCE as + /// sub-register references of the register defined by REG_SEQUENCE. + bool EliminateRegSequences(); + public: static char ID; // Pass identification, replacement for typeid TwoAddressInstructionPass() : MachineFunctionPass(&ID) {} @@ -768,7 +780,7 @@ canUpdateDeletedKills(SmallVector<unsigned, 4> &Kills, if (!LastKill) return false; - bool isModRef = LastKill->modifiesRegister(Kill); + bool isModRef = LastKill->definesRegister(Kill); NewKills.push_back(std::make_pair(std::make_pair(Kill, isModRef), LastKill)); } @@ -929,6 +941,10 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &MF) { continue; } + // Remember REG_SEQUENCE instructions, we'll deal with them later. + if (mi->isRegSequence()) + RegSequences.push_back(&*mi); + const TargetInstrDesc &TID = mi->getDesc(); bool FirstTied = true; @@ -1035,7 +1051,8 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &MF) { ReMatRegs.set(regB); ++NumReMats; } else { - bool Emitted = TII->copyRegToReg(*mbbi, mi, regA, regB, rc, rc); + bool Emitted = TII->copyRegToReg(*mbbi, mi, regA, regB, rc, rc, + mi->getDebugLoc()); (void)Emitted; assert(Emitted && "Unable to issue a copy instruction!\n"); } @@ -1110,5 +1127,211 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &MF) { VReg = ReMatRegs.find_next(VReg); } + // Eliminate REG_SEQUENCE instructions. Their whole purpose was to preseve + // SSA form. It's now safe to de-SSA. + MadeChange |= EliminateRegSequences(); + return MadeChange; } + +static void UpdateRegSequenceSrcs(unsigned SrcReg, + unsigned DstReg, unsigned SubIdx, + MachineRegisterInfo *MRI) { + for (MachineRegisterInfo::reg_iterator RI = MRI->reg_begin(SrcReg), + RE = MRI->reg_end(); RI != RE; ) { + MachineOperand &MO = RI.getOperand(); + ++RI; + MO.setReg(DstReg); + assert(MO.getSubReg() == 0); + MO.setSubReg(SubIdx); + } +} + +/// CoalesceExtSubRegs - If a number of sources of the REG_SEQUENCE are +/// EXTRACT_SUBREG from the same register and to the same virtual register +/// with different sub-register indices, attempt to combine the +/// EXTRACT_SUBREGs and pre-coalesce them. e.g. +/// %reg1026<def> = VLDMQ %reg1025<kill>, 260, pred:14, pred:%reg0 +/// %reg1029:6<def> = EXTRACT_SUBREG %reg1026, 6 +/// %reg1029:5<def> = EXTRACT_SUBREG %reg1026<kill>, 5 +/// Since D subregs 5, 6 can combine to a Q register, we can coalesce +/// reg1026 to reg1029. +void +TwoAddressInstructionPass::CoalesceExtSubRegs(SmallVector<unsigned,4> &Srcs, + unsigned DstReg) { + SmallSet<unsigned, 4> Seen; + for (unsigned i = 0, e = Srcs.size(); i != e; ++i) { + unsigned SrcReg = Srcs[i]; + if (!Seen.insert(SrcReg)) + continue; + + // If there are no other uses than extract_subreg which feed into + // the reg_sequence, then we might be able to coalesce them. + bool CanCoalesce = true; + SmallVector<unsigned, 4> SubIndices; + for (MachineRegisterInfo::use_nodbg_iterator + UI = MRI->use_nodbg_begin(SrcReg), + UE = MRI->use_nodbg_end(); UI != UE; ++UI) { + MachineInstr *UseMI = &*UI; + if (!UseMI->isExtractSubreg() || + UseMI->getOperand(0).getReg() != DstReg) { + CanCoalesce = false; + break; + } + SubIndices.push_back(UseMI->getOperand(2).getImm()); + } + + if (!CanCoalesce || SubIndices.size() < 2) + continue; + + std::sort(SubIndices.begin(), SubIndices.end()); + unsigned NewSubIdx = 0; + if (TRI->canCombinedSubRegIndex(MRI->getRegClass(SrcReg), SubIndices, + NewSubIdx)) { + bool Proceed = true; + if (NewSubIdx) + for (MachineRegisterInfo::reg_iterator RI = MRI->reg_begin(SrcReg), + RE = MRI->reg_end(); RI != RE; ) { + MachineOperand &MO = RI.getOperand(); + ++RI; + // FIXME: If the sub-registers do not combine to the whole + // super-register, i.e. NewSubIdx != 0, and any of the use has a + // sub-register index, then abort the coalescing attempt. + if (MO.getSubReg()) { + Proceed = false; + break; + } + MO.setReg(DstReg); + MO.setSubReg(NewSubIdx); + } + if (Proceed) + for (MachineRegisterInfo::reg_iterator RI = MRI->reg_begin(SrcReg), + RE = MRI->reg_end(); RI != RE; ) { + MachineOperand &MO = RI.getOperand(); + ++RI; + MO.setReg(DstReg); + if (NewSubIdx) + MO.setSubReg(NewSubIdx); + } + } + } +} + +static bool HasOtherRegSequenceUses(unsigned Reg, MachineInstr *RegSeq, + MachineRegisterInfo *MRI) { + for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(Reg), + UE = MRI->use_end(); UI != UE; ++UI) { + MachineInstr *UseMI = &*UI; + if (UseMI != RegSeq && UseMI->isRegSequence()) + return true; + } + return false; +} + +/// EliminateRegSequences - Eliminate REG_SEQUENCE instructions as part +/// of the de-ssa process. This replaces sources of REG_SEQUENCE as +/// sub-register references of the register defined by REG_SEQUENCE. e.g. +/// +/// %reg1029<def>, %reg1030<def> = VLD1q16 %reg1024<kill>, ... +/// %reg1031<def> = REG_SEQUENCE %reg1029<kill>, 5, %reg1030<kill>, 6 +/// => +/// %reg1031:5<def>, %reg1031:6<def> = VLD1q16 %reg1024<kill>, ... +bool TwoAddressInstructionPass::EliminateRegSequences() { + if (RegSequences.empty()) + return false; + + for (unsigned i = 0, e = RegSequences.size(); i != e; ++i) { + MachineInstr *MI = RegSequences[i]; + unsigned DstReg = MI->getOperand(0).getReg(); + if (MI->getOperand(0).getSubReg() || + TargetRegisterInfo::isPhysicalRegister(DstReg) || + !(MI->getNumOperands() & 1)) { + DEBUG(dbgs() << "Illegal REG_SEQUENCE instruction:" << *MI); + llvm_unreachable(0); + } + + bool IsImpDef = true; + SmallVector<unsigned, 4> RealSrcs; + SmallSet<unsigned, 4> Seen; + for (unsigned i = 1, e = MI->getNumOperands(); i < e; i += 2) { + unsigned SrcReg = MI->getOperand(i).getReg(); + if (MI->getOperand(i).getSubReg() || + TargetRegisterInfo::isPhysicalRegister(SrcReg)) { + DEBUG(dbgs() << "Illegal REG_SEQUENCE instruction:" << *MI); + llvm_unreachable(0); + } + + MachineInstr *DefMI = MRI->getVRegDef(SrcReg); + if (DefMI->isImplicitDef()) { + DefMI->eraseFromParent(); + continue; + } + IsImpDef = false; + + // Remember EXTRACT_SUBREG sources. These might be candidate for + // coalescing. + if (DefMI->isExtractSubreg()) + RealSrcs.push_back(DefMI->getOperand(1).getReg()); + + if (!Seen.insert(SrcReg) || + MI->getParent() != DefMI->getParent() || + !MI->getOperand(i).isKill() || + HasOtherRegSequenceUses(SrcReg, MI, MRI)) { + // REG_SEQUENCE cannot have duplicated operands, add a copy. + // Also add an copy if the source is live-in the block. We don't want + // to end up with a partial-redef of a livein, e.g. + // BB0: + // reg1051:10<def> = + // ... + // BB1: + // ... = reg1051:10 + // BB2: + // reg1051:9<def> = + // LiveIntervalAnalysis won't like it. + // + // If the REG_SEQUENCE doesn't kill its source, keeping live variables + // correctly up to date becomes very difficult. Insert a copy. + // + const TargetRegisterClass *RC = MRI->getRegClass(SrcReg); + unsigned NewReg = MRI->createVirtualRegister(RC); + MachineBasicBlock::iterator InsertLoc = MI; + bool Emitted = + TII->copyRegToReg(*MI->getParent(), InsertLoc, NewReg, SrcReg, RC, RC, + MI->getDebugLoc()); + (void)Emitted; + assert(Emitted && "Unable to issue a copy instruction!\n"); + MI->getOperand(i).setReg(NewReg); + if (MI->getOperand(i).isKill()) { + MachineBasicBlock::iterator CopyMI = prior(InsertLoc); + MachineOperand *KillMO = CopyMI->findRegisterUseOperand(SrcReg); + KillMO->setIsKill(); + if (LV) + // Update live variables + LV->replaceKillInstruction(SrcReg, MI, &*CopyMI); + } + } + } + + for (unsigned i = 1, e = MI->getNumOperands(); i < e; i += 2) { + unsigned SrcReg = MI->getOperand(i).getReg(); + unsigned SubIdx = MI->getOperand(i+1).getImm(); + UpdateRegSequenceSrcs(SrcReg, DstReg, SubIdx, MRI); + } + + if (IsImpDef) { + DEBUG(dbgs() << "Turned: " << *MI << " into an IMPLICIT_DEF"); + MI->setDesc(TII->get(TargetOpcode::IMPLICIT_DEF)); + for (int j = MI->getNumOperands() - 1, ee = 0; j > ee; --j) + MI->RemoveOperand(j); + } else { + DEBUG(dbgs() << "Eliminated: " << *MI); + MI->eraseFromParent(); + } + + // Try coalescing some EXTRACT_SUBREG instructions. + CoalesceExtSubRegs(RealSrcs, DstReg); + } + + RegSequences.clear(); + return true; +} diff --git a/lib/CodeGen/VirtRegRewriter.cpp b/lib/CodeGen/VirtRegRewriter.cpp index 7f0412c..871d836 100644 --- a/lib/CodeGen/VirtRegRewriter.cpp +++ b/lib/CodeGen/VirtRegRewriter.cpp @@ -907,7 +907,7 @@ unsigned ReuseInfo::GetRegForReload(const TargetRegisterClass *RC, TRI, VRM); } else { TII->loadRegFromStackSlot(*MBB, InsertLoc, NewPhysReg, - NewOp.StackSlotOrReMat, AliasRC); + NewOp.StackSlotOrReMat, AliasRC, TRI); MachineInstr *LoadMI = prior(InsertLoc); VRM.addSpillSlotUse(NewOp.StackSlotOrReMat, LoadMI); // Any stores to this stack slot are not dead anymore. @@ -1265,7 +1265,7 @@ OptimizeByUnfold2(unsigned VirtReg, int SS, ComputeReloadLoc(MII, MBB->begin(), PhysReg, TRI, false, SS, TII, MF); // Load from SS to the spare physical register. - TII->loadRegFromStackSlot(*MBB, MII, PhysReg, SS, RC); + TII->loadRegFromStackSlot(*MBB, MII, PhysReg, SS, RC, TRI); // This invalidates Phys. Spills.ClobberPhysReg(PhysReg); // Remember it's available. @@ -1308,7 +1308,7 @@ OptimizeByUnfold2(unsigned VirtReg, int SS, } while (FoldsStackSlotModRef(*NextMII, SS, PhysReg, TII, TRI, *VRM)); // Store the value back into SS. - TII->storeRegToStackSlot(*MBB, NextMII, PhysReg, true, SS, RC); + TII->storeRegToStackSlot(*MBB, NextMII, PhysReg, true, SS, RC, TRI); MachineInstr *StoreMI = prior(NextMII); VRM->addSpillSlotUse(SS, StoreMI); VRM->virtFolded(VirtReg, StoreMI, VirtRegMap::isMod); @@ -1523,7 +1523,7 @@ CommuteToFoldReload(MachineBasicBlock::iterator &MII, VRM->virtFolded(VirtReg, FoldedMI, VirtRegMap::isRef); // Insert new def MI and spill MI. const TargetRegisterClass* RC = MRI->getRegClass(VirtReg); - TII->storeRegToStackSlot(*MBB, &MI, NewReg, true, SS, RC); + TII->storeRegToStackSlot(*MBB, &MI, NewReg, true, SS, RC, TRI); MII = prior(MII); MachineInstr *StoreMI = MII; VRM->addSpillSlotUse(SS, StoreMI); @@ -1566,7 +1566,8 @@ SpillRegToStackSlot(MachineBasicBlock::iterator &MII, std::vector<MachineOperand*> &KillOps) { MachineBasicBlock::iterator oldNextMII = llvm::next(MII); - TII->storeRegToStackSlot(*MBB, llvm::next(MII), PhysReg, true, StackSlot, RC); + TII->storeRegToStackSlot(*MBB, llvm::next(MII), PhysReg, true, StackSlot, RC, + TRI); MachineInstr *StoreMI = prior(oldNextMII); VRM->addSpillSlotUse(StackSlot, StoreMI); DEBUG(dbgs() << "Store:\t" << *StoreMI); @@ -1709,7 +1710,7 @@ bool LocalRewriter::InsertEmergencySpills(MachineInstr *MI) { if (UsedSS.count(SS)) llvm_unreachable("Need to spill more than one physical registers!"); UsedSS.insert(SS); - TII->storeRegToStackSlot(*MBB, MII, PhysReg, true, SS, RC); + TII->storeRegToStackSlot(*MBB, MII, PhysReg, true, SS, RC, TRI); MachineInstr *StoreMI = prior(MII); VRM->addSpillSlotUse(SS, StoreMI); @@ -1718,7 +1719,7 @@ bool LocalRewriter::InsertEmergencySpills(MachineInstr *MI) { ComputeReloadLoc(llvm::next(MII), MBB->begin(), PhysReg, TRI, false, SS, TII, *MBB->getParent()); - TII->loadRegFromStackSlot(*MBB, InsertLoc, PhysReg, SS, RC); + TII->loadRegFromStackSlot(*MBB, InsertLoc, PhysReg, SS, RC, TRI); MachineInstr *LoadMI = prior(InsertLoc); VRM->addSpillSlotUse(SS, LoadMI); @@ -1793,7 +1794,8 @@ bool LocalRewriter::InsertRestores(MachineInstr *MI, ComputeReloadLoc(MII, MBB->begin(), Phys, TRI, DoReMat, SSorRMId, TII, *MBB->getParent()); - TII->copyRegToReg(*MBB, InsertLoc, Phys, InReg, RC, RC); + TII->copyRegToReg(*MBB, InsertLoc, Phys, InReg, RC, RC, + MI->getDebugLoc()); // This invalidates Phys. Spills.ClobberPhysReg(Phys); @@ -1821,7 +1823,7 @@ bool LocalRewriter::InsertRestores(MachineInstr *MI, ReMaterialize(*MBB, InsertLoc, Phys, VirtReg, TII, TRI, *VRM); } else { const TargetRegisterClass* RC = MRI->getRegClass(VirtReg); - TII->loadRegFromStackSlot(*MBB, InsertLoc, Phys, SSorRMId, RC); + TII->loadRegFromStackSlot(*MBB, InsertLoc, Phys, SSorRMId, RC, TRI); MachineInstr *LoadMI = prior(InsertLoc); VRM->addSpillSlotUse(SSorRMId, LoadMI); ++NumLoads; @@ -1857,7 +1859,7 @@ bool LocalRewriter::InsertSpills(MachineInstr *MI) { int StackSlot = VRM->getStackSlot(VirtReg); MachineBasicBlock::iterator oldNextMII = llvm::next(MII); TII->storeRegToStackSlot(*MBB, llvm::next(MII), Phys, isKill, StackSlot, - RC); + RC, TRI); MachineInstr *StoreMI = prior(oldNextMII); VRM->addSpillSlotUse(StackSlot, StoreMI); DEBUG(dbgs() << "Store:\t" << *StoreMI); @@ -1893,6 +1895,11 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, // Clear kill info. SmallSet<unsigned, 2> KilledMIRegs; + + // Keep track of the registers we have already spilled in case there are + // multiple defs of the same register in MI. + SmallSet<unsigned, 8> SpilledMIRegs; + RegKills.reset(); KillOps.clear(); KillOps.resize(TRI->getNumRegs(), NULL); @@ -2138,7 +2145,8 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, ComputeReloadLoc(&MI, MBB->begin(), PhysReg, TRI, DoReMat, SSorRMId, TII, MF); - TII->copyRegToReg(*MBB, InsertLoc, DesignatedReg, PhysReg, RC, RC); + TII->copyRegToReg(*MBB, InsertLoc, DesignatedReg, PhysReg, RC, RC, + MI.getDebugLoc()); MachineInstr *CopyMI = prior(InsertLoc); CopyMI->setAsmPrinterFlag(MachineInstr::ReloadReuse); @@ -2183,7 +2191,7 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, ReMaterialize(*MBB, InsertLoc, PhysReg, VirtReg, TII, TRI, *VRM); } else { const TargetRegisterClass* RC = MRI->getRegClass(VirtReg); - TII->loadRegFromStackSlot(*MBB, InsertLoc, PhysReg, SSorRMId, RC); + TII->loadRegFromStackSlot(*MBB, InsertLoc, PhysReg, SSorRMId, RC,TRI); MachineInstr *LoadMI = prior(InsertLoc); VRM->addSpillSlotUse(SSorRMId, LoadMI); ++NumLoads; @@ -2262,7 +2270,8 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, DEBUG(dbgs() << "Promoted Load To Copy: " << MI); if (DestReg != InReg) { const TargetRegisterClass *RC = MRI->getRegClass(VirtReg); - TII->copyRegToReg(*MBB, &MI, DestReg, InReg, RC, RC); + TII->copyRegToReg(*MBB, &MI, DestReg, InReg, RC, RC, + MI.getDebugLoc()); MachineOperand *DefMO = MI.findRegisterDefOperand(DestReg); unsigned SubIdx = DefMO->getSubReg(); // Revisit the copy so we make sure to notice the effects of the @@ -2408,6 +2417,7 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, } // Process all of the spilled defs. + SpilledMIRegs.clear(); for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { MachineOperand &MO = MI.getOperand(i); if (!(MO.isReg() && MO.getReg() && MO.isDef())) @@ -2421,7 +2431,8 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, // eliminate this or else the undef marker is lost and it will // confuses the scavenger. This is extremely rare. unsigned Src, Dst, SrcSR, DstSR; - if (TII->isMoveInstr(MI, Src, Dst, SrcSR, DstSR) && Src == Dst && + if (TII->isMoveInstr(MI, Src, Dst, SrcSR, DstSR) && + Src == Dst && SrcSR == DstSR && !MI.findRegisterUseOperand(Src)->isUndef()) { ++NumDCE; DEBUG(dbgs() << "Removing now-noop copy: " << MI); @@ -2500,7 +2511,7 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, MI.getOperand(i).setReg(RReg); MI.getOperand(i).setSubReg(0); - if (!MO.isDead()) { + if (!MO.isDead() && SpilledMIRegs.insert(VirtReg)) { MachineInstr *&LastStore = MaybeDeadStores[StackSlot]; SpillRegToStackSlot(MII, -1, PhysReg, StackSlot, RC, true, LastStore, Spills, ReMatDefs, RegKills, KillOps); @@ -2510,7 +2521,8 @@ LocalRewriter::RewriteMBB(LiveIntervals *LIs, // instruction before considering the dest reg to be changed. { unsigned Src, Dst, SrcSR, DstSR; - if (TII->isMoveInstr(MI, Src, Dst, SrcSR, DstSR) && Src == Dst) { + if (TII->isMoveInstr(MI, Src, Dst, SrcSR, DstSR) && + Src == Dst && SrcSR == DstSR) { ++NumDCE; DEBUG(dbgs() << "Removing now-noop copy: " << MI); InvalidateKills(MI, TRI, RegKills, KillOps); |
