diff options
Diffstat (limited to 'lib/CodeGen')
46 files changed, 2172 insertions, 936 deletions
diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/lib/CodeGen/AggressiveAntiDepBreaker.cpp index bb61682..761fbc6 100644 --- a/lib/CodeGen/AggressiveAntiDepBreaker.cpp +++ b/lib/CodeGen/AggressiveAntiDepBreaker.cpp @@ -127,11 +127,11 @@ AggressiveAntiDepBreaker(MachineFunction& MFi, CriticalPathSet |= CPSet; } - DEBUG(errs() << "AntiDep Critical-Path Registers:"); + DEBUG(dbgs() << "AntiDep Critical-Path Registers:"); DEBUG(for (int r = CriticalPathSet.find_first(); r != -1; r = CriticalPathSet.find_next(r)) - errs() << " " << TRI->getName(r)); - DEBUG(errs() << '\n'); + dbgs() << " " << TRI->getName(r)); + DEBUG(dbgs() << '\n'); } AggressiveAntiDepBreaker::~AggressiveAntiDepBreaker() { @@ -218,9 +218,9 @@ void AggressiveAntiDepBreaker::Observe(MachineInstr *MI, unsigned Count, PrescanInstruction(MI, Count, PassthruRegs); ScanInstruction(MI, Count); - DEBUG(errs() << "Observe: "); + DEBUG(dbgs() << "Observe: "); DEBUG(MI->dump()); - DEBUG(errs() << "\tRegs:"); + DEBUG(dbgs() << "\tRegs:"); unsigned *DefIndices = State->GetDefIndices(); for (unsigned Reg = 0; Reg != TRI->getNumRegs(); ++Reg) { @@ -232,14 +232,14 @@ void AggressiveAntiDepBreaker::Observe(MachineInstr *MI, unsigned Count, // schedule region). if (State->IsLive(Reg)) { DEBUG(if (State->GetGroup(Reg) != 0) - errs() << " " << TRI->getName(Reg) << "=g" << + dbgs() << " " << TRI->getName(Reg) << "=g" << State->GetGroup(Reg) << "->g0(region live-out)"); State->UnionGroups(Reg, 0); } else if ((DefIndices[Reg] < InsertPosIndex) && (DefIndices[Reg] >= Count)) { DefIndices[Reg] = Count; } } - DEBUG(errs() << '\n'); + DEBUG(dbgs() << '\n'); } bool AggressiveAntiDepBreaker::IsImplicitDefUse(MachineInstr *MI, @@ -333,8 +333,8 @@ void AggressiveAntiDepBreaker::HandleLastUse(unsigned Reg, unsigned KillIdx, RegRefs.erase(Reg); State->LeaveGroup(Reg); DEBUG(if (header != NULL) { - errs() << header << TRI->getName(Reg); header = NULL; }); - DEBUG(errs() << "->g" << State->GetGroup(Reg) << tag); + dbgs() << header << TRI->getName(Reg); header = NULL; }); + DEBUG(dbgs() << "->g" << State->GetGroup(Reg) << tag); } // Repeat for subregisters. for (const unsigned *Subreg = TRI->getSubRegisters(Reg); @@ -346,13 +346,13 @@ void AggressiveAntiDepBreaker::HandleLastUse(unsigned Reg, unsigned KillIdx, RegRefs.erase(SubregReg); State->LeaveGroup(SubregReg); DEBUG(if (header != NULL) { - errs() << header << TRI->getName(Reg); header = NULL; }); - DEBUG(errs() << " " << TRI->getName(SubregReg) << "->g" << + dbgs() << header << TRI->getName(Reg); header = NULL; }); + DEBUG(dbgs() << " " << TRI->getName(SubregReg) << "->g" << State->GetGroup(SubregReg) << tag); } } - DEBUG(if ((header == NULL) && (footer != NULL)) errs() << footer); + DEBUG(if ((header == NULL) && (footer != NULL)) dbgs() << footer); } void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, unsigned Count, @@ -375,20 +375,20 @@ void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, unsigned Cou HandleLastUse(Reg, Count + 1, "", "\tDead Def: ", "\n"); } - DEBUG(errs() << "\tDef Groups:"); + DEBUG(dbgs() << "\tDef Groups:"); for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); if (!MO.isReg() || !MO.isDef()) continue; unsigned Reg = MO.getReg(); if (Reg == 0) continue; - DEBUG(errs() << " " << TRI->getName(Reg) << "=g" << State->GetGroup(Reg)); + DEBUG(dbgs() << " " << TRI->getName(Reg) << "=g" << State->GetGroup(Reg)); // If MI's defs have a special allocation requirement, don't allow // any def registers to be changed. Also assume all registers // defined in a call must not be changed (ABI). if (MI->getDesc().isCall() || MI->getDesc().hasExtraDefRegAllocReq()) { - DEBUG(if (State->GetGroup(Reg) != 0) errs() << "->g0(alloc-req)"); + DEBUG(if (State->GetGroup(Reg) != 0) dbgs() << "->g0(alloc-req)"); State->UnionGroups(Reg, 0); } @@ -398,7 +398,7 @@ void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, unsigned Cou unsigned AliasReg = *Alias; if (State->IsLive(AliasReg)) { State->UnionGroups(Reg, AliasReg); - DEBUG(errs() << "->g" << State->GetGroup(Reg) << "(via " << + DEBUG(dbgs() << "->g" << State->GetGroup(Reg) << "(via " << TRI->getName(AliasReg) << ")"); } } @@ -411,7 +411,7 @@ void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, unsigned Cou RegRefs.insert(std::make_pair(Reg, RR)); } - DEBUG(errs() << '\n'); + DEBUG(dbgs() << '\n'); // Scan the register defs for this instruction and update // live-ranges. @@ -437,7 +437,7 @@ void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, unsigned Cou void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI, unsigned Count) { - DEBUG(errs() << "\tUse Groups:"); + DEBUG(dbgs() << "\tUse Groups:"); std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>& RegRefs = State->GetRegRefs(); @@ -449,7 +449,7 @@ void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI, unsigned Reg = MO.getReg(); if (Reg == 0) continue; - DEBUG(errs() << " " << TRI->getName(Reg) << "=g" << + DEBUG(dbgs() << " " << TRI->getName(Reg) << "=g" << State->GetGroup(Reg)); // It wasn't previously live but now it is, this is a kill. Forget @@ -461,7 +461,7 @@ void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI, // any use registers to be changed. Also assume all registers // used in a call must not be changed (ABI). if (MI->getDesc().isCall() || MI->getDesc().hasExtraSrcRegAllocReq()) { - DEBUG(if (State->GetGroup(Reg) != 0) errs() << "->g0(alloc-req)"); + DEBUG(if (State->GetGroup(Reg) != 0) dbgs() << "->g0(alloc-req)"); State->UnionGroups(Reg, 0); } @@ -473,12 +473,12 @@ void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI, RegRefs.insert(std::make_pair(Reg, RR)); } - DEBUG(errs() << '\n'); + DEBUG(dbgs() << '\n'); // Form a group of all defs and uses of a KILL instruction to ensure // that all registers are renamed as a group. if (MI->getOpcode() == TargetInstrInfo::KILL) { - DEBUG(errs() << "\tKill Group:"); + DEBUG(dbgs() << "\tKill Group:"); unsigned FirstReg = 0; for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { @@ -488,15 +488,15 @@ void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI, if (Reg == 0) continue; if (FirstReg != 0) { - DEBUG(errs() << "=" << TRI->getName(Reg)); + DEBUG(dbgs() << "=" << TRI->getName(Reg)); State->UnionGroups(FirstReg, Reg); } else { - DEBUG(errs() << " " << TRI->getName(Reg)); + DEBUG(dbgs() << " " << TRI->getName(Reg)); FirstReg = Reg; } } - DEBUG(errs() << "->g" << State->GetGroup(FirstReg) << '\n'); + DEBUG(dbgs() << "->g" << State->GetGroup(FirstReg) << '\n'); } } @@ -525,7 +525,7 @@ BitVector AggressiveAntiDepBreaker::GetRenameRegisters(unsigned Reg) { BV &= RCBV; } - DEBUG(errs() << " " << RC->getName()); + DEBUG(dbgs() << " " << RC->getName()); } return BV; @@ -552,7 +552,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters( // Find the "superest" register in the group. At the same time, // collect the BitVector of registers that can be used to rename // each register. - DEBUG(errs() << "\tRename Candidates for Group g" << AntiDepGroupIndex << ":\n"); + DEBUG(dbgs() << "\tRename Candidates for Group g" << AntiDepGroupIndex << ":\n"); std::map<unsigned, BitVector> RenameRegisterMap; unsigned SuperReg = 0; for (unsigned i = 0, e = Regs.size(); i != e; ++i) { @@ -562,15 +562,15 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters( // If Reg has any references, then collect possible rename regs if (RegRefs.count(Reg) > 0) { - DEBUG(errs() << "\t\t" << TRI->getName(Reg) << ":"); + DEBUG(dbgs() << "\t\t" << TRI->getName(Reg) << ":"); BitVector BV = GetRenameRegisters(Reg); RenameRegisterMap.insert(std::pair<unsigned, BitVector>(Reg, BV)); - DEBUG(errs() << " ::"); + DEBUG(dbgs() << " ::"); DEBUG(for (int r = BV.find_first(); r != -1; r = BV.find_next(r)) - errs() << " " << TRI->getName(r)); - DEBUG(errs() << "\n"); + dbgs() << " " << TRI->getName(r)); + DEBUG(dbgs() << "\n"); } } @@ -591,7 +591,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters( if (renamecnt++ % DebugDiv != DebugMod) return false; - errs() << "*** Performing rename " << TRI->getName(SuperReg) << + dbgs() << "*** Performing rename " << TRI->getName(SuperReg) << " for debug ***\n"; } #endif @@ -606,11 +606,11 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters( const TargetRegisterClass::iterator RB = SuperRC->allocation_order_begin(MF); const TargetRegisterClass::iterator RE = SuperRC->allocation_order_end(MF); if (RB == RE) { - DEBUG(errs() << "\tEmpty Super Regclass!!\n"); + DEBUG(dbgs() << "\tEmpty Super Regclass!!\n"); return false; } - DEBUG(errs() << "\tFind Registers:"); + DEBUG(dbgs() << "\tFind Registers:"); if (RenameOrder.count(SuperRC) == 0) RenameOrder.insert(RenameOrderType::value_type(SuperRC, RE)); @@ -625,7 +625,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters( // Don't replace a register with itself. if (NewSuperReg == SuperReg) continue; - DEBUG(errs() << " [" << TRI->getName(NewSuperReg) << ':'); + DEBUG(dbgs() << " [" << TRI->getName(NewSuperReg) << ':'); RenameMap.clear(); // For each referenced group register (which must be a SuperReg or @@ -642,12 +642,12 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters( NewReg = TRI->getSubReg(NewSuperReg, NewSubRegIdx); } - DEBUG(errs() << " " << TRI->getName(NewReg)); + DEBUG(dbgs() << " " << TRI->getName(NewReg)); // Check if Reg can be renamed to NewReg. BitVector BV = RenameRegisterMap[Reg]; if (!BV.test(NewReg)) { - DEBUG(errs() << "(no rename)"); + DEBUG(dbgs() << "(no rename)"); goto next_super_reg; } @@ -656,7 +656,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters( // must also check all aliases of NewReg, because we can't define a // register when any sub or super is already live. if (State->IsLive(NewReg) || (KillIndices[Reg] > DefIndices[NewReg])) { - DEBUG(errs() << "(live)"); + DEBUG(dbgs() << "(live)"); goto next_super_reg; } else { bool found = false; @@ -664,7 +664,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters( *Alias; ++Alias) { unsigned AliasReg = *Alias; if (State->IsLive(AliasReg) || (KillIndices[Reg] > DefIndices[AliasReg])) { - DEBUG(errs() << "(alias " << TRI->getName(AliasReg) << " live)"); + DEBUG(dbgs() << "(alias " << TRI->getName(AliasReg) << " live)"); found = true; break; } @@ -681,14 +681,14 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters( // renamed, as recorded in RenameMap. RenameOrder.erase(SuperRC); RenameOrder.insert(RenameOrderType::value_type(SuperRC, R)); - DEBUG(errs() << "]\n"); + DEBUG(dbgs() << "]\n"); return true; next_super_reg: - DEBUG(errs() << ']'); + DEBUG(dbgs() << ']'); } while (R != EndR); - DEBUG(errs() << '\n'); + DEBUG(dbgs() << '\n'); // No registers are free and available! return false; @@ -740,13 +740,13 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies( } #ifndef NDEBUG - DEBUG(errs() << "\n===== Aggressive anti-dependency breaking\n"); - DEBUG(errs() << "Available regs:"); + DEBUG(dbgs() << "\n===== Aggressive anti-dependency breaking\n"); + DEBUG(dbgs() << "Available regs:"); for (unsigned Reg = 0; Reg < TRI->getNumRegs(); ++Reg) { if (!State->IsLive(Reg)) - DEBUG(errs() << " " << TRI->getName(Reg)); + DEBUG(dbgs() << " " << TRI->getName(Reg)); } - DEBUG(errs() << '\n'); + DEBUG(dbgs() << '\n'); #endif // Attempt to break anti-dependence edges. Walk the instructions @@ -758,7 +758,7 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies( I != E; --Count) { MachineInstr *MI = --I; - DEBUG(errs() << "Anti: "); + DEBUG(dbgs() << "Anti: "); DEBUG(MI->dump()); std::set<unsigned> PassthruRegs; @@ -795,30 +795,30 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies( (Edge->getKind() != SDep::Output)) continue; unsigned AntiDepReg = Edge->getReg(); - DEBUG(errs() << "\tAntidep reg: " << TRI->getName(AntiDepReg)); + DEBUG(dbgs() << "\tAntidep reg: " << TRI->getName(AntiDepReg)); assert(AntiDepReg != 0 && "Anti-dependence on reg0?"); if (!AllocatableSet.test(AntiDepReg)) { // Don't break anti-dependencies on non-allocatable registers. - DEBUG(errs() << " (non-allocatable)\n"); + DEBUG(dbgs() << " (non-allocatable)\n"); continue; } else if ((ExcludeRegs != NULL) && ExcludeRegs->test(AntiDepReg)) { // Don't break anti-dependencies for critical path registers // if not on the critical path - DEBUG(errs() << " (not critical-path)\n"); + DEBUG(dbgs() << " (not critical-path)\n"); continue; } else if (PassthruRegs.count(AntiDepReg) != 0) { // If the anti-dep register liveness "passes-thru", then // don't try to change it. It will be changed along with // the use if required to break an earlier antidep. - DEBUG(errs() << " (passthru)\n"); + DEBUG(dbgs() << " (passthru)\n"); continue; } else { // No anti-dep breaking for implicit deps MachineOperand *AntiDepOp = MI->findRegisterDefOperand(AntiDepReg); assert(AntiDepOp != NULL && "Can't find index for defined register operand"); if ((AntiDepOp == NULL) || AntiDepOp->isImplicit()) { - DEBUG(errs() << " (implicit)\n"); + DEBUG(dbgs() << " (implicit)\n"); continue; } @@ -844,13 +844,13 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies( PE = PathSU->Preds.end(); P != PE; ++P) { if ((P->getSUnit() == NextSU) && (P->getKind() != SDep::Anti) && (P->getKind() != SDep::Output)) { - DEBUG(errs() << " (real dependency)\n"); + DEBUG(dbgs() << " (real dependency)\n"); AntiDepReg = 0; break; } else if ((P->getSUnit() != NextSU) && (P->getKind() == SDep::Data) && (P->getReg() == AntiDepReg)) { - DEBUG(errs() << " (other dependency)\n"); + DEBUG(dbgs() << " (other dependency)\n"); AntiDepReg = 0; break; } @@ -865,16 +865,16 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies( // Determine AntiDepReg's register group. const unsigned GroupIndex = State->GetGroup(AntiDepReg); if (GroupIndex == 0) { - DEBUG(errs() << " (zero group)\n"); + DEBUG(dbgs() << " (zero group)\n"); continue; } - DEBUG(errs() << '\n'); + DEBUG(dbgs() << '\n'); // Look for a suitable register to use to break the anti-dependence. std::map<unsigned, unsigned> RenameMap; if (FindSuitableFreeRegisters(GroupIndex, RenameOrder, RenameMap)) { - DEBUG(errs() << "\tBreaking anti-dependence edge on " + DEBUG(dbgs() << "\tBreaking anti-dependence edge on " << TRI->getName(AntiDepReg) << ":"); // Handle each group register... @@ -883,7 +883,7 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies( unsigned CurrReg = S->first; unsigned NewReg = S->second; - DEBUG(errs() << " " << TRI->getName(CurrReg) << "->" << + DEBUG(dbgs() << " " << TRI->getName(CurrReg) << "->" << TRI->getName(NewReg) << "(" << RegRefs.count(CurrReg) << " refs)"); @@ -917,7 +917,7 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies( } ++Broken; - DEBUG(errs() << '\n'); + DEBUG(dbgs() << '\n'); } } } diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 44fd176..6b24e24 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -236,7 +236,7 @@ namespace { const MCSection *S; unsigned Alignment; SmallVector<unsigned, 4> CPEs; - SectionCPs(const MCSection *s, unsigned a) : S(s), Alignment(a) {}; + SectionCPs(const MCSection *s, unsigned a) : S(s), Alignment(a) {} }; } @@ -1905,7 +1905,6 @@ void AsmPrinter::EmitComments(const MachineInstr &MI) const { if (Newline) O << '\n'; O.PadToColumn(MAI->getCommentColumn()); O << MAI->getCommentString() << " Reload Reuse"; - Newline = true; } } } diff --git a/lib/CodeGen/AsmPrinter/DIE.cpp b/lib/CodeGen/AsmPrinter/DIE.cpp index 0e93b98..b85e11a 100644 --- a/lib/CodeGen/AsmPrinter/DIE.cpp +++ b/lib/CodeGen/AsmPrinter/DIE.cpp @@ -16,6 +16,7 @@ #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/Target/TargetData.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" using namespace llvm; @@ -93,7 +94,7 @@ void DIEAbbrev::print(raw_ostream &O) { << '\n'; } } -void DIEAbbrev::dump() { print(errs()); } +void DIEAbbrev::dump() { print(dbgs()); } #endif //===----------------------------------------------------------------------===// @@ -164,14 +165,14 @@ void DIE::print(raw_ostream &O, unsigned IncIndent) { } void DIE::dump() { - print(errs()); + print(dbgs()); } #endif #ifndef NDEBUG void DIEValue::dump() { - print(errs()); + print(dbgs()); } #endif diff --git a/lib/CodeGen/AsmPrinter/DIE.h b/lib/CodeGen/AsmPrinter/DIE.h index cad8b89..a6dc9b6 100644 --- a/lib/CodeGen/AsmPrinter/DIE.h +++ b/lib/CodeGen/AsmPrinter/DIE.h @@ -68,6 +68,7 @@ namespace llvm { /// Data - Raw data bytes for abbreviation. /// SmallVector<DIEAbbrevData, 8> Data; + public: DIEAbbrev(unsigned T, unsigned C) : Tag(T), ChildrenFlag(C), Data() {} virtual ~DIEAbbrev() {} @@ -131,19 +132,18 @@ namespace llvm { /// std::vector<DIE *> Children; + DIE *Parent; + /// Attributes values. /// SmallVector<DIEValue*, 32> Values; - /// Abstract compile unit. - CompileUnit *AbstractCU; - // Private data for print() mutable unsigned IndentCount; public: explicit DIE(unsigned Tag) : Abbrev(Tag, dwarf::DW_CHILDREN_no), Offset(0), - Size(0), IndentCount(0) {} + Size(0), Parent (0), IndentCount(0) {} virtual ~DIE(); // Accessors. @@ -154,13 +154,12 @@ namespace llvm { unsigned getSize() const { return Size; } const std::vector<DIE *> &getChildren() const { return Children; } SmallVector<DIEValue*, 32> &getValues() { return Values; } - CompileUnit *getAbstractCompileUnit() const { return AbstractCU; } - + DIE *getParent() const { return Parent; } void setTag(unsigned Tag) { Abbrev.setTag(Tag); } void setOffset(unsigned O) { Offset = O; } void setSize(unsigned S) { Size = S; } - void setAbstractCompileUnit(CompileUnit *CU) { AbstractCU = CU; } - + void setParent(DIE *P) { Parent = P; } + /// addValue - Add a value and attributes to a DIE. /// void addValue(unsigned Attribute, unsigned Form, DIEValue *Value) { @@ -179,8 +178,13 @@ namespace llvm { /// addChild - Add a child to the DIE. /// void addChild(DIE *Child) { + if (Child->getParent()) { + assert (Child->getParent() == this && "Unexpected DIE Parent!"); + return; + } Abbrev.setChildrenFlag(dwarf::DW_CHILDREN_yes); Children.push_back(Child); + Child->setParent(this); } #ifndef NDEBUG diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index c200a46..8a3ceb6 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -30,11 +30,6 @@ #include "llvm/System/Path.h" using namespace llvm; -static TimerGroup &getDwarfTimerGroup() { - static TimerGroup DwarfTimerGroup("Dwarf Debugging"); - return DwarfTimerGroup; -} - //===----------------------------------------------------------------------===// /// Configuration values for initial hash set sizes (log2). @@ -112,7 +107,12 @@ public: /// getDIEEntry - Returns the debug information entry for the speciefied /// debug variable. - DIEEntry *getDIEEntry(MDNode *N) { return GVToDIEEntryMap.lookup(N); } + DIEEntry *getDIEEntry(MDNode *N) { + ValueMap<MDNode *, DIEEntry *>::iterator I = GVToDIEEntryMap.find(N); + if (I == GVToDIEEntryMap.end()) + return NULL; + return I->second; + } /// insertDIEEntry - Insert debug information entry into the map. void insertDIEEntry(MDNode *N, DIEEntry *E) { @@ -234,7 +234,7 @@ public: #ifndef NDEBUG void DbgScope::dump() const { - raw_ostream &err = errs(); + raw_ostream &err = dbgs(); err.indent(IndentLevel); MDNode *N = Desc.getNode(); N->dump(); @@ -269,8 +269,7 @@ DwarfDebug::DwarfDebug(raw_ostream &OS, AsmPrinter *A, const MCAsmInfo *T) SectionSourceLines(), didInitial(false), shouldEmit(false), CurrentFnDbgScope(0), DebugTimer(0) { if (TimePassesIsEnabled) - DebugTimer = new Timer("Dwarf Debug Writer", - getDwarfTimerGroup()); + DebugTimer = new Timer("Dwarf Debug Writer"); } DwarfDebug::~DwarfDebug() { for (unsigned j = 0, M = DIEValues.size(); j < M; ++j) @@ -446,6 +445,23 @@ void DwarfDebug::addSourceLine(DIE *Die, const DIType *Ty) { addUInt(Die, dwarf::DW_AT_decl_line, 0, Line); } +/// addSourceLine - Add location information to specified debug information +/// entry. +void DwarfDebug::addSourceLine(DIE *Die, const DINameSpace *NS) { + // If there is no compile unit specified, don't add a line #. + if (NS->getCompileUnit().isNull()) + return; + + unsigned Line = NS->getLineNumber(); + StringRef FN = NS->getFilename(); + StringRef Dir = NS->getDirectory(); + + unsigned FileID = GetOrCreateSourceID(Dir, FN); + assert(FileID && "Invalid file id"); + addUInt(Die, dwarf::DW_AT_decl_file, 0, FileID); + addUInt(Die, dwarf::DW_AT_decl_line, 0, Line); +} + /* Byref variables, in Blocks, are declared by the programmer as "SomeType VarName;", but the compiler creates a __Block_byref_x_VarName struct, and gives the variable VarName @@ -745,6 +761,9 @@ void DwarfDebug::addToContextOwner(DIE *Die, DIDescriptor Context) { else if (Context.isType()) { DIE *ContextDIE = getOrCreateTypeDIE(DIType(Context.getNode())); ContextDIE->addChild(Die); + } else if (Context.isNameSpace()) { + DIE *ContextDIE = getOrCreateNameSpace(DINameSpace(Context.getNode())); + ContextDIE->addChild(Die); } else if (DIE *ContextDIE = ModuleCU->getDIE(Context.getNode())) ContextDIE->addChild(Die); else @@ -781,7 +800,6 @@ void DwarfDebug::addType(DIE *Entity, DIType Ty) { // Check for pre-existence. DIEEntry *Entry = ModuleCU->getDIEEntry(Ty.getNode()); - // If it exists then use the existing value. if (Entry) { Entity->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, Entry); @@ -1030,13 +1048,6 @@ DIE *DwarfDebug::createGlobalVariableDIE(const DIGlobalVariable &GV) { addUInt(GVDie, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1); addSourceLine(GVDie, &GV); - // Add address. - DIEBlock *Block = new DIEBlock(); - addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr); - addObjectLabel(Block, 0, dwarf::DW_FORM_udata, - Asm->Mang->getMangledName(GV.getGlobal())); - addBlock(GVDie, dwarf::DW_AT_location, 0, Block); - return GVDie; } @@ -1285,7 +1296,6 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(MDNode *SPNode) { SPDie = new DIE(dwarf::DW_TAG_subprogram); addDIEEntry(SPDie, dwarf::DW_AT_specification, dwarf::DW_FORM_ref4, SPDeclDie); - ModuleCU->addDie(SPDie); } @@ -1559,6 +1569,20 @@ unsigned DwarfDebug::GetOrCreateSourceID(StringRef DirName, StringRef FileName) return SrcId; } +/// getOrCreateNameSpace - Create a DIE for DINameSpace. +DIE *DwarfDebug::getOrCreateNameSpace(DINameSpace NS) { + DIE *NDie = ModuleCU->getDIE(NS.getNode()); + if (NDie) + return NDie; + NDie = new DIE(dwarf::DW_TAG_namespace); + ModuleCU->insertDIE(NS.getNode(), NDie); + if (!NS.getName().empty()) + addString(NDie, dwarf::DW_AT_name, dwarf::DW_FORM_string, NS.getName()); + addSourceLine(NDie, &NS); + addToContextOwner(NDie, NS.getContext()); + return NDie; +} + CompileUnit *DwarfDebug::constructCompileUnit(MDNode *N) { DICompileUnit DIUnit(N); StringRef FN = DIUnit.getFilename(); @@ -1620,6 +1644,25 @@ void DwarfDebug::constructGlobalVariableDIE(MDNode *N) { ModuleCU->insertDIE(N, VariableDie); // Add to context owner. + if (DI_GV.isDefinition() + && !DI_GV.getContext().isCompileUnit()) { + // Create specification DIE. + DIE *VariableSpecDIE = new DIE(dwarf::DW_TAG_variable); + addDIEEntry(VariableSpecDIE, dwarf::DW_AT_specification, + dwarf::DW_FORM_ref4, VariableDie); + DIEBlock *Block = new DIEBlock(); + addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr); + addObjectLabel(Block, 0, dwarf::DW_FORM_udata, + Asm->Mang->getMangledName(DI_GV.getGlobal())); + addBlock(VariableSpecDIE, dwarf::DW_AT_location, 0, Block); + ModuleCU->addDie(VariableSpecDIE); + } else { + DIEBlock *Block = new DIEBlock(); + addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr); + addObjectLabel(Block, 0, dwarf::DW_FORM_udata, + Asm->Mang->getMangledName(DI_GV.getGlobal())); + addBlock(VariableDie, dwarf::DW_AT_location, 0, Block); + } addToContextOwner(VariableDie, DI_GV.getContext()); // Expose as global. FIXME - need to check external flag. @@ -1652,9 +1695,7 @@ void DwarfDebug::constructSubprogramDIE(MDNode *N) { ModuleCU->insertDIE(N, SubprogramDie); // Add to context owner. - if (SP.getContext().getNode() == SP.getCompileUnit().getNode()) - if (TopLevelDIEs.insert(SubprogramDie)) - TopLevelDIEsVector.push_back(SubprogramDie); + addToContextOwner(SubprogramDie, SP.getContext()); // Expose as global. ModuleCU->addGlobal(SP.getName(), SubprogramDie); @@ -2365,7 +2406,6 @@ void DwarfDebug::emitDebugInfo() { EmitLabel("info_end", ModuleCU->getID()); Asm->EOL(); - } /// emitAbbreviations - Emit the abbreviation section. diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.h b/lib/CodeGen/AsmPrinter/DwarfDebug.h index 12ad322..2b8164e 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.h +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.h @@ -285,6 +285,7 @@ class DwarfDebug : public Dwarf { void addSourceLine(DIE *Die, const DIGlobal *G); void addSourceLine(DIE *Die, const DISubprogram *SP); void addSourceLine(DIE *Die, const DIType *Ty); + void addSourceLine(DIE *Die, const DINameSpace *NS); /// addAddress - Add an address attribute to a die based on the location /// provided. @@ -315,6 +316,10 @@ class DwarfDebug : public Dwarf { /// addType - Add a new type attribute to the specified entity. void addType(DIE *Entity, DIType Ty); + + /// getOrCreateNameSpace - Create a DIE for DINameSpace. + DIE *getOrCreateNameSpace(DINameSpace NS); + /// getOrCreateTypeDIE - Find existing DIE or create new DIE for the /// given DIType. DIE *getOrCreateTypeDIE(DIType Ty); diff --git a/lib/CodeGen/AsmPrinter/DwarfException.cpp b/lib/CodeGen/AsmPrinter/DwarfException.cpp index 3fd077f..d01f300 100644 --- a/lib/CodeGen/AsmPrinter/DwarfException.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfException.cpp @@ -35,19 +35,13 @@ #include "llvm/ADT/StringExtras.h" using namespace llvm; -static TimerGroup &getDwarfTimerGroup() { - static TimerGroup DwarfTimerGroup("DWARF Exception"); - return DwarfTimerGroup; -} - DwarfException::DwarfException(raw_ostream &OS, AsmPrinter *A, const MCAsmInfo *T) : Dwarf(OS, A, T, "eh"), shouldEmitTable(false), shouldEmitMoves(false), shouldEmitTableModule(false), shouldEmitMovesModule(false), ExceptionTimer(0) { if (TimePassesIsEnabled) - ExceptionTimer = new Timer("DWARF Exception Writer", - getDwarfTimerGroup()); + ExceptionTimer = new Timer("DWARF Exception Writer"); } DwarfException::~DwarfException() { @@ -292,13 +286,14 @@ void DwarfException::EmitFDE(const FunctionEHFrameInfo &EHFrameInfo) { Asm->EmitULEB128Bytes(is4Byte ? 4 : 8); Asm->EOL("Augmentation size"); - // We force 32-bits here because we've encoded our LSDA in the CIE with - // `dwarf::DW_EH_PE_sdata4'. And the CIE and FDE should agree. if (EHFrameInfo.hasLandingPads) - EmitReference("exception", EHFrameInfo.Number, true, true); - else - Asm->EmitInt32((int)0); - + EmitReference("exception", EHFrameInfo.Number, true, false); + else { + if (is4Byte) + Asm->EmitInt32((int)0); + else + Asm->EmitInt64((int)0); + } Asm->EOL("Language Specific Data Area"); } else { Asm->EmitULEB128Bytes(0); diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp index 3887e6d..92849d3 100644 --- a/lib/CodeGen/BranchFolding.cpp +++ b/lib/CodeGen/BranchFolding.cpp @@ -98,7 +98,7 @@ BranchFolder::BranchFolder(bool defaultEnableTailMerge) { /// function, updating the CFG. void BranchFolder::RemoveDeadBlock(MachineBasicBlock *MBB) { assert(MBB->pred_empty() && "MBB must be dead!"); - DEBUG(errs() << "\nRemoving MBB: " << *MBB); + DEBUG(dbgs() << "\nRemoving MBB: " << *MBB); MachineFunction *MF = MBB->getParent(); // drop all successors. @@ -636,7 +636,7 @@ unsigned BranchFolder::CreateCommonTailOnlyBlock(MachineBasicBlock *&PredBB, SameTails[commonTailIndex].getTailStartPos(); MachineBasicBlock *MBB = SameTails[commonTailIndex].getBlock(); - DEBUG(errs() << "\nSplitting BB#" << MBB->getNumber() << ", size " + DEBUG(dbgs() << "\nSplitting BB#" << MBB->getNumber() << ", size " << maxCommonTailLength); MachineBasicBlock *newMBB = SplitMBBAt(*MBB, BBI); @@ -666,18 +666,18 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB, // this many instructions in common. unsigned minCommonTailLength = TailMergeSize; - DEBUG(errs() << "\nTryTailMergeBlocks: "; + DEBUG(dbgs() << "\nTryTailMergeBlocks: "; for (unsigned i = 0, e = MergePotentials.size(); i != e; ++i) - errs() << "BB#" << MergePotentials[i].getBlock()->getNumber() + dbgs() << "BB#" << MergePotentials[i].getBlock()->getNumber() << (i == e-1 ? "" : ", "); - errs() << "\n"; + dbgs() << "\n"; if (SuccBB) { - errs() << " with successor BB#" << SuccBB->getNumber() << '\n'; + dbgs() << " with successor BB#" << SuccBB->getNumber() << '\n'; if (PredBB) - errs() << " which has fall-through from BB#" + dbgs() << " which has fall-through from BB#" << PredBB->getNumber() << "\n"; } - errs() << "Looking for common tails of at least " + dbgs() << "Looking for common tails of at least " << minCommonTailLength << " instruction" << (minCommonTailLength == 1 ? "" : "s") << '\n'; ); @@ -748,19 +748,19 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB, MachineBasicBlock *MBB = SameTails[commonTailIndex].getBlock(); // MBB is common tail. Adjust all other BB's to jump to this one. // Traversal must be forwards so erases work. - DEBUG(errs() << "\nUsing common tail in BB#" << MBB->getNumber() + DEBUG(dbgs() << "\nUsing common tail in BB#" << MBB->getNumber() << " for "); for (unsigned int i=0, e = SameTails.size(); i != e; ++i) { if (commonTailIndex == i) continue; - DEBUG(errs() << "BB#" << SameTails[i].getBlock()->getNumber() + DEBUG(dbgs() << "BB#" << SameTails[i].getBlock()->getNumber() << (i == e-1 ? "" : ", ")); // Hack the end off BB i, making it jump to BB commonTailIndex instead. ReplaceTailWithBranchTo(SameTails[i].getTailStartPos(), MBB); // BB i is no longer a predecessor of SuccBB; remove it from the worklist. MergePotentials.erase(SameTails[i].getMPIter()); } - DEBUG(errs() << "\n"); + DEBUG(dbgs() << "\n"); // We leave commonTailIndex in the worklist in case there are other blocks // that match it with a smaller number of instructions. MadeChange = true; @@ -999,7 +999,7 @@ ReoptimizeBlock: if (PriorCond.empty() && !PriorTBB && MBB->pred_size() == 1 && PrevBB.succ_size() == 1 && !MBB->hasAddressTaken()) { - DEBUG(errs() << "\nMerging into block: " << PrevBB + DEBUG(dbgs() << "\nMerging into block: " << PrevBB << "From MBB: " << *MBB); PrevBB.splice(PrevBB.end(), MBB, MBB->begin(), MBB->end()); PrevBB.removeSuccessor(PrevBB.succ_begin());; @@ -1084,7 +1084,7 @@ ReoptimizeBlock: // Reverse the branch so we will fall through on the previous true cond. SmallVector<MachineOperand, 4> NewPriorCond(PriorCond); if (!TII->ReverseBranchCondition(NewPriorCond)) { - DEBUG(errs() << "\nMoving MBB: " << *MBB + DEBUG(dbgs() << "\nMoving MBB: " << *MBB << "To make fallthrough to: " << *PriorTBB << "\n"); TII->RemoveBranch(PrevBB); @@ -1222,7 +1222,7 @@ ReoptimizeBlock: // Analyze the branch at the end of the pred. MachineBasicBlock *PredBB = *PI; MachineFunction::iterator PredFallthrough = PredBB; ++PredFallthrough; - MachineBasicBlock *PredTBB, *PredFBB; + MachineBasicBlock *PredTBB = 0, *PredFBB = 0; SmallVector<MachineOperand, 4> PredCond; if (PredBB != MBB && !PredBB->canFallThrough() && !TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true) @@ -1274,7 +1274,7 @@ ReoptimizeBlock: // Okay, there is no really great place to put this block. If, however, // the block before this one would be a fall-through if this block were // removed, move this block to the end of the function. - MachineBasicBlock *PrevTBB, *PrevFBB; + MachineBasicBlock *PrevTBB = 0, *PrevFBB = 0; SmallVector<MachineOperand, 4> PrevCond; if (FallThrough != MF.end() && !TII->AnalyzeBranch(PrevBB, PrevTBB, PrevFBB, PrevCond, true) && diff --git a/lib/CodeGen/CalcSpillWeights.cpp b/lib/CodeGen/CalcSpillWeights.cpp index dcffb8a2..b8ef219 100644 --- a/lib/CodeGen/CalcSpillWeights.cpp +++ b/lib/CodeGen/CalcSpillWeights.cpp @@ -37,7 +37,7 @@ void CalculateSpillWeights::getAnalysisUsage(AnalysisUsage &au) const { bool CalculateSpillWeights::runOnMachineFunction(MachineFunction &fn) { - DEBUG(errs() << "********** Compute Spill Weights **********\n" + DEBUG(dbgs() << "********** Compute Spill Weights **********\n" << "********** Function: " << fn.getFunction()->getName() << '\n'); @@ -95,7 +95,7 @@ bool CalculateSpillWeights::runOnMachineFunction(MachineFunction &fn) { SlotIndex defIdx = lis->getInstructionIndex(mi).getDefIndex(); const LiveRange *dlr = lis->getInterval(reg).getLiveRangeContaining(defIdx); - if (dlr->end > mbbEnd) + if (dlr->end >= mbbEnd) weight *= 3.0F; } regInt.weight += weight; diff --git a/lib/CodeGen/CodePlacementOpt.cpp b/lib/CodeGen/CodePlacementOpt.cpp index ff71f6b..126700b 100644 --- a/lib/CodeGen/CodePlacementOpt.cpp +++ b/lib/CodeGen/CodePlacementOpt.cpp @@ -233,7 +233,6 @@ bool CodePlacementOpt::EliminateUnconditionalJumpsToTop(MachineFunction &MF, !BotHasFallthrough && HasFallthrough(L->getBottomBlock())) { ++NumIntraElim; - BotHasFallthrough = true; } return Changed; diff --git a/lib/CodeGen/ELF.h b/lib/CodeGen/ELF.h index e303ebb..cb5a8c0 100644 --- a/lib/CodeGen/ELF.h +++ b/lib/CodeGen/ELF.h @@ -82,14 +82,14 @@ namespace llvm { const GlobalValue *getGlobalValue() const { assert(SourceType == isGV && "This is not a global value"); return Source.GV; - }; + } // getExternalSym - If this is an external symbol which originated the // elf symbol, return a reference to it. const char *getExternalSymbol() const { assert(SourceType == isExtSym && "This is not an external symbol"); return Source.Ext; - }; + } // getGV - From a global value return a elf symbol to represent it static ELFSym *getGV(const GlobalValue *GV, unsigned Bind, diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp index 297dd31..d5fd051 100644 --- a/lib/CodeGen/LLVMTargetMachine.cpp +++ b/lib/CodeGen/LLVMTargetMachine.cpp @@ -83,7 +83,18 @@ LLVMTargetMachine::LLVMTargetMachine(const Target &T, AsmInfo = T.createAsmInfo(TargetTriple); } +// Set the default code model for the JIT for a generic target. +// FIXME: Is small right here? or .is64Bit() ? Large : Small? +void +LLVMTargetMachine::setCodeModelForJIT() { + setCodeModel(CodeModel::Small); +} +// Set the default code model for static compilation for a generic target. +void +LLVMTargetMachine::setCodeModelForStatic() { + setCodeModel(CodeModel::Small); +} FileModel::Model LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, @@ -130,6 +141,9 @@ bool LLVMTargetMachine::addAssemblyEmitter(PassManagerBase &PM, bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM, MachineCodeEmitter *MCE, CodeGenOpt::Level OptLevel) { + // Make sure the code model is set. + setCodeModelForStatic(); + if (MCE) addSimpleCodeEmitter(PM, OptLevel, *MCE); if (PrintEmittedAsm) @@ -146,6 +160,9 @@ bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM, bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM, JITCodeEmitter *JCE, CodeGenOpt::Level OptLevel) { + // Make sure the code model is set. + setCodeModelForJIT(); + if (JCE) addSimpleCodeEmitter(PM, OptLevel, *JCE); if (PrintEmittedAsm) @@ -162,6 +179,9 @@ bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM, bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM, ObjectCodeEmitter *OCE, CodeGenOpt::Level OptLevel) { + // Make sure the code model is set. + setCodeModelForStatic(); + if (OCE) addSimpleCodeEmitter(PM, OptLevel, *OCE); if (PrintEmittedAsm) @@ -181,6 +201,9 @@ bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM, bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM, MachineCodeEmitter &MCE, CodeGenOpt::Level OptLevel) { + // Make sure the code model is set. + setCodeModelForJIT(); + // Add common CodeGen passes. if (addCommonCodeGenPasses(PM, OptLevel)) return true; @@ -203,6 +226,9 @@ bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM, bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM, JITCodeEmitter &JCE, CodeGenOpt::Level OptLevel) { + // Make sure the code model is set. + setCodeModelForJIT(); + // Add common CodeGen passes. if (addCommonCodeGenPasses(PM, OptLevel)) return true; diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp index 8806439f..452f872 100644 --- a/lib/CodeGen/LiveIntervalAnalysis.cpp +++ b/lib/CodeGen/LiveIntervalAnalysis.cpp @@ -324,8 +324,7 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, // of the defining block, potentially live across some blocks, then is // live into some number of blocks, but gets killed. Start by adding a // range that goes from this definition to the end of the defining block. - LiveRange NewLR(defIndex, getMBBEndIdx(mbb).getNextIndex().getLoadIndex(), - ValNo); + LiveRange NewLR(defIndex, getMBBEndIdx(mbb), ValNo); DEBUG(errs() << " +" << NewLR); interval.addRange(NewLR); @@ -334,10 +333,8 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, // live interval. for (SparseBitVector<>::iterator I = vi.AliveBlocks.begin(), E = vi.AliveBlocks.end(); I != E; ++I) { - LiveRange LR( - getMBBStartIdx(mf_->getBlockNumbered(*I)), - getMBBEndIdx(mf_->getBlockNumbered(*I)).getNextIndex().getLoadIndex(), - ValNo); + MachineBasicBlock *aliveBlock = mf_->getBlockNumbered(*I); + LiveRange LR(getMBBStartIdx(aliveBlock), getMBBEndIdx(aliveBlock), ValNo); interval.addRange(LR); DEBUG(errs() << " +" << LR); } @@ -415,19 +412,32 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, // first redefinition of the vreg that we have seen, go back and change // the live range in the PHI block to be a different value number. if (interval.containsOneValue()) { - // Remove the old range that we now know has an incorrect number. + VNInfo *VNI = interval.getValNumInfo(0); - MachineInstr *Killer = vi.Kills[0]; - SlotIndex Start = getMBBStartIdx(Killer->getParent()); - SlotIndex End = getInstructionIndex(Killer).getDefIndex(); - DEBUG({ - errs() << " Removing [" << Start << "," << End << "] from: "; - interval.print(errs(), tri_); - errs() << "\n"; - }); - interval.removeRange(Start, End); - assert(interval.ranges.size() == 1 && - "Newly discovered PHI interval has >1 ranges."); + // Phi elimination may have reused the register for multiple identical + // phi nodes. There will be a kill per phi. Remove the old ranges that + // we now know have an incorrect number. + for (unsigned ki=0, ke=vi.Kills.size(); ki != ke; ++ki) { + MachineInstr *Killer = vi.Kills[ki]; + SlotIndex Start = getMBBStartIdx(Killer->getParent()); + SlotIndex End = getInstructionIndex(Killer).getDefIndex(); + DEBUG({ + errs() << "\n\t\trenaming [" << Start << "," << End << "] in: "; + interval.print(errs(), tri_); + }); + interval.removeRange(Start, End); + + // Replace the interval with one of a NEW value number. Note that + // this value number isn't actually defined by an instruction, weird + // huh? :) + LiveRange LR(Start, End, + interval.getNextValue(SlotIndex(Start, true), + 0, false, VNInfoAllocator)); + LR.valno->setIsPHIDef(true); + interval.addRange(LR); + LR.valno->addKill(End); + } + MachineBasicBlock *killMBB = getMBBFromIndex(VNI->def); VNI->addKill(indexes_->getTerminatorGap(killMBB)); VNI->setHasPHIKill(true); @@ -435,20 +445,6 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, errs() << " RESULT: "; interval.print(errs(), tri_); }); - - // Replace the interval with one of a NEW value number. Note that this - // value number isn't actually defined by an instruction, weird huh? :) - LiveRange LR(Start, End, - interval.getNextValue(SlotIndex(getMBBStartIdx(Killer->getParent()), true), - 0, false, VNInfoAllocator)); - LR.valno->setIsPHIDef(true); - DEBUG(errs() << " replace range with " << LR); - interval.addRange(LR); - LR.valno->addKill(End); - DEBUG({ - errs() << " RESULT: "; - interval.print(errs(), tri_); - }); } // In the case of PHI elimination, each variable definition is only @@ -468,7 +464,7 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, CopyMI = mi; ValNo = interval.getNextValue(defIndex, CopyMI, true, VNInfoAllocator); - SlotIndex killIndex = getMBBEndIdx(mbb).getNextIndex().getLoadIndex(); + SlotIndex killIndex = getMBBEndIdx(mbb); LiveRange LR(defIndex, killIndex, ValNo); interval.addRange(LR); ValNo->addKill(indexes_->getTerminatorGap(mbb)); @@ -1248,7 +1244,7 @@ bool LiveIntervals::anyKillInMBBAfterIdx(const LiveInterval &li, continue; SlotIndex KillIdx = VNI->kills[j]; - if (KillIdx > Idx && KillIdx < End) + if (KillIdx > Idx && KillIdx <= End) return true; } return false; @@ -2086,7 +2082,7 @@ LiveRange LiveIntervals::addLiveRangeToEndOfBlock(unsigned reg, VN->kills.push_back(indexes_->getTerminatorGap(startInst->getParent())); LiveRange LR( SlotIndex(getInstructionIndex(startInst).getDefIndex()), - getMBBEndIdx(startInst->getParent()).getNextIndex().getBaseIndex(), VN); + getMBBEndIdx(startInst->getParent()), VN); Interval.addRange(LR); return LR; diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp index a58286d..74a0d57 100644 --- a/lib/CodeGen/MachineBasicBlock.cpp +++ b/lib/CodeGen/MachineBasicBlock.cpp @@ -450,14 +450,29 @@ void MachineBasicBlock::ReplaceUsesOfBlockWith(MachineBasicBlock *Old, /// CorrectExtraCFGEdges - Various pieces of code can cause excess edges in the /// CFG to be inserted. If we have proven that MBB can only branch to DestA and -/// DestB, remove any other MBB successors from the CFG. DestA and DestB can -/// be null. +/// DestB, remove any other MBB successors from the CFG. DestA and DestB can be +/// null. +/// /// Besides DestA and DestB, retain other edges leading to LandingPads /// (currently there can be only one; we don't check or require that here). /// Note it is possible that DestA and/or DestB are LandingPads. bool MachineBasicBlock::CorrectExtraCFGEdges(MachineBasicBlock *DestA, MachineBasicBlock *DestB, bool isCond) { + // The values of DestA and DestB frequently come from a call to the + // 'TargetInstrInfo::AnalyzeBranch' method. We take our meaning of the initial + // values from there. + // + // 1. If both DestA and DestB are null, then the block ends with no branches + // (it falls through to its successor). + // 2. If DestA is set, DestB is null, and isCond is false, then the block ends + // with only an unconditional branch. + // 3. If DestA is set, DestB is null, and isCond is true, then the block ends + // with a conditional branch that falls through to a successor (DestB). + // 4. If DestA and DestB is set and isCond is true, then the block ends with a + // conditional branch followed by an unconditional branch. DestA is the + // 'true' destination and DestB is the 'false' destination. + bool MadeChange = false; bool AddedFallThrough = false; @@ -483,14 +498,15 @@ bool MachineBasicBlock::CorrectExtraCFGEdges(MachineBasicBlock *DestA, MachineBasicBlock::succ_iterator SI = succ_begin(); MachineBasicBlock *OrigDestA = DestA, *OrigDestB = DestB; while (SI != succ_end()) { - if (*SI == DestA) { + const MachineBasicBlock *MBB = *SI; + if (MBB == DestA) { DestA = 0; ++SI; - } else if (*SI == DestB) { + } else if (MBB == DestB) { DestB = 0; ++SI; - } else if ((*SI)->isLandingPad() && - *SI!=OrigDestA && *SI!=OrigDestB) { + } else if (MBB->isLandingPad() && + MBB != OrigDestA && MBB != OrigDestB) { ++SI; } else { // Otherwise, this is a superfluous edge, remove it. @@ -498,12 +514,12 @@ bool MachineBasicBlock::CorrectExtraCFGEdges(MachineBasicBlock *DestA, MadeChange = true; } } - if (!AddedFallThrough) { - assert(DestA == 0 && DestB == 0 && - "MachineCFG is missing edges!"); - } else if (isCond) { + + if (!AddedFallThrough) + assert(DestA == 0 && DestB == 0 && "MachineCFG is missing edges!"); + else if (isCond) assert(DestA == 0 && "MachineCFG is missing edges!"); - } + return MadeChange; } diff --git a/lib/CodeGen/MachineDominators.cpp b/lib/CodeGen/MachineDominators.cpp index 0f796f3..4088739 100644 --- a/lib/CodeGen/MachineDominators.cpp +++ b/lib/CodeGen/MachineDominators.cpp @@ -17,8 +17,10 @@ using namespace llvm; +namespace llvm { TEMPLATE_INSTANTIATION(class DomTreeNodeBase<MachineBasicBlock>); TEMPLATE_INSTANTIATION(class DominatorTreeBase<MachineBasicBlock>); +} char MachineDominatorTree::ID = 0; diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp index 12b974d..a761c2d 100644 --- a/lib/CodeGen/MachineInstr.cpp +++ b/lib/CodeGen/MachineInstr.cpp @@ -15,6 +15,7 @@ #include "llvm/Constants.h" #include "llvm/Function.h" #include "llvm/InlineAsm.h" +#include "llvm/Type.h" #include "llvm/Value.h" #include "llvm/Assembly/Writer.h" #include "llvm/CodeGen/MachineFunction.h" @@ -555,8 +556,13 @@ void MachineInstr::addOperand(const MachineOperand &Op) { Operands.back().ParentMI = this; // If the operand is a register, update the operand's use list. - if (Op.isReg()) + if (Op.isReg()) { Operands.back().AddRegOperandToRegInfo(RegInfo); + // If the register operand is flagged as early, mark the operand as such + unsigned OpNo = Operands.size() - 1; + if (TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1) + Operands[OpNo].setIsEarlyClobber(true); + } return; } } @@ -573,8 +579,12 @@ void MachineInstr::addOperand(const MachineOperand &Op) { // Do explicitly set the reginfo for this operand though, to ensure the // next/prev fields are properly nulled out. - if (Operands[OpNo].isReg()) + if (Operands[OpNo].isReg()) { Operands[OpNo].AddRegOperandToRegInfo(0); + // If the register operand is flagged as early, mark the operand as such + if (TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1) + Operands[OpNo].setIsEarlyClobber(true); + } } else if (Operands.size()+1 <= Operands.capacity()) { // Otherwise, we have to remove register operands from their register use @@ -594,8 +604,12 @@ void MachineInstr::addOperand(const MachineOperand &Op) { Operands.insert(Operands.begin()+OpNo, Op); Operands[OpNo].ParentMI = this; - if (Operands[OpNo].isReg()) + if (Operands[OpNo].isReg()) { Operands[OpNo].AddRegOperandToRegInfo(RegInfo); + // If the register operand is flagged as early, mark the operand as such + if (TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1) + Operands[OpNo].setIsEarlyClobber(true); + } // Re-add all the implicit ops. for (unsigned i = OpNo+1, e = Operands.size(); i != e; ++i) { @@ -613,6 +627,11 @@ void MachineInstr::addOperand(const MachineOperand &Op) { // Re-add all the operands. AddRegOperandsToUseLists(*RegInfo); + + // If the register operand is flagged as early, mark the operand as such + if (Operands[OpNo].isReg() + && TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1) + Operands[OpNo].setIsEarlyClobber(true); } } @@ -1141,7 +1160,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { // Briefly indicate whether any call clobbers were omitted. if (OmittedAnyCallClobbers) { - if (FirstOp) FirstOp = false; else OS << ","; + if (!FirstOp) OS << ","; OS << " ..."; } @@ -1159,7 +1178,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { } if (!debugLoc.isUnknown() && MF) { - if (!HaveSemi) OS << ";"; HaveSemi = true; + if (!HaveSemi) OS << ";"; // TODO: print InlinedAtLoc information diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp index 66de535..0a57ea1 100644 --- a/lib/CodeGen/MachineLICM.cpp +++ b/lib/CodeGen/MachineLICM.cpp @@ -322,7 +322,7 @@ bool MachineLICM::IsLoopInvariantInst(MachineInstr &I) { // If the loop contains the definition of an operand, then the instruction // isn't loop invariant. - if (CurLoop->contains(RegInfo->getVRegDef(Reg)->getParent())) + if (CurLoop->contains(RegInfo->getVRegDef(Reg))) return false; } diff --git a/lib/CodeGen/MachineLoopInfo.cpp b/lib/CodeGen/MachineLoopInfo.cpp index 63f4f18..d561a5b 100644 --- a/lib/CodeGen/MachineLoopInfo.cpp +++ b/lib/CodeGen/MachineLoopInfo.cpp @@ -19,12 +19,14 @@ #include "llvm/CodeGen/Passes.h" using namespace llvm; +namespace llvm { #define MLB class LoopBase<MachineBasicBlock, MachineLoop> TEMPLATE_INSTANTIATION(MLB); #undef MLB #define MLIB class LoopInfoBase<MachineBasicBlock, MachineLoop> TEMPLATE_INSTANTIATION(MLIB); #undef MLIB +} char MachineLoopInfo::ID = 0; static RegisterPass<MachineLoopInfo> diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp index 917d053..0772319 100644 --- a/lib/CodeGen/MachineVerifier.cpp +++ b/lib/CodeGen/MachineVerifier.cpp @@ -365,24 +365,6 @@ void MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) { const TargetInstrInfo *TII = MF->getTarget().getInstrInfo(); - // Start with minimal CFG sanity checks. - MachineFunction::const_iterator MBBI = MBB; - ++MBBI; - if (MBBI != MF->end()) { - // Block is not last in function. - if (!MBB->isSuccessor(MBBI)) { - // Block does not fall through. - if (MBB->empty()) { - report("MBB doesn't fall through but is empty!", MBB); - } - } - } else { - // Block is last in function. - if (MBB->empty()) { - report("MBB is last in function but is empty!", MBB); - } - } - // Call AnalyzeBranch. If it succeeds, there several more conditions to check. MachineBasicBlock *TBB = 0, *FBB = 0; SmallVector<MachineOperand, 4> Cond; @@ -553,7 +535,8 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { report("Explicit operand marked as implicit", MO, MONum); } } else { - if (MO->isReg() && !MO->isImplicit() && !TI.isVariadic()) + // ARM adds %reg0 operands to indicate predicates. We'll allow that. + if (MO->isReg() && !MO->isImplicit() && !TI.isVariadic() && MO->getReg()) report("Extra explicit operand on non-variadic instruction", MO, MONum); } diff --git a/lib/CodeGen/PBQP/AnnotatedGraph.h b/lib/CodeGen/PBQP/AnnotatedGraph.h index 904061c..a47dce9 100644 --- a/lib/CodeGen/PBQP/AnnotatedGraph.h +++ b/lib/CodeGen/PBQP/AnnotatedGraph.h @@ -132,19 +132,19 @@ public: } NodeData& getNodeData(const NodeIterator &nodeItr) { - return getNodeEntry(nodeItr).getNodeData(); + return PGraph::getNodeEntry(nodeItr).getNodeData(); } const NodeData& getNodeData(const NodeIterator &nodeItr) const { - return getNodeEntry(nodeItr).getNodeData(); + return PGraph::getNodeEntry(nodeItr).getNodeData(); } EdgeData& getEdgeData(const EdgeIterator &edgeItr) { - return getEdgeEntry(edgeItr).getEdgeData(); + return PGraph::getEdgeEntry(edgeItr).getEdgeData(); } const EdgeEntry& getEdgeData(const EdgeIterator &edgeItr) const { - return getEdgeEntry(edgeItr).getEdgeData(); + return PGraph::getEdgeEntry(edgeItr).getEdgeData(); } SimpleGraph toSimpleGraph() const { diff --git a/lib/CodeGen/PBQP/GraphBase.h b/lib/CodeGen/PBQP/GraphBase.h index cc3e017..0c7493b 100644 --- a/lib/CodeGen/PBQP/GraphBase.h +++ b/lib/CodeGen/PBQP/GraphBase.h @@ -298,7 +298,7 @@ public: for (ConstAdjEdgeIterator adjEdgeItr = adjEdgesBegin(node1Itr), adjEdgeEnd = adjEdgesEnd(node1Itr); - adjEdgeItr != adjEdgesEnd; ++adjEdgeItr) { + adjEdgeItr != adjEdgeEnd; ++adjEdgeItr) { if ((getEdgeNode1Itr(*adjEdgeItr) == node2Itr) || (getEdgeNode2Itr(*adjEdgeItr) == node2Itr)) { return *adjEdgeItr; diff --git a/lib/CodeGen/PBQP/HeuristicSolver.h b/lib/CodeGen/PBQP/HeuristicSolver.h index e786246..1670877 100644 --- a/lib/CodeGen/PBQP/HeuristicSolver.h +++ b/lib/CodeGen/PBQP/HeuristicSolver.h @@ -536,7 +536,7 @@ private: else reductionFinished = true; } - }; + } void processR1() { diff --git a/lib/CodeGen/PHIElimination.cpp b/lib/CodeGen/PHIElimination.cpp index c62d179..58c3dec 100644 --- a/lib/CodeGen/PHIElimination.cpp +++ b/lib/CodeGen/PHIElimination.cpp @@ -35,6 +35,7 @@ using namespace llvm; STATISTIC(NumAtomic, "Number of atomic phis lowered"); STATISTIC(NumSplits, "Number of critical edges split on demand"); +STATISTIC(NumReused, "Number of reused lowered phis"); char PHIElimination::ID = 0; static RegisterPass<PHIElimination> @@ -70,7 +71,7 @@ bool llvm::PHIElimination::runOnMachineFunction(MachineFunction &Fn) { Changed |= EliminatePHINodes(Fn, *I); // Remove dead IMPLICIT_DEF instructions. - for (SmallPtrSet<MachineInstr*,4>::iterator I = ImpDefs.begin(), + for (SmallPtrSet<MachineInstr*, 4>::iterator I = ImpDefs.begin(), E = ImpDefs.end(); I != E; ++I) { MachineInstr *DefMI = *I; unsigned DefReg = DefMI->getOperand(0).getReg(); @@ -78,6 +79,12 @@ bool llvm::PHIElimination::runOnMachineFunction(MachineFunction &Fn) { DefMI->eraseFromParent(); } + // Clean up the lowered PHI instructions. + for (LoweredPHIMap::iterator I = LoweredPHIs.begin(), E = LoweredPHIs.end(); + I != E; ++I) + Fn.DeleteMachineInstr(I->first); + + LoweredPHIs.clear(); ImpDefs.clear(); VRegPHIUseCount.clear(); return Changed; @@ -168,6 +175,7 @@ llvm::PHIElimination::FindCopyInsertPoint(MachineBasicBlock &MBB, void llvm::PHIElimination::LowerAtomicPHINode( MachineBasicBlock &MBB, MachineBasicBlock::iterator AfterPHIsIt) { + ++NumAtomic; // Unlink the PHI node from the basic block, but don't delete the PHI yet. MachineInstr *MPhi = MBB.remove(MBB.begin()); @@ -179,6 +187,7 @@ void llvm::PHIElimination::LowerAtomicPHINode( MachineFunction &MF = *MBB.getParent(); const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(DestReg); unsigned IncomingReg = 0; + bool reusedIncoming = false; // Is IncomingReg reused from an earlier PHI? // Insert a register to register copy at the top of the current block (but // after any remaining phi nodes) which copies the new incoming register @@ -190,7 +199,18 @@ void llvm::PHIElimination::LowerAtomicPHINode( BuildMI(MBB, AfterPHIsIt, MPhi->getDebugLoc(), TII->get(TargetInstrInfo::IMPLICIT_DEF), DestReg); else { - IncomingReg = MF.getRegInfo().createVirtualRegister(RC); + // Can we reuse an earlier PHI node? This only happens for critical edges, + // typically those created by tail duplication. + unsigned &entry = LoweredPHIs[MPhi]; + if (entry) { + // An identical PHI node was already lowered. Reuse the incoming register. + IncomingReg = entry; + reusedIncoming = true; + ++NumReused; + DEBUG(errs() << "Reusing %reg" << IncomingReg << " for " << *MPhi); + } else { + entry = IncomingReg = MF.getRegInfo().createVirtualRegister(RC); + } TII->copyRegToReg(MBB, AfterPHIsIt, DestReg, IncomingReg, RC, RC); } @@ -204,8 +224,20 @@ void llvm::PHIElimination::LowerAtomicPHINode( MachineInstr *PHICopy = prior(AfterPHIsIt); if (IncomingReg) { + LiveVariables::VarInfo &VI = LV->getVarInfo(IncomingReg); + // Increment use count of the newly created virtual register. - LV->getVarInfo(IncomingReg).NumUses++; + VI.NumUses++; + + // When we are reusing the incoming register, it may already have been + // killed in this block. The old kill will also have been inserted at + // AfterPHIsIt, so it appears before the current PHICopy. + if (reusedIncoming) + if (MachineInstr *OldKill = VI.findKill(&MBB)) { + DEBUG(errs() << "Remove old kill from " << *OldKill); + LV->removeVirtualRegisterKilled(IncomingReg, OldKill); + DEBUG(MBB.dump()); + } // Add information to LiveVariables to know that the incoming value is // killed. Note that because the value is defined in several places (once @@ -228,7 +260,7 @@ void llvm::PHIElimination::LowerAtomicPHINode( // Adjust the VRegPHIUseCount map to account for the removal of this PHI node. for (unsigned i = 1; i != MPhi->getNumOperands(); i += 2) - --VRegPHIUseCount[BBVRegPair(MPhi->getOperand(i + 1).getMBB(), + --VRegPHIUseCount[BBVRegPair(MPhi->getOperand(i+1).getMBB()->getNumber(), MPhi->getOperand(i).getReg())]; // Now loop over all of the incoming arguments, changing them to copy into the @@ -266,7 +298,8 @@ void llvm::PHIElimination::LowerAtomicPHINode( FindCopyInsertPoint(opBlock, MBB, SrcReg); // Insert the copy. - TII->copyRegToReg(opBlock, InsertPos, IncomingReg, SrcReg, RC, RC); + if (!reusedIncoming && IncomingReg) + TII->copyRegToReg(opBlock, InsertPos, IncomingReg, SrcReg, RC, RC); // Now update live variable information if we have it. Otherwise we're done if (!LV) continue; @@ -283,7 +316,7 @@ void llvm::PHIElimination::LowerAtomicPHINode( // point later. // Is it used by any PHI instructions in this block? - bool ValueIsUsed = VRegPHIUseCount[BBVRegPair(&opBlock, SrcReg)] != 0; + bool ValueIsUsed = VRegPHIUseCount[BBVRegPair(opBlock.getNumber(), SrcReg)]; // Okay, if we now know that the value is not live out of the block, we can // add a kill marker in this block saying that it kills the incoming value! @@ -293,11 +326,10 @@ void llvm::PHIElimination::LowerAtomicPHINode( // terminator instruction at the end of the block may also use the value. // In this case, we should mark *it* as being the killing block, not the // copy. - MachineBasicBlock::iterator KillInst = prior(InsertPos); + MachineBasicBlock::iterator KillInst; MachineBasicBlock::iterator Term = opBlock.getFirstTerminator(); - if (Term != opBlock.end()) { - if (Term->readsRegister(SrcReg)) - KillInst = Term; + if (Term != opBlock.end() && Term->readsRegister(SrcReg)) { + KillInst = Term; // Check that no other terminators use values. #ifndef NDEBUG @@ -308,7 +340,17 @@ void llvm::PHIElimination::LowerAtomicPHINode( "they are the first terminator in a block!"); } #endif + } else if (reusedIncoming || !IncomingReg) { + // We may have to rewind a bit if we didn't insert a copy this time. + KillInst = Term; + while (KillInst != opBlock.begin()) + if ((--KillInst)->readsRegister(SrcReg)) + break; + } else { + // We just inserted this copy. + KillInst = prior(InsertPos); } + assert(KillInst->readsRegister(SrcReg) && "Cannot find kill instruction"); // Finally, mark it killed. LV->addVirtualRegisterKilled(SrcReg, KillInst); @@ -319,9 +361,9 @@ void llvm::PHIElimination::LowerAtomicPHINode( } } - // Really delete the PHI instruction now! - MF.DeleteMachineInstr(MPhi); - ++NumAtomic; + // Really delete the PHI instruction now, if it is not in the LoweredPHIs map. + if (reusedIncoming || !IncomingReg) + MF.DeleteMachineInstr(MPhi); } /// analyzePHINodes - Gather information about the PHI nodes in here. In @@ -335,14 +377,15 @@ void llvm::PHIElimination::analyzePHINodes(const MachineFunction& Fn) { for (MachineBasicBlock::const_iterator BBI = I->begin(), BBE = I->end(); BBI != BBE && BBI->getOpcode() == TargetInstrInfo::PHI; ++BBI) for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2) - ++VRegPHIUseCount[BBVRegPair(BBI->getOperand(i + 1).getMBB(), + ++VRegPHIUseCount[BBVRegPair(BBI->getOperand(i+1).getMBB()->getNumber(), BBI->getOperand(i).getReg())]; } bool llvm::PHIElimination::SplitPHIEdges(MachineFunction &MF, MachineBasicBlock &MBB, LiveVariables &LV) { - if (MBB.empty() || MBB.front().getOpcode() != TargetInstrInfo::PHI) + if (MBB.empty() || MBB.front().getOpcode() != TargetInstrInfo::PHI || + MBB.isLandingPad()) return false; // Quick exit for basic blocks without PHIs. for (MachineBasicBlock::const_iterator BBI = MBB.begin(), BBE = MBB.end(); @@ -408,3 +451,34 @@ MachineBasicBlock *PHIElimination::SplitCriticalEdge(MachineBasicBlock *A, return NMBB; } + +unsigned +PHIElimination::PHINodeTraits::getHashValue(const MachineInstr *MI) { + if (!MI || MI==getEmptyKey() || MI==getTombstoneKey()) + return DenseMapInfo<MachineInstr*>::getHashValue(MI); + unsigned hash = 0; + for (unsigned ni = 1, ne = MI->getNumOperands(); ni != ne; ni += 2) + hash = hash*37 + DenseMapInfo<BBVRegPair>:: + getHashValue(BBVRegPair(MI->getOperand(ni+1).getMBB()->getNumber(), + MI->getOperand(ni).getReg())); + return hash; +} + +bool PHIElimination::PHINodeTraits::isEqual(const MachineInstr *LHS, + const MachineInstr *RHS) { + const MachineInstr *EmptyKey = getEmptyKey(); + const MachineInstr *TombstoneKey = getTombstoneKey(); + if (!LHS || !RHS || LHS==EmptyKey || RHS==EmptyKey || + LHS==TombstoneKey || RHS==TombstoneKey) + return LHS==RHS; + + unsigned ne = LHS->getNumOperands(); + if (ne != RHS->getNumOperands()) + return false; + // Ignore operand 0, the defined register. + for (unsigned ni = 1; ni != ne; ni += 2) + if (LHS->getOperand(ni).getReg() != RHS->getOperand(ni).getReg() || + LHS->getOperand(ni+1).getMBB() != RHS->getOperand(ni+1).getMBB()) + return false; + return true; +} diff --git a/lib/CodeGen/PHIElimination.h b/lib/CodeGen/PHIElimination.h index b0b71ce..1bcc9dc 100644 --- a/lib/CodeGen/PHIElimination.h +++ b/lib/CodeGen/PHIElimination.h @@ -16,8 +16,6 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/Target/TargetInstrInfo.h" -#include <map> - namespace llvm { /// Lower PHI instructions to copies. @@ -120,8 +118,8 @@ namespace llvm { return I; } - typedef std::pair<const MachineBasicBlock*, unsigned> BBVRegPair; - typedef std::map<BBVRegPair, unsigned> VRegPHIUse; + typedef std::pair<unsigned, unsigned> BBVRegPair; + typedef DenseMap<BBVRegPair, unsigned> VRegPHIUse; VRegPHIUse VRegPHIUseCount; PHIDefMap PHIDefs; @@ -129,6 +127,17 @@ namespace llvm { // Defs of PHI sources which are implicit_def. SmallPtrSet<MachineInstr*, 4> ImpDefs; + + // Lowered PHI nodes may be reused. We provide special DenseMap traits to + // match PHI nodes with identical arguments. + struct PHINodeTraits : public DenseMapInfo<MachineInstr*> { + static unsigned getHashValue(const MachineInstr *PtrVal); + static bool isEqual(const MachineInstr *LHS, const MachineInstr *RHS); + }; + + // Map reusable lowered PHI node -> incoming join register. + typedef DenseMap<MachineInstr*, unsigned, PHINodeTraits> LoweredPHIMap; + LoweredPHIMap LoweredPHIs; }; } diff --git a/lib/CodeGen/PreAllocSplitting.cpp b/lib/CodeGen/PreAllocSplitting.cpp index b0d7a47..1c5222c 100644 --- a/lib/CodeGen/PreAllocSplitting.cpp +++ b/lib/CodeGen/PreAllocSplitting.cpp @@ -378,7 +378,7 @@ PreAllocSplitting::UpdateSpillSlotInterval(VNInfo *ValNo, SlotIndex SpillIndex, SmallPtrSet<MachineBasicBlock*, 4> Processed; SlotIndex EndIdx = LIs->getMBBEndIdx(MBB); - LiveRange SLR(SpillIndex, EndIdx.getNextSlot(), CurrSValNo); + LiveRange SLR(SpillIndex, EndIdx, CurrSValNo); CurrSLI->addRange(SLR); Processed.insert(MBB); @@ -475,7 +475,7 @@ PreAllocSplitting::PerformPHIConstruction(MachineBasicBlock::iterator UseI, SlotIndex EndIndex = LIs->getMBBEndIdx(MBB); RetVNI = NewVNs[Walker]; - LI->addRange(LiveRange(DefIndex, EndIndex.getNextSlot(), RetVNI)); + LI->addRange(LiveRange(DefIndex, EndIndex, RetVNI)); } else if (!ContainsDefs && ContainsUses) { SmallPtrSet<MachineInstr*, 2>& BlockUses = Uses[MBB]; @@ -511,8 +511,7 @@ PreAllocSplitting::PerformPHIConstruction(MachineBasicBlock::iterator UseI, UseIndex = UseIndex.getUseIndex(); SlotIndex EndIndex; if (IsIntraBlock) { - EndIndex = LIs->getInstructionIndex(UseI); - EndIndex = EndIndex.getUseIndex(); + EndIndex = LIs->getInstructionIndex(UseI).getDefIndex(); } else EndIndex = LIs->getMBBEndIdx(MBB); @@ -521,7 +520,7 @@ PreAllocSplitting::PerformPHIConstruction(MachineBasicBlock::iterator UseI, RetVNI = PerformPHIConstruction(Walker, MBB, LI, Visited, Defs, Uses, NewVNs, LiveOut, Phis, false, true); - LI->addRange(LiveRange(UseIndex, EndIndex.getNextSlot(), RetVNI)); + LI->addRange(LiveRange(UseIndex, EndIndex, RetVNI)); // FIXME: Need to set kills properly for inter-block stuff. if (RetVNI->isKill(UseIndex)) RetVNI->removeKill(UseIndex); @@ -571,8 +570,7 @@ PreAllocSplitting::PerformPHIConstruction(MachineBasicBlock::iterator UseI, StartIndex = foundDef ? StartIndex.getDefIndex() : StartIndex.getUseIndex(); SlotIndex EndIndex; if (IsIntraBlock) { - EndIndex = LIs->getInstructionIndex(UseI); - EndIndex = EndIndex.getUseIndex(); + EndIndex = LIs->getInstructionIndex(UseI).getDefIndex(); } else EndIndex = LIs->getMBBEndIdx(MBB); @@ -582,7 +580,7 @@ PreAllocSplitting::PerformPHIConstruction(MachineBasicBlock::iterator UseI, RetVNI = PerformPHIConstruction(Walker, MBB, LI, Visited, Defs, Uses, NewVNs, LiveOut, Phis, false, true); - LI->addRange(LiveRange(StartIndex, EndIndex.getNextSlot(), RetVNI)); + LI->addRange(LiveRange(StartIndex, EndIndex, RetVNI)); if (foundUse && RetVNI->isKill(StartIndex)) RetVNI->removeKill(StartIndex); @@ -663,7 +661,7 @@ PreAllocSplitting::PerformPHIConstructionFallBack(MachineBasicBlock::iterator Us for (DenseMap<MachineBasicBlock*, VNInfo*>::iterator I = IncomingVNs.begin(), E = IncomingVNs.end(); I != E; ++I) { I->second->setHasPHIKill(true); - SlotIndex KillIndex = LIs->getMBBEndIdx(I->first); + SlotIndex KillIndex(LIs->getMBBEndIdx(I->first), true); if (!I->second->isKill(KillIndex)) I->second->addKill(KillIndex); } @@ -671,11 +669,10 @@ PreAllocSplitting::PerformPHIConstructionFallBack(MachineBasicBlock::iterator Us SlotIndex EndIndex; if (IsIntraBlock) { - EndIndex = LIs->getInstructionIndex(UseI); - EndIndex = EndIndex.getUseIndex(); + EndIndex = LIs->getInstructionIndex(UseI).getDefIndex(); } else EndIndex = LIs->getMBBEndIdx(MBB); - LI->addRange(LiveRange(StartIndex, EndIndex.getNextSlot(), RetVNI)); + LI->addRange(LiveRange(StartIndex, EndIndex, RetVNI)); if (IsIntraBlock) RetVNI->addKill(EndIndex); @@ -902,8 +899,6 @@ MachineInstr* PreAllocSplitting::FoldSpill(unsigned vreg, MachineBasicBlock* MBB, int& SS, SmallPtrSet<MachineInstr*, 4>& RefsInMBB) { - MachineBasicBlock::iterator Pt = MBB->begin(); - // Go top down if RefsInMBB is empty. if (RefsInMBB.empty()) return 0; diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp index e94247f..709d46a 100644 --- a/lib/CodeGen/PrologEpilogInserter.cpp +++ b/lib/CodeGen/PrologEpilogInserter.cpp @@ -860,7 +860,7 @@ void PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) { // Remove all instructions up 'til the last use, since they're // just calculating the value we already have. BB->erase(I, LastUseMI); - MI = I = LastUseMI; + I = LastUseMI; // Extend the live range of the scratch register PrevLastUseMI->getOperand(PrevLastUseOp).setIsKill(false); diff --git a/lib/CodeGen/RegAllocLinearScan.cpp b/lib/CodeGen/RegAllocLinearScan.cpp index c02d47b..9e97d89 100644 --- a/lib/CodeGen/RegAllocLinearScan.cpp +++ b/lib/CodeGen/RegAllocLinearScan.cpp @@ -891,7 +891,7 @@ namespace { const RALinScan &Allocator; public: - WeightCompare(const RALinScan &Alloc) : Allocator(Alloc) {}; + WeightCompare(const RALinScan &Alloc) : Allocator(Alloc) {} typedef std::pair<unsigned, float> RegWeightPair; bool operator()(const RegWeightPair &LHS, const RegWeightPair &RHS) const { diff --git a/lib/CodeGen/RegAllocLocal.cpp b/lib/CodeGen/RegAllocLocal.cpp index 7bb020a..aea5cff 100644 --- a/lib/CodeGen/RegAllocLocal.cpp +++ b/lib/CodeGen/RegAllocLocal.cpp @@ -233,14 +233,17 @@ namespace { /// in one of several ways: if the register is available in a physical /// register already, it uses that physical register. If the value is not /// in a physical register, and if there are physical registers available, - /// it loads it into a register. If register pressure is high, and it is - /// possible, it tries to fold the load of the virtual register into the - /// instruction itself. It avoids doing this if register pressure is low to - /// improve the chance that subsequent instructions can use the reloaded - /// value. This method returns the modified instruction. + /// it loads it into a register: PhysReg if that is an available physical + /// register, otherwise any physical register of the right class. + /// If register pressure is high, and it is possible, it tries to fold the + /// load of the virtual register into the instruction itself. It avoids + /// doing this if register pressure is low to improve the chance that + /// subsequent instructions can use the reloaded value. This method + /// returns the modified instruction. /// MachineInstr *reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI, - unsigned OpNum, SmallSet<unsigned, 4> &RRegs); + unsigned OpNum, SmallSet<unsigned, 4> &RRegs, + unsigned PhysReg); /// ComputeLocalLiveness - Computes liveness of registers within a basic /// block, setting the killed/dead flags as appropriate. @@ -471,15 +474,17 @@ unsigned RALocal::getReg(MachineBasicBlock &MBB, MachineInstr *I, /// one of several ways: if the register is available in a physical register /// already, it uses that physical register. If the value is not in a physical /// register, and if there are physical registers available, it loads it into a +/// register: PhysReg if that is an available physical register, otherwise any /// register. If register pressure is high, and it is possible, it tries to /// fold the load of the virtual register into the instruction itself. It /// avoids doing this if register pressure is low to improve the chance that -/// subsequent instructions can use the reloaded value. This method returns the -/// modified instruction. +/// subsequent instructions can use the reloaded value. This method returns +/// the modified instruction. /// MachineInstr *RALocal::reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI, unsigned OpNum, - SmallSet<unsigned, 4> &ReloadedRegs) { + SmallSet<unsigned, 4> &ReloadedRegs, + unsigned PhysReg) { unsigned VirtReg = MI->getOperand(OpNum).getReg(); // If the virtual register is already available, just update the instruction @@ -494,7 +499,11 @@ MachineInstr *RALocal::reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI, // Otherwise, we need to fold it into the current instruction, or reload it. // If we have registers available to hold the value, use them. const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(VirtReg); - unsigned PhysReg = getFreeReg(RC); + // If we already have a PhysReg (this happens when the instruction is a + // reg-to-reg copy with a PhysReg destination) use that. + if (!PhysReg || !TargetRegisterInfo::isPhysicalRegister(PhysReg) || + !isPhysRegAvailable(PhysReg)) + PhysReg = getFreeReg(RC); int FrameIndex = getStackSpaceFor(VirtReg, RC); if (PhysReg) { // Register is available, allocate it! @@ -752,6 +761,12 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) { errs() << '\n'; }); + // Determine whether this is a copy instruction. The cases where the + // source or destination are phys regs are handled specially. + unsigned SrcCopyReg, DstCopyReg, SrcCopySubReg, DstCopySubReg; + bool isCopy = TII->isMoveInstr(*MI, SrcCopyReg, DstCopyReg, + SrcCopySubReg, DstCopySubReg); + // Loop over the implicit uses, making sure that they are at the head of the // use order list, so they don't get reallocated. if (TID.ImplicitUses) { @@ -835,7 +850,8 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) { // here we are looking for only used operands (never def&use) if (MO.isReg() && !MO.isDef() && MO.getReg() && !MO.isImplicit() && TargetRegisterInfo::isVirtualRegister(MO.getReg())) - MI = reloadVirtReg(MBB, MI, i, ReloadedRegs); + MI = reloadVirtReg(MBB, MI, i, ReloadedRegs, + isCopy ? DstCopyReg : 0); } // If this instruction is the last user of this register, kill the @@ -948,8 +964,17 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) { unsigned DestPhysReg; // If DestVirtReg already has a value, use it. - if (!(DestPhysReg = getVirt2PhysRegMapSlot(DestVirtReg))) - DestPhysReg = getReg(MBB, MI, DestVirtReg); + if (!(DestPhysReg = getVirt2PhysRegMapSlot(DestVirtReg))) { + // If this is a copy, the source reg is a phys reg, and + // that reg is available, use that phys reg for DestPhysReg. + if (isCopy && + TargetRegisterInfo::isPhysicalRegister(SrcCopyReg) && + isPhysRegAvailable(SrcCopyReg)) { + DestPhysReg = SrcCopyReg; + assignVirtToPhysReg(DestVirtReg, DestPhysReg); + } else + DestPhysReg = getReg(MBB, MI, DestVirtReg); + } MF->getRegInfo().setPhysRegUsed(DestPhysReg); markVirtRegModified(DestVirtReg); getVirtRegLastUse(DestVirtReg) = std::make_pair((MachineInstr*)0, 0); @@ -995,9 +1020,9 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) { // Finally, if this is a noop copy instruction, zap it. (Except that if // the copy is dead, it must be kept to avoid messing up liveness info for // the register scavenger. See pr4100.) - unsigned SrcReg, DstReg, SrcSubReg, DstSubReg; - if (TII->isMoveInstr(*MI, SrcReg, DstReg, SrcSubReg, DstSubReg) && - SrcReg == DstReg && DeadDefs.empty()) + if (TII->isMoveInstr(*MI, SrcCopyReg, DstCopyReg, + SrcCopySubReg, DstCopySubReg) && + SrcCopyReg == DstCopyReg && DeadDefs.empty()) MBB.erase(MI); } diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 2b52187..e6aa14c 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -2755,7 +2755,34 @@ SDValue DAGCombiner::visitSRL(SDNode *N) { if (N1C && SimplifyDemandedBits(SDValue(N, 0))) return SDValue(N, 0); - return N1C ? visitShiftByConstant(N, N1C->getZExtValue()) : SDValue(); + if (N1C) { + SDValue NewSRL = visitShiftByConstant(N, N1C->getZExtValue()); + if (NewSRL.getNode()) + return NewSRL; + } + + // Here is a common situation. We want to optimize: + // + // %a = ... + // %b = and i32 %a, 2 + // %c = srl i32 %b, 1 + // brcond i32 %c ... + // + // into + // + // %a = ... + // %b = and %a, 2 + // %c = setcc eq %b, 0 + // brcond %c ... + // + // However when after the source operand of SRL is optimized into AND, the SRL + // itself may not be optimized further. Look for it and add the BRCOND into + // the worklist. + if (N->hasOneUse() && + N->use_begin()->getOpcode() == ISD::BRCOND) + AddToWorkList(*N->use_begin()); + + return SDValue(); } SDValue DAGCombiner::visitCTLZ(SDNode *N) { @@ -3202,19 +3229,6 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) { X, DAG.getConstant(Mask, VT)); } - // Fold (zext (and x, cst)) -> (and (zext x), cst) - if (N0.getOpcode() == ISD::AND && - N0.getOperand(1).getOpcode() == ISD::Constant && - N0.getOperand(0).getOpcode() != ISD::TRUNCATE && - N0.getOperand(0).hasOneUse()) { - APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue(); - Mask.zext(VT.getSizeInBits()); - return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, - DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, - N0.getOperand(0)), - DAG.getConstant(Mask, VT)); - } - // fold (zext (load x)) -> (zext (truncate (zextload x))) if (ISD::isNON_EXTLoad(N0.getNode()) && ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) || diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp index 4ead9c9..33694f2 100644 --- a/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -43,7 +43,6 @@ #include "llvm/GlobalVariable.h" #include "llvm/Instructions.h" #include "llvm/IntrinsicInst.h" -#include "llvm/LLVMContext.h" #include "llvm/CodeGen/FastISel.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfo.h" @@ -334,7 +333,7 @@ bool FastISel::SelectCall(User *I) { return true; case Intrinsic::dbg_declare: { DbgDeclareInst *DI = cast<DbgDeclareInst>(I); - if (!isValidDebugInfoIntrinsic(*DI, CodeGenOpt::None) || !DW + if (!DIDescriptor::ValidDebugInfo(DI->getVariable(), CodeGenOpt::None)||!DW || !DW->ShouldEmitDwarfDebug()) return true; @@ -349,11 +348,8 @@ bool FastISel::SelectCall(User *I) { if (SI == StaticAllocaMap.end()) break; // VLAs. int FI = SI->second; if (MMI) { - MetadataContext &TheMetadata = - DI->getParent()->getContext().getMetadata(); - unsigned MDDbgKind = TheMetadata.getMDKind("dbg"); - MDNode *Dbg = TheMetadata.getMD(MDDbgKind, DI); - MMI->setVariableDbgInfo(DI->getVariable(), FI, Dbg); + if (MDNode *Dbg = DI->getMetadata("dbg")) + MMI->setVariableDbgInfo(DI->getVariable(), FI, Dbg); } return true; } @@ -548,9 +544,6 @@ FastISel::SelectInstruction(Instruction *I) { /// the CFG. void FastISel::FastEmitBranch(MachineBasicBlock *MSucc) { - MachineFunction::iterator NextMBB = - llvm::next(MachineFunction::iterator(MBB)); - if (MBB->isLayoutSuccessor(MSucc)) { // The unconditional fall-through case, which needs no instructions. } else { diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index f9c05d0..474d833 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -187,7 +187,6 @@ SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(EVT NVT, EVT VT, DebugLoc dl, SDValue N1, SDValue N2, SmallVectorImpl<int> &Mask) const { - EVT EltVT = NVT.getVectorElementType(); unsigned NumMaskElts = VT.getVectorNumElements(); unsigned NumDestElts = NVT.getVectorNumElements(); unsigned NumEltsGrowth = NumDestElts / NumMaskElts; @@ -461,8 +460,7 @@ SDValue ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, !ST->getMemoryVT().isVector() && "Unaligned store of unknown type."); // Get the half-size VT - EVT NewStoredVT = - (MVT::SimpleValueType)(ST->getMemoryVT().getSimpleVT().SimpleTy - 1); + EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext()); int NumBits = NewStoredVT.getSizeInBits(); int IncrementSize = NumBits / 8; @@ -1170,8 +1168,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp2 = LegalizeOp(Ch); } else if (SrcWidth & (SrcWidth - 1)) { // If not loading a power-of-2 number of bits, expand as two loads. - assert(SrcVT.isExtended() && !SrcVT.isVector() && - "Unsupported extload!"); + assert(!SrcVT.isVector() && "Unsupported extload!"); unsigned RoundWidth = 1 << Log2_32(SrcWidth); assert(RoundWidth < SrcWidth); unsigned ExtraWidth = SrcWidth - RoundWidth; @@ -1384,8 +1381,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { SVOffset, NVT, isVolatile, Alignment); } else if (StWidth & (StWidth - 1)) { // If not storing a power-of-2 number of bits, expand as two stores. - assert(StVT.isExtended() && !StVT.isVector() && - "Unsupported truncstore!"); + assert(!StVT.isVector() && "Unsupported truncstore!"); unsigned RoundWidth = 1 << Log2_32(StWidth); assert(RoundWidth < StWidth); unsigned ExtraWidth = StWidth - RoundWidth; @@ -1869,7 +1865,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, 0, TLI.getLibcallCallingConv(LC), false, /*isReturnValueUsed=*/true, Callee, Args, DAG, - Node->getDebugLoc()); + Node->getDebugLoc(), DAG.GetOrdering(Node)); // Legalize the call sequence, starting with the chain. This will advance // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that @@ -2274,7 +2270,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, false, false, false, false, 0, CallingConv::C, false, /*isReturnValueUsed=*/true, DAG.getExternalSymbol("abort", TLI.getPointerTy()), - Args, DAG, dl); + Args, DAG, dl, DAG.GetOrdering(Node)); Results.push_back(CallResult.second); break; } @@ -2750,7 +2746,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, SDValue RHS = Node->getOperand(1); SDValue BottomHalf; SDValue TopHalf; - static unsigned Ops[2][3] = + static const unsigned Ops[2][3] = { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; bool isSigned = Node->getOpcode() == ISD::SMULO; @@ -2967,7 +2963,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node, break; case ISD::BSWAP: { unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); - Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Tmp1); + Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, DAG.getConstant(DiffBits, TLI.getShiftAmountTy())); diff --git a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index 84e39b4..2831617 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -637,7 +637,8 @@ void DAGTypeLegalizer::SoftenSetCCOperands(SDValue &NewLHS, SDValue &NewRHS, } } - EVT RetVT = MVT::i32; // FIXME: is this the correct return type? + // Use the target specific return value for comparions lib calls. + EVT RetVT = TLI.getCmpLibcallReturnType(); SDValue Ops[2] = { LHSInt, RHSInt }; NewLHS = MakeLibCall(LC1, RetVT, Ops, 2, false/*sign irrelevant*/, dl); NewRHS = DAG.getConstant(0, RetVT); diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 2f4457e..bd3b97a 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -2026,8 +2026,6 @@ void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, GetExpandedInteger(NewLHS, LHSLo, LHSHi); GetExpandedInteger(NewRHS, RHSLo, RHSHi); - EVT VT = NewLHS.getValueType(); - if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { if (RHSLo == RHSHi) { if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp index 003cea7..d9efd4f 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp @@ -1033,7 +1033,8 @@ SDValue DAGTypeLegalizer::MakeLibCall(RTLIB::Libcall LC, EVT RetVT, TLI.LowerCallTo(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false, false, 0, TLI.getLibcallCallingConv(LC), false, /*isReturnValueUsed=*/true, - Callee, Args, DAG, dl); + Callee, Args, DAG, dl, + DAG.GetOrdering(DAG.getEntryNode().getNode())); return CallInfo.first; } diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp index dbd3e39..a1b6ced 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp @@ -464,7 +464,6 @@ void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo, void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) { EVT LoVT, HiVT; - DebugLoc dl = N->getDebugLoc(); GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); Lo = DAG.getUNDEF(LoVT); Hi = DAG.getUNDEF(HiVT); diff --git a/lib/CodeGen/SelectionDAG/SDNodeOrdering.h b/lib/CodeGen/SelectionDAG/SDNodeOrdering.h new file mode 100644 index 0000000..f88b26d --- /dev/null +++ b/lib/CodeGen/SelectionDAG/SDNodeOrdering.h @@ -0,0 +1,54 @@ +//===-- llvm/CodeGen/SDNodeOrdering.h - SDNode Ordering ---------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declares the SDNodeOrdering class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_SDNODEORDERING_H +#define LLVM_CODEGEN_SDNODEORDERING_H + +#include "llvm/ADT/DenseMap.h" + +namespace llvm { + +class SDNode; + +/// SDNodeOrdering - Maps a unique (monotonically increasing) value to each +/// SDNode that roughly corresponds to the ordering of the original LLVM +/// instruction. This is used for turning off scheduling, because we'll forgo +/// the normal scheduling algorithms and output the instructions according to +/// this ordering. +class SDNodeOrdering { + DenseMap<const SDNode*, unsigned> OrderMap; + + void operator=(const SDNodeOrdering&); // Do not implement. + SDNodeOrdering(const SDNodeOrdering&); // Do not implement. +public: + SDNodeOrdering() {} + + void add(const SDNode *Node, unsigned O) { + OrderMap[Node] = O; + } + void remove(const SDNode *Node) { + DenseMap<const SDNode*, unsigned>::iterator Itr = OrderMap.find(Node); + if (Itr != OrderMap.end()) + OrderMap.erase(Itr); + } + void clear() { + OrderMap.clear(); + } + unsigned getOrder(const SDNode *Node) { + return OrderMap[Node]; + } +}; + +} // end llvm namespace + +#endif diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index b2ee8bb..d53de34 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -20,16 +20,10 @@ #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetSubtarget.h" -#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; -cl::opt<bool> -DisableInstScheduling("disable-inst-scheduling", - cl::init(false), - cl::desc("Disable instruction scheduling")); - ScheduleDAGSDNodes::ScheduleDAGSDNodes(MachineFunction &mf) : ScheduleDAG(mf) { } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index da55e6b..77301b0 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -10,7 +10,9 @@ // This implements the SelectionDAG class. // //===----------------------------------------------------------------------===// + #include "llvm/CodeGen/SelectionDAG.h" +#include "SDNodeOrdering.h" #include "llvm/Constants.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Function.h" @@ -48,8 +50,6 @@ #include <cmath> using namespace llvm; -extern cl::opt<bool> DisableInstScheduling; - /// makeVTList - Return an instance of the SDVTList struct initialized with the /// specified members. static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { @@ -554,9 +554,6 @@ void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes, } DeallocateNode(N); - - // Remove the ordering of this node. - if (Ordering) Ordering->remove(N); } } @@ -582,9 +579,6 @@ void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { N->DropOperands(); DeallocateNode(N); - - // Remove the ordering of this node. - if (Ordering) Ordering->remove(N); } void SelectionDAG::DeallocateNode(SDNode *N) { @@ -703,7 +697,6 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); AddNodeIDCustom(ID, N); SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); - if (Ordering) Ordering->remove(Node); return Node; } @@ -722,7 +715,6 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); AddNodeIDCustom(ID, N); SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); - if (Ordering) Ordering->remove(Node); return Node; } @@ -741,7 +733,6 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); AddNodeIDCustom(ID, N); SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); - if (Ordering) Ordering->remove(Node); return Node; } @@ -798,10 +789,8 @@ SelectionDAG::SelectionDAG(TargetLowering &tli, FunctionLoweringInfo &fli) getVTList(MVT::Other)), Root(getEntryNode()), Ordering(0) { AllNodes.push_back(&EntryNode); - if (DisableInstScheduling) { - Ordering = new NodeOrdering(); - Ordering->add(&EntryNode); - } + if (DisableScheduling) + Ordering = new SDNodeOrdering(); } void SelectionDAG::init(MachineFunction &mf, MachineModuleInfo *mmi, @@ -840,10 +829,8 @@ void SelectionDAG::clear() { EntryNode.UseList = 0; AllNodes.push_back(&EntryNode); Root = getEntryNode(); - if (DisableInstScheduling) { - Ordering = new NodeOrdering(); - Ordering->add(&EntryNode); - } + if (DisableScheduling) + Ordering = new SDNodeOrdering(); } SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { @@ -904,17 +891,15 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { ID.AddPointer(&Val); void *IP = 0; SDNode *N = NULL; - if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) { - if (Ordering) Ordering->add(N); + if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) if (!VT.isVector()) return SDValue(N, 0); - } + if (!N) { N = NodeAllocator.Allocate<ConstantSDNode>(); new (N) ConstantSDNode(isT, &Val, EltVT); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); } SDValue Result(N, 0); @@ -951,17 +936,15 @@ SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ ID.AddPointer(&V); void *IP = 0; SDNode *N = NULL; - if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) { - if (Ordering) Ordering->add(N); + if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) if (!VT.isVector()) return SDValue(N, 0); - } + if (!N) { N = NodeAllocator.Allocate<ConstantFPSDNode>(); new (N) ConstantFPSDNode(isTarget, &V, EltVT); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); } SDValue Result(N, 0); @@ -1016,15 +999,13 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, ID.AddInteger(Offset); ID.AddInteger(TargetFlags); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + SDNode *N = NodeAllocator.Allocate<GlobalAddressSDNode>(); new (N) GlobalAddressSDNode(Opc, GV, VT, Offset, TargetFlags); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1034,15 +1015,13 @@ SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); ID.AddInteger(FI); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + SDNode *N = NodeAllocator.Allocate<FrameIndexSDNode>(); new (N) FrameIndexSDNode(FI, VT, isTarget); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1056,15 +1035,13 @@ SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, ID.AddInteger(JTI); ID.AddInteger(TargetFlags); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + SDNode *N = NodeAllocator.Allocate<JumpTableSDNode>(); new (N) JumpTableSDNode(JTI, VT, isTarget, TargetFlags); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1084,15 +1061,13 @@ SDValue SelectionDAG::getConstantPool(Constant *C, EVT VT, ID.AddPointer(C); ID.AddInteger(TargetFlags); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + SDNode *N = NodeAllocator.Allocate<ConstantPoolSDNode>(); new (N) ConstantPoolSDNode(isTarget, C, VT, Offset, Alignment, TargetFlags); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1113,15 +1088,13 @@ SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, C->AddSelectionDAGCSEId(ID); ID.AddInteger(TargetFlags); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + SDNode *N = NodeAllocator.Allocate<ConstantPoolSDNode>(); new (N) ConstantPoolSDNode(isTarget, C, VT, Offset, Alignment, TargetFlags); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1130,15 +1103,13 @@ SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0); ID.AddPointer(MBB); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + SDNode *N = NodeAllocator.Allocate<BasicBlockSDNode>(); new (N) BasicBlockSDNode(MBB); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1154,7 +1125,6 @@ SDValue SelectionDAG::getValueType(EVT VT) { N = NodeAllocator.Allocate<VTSDNode>(); new (N) VTSDNode(VT); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1164,7 +1134,6 @@ SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { N = NodeAllocator.Allocate<ExternalSymbolSDNode>(); new (N) ExternalSymbolSDNode(false, Sym, 0, VT); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1177,7 +1146,6 @@ SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, N = NodeAllocator.Allocate<ExternalSymbolSDNode>(); new (N) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1190,8 +1158,8 @@ SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { new (N) CondCodeSDNode(Cond); CondCodeNodes[Cond] = N; AllNodes.push_back(N); - if (Ordering) Ordering->add(N); } + return SDValue(CondCodeNodes[Cond], 0); } @@ -1283,10 +1251,8 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, DebugLoc dl, SDValue N1, ID.AddInteger(MaskVec[i]); void* IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } // Allocate the mask array for the node out of the BumpPtrAllocator, since // SDNode doesn't have access to it. This memory will be "leaked" when @@ -1298,7 +1264,6 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, DebugLoc dl, SDValue N1, new (N) ShuffleVectorSDNode(VT, dl, N1, N2, MaskAlloc); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1316,15 +1281,13 @@ SDValue SelectionDAG::getConvertRndSat(EVT VT, DebugLoc dl, SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); void* IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + CvtRndSatSDNode *N = NodeAllocator.Allocate<CvtRndSatSDNode>(); new (N) CvtRndSatSDNode(VT, dl, Ops, 5, Code); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1333,15 +1296,13 @@ SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0); ID.AddInteger(RegNo); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + SDNode *N = NodeAllocator.Allocate<RegisterSDNode>(); new (N) RegisterSDNode(RegNo, VT); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1353,15 +1314,13 @@ SDValue SelectionDAG::getLabel(unsigned Opcode, DebugLoc dl, AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), &Ops[0], 1); ID.AddInteger(LabelID); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + SDNode *N = NodeAllocator.Allocate<LabelSDNode>(); new (N) LabelSDNode(Opcode, dl, Root, LabelID); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1375,15 +1334,13 @@ SDValue SelectionDAG::getBlockAddress(BlockAddress *BA, EVT VT, ID.AddPointer(BA); ID.AddInteger(TargetFlags); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + SDNode *N = NodeAllocator.Allocate<BlockAddressSDNode>(); new (N) BlockAddressSDNode(Opc, VT, BA, TargetFlags); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -1396,16 +1353,13 @@ SDValue SelectionDAG::getSrcValue(const Value *V) { ID.AddPointer(V); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } SDNode *N = NodeAllocator.Allocate<SrcValueSDNode>(); new (N) SrcValueSDNode(V); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -2316,16 +2270,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT) { FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + SDNode *N = NodeAllocator.Allocate<SDNode>(); new (N) SDNode(Opcode, DL, getVTList(VT)); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); #ifndef NDEBUG VerifyNode(N); #endif @@ -2549,10 +2501,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDValue Ops[1] = { Operand }; AddNodeIDNode(ID, Opcode, VTs, Ops, 1); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + N = NodeAllocator.Allocate<UnarySDNode>(); new (N) UnarySDNode(Opcode, DL, VTs, Operand); CSEMap.InsertNode(N, IP); @@ -2562,7 +2513,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, } AllNodes.push_back(N); - if (Ordering) Ordering->add(N); #ifndef NDEBUG VerifyNode(N); #endif @@ -2970,10 +2920,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTs, Ops, 2); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + N = NodeAllocator.Allocate<BinarySDNode>(); new (N) BinarySDNode(Opcode, DL, VTs, N1, N2); CSEMap.InsertNode(N, IP); @@ -2983,7 +2932,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, } AllNodes.push_back(N); - if (Ordering) Ordering->add(N); #ifndef NDEBUG VerifyNode(N); #endif @@ -3050,10 +2998,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTs, Ops, 3); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + N = NodeAllocator.Allocate<TernarySDNode>(); new (N) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); CSEMap.InsertNode(N, IP); @@ -3063,7 +3010,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, } AllNodes.push_back(N); - if (Ordering) Ordering->add(N); #ifndef NDEBUG VerifyNode(N); #endif @@ -3503,7 +3449,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, /*isReturnValueUsed=*/false, getExternalSymbol(TLI.getLibcallName(RTLIB::MEMCPY), TLI.getPointerTy()), - Args, *this, dl); + Args, *this, dl, GetOrdering(Chain.getNode())); return CallResult.second; } @@ -3552,7 +3498,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, /*isReturnValueUsed=*/false, getExternalSymbol(TLI.getLibcallName(RTLIB::MEMMOVE), TLI.getPointerTy()), - Args, *this, dl); + Args, *this, dl, GetOrdering(Chain.getNode())); return CallResult.second; } @@ -3611,7 +3557,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, /*isReturnValueUsed=*/false, getExternalSymbol(TLI.getLibcallName(RTLIB::MEMSET), TLI.getPointerTy()), - Args, *this, dl); + Args, *this, dl, GetOrdering(Chain.getNode())); return CallResult.second; } @@ -3659,14 +3605,12 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, void* IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { cast<AtomicSDNode>(E)->refineAlignment(MMO); - if (Ordering) Ordering->add(E); return SDValue(E, 0); } SDNode* N = NodeAllocator.Allocate<AtomicSDNode>(); new (N) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Cmp, Swp, MMO); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -3724,14 +3668,12 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, void* IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { cast<AtomicSDNode>(E)->refineAlignment(MMO); - if (Ordering) Ordering->add(E); return SDValue(E, 0); } SDNode* N = NodeAllocator.Allocate<AtomicSDNode>(); new (N) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Val, MMO); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -3804,7 +3746,6 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); - if (Ordering) Ordering->add(E); return SDValue(E, 0); } @@ -3816,7 +3757,6 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); } AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -3881,14 +3821,12 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { cast<LoadSDNode>(E)->refineAlignment(MMO); - if (Ordering) Ordering->add(E); return SDValue(E, 0); } SDNode *N = NodeAllocator.Allocate<LoadSDNode>(); new (N) LoadSDNode(Ops, dl, VTs, AM, ExtType, MemVT, MMO); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -3959,14 +3897,12 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { cast<StoreSDNode>(E)->refineAlignment(MMO); - if (Ordering) Ordering->add(E); return SDValue(E, 0); } SDNode *N = NodeAllocator.Allocate<StoreSDNode>(); new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, false, VT, MMO); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -4021,14 +3957,12 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { cast<StoreSDNode>(E)->refineAlignment(MMO); - if (Ordering) Ordering->add(E); return SDValue(E, 0); } SDNode *N = NodeAllocator.Allocate<StoreSDNode>(); new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, true, SVT, MMO); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -4045,17 +3979,15 @@ SelectionDAG::getIndexedStore(SDValue OrigStore, DebugLoc dl, SDValue Base, ID.AddInteger(ST->getMemoryVT().getRawBits()); ID.AddInteger(ST->getRawSubclassData()); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + SDNode *N = NodeAllocator.Allocate<StoreSDNode>(); new (N) StoreSDNode(Ops, dl, VTs, AM, ST->isTruncatingStore(), ST->getMemoryVT(), ST->getMemOperand()); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); return SDValue(N, 0); } @@ -4121,10 +4053,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } N = NodeAllocator.Allocate<SDNode>(); new (N) SDNode(Opcode, DL, VTs, Ops, NumOps); @@ -4135,7 +4065,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, } AllNodes.push_back(N); - if (Ordering) Ordering->add(N); #ifndef NDEBUG VerifyNode(N); #endif @@ -4191,10 +4120,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - } + if (NumOps == 1) { N = NodeAllocator.Allocate<UnarySDNode>(); new (N) UnarySDNode(Opcode, DL, VTList, Ops[0]); @@ -4225,7 +4153,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, } } AllNodes.push_back(N); - if (Ordering) Ordering->add(N); #ifndef NDEBUG VerifyNode(N); #endif @@ -4325,6 +4252,7 @@ SDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { case 1: return getVTList(VTs[0]); case 2: return getVTList(VTs[0], VTs[1]); case 3: return getVTList(VTs[0], VTs[1], VTs[2]); + case 4: return getVTList(VTs[0], VTs[1], VTs[2], VTs[3]); default: break; } @@ -4688,10 +4616,8 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, if (VTs.VTs[VTs.NumVTs-1] != MVT::Flag) { FoldingSetNodeID ID; AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); - if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(ON); + if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) return ON; - } } if (!RemoveNodeFromCSEMaps(N)) @@ -4755,7 +4681,6 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, if (IP) CSEMap.InsertNode(N, IP); // Memoize the new node. - if (Ordering) Ordering->add(N); return N; } @@ -4894,10 +4819,8 @@ SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, FoldingSetNodeID ID; AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return cast<MachineSDNode>(E); - } } // Allocate a new MachineSDNode. @@ -4919,7 +4842,6 @@ SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, CSEMap.InsertNode(N, IP); AllNodes.push_back(N); - if (Ordering) Ordering->add(N); #ifndef NDEBUG VerifyNode(N); #endif @@ -4956,10 +4878,8 @@ SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { - if (Ordering) Ordering->add(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return E; - } } return NULL; } @@ -5291,6 +5211,18 @@ unsigned SelectionDAG::AssignTopologicalOrder() { return DAGSize; } +/// AssignOrdering - Assign an order to the SDNode. +void SelectionDAG::AssignOrdering(SDNode *SD, unsigned Order) { + assert(SD && "Trying to assign an order to a null node!"); + if (Ordering) + Ordering->add(SD, Order); +} + +/// GetOrdering - Get the order for the SDNode. +unsigned SelectionDAG::GetOrdering(const SDNode *SD) const { + assert(SD && "Trying to get the order of a null node!"); + return Ordering ? Ordering->getOrder(SD) : 0; +} //===----------------------------------------------------------------------===// @@ -5931,6 +5863,10 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { if (unsigned int TF = BA->getTargetFlags()) OS << " [TF=" << TF << ']'; } + + if (G) + if (unsigned Order = G->GetOrdering(this)) + OS << " [ORD=" << Order << ']'; } void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const { @@ -6126,9 +6062,6 @@ void SelectionDAG::dump() const { errs() << "\n\n"; } -void SelectionDAG::NodeOrdering::dump() const { -} - void SDNode::printr(raw_ostream &OS, const SelectionDAG *G) const { print_types(OS, G); print_details(OS, G); @@ -6139,25 +6072,31 @@ static void DumpNodesr(raw_ostream &OS, const SDNode *N, unsigned indent, const SelectionDAG *G, VisitedSDNodeSet &once) { if (!once.insert(N)) // If we've been here before, return now. return; + // Dump the current SDNode, but don't end the line yet. OS << std::string(indent, ' '); N->printr(OS, G); + // Having printed this SDNode, walk the children: for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { const SDNode *child = N->getOperand(i).getNode(); + if (i) OS << ","; OS << " "; + if (child->getNumOperands() == 0) { // This child has no grandchildren; print it inline right here. child->printr(OS, G); once.insert(child); - } else { // Just the address. FIXME: also print the child's opcode + } else { // Just the address. FIXME: also print the child's opcode. OS << (void*)child; if (unsigned RN = N->getOperand(i).getResNo()) OS << ":" << RN; } } + OS << "\n"; + // Dump children that have grandchildren on their own line(s). for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { const SDNode *child = N->getOperand(i).getNode(); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 7568384..74d624f 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -17,6 +17,7 @@ #include "llvm/ADT/BitVector.h" #include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/ConstantFolding.h" #include "llvm/Constants.h" #include "llvm/CallingConv.h" #include "llvm/DerivedTypes.h" @@ -26,7 +27,6 @@ #include "llvm/Instructions.h" #include "llvm/Intrinsics.h" #include "llvm/IntrinsicInst.h" -#include "llvm/LLVMContext.h" #include "llvm/Module.h" #include "llvm/CodeGen/FastISel.h" #include "llvm/CodeGen/GCStrategy.h" @@ -70,13 +70,13 @@ LimitFPPrecision("limit-float-precision", namespace { /// RegsForValue - This struct represents the registers (physical or virtual) - /// that a particular set of values is assigned, and the type information about - /// the value. The most common situation is to represent one value at a time, - /// but struct or array values are handled element-wise as multiple values. - /// The splitting of aggregates is performed recursively, so that we never - /// have aggregate-typed registers. The values at this point do not necessarily - /// have legal types, so each value may require one or more registers of some - /// legal type. + /// that a particular set of values is assigned, and the type information + /// about the value. The most common situation is to represent one value at a + /// time, but struct or array values are handled element-wise as multiple + /// values. The splitting of aggregates is performed recursively, so that we + /// never have aggregate-typed registers. The values at this point do not + /// necessarily have legal types, so each value may require one or more + /// registers of some legal type. /// struct RegsForValue { /// TLI - The TargetLowering object. @@ -144,22 +144,23 @@ namespace { /// this value and returns the result as a ValueVTs value. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. - SDValue getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, - SDValue &Chain, SDValue *Flag) const; + SDValue getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, unsigned Order, + SDValue &Chain, SDValue *Flag) const; /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the /// specified value into the registers specified by this object. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. void getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, - SDValue &Chain, SDValue *Flag) const; + unsigned Order, SDValue &Chain, SDValue *Flag) const; /// AddInlineAsmOperands - Add this value to the specified inlineasm node /// operand list. This adds the code marker, matching input operand index /// (if applicable), and includes the number of values added into it. void AddInlineAsmOperands(unsigned Code, bool HasMatching, unsigned MatchingIdx, - SelectionDAG &DAG, std::vector<SDValue> &Ops) const; + SelectionDAG &DAG, unsigned Order, + std::vector<SDValue> &Ops) const; }; } @@ -168,13 +169,14 @@ namespace { /// larger then ValueVT then AssertOp can be used to specify whether the extra /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT /// (ISD::AssertSext). -static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, +static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, unsigned Order, const SDValue *Parts, unsigned NumParts, EVT PartVT, EVT ValueVT, ISD::NodeType AssertOp = ISD::DELETED_NODE) { assert(NumParts > 0 && "No parts to assemble!"); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue Val = Parts[0]; + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); if (NumParts > 1) { // Assemble the value from multiple parts. @@ -193,23 +195,32 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); if (RoundParts > 2) { - Lo = getCopyFromParts(DAG, dl, Parts, RoundParts/2, PartVT, HalfVT); - Hi = getCopyFromParts(DAG, dl, Parts+RoundParts/2, RoundParts/2, + Lo = getCopyFromParts(DAG, dl, Order, Parts, RoundParts / 2, PartVT, HalfVT); + Hi = getCopyFromParts(DAG, dl, Order, Parts + RoundParts / 2, + RoundParts / 2, PartVT, HalfVT); } else { Lo = DAG.getNode(ISD::BIT_CONVERT, dl, HalfVT, Parts[0]); Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HalfVT, Parts[1]); } + if (TLI.isBigEndian()) std::swap(Lo, Hi); + Val = DAG.getNode(ISD::BUILD_PAIR, dl, RoundVT, Lo, Hi); + if (DisableScheduling) { + DAG.AssignOrdering(Lo.getNode(), Order); + DAG.AssignOrdering(Hi.getNode(), Order); + DAG.AssignOrdering(Val.getNode(), Order); + } + if (RoundParts < NumParts) { // Assemble the trailing non-power-of-2 part. unsigned OddParts = NumParts - RoundParts; EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits); - Hi = getCopyFromParts(DAG, dl, - Parts+RoundParts, OddParts, PartVT, OddVT); + Hi = getCopyFromParts(DAG, dl, Order, + Parts + RoundParts, OddParts, PartVT, OddVT); // Combine the round and odd parts. Lo = Val; @@ -217,22 +228,28 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, std::swap(Lo, Hi); EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); Hi = DAG.getNode(ISD::ANY_EXTEND, dl, TotalVT, Hi); + if (DisableScheduling) DAG.AssignOrdering(Hi.getNode(), Order); Hi = DAG.getNode(ISD::SHL, dl, TotalVT, Hi, DAG.getConstant(Lo.getValueType().getSizeInBits(), TLI.getPointerTy())); + if (DisableScheduling) DAG.AssignOrdering(Hi.getNode(), Order); Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, TotalVT, Lo); + if (DisableScheduling) DAG.AssignOrdering(Lo.getNode(), Order); Val = DAG.getNode(ISD::OR, dl, TotalVT, Lo, Hi); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); } } else if (ValueVT.isVector()) { // Handle a multi-element vector. EVT IntermediateVT, RegisterVT; unsigned NumIntermediates; unsigned NumRegs = - TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, + TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, NumIntermediates, RegisterVT); - assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); + assert(NumRegs == NumParts + && "Part count doesn't match vector breakdown!"); NumParts = NumRegs; // Silence a compiler warning. - assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); + assert(RegisterVT == PartVT + && "Part type doesn't match vector breakdown!"); assert(RegisterVT == Parts[0].getValueType() && "Part type doesn't match part!"); @@ -242,24 +259,25 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, // If the register was not expanded, truncate or copy the value, // as appropriate. for (unsigned i = 0; i != NumParts; ++i) - Ops[i] = getCopyFromParts(DAG, dl, &Parts[i], 1, + Ops[i] = getCopyFromParts(DAG, dl, Order, &Parts[i], 1, PartVT, IntermediateVT); } else if (NumParts > 0) { - // If the intermediate type was expanded, build the intermediate operands - // from the parts. + // If the intermediate type was expanded, build the intermediate + // operands from the parts. assert(NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"); unsigned Factor = NumParts / NumIntermediates; for (unsigned i = 0; i != NumIntermediates; ++i) - Ops[i] = getCopyFromParts(DAG, dl, &Parts[i * Factor], Factor, + Ops[i] = getCopyFromParts(DAG, dl, Order, &Parts[i * Factor], Factor, PartVT, IntermediateVT); } - // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the intermediate - // operands. + // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the + // intermediate operands. Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR, dl, ValueVT, &Ops[0], NumIntermediates); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); } else if (PartVT.isFloatingPoint()) { // FP split into multiple FP parts (for ppcf128) assert(ValueVT == EVT(MVT::ppcf128) && PartVT == EVT(MVT::f64) && @@ -270,12 +288,18 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, if (TLI.isBigEndian()) std::swap(Lo, Hi); Val = DAG.getNode(ISD::BUILD_PAIR, dl, ValueVT, Lo, Hi); + + if (DisableScheduling) { + DAG.AssignOrdering(Hi.getNode(), Order); + DAG.AssignOrdering(Lo.getNode(), Order); + DAG.AssignOrdering(Val.getNode(), Order); + } } else { // FP split into integer parts (soft fp) assert(ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"); EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); - Val = getCopyFromParts(DAG, dl, Parts, NumParts, PartVT, IntVT); + Val = getCopyFromParts(DAG, dl, Order, Parts, NumParts, PartVT, IntVT); } } @@ -287,14 +311,20 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, if (PartVT.isVector()) { assert(ValueVT.isVector() && "Unknown vector conversion!"); - return DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val); + SDValue Res = DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); + return Res; } if (ValueVT.isVector()) { assert(ValueVT.getVectorElementType() == PartVT && ValueVT.getVectorNumElements() == 1 && "Only trivial scalar-to-vector conversions should get here!"); - return DAG.getNode(ISD::BUILD_VECTOR, dl, ValueVT, Val); + SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, dl, ValueVT, Val); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); + return Res; } if (PartVT.isInteger() && @@ -306,22 +336,36 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, if (AssertOp != ISD::DELETED_NODE) Val = DAG.getNode(AssertOp, dl, PartVT, Val, DAG.getValueType(ValueVT)); - return DAG.getNode(ISD::TRUNCATE, dl, ValueVT, Val); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + Val = DAG.getNode(ISD::TRUNCATE, dl, ValueVT, Val); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + return Val; } else { - return DAG.getNode(ISD::ANY_EXTEND, dl, ValueVT, Val); + Val = DAG.getNode(ISD::ANY_EXTEND, dl, ValueVT, Val); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + return Val; } } if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { - if (ValueVT.bitsLT(Val.getValueType())) + if (ValueVT.bitsLT(Val.getValueType())) { // FP_ROUND's are always exact here. - return DAG.getNode(ISD::FP_ROUND, dl, ValueVT, Val, - DAG.getIntPtrConstant(1)); - return DAG.getNode(ISD::FP_EXTEND, dl, ValueVT, Val); + Val = DAG.getNode(ISD::FP_ROUND, dl, ValueVT, Val, + DAG.getIntPtrConstant(1)); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + return Val; + } + + Val = DAG.getNode(ISD::FP_EXTEND, dl, ValueVT, Val); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + return Val; } - if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) - return DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val); + if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { + Val = DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val); + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + return Val; + } llvm_unreachable("Unknown mismatch!"); return SDValue(); @@ -330,8 +374,9 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, /// getCopyToParts - Create a series of nodes that contain the specified value /// split into legal parts. If the parts contain more bits than Val, then, for /// integers, ExtendKind can be used to specify how to generate the extra bits. -static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, - SDValue *Parts, unsigned NumParts, EVT PartVT, +static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, unsigned Order, + SDValue Val, SDValue *Parts, unsigned NumParts, + EVT PartVT, ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT PtrVT = TLI.getPointerTy(); @@ -375,6 +420,8 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, } } + if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order); + // The value may have changed - recompute ValueVT. ValueVT = Val.getValueType(); assert(NumParts * PartBits == ValueVT.getSizeInBits() && @@ -397,20 +444,33 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, SDValue OddVal = DAG.getNode(ISD::SRL, dl, ValueVT, Val, DAG.getConstant(RoundBits, TLI.getPointerTy())); - getCopyToParts(DAG, dl, OddVal, Parts + RoundParts, OddParts, PartVT); + getCopyToParts(DAG, dl, Order, OddVal, Parts + RoundParts, + OddParts, PartVT); + if (TLI.isBigEndian()) // The odd parts were reversed by getCopyToParts - unreverse them. std::reverse(Parts + RoundParts, Parts + NumParts); + NumParts = RoundParts; ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); Val = DAG.getNode(ISD::TRUNCATE, dl, ValueVT, Val); + + if (DisableScheduling) { + DAG.AssignOrdering(OddVal.getNode(), Order); + DAG.AssignOrdering(Val.getNode(), Order); + } } // The number of parts is a power of 2. Repeatedly bisect the value using // EXTRACT_ELEMENT. Parts[0] = DAG.getNode(ISD::BIT_CONVERT, dl, - EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()), + EVT::getIntegerVT(*DAG.getContext(), + ValueVT.getSizeInBits()), Val); + + if (DisableScheduling) + DAG.AssignOrdering(Parts[0].getNode(), Order); + for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { for (unsigned i = 0; i < NumParts; i += StepSize) { unsigned ThisBits = StepSize * PartBits / 2; @@ -425,11 +485,20 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, ThisVT, Part0, DAG.getConstant(0, PtrVT)); + if (DisableScheduling) { + DAG.AssignOrdering(Part0.getNode(), Order); + DAG.AssignOrdering(Part1.getNode(), Order); + } + if (ThisBits == PartBits && ThisVT != PartVT) { Part0 = DAG.getNode(ISD::BIT_CONVERT, dl, PartVT, Part0); Part1 = DAG.getNode(ISD::BIT_CONVERT, dl, PartVT, Part1); + if (DisableScheduling) { + DAG.AssignOrdering(Part0.getNode(), Order); + DAG.AssignOrdering(Part1.getNode(), Order); + } } } } @@ -443,7 +512,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, // Vector ValueVT. if (NumParts == 1) { if (PartVT != ValueVT) { - if (PartVT.isVector()) { + if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { Val = DAG.getNode(ISD::BIT_CONVERT, dl, PartVT, Val); } else { assert(ValueVT.getVectorElementType() == PartVT && @@ -455,6 +524,9 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, } } + if (DisableScheduling) + DAG.AssignOrdering(Val.getNode(), Order); + Parts[0] = Val; return; } @@ -472,7 +544,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, // Split the vector into intermediate operands. SmallVector<SDValue, 8> Ops(NumIntermediates); - for (unsigned i = 0; i != NumIntermediates; ++i) + for (unsigned i = 0; i != NumIntermediates; ++i) { if (IntermediateVT.isVector()) Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, IntermediateVT, Val, @@ -483,12 +555,16 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, IntermediateVT, Val, DAG.getConstant(i, PtrVT)); + if (DisableScheduling) + DAG.AssignOrdering(Ops[i].getNode(), Order); + } + // Split the intermediate operands into legal parts. if (NumParts == NumIntermediates) { // If the register was not expanded, promote or copy the value, // as appropriate. for (unsigned i = 0; i != NumParts; ++i) - getCopyToParts(DAG, dl, Ops[i], &Parts[i], 1, PartVT); + getCopyToParts(DAG, dl, Order, Ops[i], &Parts[i], 1, PartVT); } else if (NumParts > 0) { // If the intermediate type was expanded, split each the value into // legal parts. @@ -496,7 +572,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val, "Must expand into a divisible number of parts!"); unsigned Factor = NumParts / NumIntermediates; for (unsigned i = 0; i != NumIntermediates; ++i) - getCopyToParts(DAG, dl, Ops[i], &Parts[i * Factor], Factor, PartVT); + getCopyToParts(DAG, dl, Order, Ops[i], &Parts[i*Factor], Factor, PartVT); } } @@ -583,8 +659,8 @@ void SelectionDAGBuilder::visit(Instruction &I) { } void SelectionDAGBuilder::visit(unsigned Opcode, User &I) { - // Tell the DAG that we're processing a new instruction. - DAG.NewInst(); + // We're processing a new instruction. + ++SDNodeOrder; // Note: this doesn't use InstVisitor, because it has to work with // ConstantExpr's in addition to instructions. @@ -592,7 +668,7 @@ void SelectionDAGBuilder::visit(unsigned Opcode, User &I) { default: llvm_unreachable("Unknown instruction type encountered!"); // Build the switch statement using the Instruction.def file. #define HANDLE_INST(NUM, OPCODE, CLASS) \ - case Instruction::OPCODE:return visit##OPCODE((CLASS&)I); + case Instruction::OPCODE: return visit##OPCODE((CLASS&)I); #include "llvm/Instruction.def" } } @@ -638,8 +714,12 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) Constants.push_back(SDValue(Val, i)); } - return DAG.getMergeValues(&Constants[0], Constants.size(), - getCurDebugLoc()); + + SDValue Res = DAG.getMergeValues(&Constants[0], Constants.size(), + getCurDebugLoc()); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + return Res; } if (isa<StructType>(C->getType()) || isa<ArrayType>(C->getType())) { @@ -661,7 +741,12 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { else Constants[i] = DAG.getConstant(0, EltVT); } - return DAG.getMergeValues(&Constants[0], NumElts, getCurDebugLoc()); + + SDValue Res = DAG.getMergeValues(&Constants[0], NumElts, + getCurDebugLoc()); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + return Res; } if (BlockAddress *BA = dyn_cast<BlockAddress>(C)) @@ -689,8 +774,12 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { } // Create a BUILD_VECTOR node. - return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurDebugLoc(), - VT, &Ops[0], Ops.size()); + SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, getCurDebugLoc(), + VT, &Ops[0], Ops.size()); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + + return NodeMap[V] = Res; } // If this is a static alloca, generate it as the frameindex instead of @@ -707,7 +796,8 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { RegsForValue RFV(*DAG.getContext(), TLI, InReg, V->getType()); SDValue Chain = DAG.getEntryNode(); - return RFV.getCopyFromRegs(DAG, getCurDebugLoc(), Chain, NULL); + return RFV.getCopyFromRegs(DAG, getCurDebugLoc(), + SDNodeOrder, Chain, NULL); } /// Get the EVTs and ArgFlags collections that represent the return type @@ -766,7 +856,7 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) { SDValue Chain = getControlRoot(); SmallVector<ISD::OutputArg, 8> Outs; FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); - + if (!FLI.CanLowerReturn) { unsigned DemoteReg = FLI.DemoteRegister; const Function *F = I.getParent()->getParent(); @@ -775,12 +865,12 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) { // Leave Outs empty so that LowerReturn won't try to load return // registers the usual way. SmallVector<EVT, 1> PtrValueVTs; - ComputeValueVTs(TLI, PointerType::getUnqual(F->getReturnType()), + ComputeValueVTs(TLI, PointerType::getUnqual(F->getReturnType()), PtrValueVTs); SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]); SDValue RetOp = getValue(I.getOperand(0)); - + SmallVector<EVT, 4> ValueVTs; SmallVector<uint64_t, 4> Offsets; ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets); @@ -788,22 +878,32 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) { SmallVector<SDValue, 4> Chains(NumValues); EVT PtrVT = PtrValueVTs[0]; - for (unsigned i = 0; i != NumValues; ++i) - Chains[i] = DAG.getStore(Chain, getCurDebugLoc(), - SDValue(RetOp.getNode(), RetOp.getResNo() + i), - DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, RetPtr, - DAG.getConstant(Offsets[i], PtrVT)), - NULL, Offsets[i], false, 0); + for (unsigned i = 0; i != NumValues; ++i) { + SDValue Add = DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, RetPtr, + DAG.getConstant(Offsets[i], PtrVT)); + Chains[i] = + DAG.getStore(Chain, getCurDebugLoc(), + SDValue(RetOp.getNode(), RetOp.getResNo() + i), + Add, NULL, Offsets[i], false, 0); + + if (DisableScheduling) { + DAG.AssignOrdering(Add.getNode(), SDNodeOrder); + DAG.AssignOrdering(Chains[i].getNode(), SDNodeOrder); + } + } + Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), MVT::Other, &Chains[0], NumValues); - } - else { + + if (DisableScheduling) + DAG.AssignOrdering(Chain.getNode(), SDNodeOrder); + } else { for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { SmallVector<EVT, 4> ValueVTs; ComputeValueVTs(TLI, I.getOperand(i)->getType(), ValueVTs); unsigned NumValues = ValueVTs.size(); if (NumValues == 0) continue; - + SDValue RetOp = getValue(I.getOperand(i)); for (unsigned j = 0, f = NumValues; j != f; ++j) { EVT VT = ValueVTs[j]; @@ -816,8 +916,8 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) { else if (F->paramHasAttr(0, Attribute::ZExt)) ExtendKind = ISD::ZERO_EXTEND; - // FIXME: C calling convention requires the return type to be promoted to - // at least 32-bit. But this is not necessary for non-C calling + // FIXME: C calling convention requires the return type to be promoted + // to at least 32-bit. But this is not necessary for non-C calling // conventions. The frontend should mark functions whose return values // require promoting with signext or zeroext attributes. if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) { @@ -829,7 +929,7 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) { unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), VT); EVT PartVT = TLI.getRegisterType(*DAG.getContext(), VT); SmallVector<SDValue, 4> Parts(NumParts); - getCopyToParts(DAG, getCurDebugLoc(), + getCopyToParts(DAG, getCurDebugLoc(), SDNodeOrder, SDValue(RetOp.getNode(), RetOp.getResNo() + j), &Parts[0], NumParts, PartVT, ExtendKind); @@ -862,6 +962,9 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) { // Update the DAG with the new chain value resulting from return lowering. DAG.setRoot(Chain); + + if (DisableScheduling) + DAG.AssignOrdering(Chain.getNode(), SDNodeOrder); } /// CopyToExportRegsIfNeeded - If the given value has virtual registers @@ -1110,10 +1213,16 @@ void SelectionDAGBuilder::visitBr(BranchInst &I) { CurMBB->addSuccessor(Succ0MBB); // If this is not a fall-through branch, emit the branch. - if (Succ0MBB != NextBlock) - DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(), + if (Succ0MBB != NextBlock) { + SDValue V = DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, getControlRoot(), - DAG.getBasicBlock(Succ0MBB))); + DAG.getBasicBlock(Succ0MBB)); + DAG.setRoot(V); + + if (DisableScheduling) + DAG.AssignOrdering(V.getNode(), SDNodeOrder); + } + return; } @@ -1172,6 +1281,7 @@ void SelectionDAGBuilder::visitBr(BranchInst &I) { // Create a CaseBlock record representing this branch. CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()), NULL, Succ0MBB, Succ1MBB, CurMBB); + // Use visitSwitchCase to actually insert the fast branch sequence for this // cond branch. visitSwitchCase(CB); @@ -1217,6 +1327,9 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB) { } } + if (DisableScheduling) + DAG.AssignOrdering(Cond.getNode(), SDNodeOrder); + // Update successor info CurMBB->addSuccessor(CB.TrueBB); CurMBB->addSuccessor(CB.FalseBB); @@ -1234,26 +1347,36 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB) { std::swap(CB.TrueBB, CB.FalseBB); SDValue True = DAG.getConstant(1, Cond.getValueType()); Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True); + + if (DisableScheduling) + DAG.AssignOrdering(Cond.getNode(), SDNodeOrder); } + SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(), Cond, DAG.getBasicBlock(CB.TrueBB)); + if (DisableScheduling) + DAG.AssignOrdering(BrCond.getNode(), SDNodeOrder); + // If the branch was constant folded, fix up the CFG. if (BrCond.getOpcode() == ISD::BR) { CurMBB->removeSuccessor(CB.FalseBB); - DAG.setRoot(BrCond); } else { // Otherwise, go ahead and insert the false branch. if (BrCond == getControlRoot()) CurMBB->removeSuccessor(CB.TrueBB); - if (CB.FalseBB == NextBlock) - DAG.setRoot(BrCond); - else - DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, - DAG.getBasicBlock(CB.FalseBB))); + if (CB.FalseBB != NextBlock) { + BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, + DAG.getBasicBlock(CB.FalseBB)); + + if (DisableScheduling) + DAG.AssignOrdering(BrCond.getNode(), SDNodeOrder); + } } + + DAG.setRoot(BrCond); } /// visitJumpTable - Emit JumpTable node in the current MBB @@ -1264,9 +1387,16 @@ void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) { SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurDebugLoc(), JT.Reg, PTy); SDValue Table = DAG.getJumpTable(JT.JTI, PTy); - DAG.setRoot(DAG.getNode(ISD::BR_JT, getCurDebugLoc(), - MVT::Other, Index.getValue(1), - Table, Index)); + SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurDebugLoc(), + MVT::Other, Index.getValue(1), + Table, Index); + DAG.setRoot(BrJumpTable); + + if (DisableScheduling) { + DAG.AssignOrdering(Index.getNode(), SDNodeOrder); + DAG.AssignOrdering(Table.getNode(), SDNodeOrder); + DAG.AssignOrdering(BrJumpTable.getNode(), SDNodeOrder); + } } /// visitJumpTableHeader - This function emits necessary code to produce index @@ -1278,7 +1408,7 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, // difference between smallest and largest cases. SDValue SwitchOp = getValue(JTH.SValue); EVT VT = SwitchOp.getValueType(); - SDValue SUB = DAG.getNode(ISD::SUB, getCurDebugLoc(), VT, SwitchOp, + SDValue Sub = DAG.getNode(ISD::SUB, getCurDebugLoc(), VT, SwitchOp, DAG.getConstant(JTH.First, VT)); // The SDNode we just created, which holds the value being switched on minus @@ -1286,7 +1416,7 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, // can be used as an index into the jump table in a subsequent basic block. // This value may be smaller or larger than the target's pointer type, and // therefore require extension or truncating. - SwitchOp = DAG.getZExtOrTrunc(SUB, getCurDebugLoc(), TLI.getPointerTy()); + SwitchOp = DAG.getZExtOrTrunc(Sub, getCurDebugLoc(), TLI.getPointerTy()); unsigned JumpTableReg = FuncInfo.MakeReg(TLI.getPointerTy()); SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurDebugLoc(), @@ -1297,14 +1427,22 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, // for the switch statement if the value being switched on exceeds the largest // case in the switch. SDValue CMP = DAG.getSetCC(getCurDebugLoc(), - TLI.getSetCCResultType(SUB.getValueType()), SUB, + TLI.getSetCCResultType(Sub.getValueType()), Sub, DAG.getConstant(JTH.Last-JTH.First,VT), ISD::SETUGT); + if (DisableScheduling) { + DAG.AssignOrdering(Sub.getNode(), SDNodeOrder); + DAG.AssignOrdering(SwitchOp.getNode(), SDNodeOrder); + DAG.AssignOrdering(CopyTo.getNode(), SDNodeOrder); + DAG.AssignOrdering(CMP.getNode(), SDNodeOrder); + } + // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; MachineFunction::iterator BBI = CurMBB; + if (++BBI != FuncInfo.MF->end()) NextBlock = BBI; @@ -1312,11 +1450,18 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, MVT::Other, CopyTo, CMP, DAG.getBasicBlock(JT.Default)); - if (JT.MBB == NextBlock) - DAG.setRoot(BrCond); - else - DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, BrCond, - DAG.getBasicBlock(JT.MBB))); + if (DisableScheduling) + DAG.AssignOrdering(BrCond.getNode(), SDNodeOrder); + + if (JT.MBB != NextBlock) { + BrCond = DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, BrCond, + DAG.getBasicBlock(JT.MBB)); + + if (DisableScheduling) + DAG.AssignOrdering(BrCond.getNode(), SDNodeOrder); + } + + DAG.setRoot(BrCond); } /// visitBitTestHeader - This function emits necessary code to produce value @@ -1325,21 +1470,29 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B) { // Subtract the minimum value SDValue SwitchOp = getValue(B.SValue); EVT VT = SwitchOp.getValueType(); - SDValue SUB = DAG.getNode(ISD::SUB, getCurDebugLoc(), VT, SwitchOp, + SDValue Sub = DAG.getNode(ISD::SUB, getCurDebugLoc(), VT, SwitchOp, DAG.getConstant(B.First, VT)); // Check range SDValue RangeCmp = DAG.getSetCC(getCurDebugLoc(), - TLI.getSetCCResultType(SUB.getValueType()), - SUB, DAG.getConstant(B.Range, VT), + TLI.getSetCCResultType(Sub.getValueType()), + Sub, DAG.getConstant(B.Range, VT), ISD::SETUGT); - SDValue ShiftOp = DAG.getZExtOrTrunc(SUB, getCurDebugLoc(), TLI.getPointerTy()); + SDValue ShiftOp = DAG.getZExtOrTrunc(Sub, getCurDebugLoc(), + TLI.getPointerTy()); B.Reg = FuncInfo.MakeReg(TLI.getPointerTy()); SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurDebugLoc(), B.Reg, ShiftOp); + if (DisableScheduling) { + DAG.AssignOrdering(Sub.getNode(), SDNodeOrder); + DAG.AssignOrdering(RangeCmp.getNode(), SDNodeOrder); + DAG.AssignOrdering(ShiftOp.getNode(), SDNodeOrder); + DAG.AssignOrdering(CopyTo.getNode(), SDNodeOrder); + } + // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; @@ -1356,11 +1509,18 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B) { MVT::Other, CopyTo, RangeCmp, DAG.getBasicBlock(B.Default)); - if (MBB == NextBlock) - DAG.setRoot(BrRange); - else - DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, CopyTo, - DAG.getBasicBlock(MBB))); + if (DisableScheduling) + DAG.AssignOrdering(BrRange.getNode(), SDNodeOrder); + + if (MBB != NextBlock) { + BrRange = DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, CopyTo, + DAG.getBasicBlock(MBB)); + + if (DisableScheduling) + DAG.AssignOrdering(BrRange.getNode(), SDNodeOrder); + } + + DAG.setRoot(BrRange); } /// visitBitTestCase - this function produces one "bit test" @@ -1384,6 +1544,13 @@ void SelectionDAGBuilder::visitBitTestCase(MachineBasicBlock* NextMBB, AndOp, DAG.getConstant(0, TLI.getPointerTy()), ISD::SETNE); + if (DisableScheduling) { + DAG.AssignOrdering(ShiftOp.getNode(), SDNodeOrder); + DAG.AssignOrdering(SwitchVal.getNode(), SDNodeOrder); + DAG.AssignOrdering(AndOp.getNode(), SDNodeOrder); + DAG.AssignOrdering(AndCmp.getNode(), SDNodeOrder); + } + CurMBB->addSuccessor(B.TargetBB); CurMBB->addSuccessor(NextMBB); @@ -1391,6 +1558,9 @@ void SelectionDAGBuilder::visitBitTestCase(MachineBasicBlock* NextMBB, MVT::Other, getControlRoot(), AndCmp, DAG.getBasicBlock(B.TargetBB)); + if (DisableScheduling) + DAG.AssignOrdering(BrAnd.getNode(), SDNodeOrder); + // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; @@ -1398,11 +1568,15 @@ void SelectionDAGBuilder::visitBitTestCase(MachineBasicBlock* NextMBB, if (++BBI != FuncInfo.MF->end()) NextBlock = BBI; - if (NextMBB == NextBlock) - DAG.setRoot(BrAnd); - else - DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, BrAnd, - DAG.getBasicBlock(NextMBB))); + if (NextMBB != NextBlock) { + BrAnd = DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, BrAnd, + DAG.getBasicBlock(NextMBB)); + + if (DisableScheduling) + DAG.AssignOrdering(BrAnd.getNode(), SDNodeOrder); + } + + DAG.setRoot(BrAnd); } void SelectionDAGBuilder::visitInvoke(InvokeInst &I) { @@ -1425,9 +1599,13 @@ void SelectionDAGBuilder::visitInvoke(InvokeInst &I) { CurMBB->addSuccessor(LandingPad); // Drop into normal successor. - DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(), - MVT::Other, getControlRoot(), - DAG.getBasicBlock(Return))); + SDValue Branch = DAG.getNode(ISD::BR, getCurDebugLoc(), + MVT::Other, getControlRoot(), + DAG.getBasicBlock(Return)); + DAG.setRoot(Branch); + + if (DisableScheduling) + DAG.AssignOrdering(Branch.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitUnwind(UnwindInst &I) { @@ -1669,7 +1847,7 @@ bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR, APInt Range = ComputeRange(LEnd, RBegin); assert((Range - 2ULL).isNonNegative() && "Invalid case distance"); - double LDensity = (double)LSize.roundToDouble() / + double LDensity = (double)LSize.roundToDouble() / (LEnd - First + 1ULL).roundToDouble(); double RDensity = (double)RSize.roundToDouble() / (Last - RBegin + 1ULL).roundToDouble(); @@ -1787,8 +1965,9 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, // Don't bother the code below, if there are too much unique destinations return false; } - DEBUG(errs() << "Total number of unique destinations: " << Dests.size() << '\n' - << "Total number of comparisons: " << numCmps << '\n'); + DEBUG(errs() << "Total number of unique destinations: " + << Dests.size() << '\n' + << "Total number of comparisons: " << numCmps << '\n'); // Compute span of values. const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue(); @@ -1883,7 +2062,6 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, return true; } - /// Clusterify - Transform simple list of Cases into list of CaseRange's size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases, const SwitchInst& SI) { @@ -1930,7 +2108,6 @@ size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases, void SelectionDAGBuilder::visitSwitch(SwitchInst &SI) { // Figure out which block is immediately after the current one. MachineBasicBlock *NextBlock = 0; - MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()]; // If there is only the default destination, branch to it if it is not the @@ -1940,10 +2117,16 @@ void SelectionDAGBuilder::visitSwitch(SwitchInst &SI) { // If this is not a fall-through branch, emit the branch. CurMBB->addSuccessor(Default); - if (Default != NextBlock) - DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(), - MVT::Other, getControlRoot(), - DAG.getBasicBlock(Default))); + if (Default != NextBlock) { + SDValue Res = DAG.getNode(ISD::BR, getCurDebugLoc(), + MVT::Other, getControlRoot(), + DAG.getBasicBlock(Default)); + DAG.setRoot(Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + } + return; } @@ -1995,11 +2178,14 @@ void SelectionDAGBuilder::visitIndirectBr(IndirectBrInst &I) { for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) CurMBB->addSuccessor(FuncInfo.MBBMap[I.getSuccessor(i)]); - DAG.setRoot(DAG.getNode(ISD::BRIND, getCurDebugLoc(), - MVT::Other, getControlRoot(), - getValue(I.getAddress()))); -} + SDValue Res = DAG.getNode(ISD::BRIND, getCurDebugLoc(), + MVT::Other, getControlRoot(), + getValue(I.getAddress())); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); +} void SelectionDAGBuilder::visitFSub(User &I) { // -0.0 - X --> fneg @@ -2013,17 +2199,28 @@ void SelectionDAGBuilder::visitFSub(User &I) { Constant *CNZ = ConstantVector::get(&NZ[0], NZ.size()); if (CV == CNZ) { SDValue Op2 = getValue(I.getOperand(1)); - setValue(&I, DAG.getNode(ISD::FNEG, getCurDebugLoc(), - Op2.getValueType(), Op2)); + SDValue Res = DAG.getNode(ISD::FNEG, getCurDebugLoc(), + Op2.getValueType(), Op2); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + return; } } } + if (ConstantFP *CFP = dyn_cast<ConstantFP>(I.getOperand(0))) if (CFP->isExactlyValue(ConstantFP::getNegativeZero(Ty)->getValueAPF())) { SDValue Op2 = getValue(I.getOperand(1)); - setValue(&I, DAG.getNode(ISD::FNEG, getCurDebugLoc(), - Op2.getValueType(), Op2)); + SDValue Res = DAG.getNode(ISD::FNEG, getCurDebugLoc(), + Op2.getValueType(), Op2); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + return; } @@ -2033,9 +2230,12 @@ void SelectionDAGBuilder::visitFSub(User &I) { void SelectionDAGBuilder::visitBinary(User &I, unsigned OpCode) { SDValue Op1 = getValue(I.getOperand(0)); SDValue Op2 = getValue(I.getOperand(1)); + SDValue Res = DAG.getNode(OpCode, getCurDebugLoc(), + Op1.getValueType(), Op1, Op2); + setValue(&I, Res); - setValue(&I, DAG.getNode(OpCode, getCurDebugLoc(), - Op1.getValueType(), Op1, Op2)); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitShift(User &I, unsigned Opcode) { @@ -2068,8 +2268,15 @@ void SelectionDAGBuilder::visitShift(User &I, unsigned Opcode) { TLI.getPointerTy(), Op2); } - setValue(&I, DAG.getNode(Opcode, getCurDebugLoc(), - Op1.getValueType(), Op1, Op2)); + SDValue Res = DAG.getNode(Opcode, getCurDebugLoc(), + Op1.getValueType(), Op1, Op2); + setValue(&I, Res); + + if (DisableScheduling) { + DAG.AssignOrdering(Op1.getNode(), SDNodeOrder); + DAG.AssignOrdering(Op2.getNode(), SDNodeOrder); + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + } } void SelectionDAGBuilder::visitICmp(User &I) { @@ -2081,9 +2288,13 @@ void SelectionDAGBuilder::visitICmp(User &I) { SDValue Op1 = getValue(I.getOperand(0)); SDValue Op2 = getValue(I.getOperand(1)); ISD::CondCode Opcode = getICmpCondCode(predicate); - + EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getSetCC(getCurDebugLoc(), DestVT, Op1, Op2, Opcode)); + SDValue Res = DAG.getSetCC(getCurDebugLoc(), DestVT, Op1, Op2, Opcode); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitFCmp(User &I) { @@ -2096,37 +2307,54 @@ void SelectionDAGBuilder::visitFCmp(User &I) { SDValue Op2 = getValue(I.getOperand(1)); ISD::CondCode Condition = getFCmpCondCode(predicate); EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getSetCC(getCurDebugLoc(), DestVT, Op1, Op2, Condition)); + SDValue Res = DAG.getSetCC(getCurDebugLoc(), DestVT, Op1, Op2, Condition); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitSelect(User &I) { SmallVector<EVT, 4> ValueVTs; ComputeValueVTs(TLI, I.getType(), ValueVTs); unsigned NumValues = ValueVTs.size(); - if (NumValues != 0) { - SmallVector<SDValue, 4> Values(NumValues); - SDValue Cond = getValue(I.getOperand(0)); - SDValue TrueVal = getValue(I.getOperand(1)); - SDValue FalseVal = getValue(I.getOperand(2)); + if (NumValues == 0) return; + + SmallVector<SDValue, 4> Values(NumValues); + SDValue Cond = getValue(I.getOperand(0)); + SDValue TrueVal = getValue(I.getOperand(1)); + SDValue FalseVal = getValue(I.getOperand(2)); - for (unsigned i = 0; i != NumValues; ++i) - Values[i] = DAG.getNode(ISD::SELECT, getCurDebugLoc(), - TrueVal.getNode()->getValueType(i), Cond, - SDValue(TrueVal.getNode(), TrueVal.getResNo() + i), - SDValue(FalseVal.getNode(), FalseVal.getResNo() + i)); + for (unsigned i = 0; i != NumValues; ++i) { + Values[i] = DAG.getNode(ISD::SELECT, getCurDebugLoc(), + TrueVal.getNode()->getValueType(i), Cond, + SDValue(TrueVal.getNode(), + TrueVal.getResNo() + i), + SDValue(FalseVal.getNode(), + FalseVal.getResNo() + i)); - setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(), - DAG.getVTList(&ValueVTs[0], NumValues), - &Values[0], NumValues)); + if (DisableScheduling) + DAG.AssignOrdering(Values[i].getNode(), SDNodeOrder); } -} + SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(), + DAG.getVTList(&ValueVTs[0], NumValues), + &Values[0], NumValues); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); +} void SelectionDAGBuilder::visitTrunc(User &I) { // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurDebugLoc(), DestVT, N)); + SDValue Res = DAG.getNode(ISD::TRUNCATE, getCurDebugLoc(), DestVT, N); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitZExt(User &I) { @@ -2134,7 +2362,11 @@ void SelectionDAGBuilder::visitZExt(User &I) { // ZExt also can't be a cast to bool for same reason. So, nothing much to do SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurDebugLoc(), DestVT, N)); + SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, getCurDebugLoc(), DestVT, N); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitSExt(User &I) { @@ -2142,50 +2374,78 @@ void SelectionDAGBuilder::visitSExt(User &I) { // SExt also can't be a cast to bool for same reason. So, nothing much to do SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurDebugLoc(), DestVT, N)); + SDValue Res = DAG.getNode(ISD::SIGN_EXTEND, getCurDebugLoc(), DestVT, N); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitFPTrunc(User &I) { // FPTrunc is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurDebugLoc(), - DestVT, N, DAG.getIntPtrConstant(0))); + SDValue Res = DAG.getNode(ISD::FP_ROUND, getCurDebugLoc(), + DestVT, N, DAG.getIntPtrConstant(0)); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitFPExt(User &I){ // FPTrunc is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurDebugLoc(), DestVT, N)); + SDValue Res = DAG.getNode(ISD::FP_EXTEND, getCurDebugLoc(), DestVT, N); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitFPToUI(User &I) { // FPToUI is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurDebugLoc(), DestVT, N)); + SDValue Res = DAG.getNode(ISD::FP_TO_UINT, getCurDebugLoc(), DestVT, N); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitFPToSI(User &I) { // FPToSI is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurDebugLoc(), DestVT, N)); + SDValue Res = DAG.getNode(ISD::FP_TO_SINT, getCurDebugLoc(), DestVT, N); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitUIToFP(User &I) { // UIToFP is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurDebugLoc(), DestVT, N)); + SDValue Res = DAG.getNode(ISD::UINT_TO_FP, getCurDebugLoc(), DestVT, N); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitSIToFP(User &I){ // SIToFP is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurDebugLoc(), DestVT, N)); + SDValue Res = DAG.getNode(ISD::SINT_TO_FP, getCurDebugLoc(), DestVT, N); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitPtrToInt(User &I) { @@ -2194,8 +2454,11 @@ void SelectionDAGBuilder::visitPtrToInt(User &I) { SDValue N = getValue(I.getOperand(0)); EVT SrcVT = N.getValueType(); EVT DestVT = TLI.getValueType(I.getType()); - SDValue Result = DAG.getZExtOrTrunc(N, getCurDebugLoc(), DestVT); - setValue(&I, Result); + SDValue Res = DAG.getZExtOrTrunc(N, getCurDebugLoc(), DestVT); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitIntToPtr(User &I) { @@ -2204,41 +2467,61 @@ void SelectionDAGBuilder::visitIntToPtr(User &I) { SDValue N = getValue(I.getOperand(0)); EVT SrcVT = N.getValueType(); EVT DestVT = TLI.getValueType(I.getType()); - setValue(&I, DAG.getZExtOrTrunc(N, getCurDebugLoc(), DestVT)); + SDValue Res = DAG.getZExtOrTrunc(N, getCurDebugLoc(), DestVT); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitBitCast(User &I) { SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); - // BitCast assures us that source and destination are the same size so this - // is either a BIT_CONVERT or a no-op. - if (DestVT != N.getValueType()) - setValue(&I, DAG.getNode(ISD::BIT_CONVERT, getCurDebugLoc(), - DestVT, N)); // convert types - else - setValue(&I, N); // noop cast. + // BitCast assures us that source and destination are the same size so this is + // either a BIT_CONVERT or a no-op. + if (DestVT != N.getValueType()) { + SDValue Res = DAG.getNode(ISD::BIT_CONVERT, getCurDebugLoc(), + DestVT, N); // convert types. + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + } else { + setValue(&I, N); // noop cast. + } } void SelectionDAGBuilder::visitInsertElement(User &I) { SDValue InVec = getValue(I.getOperand(0)); SDValue InVal = getValue(I.getOperand(1)); SDValue InIdx = DAG.getNode(ISD::ZERO_EXTEND, getCurDebugLoc(), - TLI.getPointerTy(), - getValue(I.getOperand(2))); + TLI.getPointerTy(), + getValue(I.getOperand(2))); + SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurDebugLoc(), + TLI.getValueType(I.getType()), + InVec, InVal, InIdx); + setValue(&I, Res); - setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurDebugLoc(), - TLI.getValueType(I.getType()), - InVec, InVal, InIdx)); + if (DisableScheduling) { + DAG.AssignOrdering(InIdx.getNode(), SDNodeOrder); + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + } } void SelectionDAGBuilder::visitExtractElement(User &I) { SDValue InVec = getValue(I.getOperand(0)); SDValue InIdx = DAG.getNode(ISD::ZERO_EXTEND, getCurDebugLoc(), - TLI.getPointerTy(), - getValue(I.getOperand(1))); - setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(), - TLI.getValueType(I.getType()), InVec, InIdx)); + TLI.getPointerTy(), + getValue(I.getOperand(1))); + SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(), + TLI.getValueType(I.getType()), InVec, InIdx); + setValue(&I, Res); + + if (DisableScheduling) { + DAG.AssignOrdering(InIdx.getNode(), SDNodeOrder); + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + } } @@ -2260,7 +2543,7 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) { // Convert the ConstantVector mask operand into an array of ints, with -1 // representing undef values. SmallVector<Constant*, 8> MaskElts; - cast<Constant>(I.getOperand(2))->getVectorElements(*DAG.getContext(), + cast<Constant>(I.getOperand(2))->getVectorElements(*DAG.getContext(), MaskElts); unsigned MaskNumElts = MaskElts.size(); for (unsigned i = 0; i != MaskNumElts; ++i) { @@ -2269,14 +2552,19 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) { else Mask.push_back(cast<ConstantInt>(MaskElts[i])->getSExtValue()); } - + EVT VT = TLI.getValueType(I.getType()); EVT SrcVT = Src1.getValueType(); unsigned SrcNumElts = SrcVT.getVectorNumElements(); if (SrcNumElts == MaskNumElts) { - setValue(&I, DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2, - &Mask[0])); + SDValue Res = DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2, + &Mask[0]); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + return; } @@ -2287,8 +2575,13 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) { // lengths match. if (SrcNumElts*2 == MaskNumElts && SequentialMask(Mask, 0)) { // The shuffle is concatenating two vectors together. - setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurDebugLoc(), - VT, Src1, Src2)); + SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, getCurDebugLoc(), + VT, Src1, Src2); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + return; } @@ -2302,12 +2595,12 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) { SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal); MOps1[0] = Src1; MOps2[0] = Src2; - - Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS, - getCurDebugLoc(), VT, + + Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS, + getCurDebugLoc(), VT, &MOps1[0], NumConcat); Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS, - getCurDebugLoc(), VT, + getCurDebugLoc(), VT, &MOps2[0], NumConcat); // Readjust mask for new input vector length. @@ -2319,8 +2612,17 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) { else MappedOps.push_back(Idx + MaskNumElts - SrcNumElts); } - setValue(&I, DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2, - &MappedOps[0])); + + SDValue Res = DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2, + &MappedOps[0]); + setValue(&I, Res); + + if (DisableScheduling) { + DAG.AssignOrdering(Src1.getNode(), SDNodeOrder); + DAG.AssignOrdering(Src2.getNode(), SDNodeOrder); + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + } + return; } @@ -2336,7 +2638,7 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) { int Input = 0; if (Idx < 0) continue; - + if (Idx >= (int)SrcNumElts) { Input = 1; Idx -= SrcNumElts; @@ -2349,7 +2651,8 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) { // Check if the access is smaller than the vector size and can we find // a reasonable extract index. - int RangeUse[2] = { 2, 2 }; // 0 = Unused, 1 = Extract, 2 = Can not Extract. + int RangeUse[2] = { 2, 2 }; // 0 = Unused, 1 = Extract, 2 = Can not + // Extract. int StartIdx[2]; // StartIdx to extract from for (int Input=0; Input < 2; ++Input) { if (MinRange[Input] == (int)(SrcNumElts+1) && MaxRange[Input] == -1) { @@ -2371,20 +2674,28 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) { } if (RangeUse[0] == 0 && RangeUse[1] == 0) { - setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used. + SDValue Res = DAG.getUNDEF(VT); + setValue(&I, Res); // Vectors are not used. + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + return; } else if (RangeUse[0] < 2 && RangeUse[1] < 2) { // Extract appropriate subvector and generate a vector shuffle for (int Input=0; Input < 2; ++Input) { - SDValue& Src = Input == 0 ? Src1 : Src2; - if (RangeUse[Input] == 0) { + SDValue &Src = Input == 0 ? Src1 : Src2; + if (RangeUse[Input] == 0) Src = DAG.getUNDEF(VT); - } else { + else Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, getCurDebugLoc(), VT, Src, DAG.getIntPtrConstant(StartIdx[Input])); - } + + if (DisableScheduling) + DAG.AssignOrdering(Src.getNode(), SDNodeOrder); } + // Calculate new mask. SmallVector<int, 8> MappedOps; for (unsigned i = 0; i != MaskNumElts; ++i) { @@ -2396,8 +2707,14 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) { else MappedOps.push_back(Idx - SrcNumElts - StartIdx[1] + MaskNumElts); } - setValue(&I, DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2, - &MappedOps[0])); + + SDValue Res = DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2, + &MappedOps[0]); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + return; } } @@ -2413,17 +2730,29 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) { Ops.push_back(DAG.getUNDEF(EltVT)); } else { int Idx = Mask[i]; + SDValue Res; + if (Idx < (int)SrcNumElts) - Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(), - EltVT, Src1, DAG.getConstant(Idx, PtrVT))); + Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(), + EltVT, Src1, DAG.getConstant(Idx, PtrVT)); else - Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(), - EltVT, Src2, - DAG.getConstant(Idx - SrcNumElts, PtrVT))); + Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(), + EltVT, Src2, + DAG.getConstant(Idx - SrcNumElts, PtrVT)); + + Ops.push_back(Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } } - setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurDebugLoc(), - VT, &Ops[0], Ops.size())); + + SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, getCurDebugLoc(), + VT, &Ops[0], Ops.size()); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitInsertValue(InsertValueInst &I) { @@ -2462,9 +2791,13 @@ void SelectionDAGBuilder::visitInsertValue(InsertValueInst &I) { Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : SDValue(Agg.getNode(), Agg.getResNo() + i); - setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(), - DAG.getVTList(&AggValueVTs[0], NumAggValues), - &Values[0], NumAggValues)); + SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(), + DAG.getVTList(&AggValueVTs[0], NumAggValues), + &Values[0], NumAggValues); + setValue(&I, Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } void SelectionDAGBuilder::visitExtractValue(ExtractValueInst &I) { @@ -2490,11 +2823,14 @@ void SelectionDAGBuilder::visitExtractValue(ExtractValueInst &I) { DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) : SDValue(Agg.getNode(), Agg.getResNo() + i); - setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(), - DAG.getVTList(&ValValueVTs[0], NumValValues), - &Values[0], NumValValues)); -} + SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(), + DAG.getVTList(&ValValueVTs[0], NumValValues), + &Values[0], NumValValues); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); +} void SelectionDAGBuilder::visitGetElementPtr(User &I) { SDValue N = getValue(I.getOperand(0)); @@ -2510,7 +2846,11 @@ void SelectionDAGBuilder::visitGetElementPtr(User &I) { uint64_t Offset = TD->getStructLayout(StTy)->getElementOffset(Field); N = DAG.getNode(ISD::ADD, getCurDebugLoc(), N.getValueType(), N, DAG.getIntPtrConstant(Offset)); + + if (DisableScheduling) + DAG.AssignOrdering(N.getNode(), SDNodeOrder); } + Ty = StTy->getElementType(Field); } else { Ty = cast<SequentialType>(Ty)->getElementType(); @@ -2523,14 +2863,21 @@ void SelectionDAGBuilder::visitGetElementPtr(User &I) { SDValue OffsVal; EVT PTy = TLI.getPointerTy(); unsigned PtrBits = PTy.getSizeInBits(); - if (PtrBits < 64) { + if (PtrBits < 64) OffsVal = DAG.getNode(ISD::TRUNCATE, getCurDebugLoc(), TLI.getPointerTy(), DAG.getConstant(Offs, MVT::i64)); - } else + else OffsVal = DAG.getIntPtrConstant(Offs); + N = DAG.getNode(ISD::ADD, getCurDebugLoc(), N.getValueType(), N, OffsVal); + + if (DisableScheduling) { + DAG.AssignOrdering(OffsVal.getNode(), SDNodeOrder); + DAG.AssignOrdering(N.getNode(), SDNodeOrder); + } + continue; } @@ -2556,12 +2903,19 @@ void SelectionDAGBuilder::visitGetElementPtr(User &I) { IdxN = DAG.getNode(ISD::MUL, getCurDebugLoc(), N.getValueType(), IdxN, Scale); } + + if (DisableScheduling) + DAG.AssignOrdering(IdxN.getNode(), SDNodeOrder); } N = DAG.getNode(ISD::ADD, getCurDebugLoc(), N.getValueType(), N, IdxN); + + if (DisableScheduling) + DAG.AssignOrdering(N.getNode(), SDNodeOrder); } } + setValue(&I, N); } @@ -2578,16 +2932,20 @@ void SelectionDAGBuilder::visitAlloca(AllocaInst &I) { I.getAlignment()); SDValue AllocSize = getValue(I.getArraySize()); - + AllocSize = DAG.getNode(ISD::MUL, getCurDebugLoc(), AllocSize.getValueType(), AllocSize, DAG.getConstant(TySize, AllocSize.getValueType())); - - - + + if (DisableScheduling) + DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder); + EVT IntPtr = TLI.getPointerTy(); AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurDebugLoc(), IntPtr); + if (DisableScheduling) + DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder); + // Handle alignment. If the requested alignment is less than or equal to // the stack alignment, ignore it. If the size is greater than or equal to // the stack alignment, we note this in the DYNAMIC_STACKALLOC node. @@ -2601,10 +2959,15 @@ void SelectionDAGBuilder::visitAlloca(AllocaInst &I) { AllocSize = DAG.getNode(ISD::ADD, getCurDebugLoc(), AllocSize.getValueType(), AllocSize, DAG.getIntPtrConstant(StackAlign-1)); + if (DisableScheduling) + DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder); + // Mask out the low bits for alignment purposes. AllocSize = DAG.getNode(ISD::AND, getCurDebugLoc(), AllocSize.getValueType(), AllocSize, DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1))); + if (DisableScheduling) + DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder); SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) }; SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other); @@ -2613,6 +2976,9 @@ void SelectionDAGBuilder::visitAlloca(AllocaInst &I) { setValue(&I, DSA); DAG.setRoot(DSA.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(DSA.getNode(), SDNodeOrder); + // Inform the Frame Information that we have just allocated a variable-sized // object. FuncInfo.MF->getFrameInfo()->CreateVariableSizedObject(); @@ -2651,30 +3017,41 @@ void SelectionDAGBuilder::visitLoad(LoadInst &I) { SmallVector<SDValue, 4> Chains(NumValues); EVT PtrVT = Ptr.getValueType(); for (unsigned i = 0; i != NumValues; ++i) { + SDValue A = DAG.getNode(ISD::ADD, getCurDebugLoc(), + PtrVT, Ptr, + DAG.getConstant(Offsets[i], PtrVT)); SDValue L = DAG.getLoad(ValueVTs[i], getCurDebugLoc(), Root, - DAG.getNode(ISD::ADD, getCurDebugLoc(), - PtrVT, Ptr, - DAG.getConstant(Offsets[i], PtrVT)), - SV, Offsets[i], isVolatile, Alignment); + A, SV, Offsets[i], isVolatile, Alignment); + Values[i] = L; Chains[i] = L.getValue(1); + + if (DisableScheduling) { + DAG.AssignOrdering(A.getNode(), SDNodeOrder); + DAG.AssignOrdering(L.getNode(), SDNodeOrder); + } } if (!ConstantMemory) { SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), - MVT::Other, - &Chains[0], NumValues); + MVT::Other, &Chains[0], NumValues); if (isVolatile) DAG.setRoot(Chain); else PendingLoads.push_back(Chain); + + if (DisableScheduling) + DAG.AssignOrdering(Chain.getNode(), SDNodeOrder); } - setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(), - DAG.getVTList(&ValueVTs[0], NumValues), - &Values[0], NumValues)); -} + SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(), + DAG.getVTList(&ValueVTs[0], NumValues), + &Values[0], NumValues); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); +} void SelectionDAGBuilder::visitStore(StoreInst &I) { Value *SrcV = I.getOperand(0); @@ -2698,16 +3075,26 @@ void SelectionDAGBuilder::visitStore(StoreInst &I) { EVT PtrVT = Ptr.getValueType(); bool isVolatile = I.isVolatile(); unsigned Alignment = I.getAlignment(); - for (unsigned i = 0; i != NumValues; ++i) + + for (unsigned i = 0; i != NumValues; ++i) { + SDValue Add = DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, Ptr, + DAG.getConstant(Offsets[i], PtrVT)); Chains[i] = DAG.getStore(Root, getCurDebugLoc(), SDValue(Src.getNode(), Src.getResNo() + i), - DAG.getNode(ISD::ADD, getCurDebugLoc(), - PtrVT, Ptr, - DAG.getConstant(Offsets[i], PtrVT)), - PtrV, Offsets[i], isVolatile, Alignment); + Add, PtrV, Offsets[i], isVolatile, Alignment); + + if (DisableScheduling) { + DAG.AssignOrdering(Add.getNode(), SDNodeOrder); + DAG.AssignOrdering(Chains[i].getNode(), SDNodeOrder); + } + } + + SDValue Res = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), + MVT::Other, &Chains[0], NumValues); + DAG.setRoot(Res); - DAG.setRoot(DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), - MVT::Other, &Chains[0], NumValues)); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC @@ -2752,6 +3139,7 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I, "Intrinsic uses a non-legal type?"); } #endif // NDEBUG + if (HasChain) ValueVTs.push_back(MVT::Other); @@ -2766,16 +3154,19 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I, Info.memVT, Info.ptrVal, Info.offset, Info.align, Info.vol, Info.readMem, Info.writeMem); - } - else if (!HasChain) + } else if (!HasChain) { Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurDebugLoc(), VTs, &Ops[0], Ops.size()); - else if (I.getType() != Type::getVoidTy(*DAG.getContext())) + } else if (I.getType() != Type::getVoidTy(*DAG.getContext())) { Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurDebugLoc(), VTs, &Ops[0], Ops.size()); - else + } else { Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurDebugLoc(), VTs, &Ops[0], Ops.size()); + } + + if (DisableScheduling) + DAG.AssignOrdering(Result.getNode(), SDNodeOrder); if (HasChain) { SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1); @@ -2784,11 +3175,16 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I, else DAG.setRoot(Chain); } + if (I.getType() != Type::getVoidTy(*DAG.getContext())) { if (const VectorType *PTy = dyn_cast<VectorType>(I.getType())) { EVT VT = TLI.getValueType(PTy); Result = DAG.getNode(ISD::BIT_CONVERT, getCurDebugLoc(), VT, Result); + + if (DisableScheduling) + DAG.AssignOrdering(Result.getNode(), SDNodeOrder); } + setValue(&I, Result); } } @@ -2800,12 +3196,20 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I, /// /// where Op is the hexidecimal representation of floating point value. static SDValue -GetSignificand(SelectionDAG &DAG, SDValue Op, DebugLoc dl) { +GetSignificand(SelectionDAG &DAG, SDValue Op, DebugLoc dl, unsigned Order) { SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, DAG.getConstant(0x007fffff, MVT::i32)); SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1, DAG.getConstant(0x3f800000, MVT::i32)); - return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t2); + SDValue Res = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t2); + + if (DisableScheduling) { + DAG.AssignOrdering(t1.getNode(), Order); + DAG.AssignOrdering(t2.getNode(), Order); + DAG.AssignOrdering(Res.getNode(), Order); + } + + return Res; } /// GetExponent - Get the exponent: @@ -2815,14 +3219,23 @@ GetSignificand(SelectionDAG &DAG, SDValue Op, DebugLoc dl) { /// where Op is the hexidecimal representation of floating point value. static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, - DebugLoc dl) { + DebugLoc dl, unsigned Order) { SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, DAG.getConstant(0x7f800000, MVT::i32)); SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0, DAG.getConstant(23, TLI.getPointerTy())); SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1, DAG.getConstant(127, MVT::i32)); - return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); + SDValue Res = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), Order); + DAG.AssignOrdering(t1.getNode(), Order); + DAG.AssignOrdering(t2.getNode(), Order); + DAG.AssignOrdering(Res.getNode(), Order); + } + + return Res; } /// getF32Constant - Get 32-bit floating point constant. @@ -2846,6 +3259,10 @@ SelectionDAGBuilder::implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op) { I.getOperand(1)); setValue(&I, L); DAG.setRoot(L.getValue(1)); + + if (DisableScheduling) + DAG.AssignOrdering(L.getNode(), SDNodeOrder); + return 0; } @@ -2859,6 +3276,10 @@ SelectionDAGBuilder::implVisitAluOverflow(CallInst &I, ISD::NodeType Op) { SDValue Result = DAG.getNode(Op, getCurDebugLoc(), VTs, Op1, Op2); setValue(&I, Result); + + if (DisableScheduling) + DAG.AssignOrdering(Result.getNode(), SDNodeOrder); + return 0; } @@ -2886,10 +3307,20 @@ SelectionDAGBuilder::visitExp(CallInst &I) { SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(X.getNode(), SDNodeOrder); + } + // IntegerPartOfX <<= 23; IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, DAG.getConstant(23, TLI.getPointerTy())); + if (DisableScheduling) + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -2912,6 +3343,16 @@ SelectionDAGBuilder::visitExp(CallInst &I) { TwoToFracPartOfX, IntegerPartOfX); result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t6); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFracPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -2938,6 +3379,18 @@ SelectionDAGBuilder::visitExp(CallInst &I) { TwoToFracPartOfX, IntegerPartOfX); result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t8); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFracPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -2977,12 +3430,32 @@ SelectionDAGBuilder::visitExp(CallInst &I) { TwoToFracPartOfX, IntegerPartOfX); result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t14); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(t9.getNode(), SDNodeOrder); + DAG.AssignOrdering(t10.getNode(), SDNodeOrder); + DAG.AssignOrdering(t11.getNode(), SDNodeOrder); + DAG.AssignOrdering(t12.getNode(), SDNodeOrder); + DAG.AssignOrdering(t13.getNode(), SDNodeOrder); + DAG.AssignOrdering(t14.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFracPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. result = DAG.getNode(ISD::FEXP, dl, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1))); + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); @@ -3000,14 +3473,20 @@ SelectionDAGBuilder::visitLog(CallInst &I) { SDValue Op = getValue(I.getOperand(1)); SDValue Op1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Op); + if (DisableScheduling) + DAG.AssignOrdering(Op1.getNode(), SDNodeOrder); + // Scale the exponent by log(2) [0.69314718f]. - SDValue Exp = GetExponent(DAG, Op1, TLI, dl); + SDValue Exp = GetExponent(DAG, Op1, TLI, dl, SDNodeOrder); SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, getF32Constant(DAG, 0x3f317218)); + if (DisableScheduling) + DAG.AssignOrdering(LogOfExponent.getNode(), SDNodeOrder); + // Get the significand and build it into a floating-point number with // exponent of 1. - SDValue X = GetSignificand(DAG, Op1, dl); + SDValue X = GetSignificand(DAG, Op1, dl, SDNodeOrder); if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: @@ -3027,6 +3506,14 @@ SelectionDAGBuilder::visitLog(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(LogOfMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -3053,6 +3540,18 @@ SelectionDAGBuilder::visitLog(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(LogOfMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -3087,12 +3586,31 @@ SelectionDAGBuilder::visitLog(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(t9.getNode(), SDNodeOrder); + DAG.AssignOrdering(t10.getNode(), SDNodeOrder); + DAG.AssignOrdering(LogOfMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. result = DAG.getNode(ISD::FLOG, dl, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1))); + + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); @@ -3110,12 +3628,18 @@ SelectionDAGBuilder::visitLog2(CallInst &I) { SDValue Op = getValue(I.getOperand(1)); SDValue Op1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Op); + if (DisableScheduling) + DAG.AssignOrdering(Op1.getNode(), SDNodeOrder); + // Get the exponent. - SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl); + SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl, SDNodeOrder); + + if (DisableScheduling) + DAG.AssignOrdering(LogOfExponent.getNode(), SDNodeOrder); // Get the significand and build it into a floating-point number with // exponent of 1. - SDValue X = GetSignificand(DAG, Op1, dl); + SDValue X = GetSignificand(DAG, Op1, dl, SDNodeOrder); // Different possible minimax approximations of significand in // floating-point for various degrees of accuracy over [1,2]. @@ -3135,6 +3659,14 @@ SelectionDAGBuilder::visitLog2(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log2ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -3161,6 +3693,18 @@ SelectionDAGBuilder::visitLog2(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log2ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -3196,12 +3740,31 @@ SelectionDAGBuilder::visitLog2(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(t9.getNode(), SDNodeOrder); + DAG.AssignOrdering(t10.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log2ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. result = DAG.getNode(ISD::FLOG2, dl, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1))); + + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); @@ -3219,14 +3782,20 @@ SelectionDAGBuilder::visitLog10(CallInst &I) { SDValue Op = getValue(I.getOperand(1)); SDValue Op1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Op); + if (DisableScheduling) + DAG.AssignOrdering(Op1.getNode(), SDNodeOrder); + // Scale the exponent by log10(2) [0.30102999f]. - SDValue Exp = GetExponent(DAG, Op1, TLI, dl); + SDValue Exp = GetExponent(DAG, Op1, TLI, dl, SDNodeOrder); SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, getF32Constant(DAG, 0x3e9a209a)); + if (DisableScheduling) + DAG.AssignOrdering(LogOfExponent.getNode(), SDNodeOrder); + // Get the significand and build it into a floating-point number with // exponent of 1. - SDValue X = GetSignificand(DAG, Op1, dl); + SDValue X = GetSignificand(DAG, Op1, dl, SDNodeOrder); if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: @@ -3246,6 +3815,14 @@ SelectionDAGBuilder::visitLog10(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log10ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -3268,6 +3845,16 @@ SelectionDAGBuilder::visitLog10(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log10ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -3298,12 +3885,29 @@ SelectionDAGBuilder::visitLog10(CallInst &I) { result = DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); + + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(Log10ofMantissa.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. result = DAG.getNode(ISD::FLOG10, dl, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1))); + + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); @@ -3322,6 +3926,9 @@ SelectionDAGBuilder::visitExp2(CallInst &I) { SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op); + if (DisableScheduling) + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + // FractionalPartOfX = x - (float)IntegerPartOfX; SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1); @@ -3330,6 +3937,12 @@ SelectionDAGBuilder::visitExp2(CallInst &I) { IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, DAG.getConstant(23, TLI.getPointerTy())); + if (DisableScheduling) { + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(X.getNode(), SDNodeOrder); + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + } + if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -3351,6 +3964,16 @@ SelectionDAGBuilder::visitExp2(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -3376,6 +3999,18 @@ SelectionDAGBuilder::visitExp2(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -3412,12 +4047,33 @@ SelectionDAGBuilder::visitExp2(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(t9.getNode(), SDNodeOrder); + DAG.AssignOrdering(t10.getNode(), SDNodeOrder); + DAG.AssignOrdering(t11.getNode(), SDNodeOrder); + DAG.AssignOrdering(t12.getNode(), SDNodeOrder); + DAG.AssignOrdering(t13.getNode(), SDNodeOrder); + DAG.AssignOrdering(t14.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. result = DAG.getNode(ISD::FEXP2, dl, getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1))); + + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); @@ -3459,10 +4115,20 @@ SelectionDAGBuilder::visitPow(CallInst &I) { SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); + if (DisableScheduling) { + DAG.AssignOrdering(t0.getNode(), SDNodeOrder); + DAG.AssignOrdering(t1.getNode(), SDNodeOrder); + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(X.getNode(), SDNodeOrder); + } + // IntegerPartOfX <<= 23; IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, DAG.getConstant(23, TLI.getPointerTy())); + if (DisableScheduling) + DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder); + if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -3484,6 +4150,16 @@ SelectionDAGBuilder::visitPow(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { // For floating-point precision of 12: // @@ -3509,6 +4185,18 @@ SelectionDAGBuilder::visitPow(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 // For floating-point precision of 18: // @@ -3545,6 +4233,24 @@ SelectionDAGBuilder::visitPow(CallInst &I) { result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, TwoToFractionalPartOfX); + + if (DisableScheduling) { + DAG.AssignOrdering(t2.getNode(), SDNodeOrder); + DAG.AssignOrdering(t3.getNode(), SDNodeOrder); + DAG.AssignOrdering(t4.getNode(), SDNodeOrder); + DAG.AssignOrdering(t5.getNode(), SDNodeOrder); + DAG.AssignOrdering(t6.getNode(), SDNodeOrder); + DAG.AssignOrdering(t7.getNode(), SDNodeOrder); + DAG.AssignOrdering(t8.getNode(), SDNodeOrder); + DAG.AssignOrdering(t9.getNode(), SDNodeOrder); + DAG.AssignOrdering(t10.getNode(), SDNodeOrder); + DAG.AssignOrdering(t11.getNode(), SDNodeOrder); + DAG.AssignOrdering(t12.getNode(), SDNodeOrder); + DAG.AssignOrdering(t13.getNode(), SDNodeOrder); + DAG.AssignOrdering(t14.getNode(), SDNodeOrder); + DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder); + DAG.AssignOrdering(result.getNode(), SDNodeOrder); + } } } else { // No special expansion. @@ -3552,17 +4258,76 @@ SelectionDAGBuilder::visitPow(CallInst &I) { getValue(I.getOperand(1)).getValueType(), getValue(I.getOperand(1)), getValue(I.getOperand(2))); + + if (DisableScheduling) + DAG.AssignOrdering(result.getNode(), SDNodeOrder); } setValue(&I, result); } + +/// ExpandPowI - Expand a llvm.powi intrinsic. +static SDValue ExpandPowI(DebugLoc DL, SDValue LHS, SDValue RHS, + SelectionDAG &DAG) { + // If RHS is a constant, we can expand this out to a multiplication tree, + // otherwise we end up lowering to a call to __powidf2 (for example). When + // optimizing for size, we only want to do this if the expansion would produce + // a small number of multiplies, otherwise we do the full expansion. + if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { + // Get the exponent as a positive value. + unsigned Val = RHSC->getSExtValue(); + if ((int)Val < 0) Val = -Val; + + // powi(x, 0) -> 1.0 + if (Val == 0) + return DAG.getConstantFP(1.0, LHS.getValueType()); + + Function *F = DAG.getMachineFunction().getFunction(); + if (!F->hasFnAttr(Attribute::OptimizeForSize) || + // If optimizing for size, don't insert too many multiplies. This + // inserts up to 5 multiplies. + CountPopulation_32(Val)+Log2_32(Val) < 7) { + // We use the simple binary decomposition method to generate the multiply + // sequence. There are more optimal ways to do this (for example, + // powi(x,15) generates one more multiply than it should), but this has + // the benefit of being both really simple and much better than a libcall. + SDValue Res; // Logically starts equal to 1.0 + SDValue CurSquare = LHS; + while (Val) { + if (Val & 1) { + if (Res.getNode()) + Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare); + else + Res = CurSquare; // 1.0*CurSquare. + } + + CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(), + CurSquare, CurSquare); + Val >>= 1; + } + + // If the original was negative, invert the result, producing 1/(x*x*x). + if (RHSC->getSExtValue() < 0) + Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(), + DAG.getConstantFP(1.0, LHS.getValueType()), Res); + return Res; + } + } + + // Otherwise, expand to a libcall. + return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); +} + + /// visitIntrinsicCall - Lower the call to the specified intrinsic function. If /// we want to emit this as a call to a named external function, return the name /// otherwise lower it and return null. const char * SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DebugLoc dl = getCurDebugLoc(); + SDValue Res; + switch (Intrinsic) { default: // By default, turn this into a target intrinsic node. @@ -3572,26 +4337,33 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::vaend: visitVAEnd(I); return 0; case Intrinsic::vacopy: visitVACopy(I); return 0; case Intrinsic::returnaddress: - setValue(&I, DAG.getNode(ISD::RETURNADDR, dl, TLI.getPointerTy(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::RETURNADDR, dl, TLI.getPointerTy(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::frameaddress: - setValue(&I, DAG.getNode(ISD::FRAMEADDR, dl, TLI.getPointerTy(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::FRAMEADDR, dl, TLI.getPointerTy(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::setjmp: return "_setjmp"+!TLI.usesUnderscoreSetJmp(); - break; case Intrinsic::longjmp: return "_longjmp"+!TLI.usesUnderscoreLongJmp(); - break; case Intrinsic::memcpy: { SDValue Op1 = getValue(I.getOperand(1)); SDValue Op2 = getValue(I.getOperand(2)); SDValue Op3 = getValue(I.getOperand(3)); unsigned Align = cast<ConstantInt>(I.getOperand(4))->getZExtValue(); - DAG.setRoot(DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false, - I.getOperand(1), 0, I.getOperand(2), 0)); + Res = DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false, + I.getOperand(1), 0, I.getOperand(2), 0); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::memset: { @@ -3599,8 +4371,11 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { SDValue Op2 = getValue(I.getOperand(2)); SDValue Op3 = getValue(I.getOperand(3)); unsigned Align = cast<ConstantInt>(I.getOperand(4))->getZExtValue(); - DAG.setRoot(DAG.getMemset(getRoot(), dl, Op1, Op2, Op3, Align, - I.getOperand(1), 0)); + Res = DAG.getMemset(getRoot(), dl, Op1, Op2, Op3, Align, + I.getOperand(1), 0); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::memmove: { @@ -3616,30 +4391,36 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { Size = C->getZExtValue(); if (AA->alias(I.getOperand(1), Size, I.getOperand(2), Size) == AliasAnalysis::NoAlias) { - DAG.setRoot(DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false, - I.getOperand(1), 0, I.getOperand(2), 0)); + Res = DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false, + I.getOperand(1), 0, I.getOperand(2), 0); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } - DAG.setRoot(DAG.getMemmove(getRoot(), dl, Op1, Op2, Op3, Align, - I.getOperand(1), 0, I.getOperand(2), 0)); + Res = DAG.getMemmove(getRoot(), dl, Op1, Op2, Op3, Align, + I.getOperand(1), 0, I.getOperand(2), 0); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } - case Intrinsic::dbg_stoppoint: + case Intrinsic::dbg_stoppoint: case Intrinsic::dbg_region_start: case Intrinsic::dbg_region_end: case Intrinsic::dbg_func_start: // FIXME - Remove this instructions once the dust settles. return 0; case Intrinsic::dbg_declare: { - if (OptLevel != CodeGenOpt::None) + if (OptLevel != CodeGenOpt::None) // FIXME: Variable debug info is not supported here. return 0; DwarfWriter *DW = DAG.getDwarfWriter(); if (!DW) return 0; DbgDeclareInst &DI = cast<DbgDeclareInst>(I); - if (!isValidDebugInfoIntrinsic(DI, CodeGenOpt::None)) + if (!DIDescriptor::ValidDebugInfo(DI.getVariable(), CodeGenOpt::None)) return 0; MDNode *Variable = DI.getVariable(); @@ -3652,18 +4433,13 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return 0; DenseMap<const AllocaInst*, int>::iterator SI = FuncInfo.StaticAllocaMap.find(AI); - if (SI == FuncInfo.StaticAllocaMap.end()) + if (SI == FuncInfo.StaticAllocaMap.end()) return 0; // VLAs. int FI = SI->second; - MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - if (MMI) { - MetadataContext &TheMetadata = - DI.getParent()->getContext().getMetadata(); - unsigned MDDbgKind = TheMetadata.getMDKind("dbg"); - MDNode *Dbg = TheMetadata.getMD(MDDbgKind, &DI); - MMI->setVariableDbgInfo(Variable, FI, Dbg); - } + if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo()) + if (MDNode *Dbg = DI.getMetadata("dbg")) + MMI->setVariableDbgInfo(Variable, FI, Dbg); return 0; } case Intrinsic::eh_exception: { @@ -3675,6 +4451,8 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { SDValue Op = DAG.getNode(ISD::EXCEPTIONADDR, dl, VTs, Ops, 1); setValue(&I, Op); DAG.setRoot(Op.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(Op.getNode(), SDNodeOrder); return 0; } @@ -3701,7 +4479,12 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DAG.setRoot(Op.getValue(1)); - setValue(&I, DAG.getSExtOrTrunc(Op, dl, MVT::i32)); + Res = DAG.getSExtOrTrunc(Op, dl, MVT::i32); + setValue(&I, Res); + if (DisableScheduling) { + DAG.AssignOrdering(Op.getNode(), SDNodeOrder); + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + } return 0; } @@ -3711,14 +4494,16 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { if (MMI) { // Find the type id for the given typeinfo. GlobalVariable *GV = ExtractTypeInfo(I.getOperand(1)); - unsigned TypeID = MMI->getTypeIDFor(GV); - setValue(&I, DAG.getConstant(TypeID, MVT::i32)); + Res = DAG.getConstant(TypeID, MVT::i32); } else { // Return something different to eh_selector. - setValue(&I, DAG.getConstant(1, MVT::i32)); + Res = DAG.getConstant(1, MVT::i32); } + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } @@ -3726,11 +4511,14 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::eh_return_i64: if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo()) { MMI->setCallsEHReturn(true); - DAG.setRoot(DAG.getNode(ISD::EH_RETURN, dl, - MVT::Other, - getControlRoot(), - getValue(I.getOperand(1)), - getValue(I.getOperand(2)))); + Res = DAG.getNode(ISD::EH_RETURN, dl, + MVT::Other, + getControlRoot(), + getValue(I.getOperand(1)), + getValue(I.getOperand(2))); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); } else { setValue(&I, DAG.getConstant(0, TLI.getPointerTy())); } @@ -3740,26 +4528,28 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo()) { MMI->setCallsUnwindInit(true); } - return 0; - case Intrinsic::eh_dwarf_cfa: { EVT VT = getValue(I.getOperand(1)).getValueType(); SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), dl, TLI.getPointerTy()); - SDValue Offset = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, TLI.getPointerTy()), CfaArg); - setValue(&I, DAG.getNode(ISD::ADD, dl, + SDValue FA = DAG.getNode(ISD::FRAMEADDR, dl, TLI.getPointerTy(), - DAG.getNode(ISD::FRAMEADDR, dl, - TLI.getPointerTy(), - DAG.getConstant(0, - TLI.getPointerTy())), - Offset)); + DAG.getConstant(0, TLI.getPointerTy())); + Res = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), + FA, Offset); + setValue(&I, Res); + if (DisableScheduling) { + DAG.AssignOrdering(CfaArg.getNode(), SDNodeOrder); + DAG.AssignOrdering(Offset.getNode(), SDNodeOrder); + DAG.AssignOrdering(FA.getNode(), SDNodeOrder); + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); + } return 0; } case Intrinsic::convertff: @@ -3784,36 +4574,48 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::convertuu: Code = ISD::CVT_UU; break; } EVT DestVT = TLI.getValueType(I.getType()); - Value* Op1 = I.getOperand(1); - setValue(&I, DAG.getConvertRndSat(DestVT, getCurDebugLoc(), getValue(Op1), - DAG.getValueType(DestVT), - DAG.getValueType(getValue(Op1).getValueType()), - getValue(I.getOperand(2)), - getValue(I.getOperand(3)), - Code)); + Value *Op1 = I.getOperand(1); + Res = DAG.getConvertRndSat(DestVT, getCurDebugLoc(), getValue(Op1), + DAG.getValueType(DestVT), + DAG.getValueType(getValue(Op1).getValueType()), + getValue(I.getOperand(2)), + getValue(I.getOperand(3)), + Code); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } - case Intrinsic::sqrt: - setValue(&I, DAG.getNode(ISD::FSQRT, dl, - getValue(I.getOperand(1)).getValueType(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::FSQRT, dl, + getValue(I.getOperand(1)).getValueType(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::powi: - setValue(&I, DAG.getNode(ISD::FPOWI, dl, - getValue(I.getOperand(1)).getValueType(), - getValue(I.getOperand(1)), - getValue(I.getOperand(2)))); + Res = ExpandPowI(dl, getValue(I.getOperand(1)), getValue(I.getOperand(2)), + DAG); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::sin: - setValue(&I, DAG.getNode(ISD::FSIN, dl, - getValue(I.getOperand(1)).getValueType(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::FSIN, dl, + getValue(I.getOperand(1)).getValueType(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::cos: - setValue(&I, DAG.getNode(ISD::FCOS, dl, - getValue(I.getOperand(1)).getValueType(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::FCOS, dl, + getValue(I.getOperand(1)).getValueType(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::log: visitLog(I); @@ -3835,55 +4637,74 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return 0; case Intrinsic::pcmarker: { SDValue Tmp = getValue(I.getOperand(1)); - DAG.setRoot(DAG.getNode(ISD::PCMARKER, dl, MVT::Other, getRoot(), Tmp)); + Res = DAG.getNode(ISD::PCMARKER, dl, MVT::Other, getRoot(), Tmp); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::readcyclecounter: { SDValue Op = getRoot(); - SDValue Tmp = DAG.getNode(ISD::READCYCLECOUNTER, dl, - DAG.getVTList(MVT::i64, MVT::Other), - &Op, 1); - setValue(&I, Tmp); - DAG.setRoot(Tmp.getValue(1)); + Res = DAG.getNode(ISD::READCYCLECOUNTER, dl, + DAG.getVTList(MVT::i64, MVT::Other), + &Op, 1); + setValue(&I, Res); + DAG.setRoot(Res.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::bswap: - setValue(&I, DAG.getNode(ISD::BSWAP, dl, - getValue(I.getOperand(1)).getValueType(), - getValue(I.getOperand(1)))); + Res = DAG.getNode(ISD::BSWAP, dl, + getValue(I.getOperand(1)).getValueType(), + getValue(I.getOperand(1))); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::cttz: { SDValue Arg = getValue(I.getOperand(1)); EVT Ty = Arg.getValueType(); - SDValue result = DAG.getNode(ISD::CTTZ, dl, Ty, Arg); - setValue(&I, result); + Res = DAG.getNode(ISD::CTTZ, dl, Ty, Arg); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::ctlz: { SDValue Arg = getValue(I.getOperand(1)); EVT Ty = Arg.getValueType(); - SDValue result = DAG.getNode(ISD::CTLZ, dl, Ty, Arg); - setValue(&I, result); + Res = DAG.getNode(ISD::CTLZ, dl, Ty, Arg); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::ctpop: { SDValue Arg = getValue(I.getOperand(1)); EVT Ty = Arg.getValueType(); - SDValue result = DAG.getNode(ISD::CTPOP, dl, Ty, Arg); - setValue(&I, result); + Res = DAG.getNode(ISD::CTPOP, dl, Ty, Arg); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::stacksave: { SDValue Op = getRoot(); - SDValue Tmp = DAG.getNode(ISD::STACKSAVE, dl, - DAG.getVTList(TLI.getPointerTy(), MVT::Other), &Op, 1); - setValue(&I, Tmp); - DAG.setRoot(Tmp.getValue(1)); + Res = DAG.getNode(ISD::STACKSAVE, dl, + DAG.getVTList(TLI.getPointerTy(), MVT::Other), &Op, 1); + setValue(&I, Res); + DAG.setRoot(Res.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::stackrestore: { - SDValue Tmp = getValue(I.getOperand(1)); - DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, dl, MVT::Other, getRoot(), Tmp)); + Res = getValue(I.getOperand(1)); + Res = DAG.getNode(ISD::STACKRESTORE, dl, MVT::Other, getRoot(), Res); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::stackprotector: { @@ -3901,11 +4722,13 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { SDValue FIN = DAG.getFrameIndex(FI, PtrTy); // Store the stack protector onto the stack. - SDValue Result = DAG.getStore(getRoot(), getCurDebugLoc(), Src, FIN, - PseudoSourceValue::getFixedStack(FI), - 0, true); - setValue(&I, Result); - DAG.setRoot(Result); + Res = DAG.getStore(getRoot(), getCurDebugLoc(), Src, FIN, + PseudoSourceValue::getFixedStack(FI), + 0, true); + setValue(&I, Res); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::objectsize: { @@ -3917,10 +4740,14 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { SDValue Arg = getValue(I.getOperand(0)); EVT Ty = Arg.getValueType(); - if (CI->getZExtValue() < 2) - setValue(&I, DAG.getConstant(-1ULL, Ty)); + if (CI->getZExtValue() == 0) + Res = DAG.getConstant(-1ULL, Ty); else - setValue(&I, DAG.getConstant(0, Ty)); + Res = DAG.getConstant(0, Ty); + + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::var_annotation: @@ -3938,15 +4765,16 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { Ops[4] = DAG.getSrcValue(I.getOperand(1)); Ops[5] = DAG.getSrcValue(F); - SDValue Tmp = DAG.getNode(ISD::TRAMPOLINE, dl, - DAG.getVTList(TLI.getPointerTy(), MVT::Other), - Ops, 6); + Res = DAG.getNode(ISD::TRAMPOLINE, dl, + DAG.getVTList(TLI.getPointerTy(), MVT::Other), + Ops, 6); - setValue(&I, Tmp); - DAG.setRoot(Tmp.getValue(1)); + setValue(&I, Res); + DAG.setRoot(Res.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } - case Intrinsic::gcroot: if (GFI) { Value *Alloca = I.getOperand(1); @@ -3956,22 +4784,22 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { GFI->addStackRoot(FI->getIndex(), TypeMap); } return 0; - case Intrinsic::gcread: case Intrinsic::gcwrite: llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!"); return 0; - - case Intrinsic::flt_rounds: { - setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, dl, MVT::i32)); + case Intrinsic::flt_rounds: + Res = DAG.getNode(ISD::FLT_ROUNDS_, dl, MVT::i32); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; - } - - case Intrinsic::trap: { - DAG.setRoot(DAG.getNode(ISD::TRAP, dl,MVT::Other, getRoot())); + case Intrinsic::trap: + Res = DAG.getNode(ISD::TRAP, dl,MVT::Other, getRoot()); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; - } - case Intrinsic::uadd_with_overflow: return implVisitAluOverflow(I, ISD::UADDO); case Intrinsic::sadd_with_overflow: @@ -3991,7 +4819,10 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { Ops[1] = getValue(I.getOperand(1)); Ops[2] = getValue(I.getOperand(2)); Ops[3] = getValue(I.getOperand(3)); - DAG.setRoot(DAG.getNode(ISD::PREFETCH, dl, MVT::Other, &Ops[0], 4)); + Res = DAG.getNode(ISD::PREFETCH, dl, MVT::Other, &Ops[0], 4); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } @@ -4001,7 +4832,10 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { for (int x = 1; x < 6; ++x) Ops[x] = getValue(I.getOperand(x)); - DAG.setRoot(DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, &Ops[0], 6)); + Res = DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, &Ops[0], 6); + DAG.setRoot(Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; } case Intrinsic::atomic_cmp_swap: { @@ -4016,6 +4850,8 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { I.getOperand(1)); setValue(&I, L); DAG.setRoot(L.getValue(1)); + if (DisableScheduling) + DAG.AssignOrdering(L.getNode(), SDNodeOrder); return 0; } case Intrinsic::atomic_load_add: @@ -4044,7 +4880,10 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::invariant_start: case Intrinsic::lifetime_start: // Discard region information. - setValue(&I, DAG.getUNDEF(TLI.getPointerTy())); + Res = DAG.getUNDEF(TLI.getPointerTy()); + setValue(&I, Res); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), SDNodeOrder); return 0; case Intrinsic::invariant_end: case Intrinsic::lifetime_end: @@ -4143,11 +4982,10 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee, SmallVector<EVT, 4> OutVTs; SmallVector<ISD::ArgFlagsTy, 4> OutsFlags; SmallVector<uint64_t, 4> Offsets; - getReturnInfo(RetTy, CS.getAttributes().getRetAttributes(), - OutVTs, OutsFlags, TLI, &Offsets); - + getReturnInfo(RetTy, CS.getAttributes().getRetAttributes(), + OutVTs, OutsFlags, TLI, &Offsets); - bool CanLowerReturn = TLI.CanLowerReturn(CS.getCallingConv(), + bool CanLowerReturn = TLI.CanLowerReturn(CS.getCallingConv(), FTy->isVarArg(), OutVTs, OutsFlags, DAG); SDValue DemoteStackSlot; @@ -4219,14 +5057,16 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee, CS.getCallingConv(), isTailCall, !CS.getInstruction()->use_empty(), - Callee, Args, DAG, getCurDebugLoc()); + Callee, Args, DAG, getCurDebugLoc(), SDNodeOrder); assert((isTailCall || Result.second.getNode()) && "Non-null chain expected with non-tail call!"); assert((Result.second.getNode() || !Result.first.getNode()) && "Null value expected with tail call!"); - if (Result.first.getNode()) + if (Result.first.getNode()) { setValue(CS.getInstruction(), Result.first); - else if (!CanLowerReturn && Result.second.getNode()) { + if (DisableScheduling) + DAG.AssignOrdering(Result.first.getNode(), SDNodeOrder); + } else if (!CanLowerReturn && Result.second.getNode()) { // The instruction result is the result of loading from the // hidden sret parameter. SmallVector<EVT, 1> PVTs; @@ -4240,27 +5080,40 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee, SmallVector<SDValue, 4> Chains(NumValues); for (unsigned i = 0; i < NumValues; ++i) { + SDValue Add = DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, + DemoteStackSlot, + DAG.getConstant(Offsets[i], PtrVT)); SDValue L = DAG.getLoad(OutVTs[i], getCurDebugLoc(), Result.second, - DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, DemoteStackSlot, - DAG.getConstant(Offsets[i], PtrVT)), - NULL, Offsets[i], false, 1); + Add, NULL, Offsets[i], false, 1); Values[i] = L; Chains[i] = L.getValue(1); } + SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), MVT::Other, &Chains[0], NumValues); PendingLoads.push_back(Chain); - setValue(CS.getInstruction(), DAG.getNode(ISD::MERGE_VALUES, - getCurDebugLoc(), DAG.getVTList(&OutVTs[0], NumValues), - &Values[0], NumValues)); + SDValue MV = DAG.getNode(ISD::MERGE_VALUES, + getCurDebugLoc(), + DAG.getVTList(&OutVTs[0], NumValues), + &Values[0], NumValues); + setValue(CS.getInstruction(), MV); + + if (DisableScheduling) { + DAG.AssignOrdering(Chain.getNode(), SDNodeOrder); + DAG.AssignOrdering(MV.getNode(), SDNodeOrder); + } } - // As a special case, a null chain means that a tail call has - // been emitted and the DAG root is already updated. - if (Result.second.getNode()) + + // As a special case, a null chain means that a tail call has been emitted and + // the DAG root is already updated. + if (Result.second.getNode()) { DAG.setRoot(Result.second); - else + if (DisableScheduling) + DAG.AssignOrdering(Result.second.getNode(), SDNodeOrder); + } else { HasTailCall = true; + } if (LandingPad && MMI) { // Insert a label at the end of the invoke call to mark the try range. This @@ -4274,6 +5127,140 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee, } } +/// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the +/// value is equal or not-equal to zero. +static bool IsOnlyUsedInZeroEqualityComparison(Value *V) { + for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); + UI != E; ++UI) { + if (ICmpInst *IC = dyn_cast<ICmpInst>(*UI)) + if (IC->isEquality()) + if (Constant *C = dyn_cast<Constant>(IC->getOperand(1))) + if (C->isNullValue()) + continue; + // Unknown instruction. + return false; + } + return true; +} + +static SDValue getMemCmpLoad(Value *PtrVal, MVT LoadVT, const Type *LoadTy, + SelectionDAGBuilder &Builder) { + + // Check to see if this load can be trivially constant folded, e.g. if the + // input is from a string literal. + if (Constant *LoadInput = dyn_cast<Constant>(PtrVal)) { + // Cast pointer to the type we really want to load. + LoadInput = ConstantExpr::getBitCast(LoadInput, + PointerType::getUnqual(LoadTy)); + + if (Constant *LoadCst = ConstantFoldLoadFromConstPtr(LoadInput, Builder.TD)) + return Builder.getValue(LoadCst); + } + + // Otherwise, we have to emit the load. If the pointer is to unfoldable but + // still constant memory, the input chain can be the entry node. + SDValue Root; + bool ConstantMemory = false; + + // Do not serialize (non-volatile) loads of constant memory with anything. + if (Builder.AA->pointsToConstantMemory(PtrVal)) { + Root = Builder.DAG.getEntryNode(); + ConstantMemory = true; + } else { + // Do not serialize non-volatile loads against each other. + Root = Builder.DAG.getRoot(); + } + + SDValue Ptr = Builder.getValue(PtrVal); + SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurDebugLoc(), Root, + Ptr, PtrVal /*SrcValue*/, 0/*SVOffset*/, + false /*volatile*/, 1 /* align=1 */); + + if (!ConstantMemory) + Builder.PendingLoads.push_back(LoadVal.getValue(1)); + return LoadVal; +} + + +/// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form. +/// If so, return true and lower it, otherwise return false and it will be +/// lowered like a normal call. +bool SelectionDAGBuilder::visitMemCmpCall(CallInst &I) { + // Verify that the prototype makes sense. int memcmp(void*,void*,size_t) + if (I.getNumOperands() != 4) + return false; + + Value *LHS = I.getOperand(1), *RHS = I.getOperand(2); + if (!isa<PointerType>(LHS->getType()) || !isa<PointerType>(RHS->getType()) || + !isa<IntegerType>(I.getOperand(3)->getType()) || + !isa<IntegerType>(I.getType())) + return false; + + ConstantInt *Size = dyn_cast<ConstantInt>(I.getOperand(3)); + + // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0 + // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0 + if (Size && IsOnlyUsedInZeroEqualityComparison(&I)) { + bool ActuallyDoIt = true; + MVT LoadVT; + const Type *LoadTy; + switch (Size->getZExtValue()) { + default: + LoadVT = MVT::Other; + LoadTy = 0; + ActuallyDoIt = false; + break; + case 2: + LoadVT = MVT::i16; + LoadTy = Type::getInt16Ty(Size->getContext()); + break; + case 4: + LoadVT = MVT::i32; + LoadTy = Type::getInt32Ty(Size->getContext()); + break; + case 8: + LoadVT = MVT::i64; + LoadTy = Type::getInt64Ty(Size->getContext()); + break; + /* + case 16: + LoadVT = MVT::v4i32; + LoadTy = Type::getInt32Ty(Size->getContext()); + LoadTy = VectorType::get(LoadTy, 4); + break; + */ + } + + // This turns into unaligned loads. We only do this if the target natively + // supports the MVT we'll be loading or if it is small enough (<= 4) that + // we'll only produce a small number of byte loads. + + // Require that we can find a legal MVT, and only do this if the target + // supports unaligned loads of that type. Expanding into byte loads would + // bloat the code. + if (ActuallyDoIt && Size->getZExtValue() > 4) { + // TODO: Handle 5 byte compare as 4-byte + 1 byte. + // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads. + if (!TLI.isTypeLegal(LoadVT) ||!TLI.allowsUnalignedMemoryAccesses(LoadVT)) + ActuallyDoIt = false; + } + + if (ActuallyDoIt) { + SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this); + SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this); + + SDValue Res = DAG.getSetCC(getCurDebugLoc(), MVT::i1, LHSVal, RHSVal, + ISD::SETNE); + EVT CallVT = TLI.getValueType(I.getType(), true); + setValue(&I, DAG.getZExtOrTrunc(Res, getCurDebugLoc(), CallVT)); + return true; + } + } + + + return false; +} + void SelectionDAGBuilder::visitCall(CallInst &I) { const char *RenameFn = 0; @@ -4348,6 +5335,9 @@ void SelectionDAGBuilder::visitCall(CallInst &I) { Tmp.getValueType(), Tmp)); return; } + } else if (Name == "memcmp") { + if (visitMemCmpCall(I)) + return; } } } else if (isa<InlineAsm>(I.getOperand(0))) { @@ -4361,21 +5351,19 @@ void SelectionDAGBuilder::visitCall(CallInst &I) { else Callee = DAG.getExternalSymbol(RenameFn, TLI.getPointerTy()); - // Check if we can potentially perform a tail call. More detailed - // checking is be done within LowerCallTo, after more information - // about the call is known. + // Check if we can potentially perform a tail call. More detailed checking is + // be done within LowerCallTo, after more information about the call is known. bool isTailCall = PerformTailCallOpt && I.isTailCall(); LowerCallTo(&I, Callee, isTailCall); } - /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from /// this value and returns the result as a ValueVT value. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, - SDValue &Chain, + unsigned Order, SDValue &Chain, SDValue *Flag) const { // Assemble the legal parts into the final values. SmallVector<SDValue, 4> Values(ValueVTs.size()); @@ -4389,14 +5377,18 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, Parts.resize(NumRegs); for (unsigned i = 0; i != NumRegs; ++i) { SDValue P; - if (Flag == 0) + if (Flag == 0) { P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); - else { + } else { P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag); *Flag = P.getValue(2); } + Chain = P.getValue(1); + if (DisableScheduling) + DAG.AssignOrdering(P.getNode(), Order); + // If the source register was virtual and if we know something about it, // add an assert node. if (TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) && @@ -4435,6 +5427,8 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, P = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl, RegisterVT, P, DAG.getValueType(FromVT)); + if (DisableScheduling) + DAG.AssignOrdering(P.getNode(), Order); } } } @@ -4442,15 +5436,20 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, Parts[i] = P; } - Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), + Values[Value] = getCopyFromParts(DAG, dl, Order, Parts.begin(), NumRegs, RegisterVT, ValueVT); + if (DisableScheduling) + DAG.AssignOrdering(Values[Value].getNode(), Order); Part += NumRegs; Parts.clear(); } - return DAG.getNode(ISD::MERGE_VALUES, dl, - DAG.getVTList(&ValueVTs[0], ValueVTs.size()), - &Values[0], ValueVTs.size()); + SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl, + DAG.getVTList(&ValueVTs[0], ValueVTs.size()), + &Values[0], ValueVTs.size()); + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); + return Res; } /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the @@ -4458,7 +5457,8 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, - SDValue &Chain, SDValue *Flag) const { + unsigned Order, SDValue &Chain, + SDValue *Flag) const { // Get the list of the values's legal parts. unsigned NumRegs = Regs.size(); SmallVector<SDValue, 8> Parts(NumRegs); @@ -4467,7 +5467,8 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, unsigned NumParts = TLI->getNumRegisters(*DAG.getContext(), ValueVT); EVT RegisterVT = RegVTs[Value]; - getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), + getCopyToParts(DAG, dl, Order, + Val.getValue(Val.getResNo() + Value), &Parts[Part], NumParts, RegisterVT); Part += NumParts; } @@ -4476,13 +5477,17 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, SmallVector<SDValue, 8> Chains(NumRegs); for (unsigned i = 0; i != NumRegs; ++i) { SDValue Part; - if (Flag == 0) + if (Flag == 0) { Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]); - else { + } else { Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag); *Flag = Part.getValue(1); } + Chains[i] = Part.getValue(0); + + if (DisableScheduling) + DAG.AssignOrdering(Part.getNode(), Order); } if (NumRegs == 1 || Flag) @@ -4499,6 +5504,9 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, Chain = Chains[NumRegs-1]; else Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Chains[0], NumRegs); + + if (DisableScheduling) + DAG.AssignOrdering(Chain.getNode(), Order); } /// AddInlineAsmOperands - Add this value to the specified inlineasm node @@ -4506,20 +5514,28 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, /// values added into it. void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,unsigned MatchingIdx, - SelectionDAG &DAG, + SelectionDAG &DAG, unsigned Order, std::vector<SDValue> &Ops) const { - EVT IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); assert(Regs.size() < (1 << 13) && "Too many inline asm outputs!"); unsigned Flag = Code | (Regs.size() << 3); if (HasMatching) Flag |= 0x80000000 | (MatchingIdx << 16); - Ops.push_back(DAG.getTargetConstant(Flag, IntPtrTy)); + SDValue Res = DAG.getTargetConstant(Flag, MVT::i32); + Ops.push_back(Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); + for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { unsigned NumRegs = TLI->getNumRegisters(*DAG.getContext(), ValueVTs[Value]); EVT RegisterVT = RegVTs[Value]; for (unsigned i = 0; i != NumRegs; ++i) { assert(Reg < Regs.size() && "Mismatch in # registers expected"); - Ops.push_back(DAG.getRegister(Regs[Reg++], RegisterVT)); + SDValue Res = DAG.getRegister(Regs[Reg++], RegisterVT); + Ops.push_back(Res); + + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); } } } @@ -4611,7 +5627,7 @@ public: /// getCallOperandValEVT - Return the EVT of the Value* that this operand /// corresponds to. If there is no Value* for this operand, it returns /// MVT::Other. - EVT getCallOperandValEVT(LLVMContext &Context, + EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI, const TargetData *TD) const { if (CallOperandVal == 0) return MVT::Other; @@ -4623,8 +5639,12 @@ public: // If this is an indirect operand, the operand is a pointer to the // accessed type. - if (isIndirect) - OpTy = cast<PointerType>(OpTy)->getElementType(); + if (isIndirect) { + const llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy); + if (!PtrTy) + llvm_report_error("Indirect operand for inline asm not a pointer!"); + OpTy = PtrTy->getElementType(); + } // If OpTy is not a single value, it may be a struct/union that we // can tile with integers. @@ -4663,8 +5683,8 @@ private: /// GetRegistersForValue - Assign registers (virtual or physical) for the /// specified operand. We prefer to assign virtual registers, to allow the -/// register allocator handle the assignment process. However, if the asm uses -/// features that we can't model on machineinstrs, we have SDISel do the +/// register allocator to handle the assignment process. However, if the asm +/// uses features that we can't model on machineinstrs, we have SDISel do the /// allocation. This produces generally horrible, but correct, code. /// /// OpInfo describes the operand. @@ -4728,12 +5748,15 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, // bitcast to the corresponding integer type. This turns an f64 value // into i64, which can be passed with two i32 values on a 32-bit // machine. - RegVT = EVT::getIntegerVT(Context, + RegVT = EVT::getIntegerVT(Context, OpInfo.ConstraintVT.getSizeInBits()); OpInfo.CallOperand = DAG.getNode(ISD::BIT_CONVERT, getCurDebugLoc(), RegVT, OpInfo.CallOperand); OpInfo.ConstraintVT = RegVT; } + + if (DisableScheduling) + DAG.AssignOrdering(OpInfo.CallOperand.getNode(), SDNodeOrder); } NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT); @@ -4770,6 +5793,7 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, Regs.push_back(*I); } } + OpInfo.AssignedRegs = RegsForValue(TLI, Regs, RegVT, ValueVT); const TargetRegisterInfo *TRI = DAG.getTarget().getRegisterInfo(); OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs, *TRI); @@ -4791,7 +5815,7 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, OpInfo.AssignedRegs = RegsForValue(TLI, Regs, RegVT, ValueVT); return; } - + // This is a reference to a register class that doesn't directly correspond // to an LLVM register class. Allocate NumRegs consecutive, available, // registers from the class. @@ -4853,7 +5877,7 @@ hasInlineAsmMemConstraint(std::vector<InlineAsm::ConstraintInfo> &CInfos, if (CType == TargetLowering::C_Memory) return true; } - + // Indirect operand accesses access memory. if (CI.isIndirect) return true; @@ -4878,9 +5902,9 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { ConstraintInfos = IA->ParseConstraints(); bool hasMemory = hasInlineAsmMemConstraint(ConstraintInfos, TLI); - + SDValue Chain, Flag; - + // We won't need to flush pending loads if this asm doesn't touch // memory and is nonvolatile. if (hasMemory || IA->hasSideEffects()) @@ -5004,6 +6028,7 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // There is no longer a Value* corresponding to this operand. OpInfo.CallOperandVal = 0; + // It is now an indirect operand. OpInfo.isIndirect = true; } @@ -5013,8 +6038,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { if (OpInfo.ConstraintType == TargetLowering::C_Register) GetRegistersForValue(OpInfo, OutputRegs, InputRegs); } - ConstraintInfos.clear(); + ConstraintInfos.clear(); // Second pass - Loop over all of the operands, assigning virtual or physregs // to register class operands. @@ -5088,7 +6113,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { 2 /* REGDEF */ , false, 0, - DAG, AsmNodeOperands); + DAG, SDNodeOrder, + AsmNodeOperands); break; } case InlineAsm::isInput: { @@ -5130,15 +6156,15 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo(); for (unsigned i = 0, e = InlineAsm::getNumOperandRegisters(OpFlag); i != e; ++i) - MatchedRegs.Regs. - push_back(RegInfo.createVirtualRegister(TLI.getRegClassFor(RegVT))); + MatchedRegs.Regs.push_back + (RegInfo.createVirtualRegister(TLI.getRegClassFor(RegVT))); // Use the produced MatchedRegs object to MatchedRegs.getCopyToRegs(InOperandVal, DAG, getCurDebugLoc(), - Chain, &Flag); + SDNodeOrder, Chain, &Flag); MatchedRegs.AddInlineAsmOperands(1 /*REGUSE*/, true, OpInfo.getMatchedOperand(), - DAG, AsmNodeOperands); + DAG, SDNodeOrder, AsmNodeOperands); break; } else { assert(((OpFlag & 7) == 4) && "Unknown matching constraint!"); @@ -5198,10 +6224,11 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { } OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, getCurDebugLoc(), - Chain, &Flag); + SDNodeOrder, Chain, &Flag); OpInfo.AssignedRegs.AddInlineAsmOperands(1/*REGUSE*/, false, 0, - DAG, AsmNodeOperands); + DAG, SDNodeOrder, + AsmNodeOperands); break; } case InlineAsm::isClobber: { @@ -5209,7 +6236,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // allocator is aware that the physreg got clobbered. if (!OpInfo.AssignedRegs.Regs.empty()) OpInfo.AssignedRegs.AddInlineAsmOperands(6 /* EARLYCLOBBER REGDEF */, - false, 0, DAG,AsmNodeOperands); + false, 0, DAG, SDNodeOrder, + AsmNodeOperands); break; } } @@ -5228,7 +6256,7 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // and set it as the value of the call. if (!RetValRegs.Regs.empty()) { SDValue Val = RetValRegs.getCopyFromRegs(DAG, getCurDebugLoc(), - Chain, &Flag); + SDNodeOrder, Chain, &Flag); // FIXME: Why don't we do this for inline asms with MRVs? if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) { @@ -5268,21 +6296,25 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { RegsForValue &OutRegs = IndirectStoresToEmit[i].first; Value *Ptr = IndirectStoresToEmit[i].second; SDValue OutVal = OutRegs.getCopyFromRegs(DAG, getCurDebugLoc(), - Chain, &Flag); + SDNodeOrder, Chain, &Flag); StoresToEmit.push_back(std::make_pair(OutVal, Ptr)); } // Emit the non-flagged stores from the physregs. SmallVector<SDValue, 8> OutChains; - for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) - OutChains.push_back(DAG.getStore(Chain, getCurDebugLoc(), - StoresToEmit[i].first, - getValue(StoresToEmit[i].second), - StoresToEmit[i].second, 0)); + for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) { + SDValue Val = DAG.getStore(Chain, getCurDebugLoc(), + StoresToEmit[i].first, + getValue(StoresToEmit[i].second), + StoresToEmit[i].second, 0); + OutChains.push_back(Val); + } + if (!OutChains.empty()) Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), MVT::Other, &OutChains[0], OutChains.size()); + DAG.setRoot(Chain); } @@ -5328,8 +6360,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, CallingConv::ID CallConv, bool isTailCall, bool isReturnValueUsed, SDValue Callee, - ArgListTy &Args, SelectionDAG &DAG, DebugLoc dl) { - + ArgListTy &Args, SelectionDAG &DAG, DebugLoc dl, + unsigned Order) { assert((!isTailCall || PerformTailCallOpt) && "isTailCall set when tail-call optimizations are disabled!"); @@ -5383,7 +6415,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, else if (Args[i].isZExt) ExtendKind = ISD::ZERO_EXTEND; - getCopyToParts(DAG, dl, Op, &Parts[0], NumParts, PartVT, ExtendKind); + getCopyToParts(DAG, dl, Order, Op, &Parts[0], NumParts, + PartVT, ExtendKind); for (unsigned j = 0; j != NumParts; ++j) { // if it isn't first piece, alignment must be 1 @@ -5444,6 +6477,9 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, "LowerCall emitted a value with the wrong type!"); }); + if (DisableScheduling) + DAG.AssignOrdering(Chain.getNode(), Order); + // For a tail call, the return value is merely live-out and there aren't // any nodes in the DAG representing it. Return a special value to // indicate that a tail call has been emitted and no more Instructions @@ -5468,9 +6504,11 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, unsigned NumRegs = getNumRegisters(RetTy->getContext(), VT); SDValue ReturnValue = - getCopyFromParts(DAG, dl, &InVals[CurReg], NumRegs, RegisterVT, VT, - AssertOp); + getCopyFromParts(DAG, dl, Order, &InVals[CurReg], NumRegs, + RegisterVT, VT, AssertOp); ReturnValues.push_back(ReturnValue); + if (DisableScheduling) + DAG.AssignOrdering(ReturnValue.getNode(), Order); CurReg += NumRegs; } @@ -5483,7 +6521,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(&RetTys[0], RetTys.size()), &ReturnValues[0], ReturnValues.size()); - + if (DisableScheduling) + DAG.AssignOrdering(Res.getNode(), Order); return std::make_pair(Res, Chain); } @@ -5500,7 +6539,6 @@ SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) { return SDValue(); } - void SelectionDAGBuilder::CopyValueToVirtualRegister(Value *V, unsigned Reg) { SDValue Op = getValue(V); assert((Op.getOpcode() != ISD::CopyFromReg || @@ -5510,7 +6548,7 @@ void SelectionDAGBuilder::CopyValueToVirtualRegister(Value *V, unsigned Reg) { RegsForValue RFV(V->getContext(), TLI, Reg, V->getType()); SDValue Chain = DAG.getEntryNode(); - RFV.getCopyToRegs(Op, DAG, getCurDebugLoc(), Chain, 0); + RFV.getCopyToRegs(Op, DAG, getCurDebugLoc(), SDNodeOrder, Chain, 0); PendingExports.push_back(Chain); } @@ -5528,12 +6566,12 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { // Check whether the function can return without sret-demotion. SmallVector<EVT, 4> OutVTs; SmallVector<ISD::ArgFlagsTy, 4> OutsFlags; - getReturnInfo(F.getReturnType(), F.getAttributes().getRetAttributes(), + getReturnInfo(F.getReturnType(), F.getAttributes().getRetAttributes(), OutVTs, OutsFlags, TLI); FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); - FLI.CanLowerReturn = TLI.CanLowerReturn(F.getCallingConv(), F.isVarArg(), - OutVTs, OutsFlags, DAG); + FLI.CanLowerReturn = TLI.CanLowerReturn(F.getCallingConv(), F.isVarArg(), + OutVTs, OutsFlags, DAG); if (!FLI.CanLowerReturn) { // Put in an sret pointer parameter before all the other parameters. SmallVector<EVT, 1> ValueVTs; @@ -5613,12 +6651,14 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { "LowerFormalArguments didn't return a valid chain!"); assert(InVals.size() == Ins.size() && "LowerFormalArguments didn't emit the correct number of values!"); - DEBUG(for (unsigned i = 0, e = Ins.size(); i != e; ++i) { - assert(InVals[i].getNode() && - "LowerFormalArguments emitted a null value!"); - assert(Ins[i].VT == InVals[i].getValueType() && - "LowerFormalArguments emitted a value with the wrong type!"); - }); + DEBUG({ + for (unsigned i = 0, e = Ins.size(); i != e; ++i) { + assert(InVals[i].getNode() && + "LowerFormalArguments emitted a null value!"); + assert(Ins[i].VT == InVals[i].getValueType() && + "LowerFormalArguments emitted a value with the wrong type!"); + } + }); // Update the DAG with the new chain value resulting from argument lowering. DAG.setRoot(NewRoot); @@ -5634,20 +6674,22 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { EVT VT = ValueVTs[0]; EVT RegVT = TLI.getRegisterType(*CurDAG->getContext(), VT); ISD::NodeType AssertOp = ISD::DELETED_NODE; - SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, - VT, AssertOp); + SDValue ArgValue = getCopyFromParts(DAG, dl, 0, &InVals[0], 1, + RegVT, VT, AssertOp); MachineFunction& MF = SDB->DAG.getMachineFunction(); MachineRegisterInfo& RegInfo = MF.getRegInfo(); unsigned SRetReg = RegInfo.createVirtualRegister(TLI.getRegClassFor(RegVT)); FLI.DemoteRegister = SRetReg; - NewRoot = SDB->DAG.getCopyToReg(NewRoot, SDB->getCurDebugLoc(), SRetReg, ArgValue); + NewRoot = SDB->DAG.getCopyToReg(NewRoot, SDB->getCurDebugLoc(), + SRetReg, ArgValue); DAG.setRoot(NewRoot); - + // i indexes lowered arguments. Bump it past the hidden sret argument. // Idx indexes LLVM arguments. Don't touch it. ++i; } + for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++Idx) { SmallVector<SDValue, 4> ArgValues; @@ -5666,19 +6708,25 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { else if (F.paramHasAttr(Idx, Attribute::ZExt)) AssertOp = ISD::AssertZext; - ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts, - PartVT, VT, AssertOp)); + ArgValues.push_back(getCopyFromParts(DAG, dl, 0, &InVals[i], + NumParts, PartVT, VT, + AssertOp)); } + i += NumParts; } + if (!I->use_empty()) { - SDB->setValue(I, DAG.getMergeValues(&ArgValues[0], NumValues, - SDB->getCurDebugLoc())); + SDValue Res = DAG.getMergeValues(&ArgValues[0], NumValues, + SDB->getCurDebugLoc()); + SDB->setValue(I, Res); + // If this argument is live outside of the entry block, insert a copy from // whereever we got it to the vreg that other BB's will reference it as. SDB->CopyToExportRegsIfNeeded(I); } } + assert(i == InVals.size() && "Argument register count mismatch!"); // Finally, if the target has anything special to do, allow it to do so. diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 244f9b5..88a2017 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -91,11 +91,13 @@ class SelectionDAGBuilder { DenseMap<const Value*, SDValue> NodeMap; +public: /// PendingLoads - Loads are not emitted to the program immediately. We bunch /// them up and then emit token factor nodes when possible. This allows us to /// get simple disambiguation between loads without worrying about alias /// analysis. SmallVector<SDValue, 8> PendingLoads; +private: /// PendingExports - CopyToReg nodes that copy values to virtual registers /// for export to other blocks need to be emitted before any terminator @@ -104,6 +106,10 @@ class SelectionDAGBuilder { /// instructions. SmallVector<SDValue, 8> PendingExports; + /// SDNodeOrder - A unique monotonically increasing number used to order the + /// SDNodes we create. + unsigned SDNodeOrder; + /// Case - A struct to record the Value for a switch case, and the /// case's target basic block. struct Case { @@ -300,7 +306,7 @@ public: SelectionDAGBuilder(SelectionDAG &dag, TargetLowering &tli, FunctionLoweringInfo &funcinfo, CodeGenOpt::Level ol) - : CurDebugLoc(DebugLoc::getUnknownLoc()), + : CurDebugLoc(DebugLoc::getUnknownLoc()), SDNodeOrder(0), TLI(tli), DAG(dag), FuncInfo(funcinfo), OptLevel(ol), HasTailCall(false), Context(dag.getContext()) { @@ -332,6 +338,8 @@ public: DebugLoc getCurDebugLoc() const { return CurDebugLoc; } void setCurDebugLoc(DebugLoc dl) { CurDebugLoc = dl; } + unsigned getSDNodeOrder() const { return SDNodeOrder; } + void CopyValueToVirtualRegister(Value *V, unsigned Reg); void visit(Instruction &I); @@ -455,6 +463,8 @@ private: void visitStore(StoreInst &I); void visitPHI(PHINode &I) { } // PHI nodes are handled specially. void visitCall(CallInst &I); + bool visitMemCmpCall(CallInst &I); + void visitInlineAsm(CallSite CS); const char *visitIntrinsicCall(CallInst &I, unsigned Intrinsic); void visitTargetIntrinsic(CallInst &I, unsigned Intrinsic); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index a640c7d..05669c0 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -362,32 +362,29 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { /// SetDebugLoc - Update MF's and SDB's DebugLocs if debug information is /// attached with this instruction. -static void SetDebugLoc(unsigned MDDbgKind, - MetadataContext &TheMetadata, - Instruction *I, +static void SetDebugLoc(unsigned MDDbgKind, Instruction *I, SelectionDAGBuilder *SDB, - FastISel *FastIS, - MachineFunction *MF) { - if (!isa<DbgInfoIntrinsic>(I)) - if (MDNode *Dbg = TheMetadata.getMD(MDDbgKind, I)) { - DILocation DILoc(Dbg); - DebugLoc Loc = ExtractDebugLocation(DILoc, MF->getDebugLocInfo()); - - SDB->setCurDebugLoc(Loc); - - if (FastIS) - FastIS->setCurDebugLoc(Loc); - - // If the function doesn't have a default debug location yet, set - // it. This is kind of a hack. - if (MF->getDefaultDebugLoc().isUnknown()) - MF->setDefaultDebugLoc(Loc); - } + FastISel *FastIS, MachineFunction *MF) { + if (isa<DbgInfoIntrinsic>(I)) return; + + if (MDNode *Dbg = I->getMetadata(MDDbgKind)) { + DILocation DILoc(Dbg); + DebugLoc Loc = ExtractDebugLocation(DILoc, MF->getDebugLocInfo()); + + SDB->setCurDebugLoc(Loc); + + if (FastIS) + FastIS->setCurDebugLoc(Loc); + + // If the function doesn't have a default debug location yet, set + // it. This is kind of a hack. + if (MF->getDefaultDebugLoc().isUnknown()) + MF->setDefaultDebugLoc(Loc); + } } /// ResetDebugLoc - Set MF's and SDB's DebugLocs to Unknown. -static void ResetDebugLoc(SelectionDAGBuilder *SDB, - FastISel *FastIS) { +static void ResetDebugLoc(SelectionDAGBuilder *SDB, FastISel *FastIS) { SDB->setCurDebugLoc(DebugLoc::getUnknownLoc()); if (FastIS) FastIS->setCurDebugLoc(DebugLoc::getUnknownLoc()); @@ -398,14 +395,12 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, BasicBlock::iterator End, bool &HadTailCall) { SDB->setCurrentBasicBlock(BB); - MetadataContext &TheMetadata = LLVMBB->getParent()->getContext().getMetadata(); - unsigned MDDbgKind = TheMetadata.getMDKind("dbg"); + unsigned MDDbgKind = LLVMBB->getContext().getMDKindID("dbg"); // Lower all of the non-terminator instructions. If a call is emitted // as a tail call, cease emitting nodes for this block. for (BasicBlock::iterator I = Begin; I != End && !SDB->HasTailCall; ++I) { - if (MDDbgKind) - SetDebugLoc(MDDbgKind, TheMetadata, I, SDB, 0, MF); + SetDebugLoc(MDDbgKind, I, SDB, 0, MF); if (!isa<TerminatorInst>(I)) { SDB->visit(*I); @@ -428,7 +423,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, HandlePHINodesInSuccessorBlocks(LLVMBB); // Lower the terminator after the copies are emitted. - SetDebugLoc(MDDbgKind, TheMetadata, LLVMBB->getTerminator(), SDB, 0, MF); + SetDebugLoc(MDDbgKind, LLVMBB->getTerminator(), SDB, 0, MF); SDB->visit(*LLVMBB->getTerminator()); ResetDebugLoc(SDB, 0); } @@ -567,9 +562,9 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { if (Changed) { if (TimePassesIsEnabled) { NamedRegionTimer T("Type Legalization 2", GroupName); - Changed = CurDAG->LegalizeTypes(); + CurDAG->LegalizeTypes(); } else { - Changed = CurDAG->LegalizeTypes(); + CurDAG->LegalizeTypes(); } if (ViewDAGCombineLT) @@ -680,8 +675,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, #endif ); - MetadataContext &TheMetadata = Fn.getContext().getMetadata(); - unsigned MDDbgKind = TheMetadata.getMDKind("dbg"); + unsigned MDDbgKind = Fn.getContext().getMDKindID("dbg"); // Iterate over all basic blocks in the function. for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) { @@ -779,8 +773,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, break; } - if (MDDbgKind) - SetDebugLoc(MDDbgKind, TheMetadata, BI, SDB, FastIS, &MF); + SetDebugLoc(MDDbgKind, BI, SDB, FastIS, &MF); // First try normal tablegen-generated "fast" selection. if (FastIS->SelectInstruction(BI)) { @@ -1182,9 +1175,8 @@ SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops) { } // Add this to the output node. - EVT IntPtrTy = TLI.getPointerTy(); Ops.push_back(CurDAG->getTargetConstant(4/*MEM*/ | (SelOps.size()<< 3), - IntPtrTy)); + MVT::i32)); Ops.insert(Ops.end(), SelOps.begin(), SelOps.end()); i += 2; } diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 1026169..d9a5a13 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -713,6 +713,10 @@ MVT::SimpleValueType TargetLowering::getSetCCResultType(EVT VT) const { return PointerTy.SimpleTy; } +MVT::SimpleValueType TargetLowering::getCmpLibcallReturnType() const { + return MVT::i32; // return the default value +} + /// getVectorTypeBreakdown - Vector types are broken down into some number of /// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32 /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack. diff --git a/lib/CodeGen/SimpleRegisterCoalescing.cpp b/lib/CodeGen/SimpleRegisterCoalescing.cpp index ed407eb..6314331 100644 --- a/lib/CodeGen/SimpleRegisterCoalescing.cpp +++ b/lib/CodeGen/SimpleRegisterCoalescing.cpp @@ -1065,7 +1065,7 @@ SimpleRegisterCoalescing::isWinToJoinVRWithSrcPhysReg(MachineInstr *CopyMI, if (SuccMBB == CopyMBB) continue; if (DstInt.overlaps(li_->getMBBStartIdx(SuccMBB), - li_->getMBBEndIdx(SuccMBB).getNextIndex().getBaseIndex())) + li_->getMBBEndIdx(SuccMBB))) return false; } } @@ -1121,7 +1121,7 @@ SimpleRegisterCoalescing::isWinToJoinVRWithDstPhysReg(MachineInstr *CopyMI, if (PredMBB == SMBB) continue; if (SrcInt.overlaps(li_->getMBBStartIdx(PredMBB), - li_->getMBBEndIdx(PredMBB).getNextIndex().getBaseIndex())) + li_->getMBBEndIdx(PredMBB))) return false; } } @@ -2246,8 +2246,9 @@ SimpleRegisterCoalescing::JoinIntervals(LiveInterval &LHS, LiveInterval &RHS, continue; // Figure out the value # from the RHS. - LHSValsDefinedFromRHS[VNI]= - RHS.getLiveRangeContaining(VNI->def.getPrevSlot())->valno; + LiveRange *lr = RHS.getLiveRangeContaining(VNI->def.getPrevSlot()); + assert(lr && "Cannot find live range"); + LHSValsDefinedFromRHS[VNI] = lr->valno; } // Loop over the value numbers of the RHS, seeing if any are defined from @@ -2264,8 +2265,9 @@ SimpleRegisterCoalescing::JoinIntervals(LiveInterval &LHS, LiveInterval &RHS, continue; // Figure out the value # from the LHS. - RHSValsDefinedFromLHS[VNI]= - LHS.getLiveRangeContaining(VNI->def.getPrevSlot())->valno; + LiveRange *lr = LHS.getLiveRangeContaining(VNI->def.getPrevSlot()); + assert(lr && "Cannot find live range"); + RHSValsDefinedFromLHS[VNI] = lr->valno; } LHSValNoAssignments.resize(LHS.getNumValNums(), -1); diff --git a/lib/CodeGen/SimpleRegisterCoalescing.h b/lib/CodeGen/SimpleRegisterCoalescing.h index 605a740..f668064 100644 --- a/lib/CodeGen/SimpleRegisterCoalescing.h +++ b/lib/CodeGen/SimpleRegisterCoalescing.h @@ -33,7 +33,7 @@ namespace llvm { MachineInstr *MI; unsigned LoopDepth; CopyRec(MachineInstr *mi, unsigned depth) - : MI(mi), LoopDepth(depth) {}; + : MI(mi), LoopDepth(depth) {} }; class SimpleRegisterCoalescing : public MachineFunctionPass, @@ -85,7 +85,7 @@ namespace llvm { bool coalesceFunction(MachineFunction &mf, RegallocQuery &) { // This runs as an independent pass, so don't do anything. return false; - }; + } /// print - Implement the dump method. virtual void print(raw_ostream &O, const Module* = 0) const; diff --git a/lib/CodeGen/SlotIndexes.cpp b/lib/CodeGen/SlotIndexes.cpp index f85384b..782af12 100644 --- a/lib/CodeGen/SlotIndexes.cpp +++ b/lib/CodeGen/SlotIndexes.cpp @@ -92,13 +92,14 @@ bool SlotIndexes::runOnMachineFunction(MachineFunction &fn) { functionSize = 0; unsigned index = 0; + push_back(createEntry(0, index)); + // Iterate over the the function. for (MachineFunction::iterator mbbItr = mf->begin(), mbbEnd = mf->end(); mbbItr != mbbEnd; ++mbbItr) { MachineBasicBlock *mbb = &*mbbItr; // Insert an index for the MBB start. - push_back(createEntry(0, index)); SlotIndex blockStartIndex(back(), SlotIndex::LOAD); index += SlotIndex::NUM; @@ -137,16 +138,16 @@ bool SlotIndexes::runOnMachineFunction(MachineFunction &fn) { index += SlotIndex::NUM; } - SlotIndex blockEndIndex(back(), SlotIndex::STORE); + // One blank instruction at the end. + push_back(createEntry(0, index)); + + SlotIndex blockEndIndex(back(), SlotIndex::LOAD); mbb2IdxMap.insert( std::make_pair(mbb, std::make_pair(blockStartIndex, blockEndIndex))); idx2MBBMap.push_back(IdxMBBPair(blockStartIndex, mbb)); } - // One blank instruction at the end. - push_back(createEntry(0, index)); - // Sort the Idx2MBBMap std::sort(idx2MBBMap.begin(), idx2MBBMap.end(), Idx2MBBCompare()); diff --git a/lib/CodeGen/Spiller.cpp b/lib/CodeGen/Spiller.cpp index bc246c1..bec9294 100644 --- a/lib/CodeGen/Spiller.cpp +++ b/lib/CodeGen/Spiller.cpp @@ -486,10 +486,10 @@ private: SlotIndex newKillRangeEnd = oldKillRange->end; oldKillRange->end = copyIdx.getDefIndex(); - if (newKillRangeEnd != lis->getMBBEndIdx(killMBB).getNextSlot()) { - assert(newKillRangeEnd > lis->getMBBEndIdx(killMBB).getNextSlot() && + if (newKillRangeEnd != lis->getMBBEndIdx(killMBB)) { + assert(newKillRangeEnd > lis->getMBBEndIdx(killMBB) && "PHI kill range doesn't reach kill-block end. Not sane."); - newLI->addRange(LiveRange(lis->getMBBEndIdx(killMBB).getNextSlot(), + newLI->addRange(LiveRange(lis->getMBBEndIdx(killMBB), newKillRangeEnd, newVNI)); } @@ -500,7 +500,7 @@ private: newKillVNI->addKill(lis->getMBBTerminatorGap(killMBB)); newKillVNI->setHasPHIKill(true); li->addRange(LiveRange(copyIdx.getDefIndex(), - lis->getMBBEndIdx(killMBB).getNextSlot(), + lis->getMBBEndIdx(killMBB), newKillVNI)); } |
