diff options
Diffstat (limited to 'contrib/llvm/lib/CodeGen')
154 files changed, 11450 insertions, 8459 deletions
diff --git a/contrib/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/contrib/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp index 7a1c049..c50f8b5 100644 --- a/contrib/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp +++ b/contrib/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp @@ -20,14 +20,13 @@ #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/RegisterClassInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; // If DebugDiv > 0 then only break antidep with (ID % DebugDiv) == DebugMod @@ -152,23 +151,7 @@ void AggressiveAntiDepBreaker::StartBlock(MachineBasicBlock *BB) { std::vector<unsigned> &KillIndices = State->GetKillIndices(); std::vector<unsigned> &DefIndices = State->GetDefIndices(); - // Determine the live-out physregs for this block. - if (IsReturnBlock) { - // In a return block, examine the function live-out regs. - for (MachineRegisterInfo::liveout_iterator I = MRI.liveout_begin(), - E = MRI.liveout_end(); I != E; ++I) { - for (MCRegAliasIterator AI(*I, TRI, true); AI.isValid(); ++AI) { - unsigned Reg = *AI; - State->UnionGroups(Reg, 0); - KillIndices[Reg] = BB->size(); - DefIndices[Reg] = ~0u; - } - } - } - - // In a non-return block, examine the live-in regs of all successors. - // Note a return block can have successors if the return instruction is - // predicated. + // Examine the live-in regs of all successors. for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), SE = BB->succ_end(); SI != SE; ++SI) for (MachineBasicBlock::livein_iterator I = (*SI)->livein_begin(), @@ -616,7 +599,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters( const TargetRegisterClass *SuperRC = TRI->getMinimalPhysRegClass(SuperReg, MVT::Other); - ArrayRef<unsigned> Order = RegClassInfo.getOrder(SuperRC); + ArrayRef<MCPhysReg> Order = RegClassInfo.getOrder(SuperRC); if (Order.empty()) { DEBUG(dbgs() << "\tEmpty Super Regclass!!\n"); return false; diff --git a/contrib/llvm/lib/CodeGen/AggressiveAntiDepBreaker.h b/contrib/llvm/lib/CodeGen/AggressiveAntiDepBreaker.h index 7067784..6683630 100644 --- a/contrib/llvm/lib/CodeGen/AggressiveAntiDepBreaker.h +++ b/contrib/llvm/lib/CodeGen/AggressiveAntiDepBreaker.h @@ -18,15 +18,15 @@ #define LLVM_CODEGEN_AGGRESSIVEANTIDEPBREAKER_H #include "AntiDepBreaker.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/ScheduleDAG.h" -#include "llvm/Target/TargetSubtargetInfo.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/ADT/BitVector.h" -#include "llvm/ADT/SmallSet.h" +#include "llvm/Target/TargetSubtargetInfo.h" #include <map> namespace llvm { diff --git a/contrib/llvm/lib/CodeGen/AllocationOrder.cpp b/contrib/llvm/lib/CodeGen/AllocationOrder.cpp index 7cde136..3fa1f8f 100644 --- a/contrib/llvm/lib/CodeGen/AllocationOrder.cpp +++ b/contrib/llvm/lib/CodeGen/AllocationOrder.cpp @@ -14,10 +14,14 @@ // //===----------------------------------------------------------------------===// +#define DEBUG_TYPE "regalloc" #include "AllocationOrder.h" -#include "VirtRegMap.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RegisterClassInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -25,56 +29,24 @@ using namespace llvm; AllocationOrder::AllocationOrder(unsigned VirtReg, const VirtRegMap &VRM, const RegisterClassInfo &RegClassInfo) - : Begin(0), End(0), Pos(0), RCI(RegClassInfo), OwnedBegin(false) { - const TargetRegisterClass *RC = VRM.getRegInfo().getRegClass(VirtReg); - std::pair<unsigned, unsigned> HintPair = - VRM.getRegInfo().getRegAllocationHint(VirtReg); - const MachineRegisterInfo &MRI = VRM.getRegInfo(); - - // HintPair.second is a register, phys or virt. - Hint = HintPair.second; - - // Translate to physreg, or 0 if not assigned yet. - if (TargetRegisterInfo::isVirtualRegister(Hint)) - Hint = VRM.getPhys(Hint); - - // The first hint pair component indicates a target-specific hint. - if (HintPair.first) { - const TargetRegisterInfo &TRI = VRM.getTargetRegInfo(); - // The remaining allocation order may depend on the hint. - ArrayRef<uint16_t> Order = - TRI.getRawAllocationOrder(RC, HintPair.first, Hint, - VRM.getMachineFunction()); - if (Order.empty()) - return; - - // Copy the allocation order with reserved registers removed. - OwnedBegin = true; - unsigned *P = new unsigned[Order.size()]; - Begin = P; - for (unsigned i = 0; i != Order.size(); ++i) - if (!MRI.isReserved(Order[i])) - *P++ = Order[i]; - End = P; - - // Target-dependent hints require resolution. - Hint = TRI.ResolveRegAllocHint(HintPair.first, Hint, - VRM.getMachineFunction()); - } else { - // If there is no hint or just a normal hint, use the cached allocation - // order from RegisterClassInfo. - ArrayRef<unsigned> O = RCI.getOrder(RC); - Begin = O.begin(); - End = O.end(); - } - - // The hint must be a valid physreg for allocation. - if (Hint && (!TargetRegisterInfo::isPhysicalRegister(Hint) || - !RC->contains(Hint) || MRI.isReserved(Hint))) - Hint = 0; -} - -AllocationOrder::~AllocationOrder() { - if (OwnedBegin) - delete [] Begin; + : Pos(0) { + const MachineFunction &MF = VRM.getMachineFunction(); + const TargetRegisterInfo *TRI = &VRM.getTargetRegInfo(); + Order = RegClassInfo.getOrder(MF.getRegInfo().getRegClass(VirtReg)); + TRI->getRegAllocationHints(VirtReg, Order, Hints, MF, &VRM); + rewind(); + + DEBUG({ + if (!Hints.empty()) { + dbgs() << "hints:"; + for (unsigned I = 0, E = Hints.size(); I != E; ++I) + dbgs() << ' ' << PrintReg(Hints[I], TRI); + dbgs() << '\n'; + } + }); +#ifndef NDEBUG + for (unsigned I = 0, E = Hints.size(); I != E; ++I) + assert(std::find(Order.begin(), Order.end(), Hints[I]) != Order.end() && + "Target hint is outside allocation order."); +#endif } diff --git a/contrib/llvm/lib/CodeGen/AllocationOrder.h b/contrib/llvm/lib/CodeGen/AllocationOrder.h index 0ce7e0c..aed461a 100644 --- a/contrib/llvm/lib/CodeGen/AllocationOrder.h +++ b/contrib/llvm/lib/CodeGen/AllocationOrder.h @@ -17,21 +17,21 @@ #ifndef LLVM_CODEGEN_ALLOCATIONORDER_H #define LLVM_CODEGEN_ALLOCATIONORDER_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/MC/MCRegisterInfo.h" + namespace llvm { class RegisterClassInfo; class VirtRegMap; class AllocationOrder { - const unsigned *Begin; - const unsigned *End; - const unsigned *Pos; - const RegisterClassInfo &RCI; - unsigned Hint; - bool OwnedBegin; -public: + SmallVector<MCPhysReg, 16> Hints; + ArrayRef<MCPhysReg> Order; + int Pos; - /// AllocationOrder - Create a new AllocationOrder for VirtReg. +public: + /// Create a new AllocationOrder for VirtReg. /// @param VirtReg Virtual register to allocate for. /// @param VRM Virtual register map for function. /// @param RegClassInfo Information about reserved and allocatable registers. @@ -39,32 +39,45 @@ public: const VirtRegMap &VRM, const RegisterClassInfo &RegClassInfo); - ~AllocationOrder(); + /// Get the allocation order without reordered hints. + ArrayRef<MCPhysReg> getOrder() const { return Order; } - /// next - Return the next physical register in the allocation order, or 0. - /// It is safe to call next again after it returned 0. - /// It will keep returning 0 until rewind() is called. + /// Return the next physical register in the allocation order, or 0. + /// It is safe to call next() again after it returned 0, it will keep + /// returning 0 until rewind() is called. unsigned next() { - // First take the hint. - if (!Pos) { - Pos = Begin; - if (Hint) - return Hint; - } - // Then look at the order from TRI. - while (Pos != End) { - unsigned Reg = *Pos++; - if (Reg != Hint) + if (Pos < 0) + return Hints.end()[Pos++]; + while (Pos < int(Order.size())) { + unsigned Reg = Order[Pos++]; + if (!isHint(Reg)) return Reg; } return 0; } - /// rewind - Start over from the beginning. - void rewind() { Pos = 0; } + /// As next(), but allow duplicates to be returned, and stop before the + /// Limit'th register in the RegisterClassInfo allocation order. + /// + /// This can produce more than Limit registers if there are hints. + unsigned nextWithDups(unsigned Limit) { + if (Pos < 0) + return Hints.end()[Pos++]; + if (Pos < int(Limit)) + return Order[Pos++]; + return 0; + } + + /// Start over from the beginning. + void rewind() { Pos = -int(Hints.size()); } - /// isHint - Return true if PhysReg is a preferred register. - bool isHint(unsigned PhysReg) const { return PhysReg == Hint; } + /// Return true if the last register returned from next() was a preferred register. + bool isHint() const { return Pos <= 0; } + + /// Return true if PhysReg is a preferred register. + bool isHint(unsigned PhysReg) const { + return std::find(Hints.begin(), Hints.end(), PhysReg) != Hints.end(); + } }; } // end namespace llvm diff --git a/contrib/llvm/lib/CodeGen/Analysis.cpp b/contrib/llvm/lib/CodeGen/Analysis.cpp index 5162ad7..dd7282c 100644 --- a/contrib/llvm/lib/CodeGen/Analysis.cpp +++ b/contrib/llvm/lib/CodeGen/Analysis.cpp @@ -13,19 +13,17 @@ #include "llvm/CodeGen/Analysis.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/DerivedTypes.h" -#include "llvm/Function.h" -#include "llvm/Instructions.h" -#include "llvm/IntrinsicInst.h" -#include "llvm/LLVMContext.h" -#include "llvm/Module.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetOptions.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Target/TargetLowering.h" using namespace llvm; /// ComputeLinearIndex - Given an LLVM IR aggregate type and a sequence @@ -266,8 +264,7 @@ static const Value *getNoopInput(const Value *V, const TargetLowering &TLI) { /// between it and the return. /// /// This function only tests target-independent requirements. -bool llvm::isInTailCallPosition(ImmutableCallSite CS, Attributes CalleeRetAttr, - const TargetLowering &TLI) { +bool llvm::isInTailCallPosition(ImmutableCallSite CS,const TargetLowering &TLI){ const Instruction *I = CS.getInstruction(); const BasicBlock *ExitBB = I->getParent(); const TerminatorInst *Term = ExitBB->getTerminator(); @@ -313,14 +310,16 @@ bool llvm::isInTailCallPosition(ImmutableCallSite CS, Attributes CalleeRetAttr, // Conservatively require the attributes of the call to match those of // the return. Ignore noalias because it doesn't affect the call sequence. const Function *F = ExitBB->getParent(); - Attributes CallerRetAttr = F->getAttributes().getRetAttributes(); - if (AttrBuilder(CalleeRetAttr).removeAttribute(Attributes::NoAlias) != - AttrBuilder(CallerRetAttr).removeAttribute(Attributes::NoAlias)) + AttributeSet CallerAttrs = F->getAttributes(); + if (AttrBuilder(CallerAttrs, AttributeSet::ReturnIndex). + removeAttribute(Attribute::NoAlias) != + AttrBuilder(CallerAttrs, AttributeSet::ReturnIndex). + removeAttribute(Attribute::NoAlias)) return false; // It's not safe to eliminate the sign / zero extension of the return value. - if (CallerRetAttr.hasAttribute(Attributes::ZExt) || - CallerRetAttr.hasAttribute(Attributes::SExt)) + if (CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::ZExt) || + CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::SExt)) return false; // Otherwise, make sure the unmodified return value of I is the return value. @@ -348,23 +347,3 @@ bool llvm::isInTailCallPosition(ImmutableCallSite CS, Attributes CalleeRetAttr, return true; } - -bool llvm::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, - SDValue &Chain, const TargetLowering &TLI) { - const Function *F = DAG.getMachineFunction().getFunction(); - - // Conservatively require the attributes of the call to match those of - // the return. Ignore noalias because it doesn't affect the call sequence. - Attributes CallerRetAttr = F->getAttributes().getRetAttributes(); - if (AttrBuilder(CallerRetAttr) - .removeAttribute(Attributes::NoAlias).hasAttributes()) - return false; - - // It's not safe to eliminate the sign / zero extension of the return value. - if (CallerRetAttr.hasAttribute(Attributes::ZExt) || - CallerRetAttr.hasAttribute(Attributes::SExt)) - return false; - - // Check if the only use is a function return node. - return TLI.isUsedByReturnOnly(Node, Chain); -} diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp index b2ebf04..188047d 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/ARMException.cpp @@ -12,32 +12,31 @@ //===----------------------------------------------------------------------===// #include "DwarfException.h" -#include "llvm/Module.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/Twine.h" #include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Dwarf.h" +#include "llvm/Support/FormattedStream.h" #include "llvm/Target/Mangler.h" -#include "llvm/DataLayout.h" #include "llvm/Target/TargetFrameLowering.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Dwarf.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/Twine.h" using namespace llvm; -cl::opt<bool> +static cl::opt<bool> EnableARMEHABIDescriptors("arm-enable-ehabi-descriptors", cl::Hidden, cl::desc("Generate ARM EHABI tables with unwinding descriptors"), cl::init(false)); @@ -69,24 +68,69 @@ void ARMException::EndFunction() { Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("eh_func_end", Asm->getFunctionNumber())); - // Emit references to personality. - if (const Function * Personality = - MMI->getPersonalities()[MMI->getPersonalityIndex()]) { - MCSymbol *PerSym = Asm->Mang->getSymbol(Personality); - Asm->OutStreamer.EmitSymbolAttribute(PerSym, MCSA_Global); - Asm->OutStreamer.EmitPersonality(PerSym); - } - if (EnableARMEHABIDescriptors) { // Map all labels and get rid of any dead landing pads. MMI->TidyLandingPads(); - Asm->OutStreamer.EmitHandlerData(); + if (!MMI->getLandingPads().empty()) { + // Emit references to personality. + if (const Function * Personality = + MMI->getPersonalities()[MMI->getPersonalityIndex()]) { + MCSymbol *PerSym = Asm->Mang->getSymbol(Personality); + Asm->OutStreamer.EmitSymbolAttribute(PerSym, MCSA_Global); + Asm->OutStreamer.EmitPersonality(PerSym); + } + + // Emit .handlerdata directive. + Asm->OutStreamer.EmitHandlerData(); - // Emit actual exception table - EmitExceptionTable(); + // Emit actual exception table + EmitExceptionTable(); + } } } Asm->OutStreamer.EmitFnEnd(); } + +void ARMException::EmitTypeInfos(unsigned TTypeEncoding) { + const std::vector<const GlobalVariable *> &TypeInfos = MMI->getTypeInfos(); + const std::vector<unsigned> &FilterIds = MMI->getFilterIds(); + + bool VerboseAsm = Asm->OutStreamer.isVerboseAsm(); + + int Entry = 0; + // Emit the Catch TypeInfos. + if (VerboseAsm && !TypeInfos.empty()) { + Asm->OutStreamer.AddComment(">> Catch TypeInfos <<"); + Asm->OutStreamer.AddBlankLine(); + Entry = TypeInfos.size(); + } + + for (std::vector<const GlobalVariable *>::const_reverse_iterator + I = TypeInfos.rbegin(), E = TypeInfos.rend(); I != E; ++I) { + const GlobalVariable *GV = *I; + if (VerboseAsm) + Asm->OutStreamer.AddComment("TypeInfo " + Twine(Entry--)); + Asm->EmitTTypeReference(GV, TTypeEncoding); + } + + // Emit the Exception Specifications. + if (VerboseAsm && !FilterIds.empty()) { + Asm->OutStreamer.AddComment(">> Filter TypeInfos <<"); + Asm->OutStreamer.AddBlankLine(); + Entry = 0; + } + for (std::vector<unsigned>::const_iterator + I = FilterIds.begin(), E = FilterIds.end(); I < E; ++I) { + unsigned TypeID = *I; + if (VerboseAsm) { + --Entry; + if (TypeID != 0) + Asm->OutStreamer.AddComment("FilterInfo " + Twine(Entry)); + } + + Asm->EmitTTypeReference((TypeID == 0 ? 0 : TypeInfos[TypeID - 1]), + TTypeEncoding); + } +} diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index d74a703..d4a745d 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -15,8 +15,10 @@ #include "llvm/CodeGen/AsmPrinter.h" #include "DwarfDebug.h" #include "DwarfException.h" -#include "llvm/DebugInfo.h" -#include "llvm/Module.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/ConstantFolding.h" +#include "llvm/Assembly/Writer.h" #include "llvm/CodeGen/GCMetadataPrinter.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -24,7 +26,10 @@ #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" -#include "llvm/Analysis/ConstantFolding.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Operator.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" @@ -32,20 +37,16 @@ #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/Timer.h" #include "llvm/Target/Mangler.h" -#include "llvm/DataLayout.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Assembly/Writer.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Format.h" -#include "llvm/Support/MathExtras.h" -#include "llvm/Support/Timer.h" using namespace llvm; static const char *DWARFGroupName = "DWARF Emission"; @@ -90,9 +91,6 @@ static unsigned getGVAlignmentLog2(const GlobalValue *GV, const DataLayout &TD, return NumBits; } - - - AsmPrinter::AsmPrinter(TargetMachine &tm, MCStreamer &Streamer) : MachineFunctionPass(ID), TM(tm), MAI(tm.getMCAsmInfo()), @@ -130,7 +128,6 @@ const TargetLoweringObjectFile &AsmPrinter::getObjFileLowering() const { return TM.getTargetLowering()->getObjFileLowering(); } - /// getDataLayout - Return information about data layout. const DataLayout &AsmPrinter::getDataLayout() const { return *TM.getDataLayout(); @@ -153,6 +150,8 @@ void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const { } bool AsmPrinter::doInitialization(Module &M) { + OutStreamer.InitStreamer(); + MMI = getAnalysisIfAvailable<MachineModuleInfo>(); MMI->AnalyzeModule(M); @@ -312,8 +311,13 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { return; } - if (Align == 1 || - MAI->getLCOMMDirectiveAlignmentType() != LCOMM::NoAlignment) { + // Use .lcomm only if it supports user-specified alignment. + // Otherwise, while it would still be correct to use .lcomm in some + // cases (e.g. when Align == 1), the external assembler might enfore + // some -unknown- default alignment behavior, which could cause + // spurious differences between external and integrated assembler. + // Prefer to simply fall back to .local / .comm in this case. + if (MAI->getLCOMMDirectiveAlignmentType() != LCOMM::NoAlignment) { // .lcomm _foo, 42 OutStreamer.EmitLocalCommonSymbol(GVSym, Size, Align); return; @@ -387,9 +391,9 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { // - pointer to mangled symbol above with initializer unsigned PtrSize = TD->getPointerSizeInBits()/8; OutStreamer.EmitSymbolValue(GetExternalSymbolSymbol("_tlv_bootstrap"), - PtrSize, 0); - OutStreamer.EmitIntValue(0, PtrSize, 0); - OutStreamer.EmitSymbolValue(MangSym, PtrSize, 0); + PtrSize); + OutStreamer.EmitIntValue(0, PtrSize); + OutStreamer.EmitSymbolValue(MangSym, PtrSize); OutStreamer.AddBlankLine(); return; @@ -943,6 +947,8 @@ bool AsmPrinter::doFinalization(Module &M) { MMI = 0; OutStreamer.Finish(); + OutStreamer.reset(); + return false; } @@ -1034,7 +1040,7 @@ void AsmPrinter::EmitConstantPool() { // Emit inter-object padding for alignment. unsigned AlignMask = CPE.getAlignment() - 1; unsigned NewOffset = (Offset + AlignMask) & ~AlignMask; - OutStreamer.EmitFill(NewOffset - Offset, 0/*fillval*/, 0/*addrspace*/); + OutStreamer.EmitZeros(NewOffset - Offset); Type *Ty = CPE.getType(); Offset = NewOffset + TM.getDataLayout()->getTypeAllocSize(Ty); @@ -1197,7 +1203,7 @@ void AsmPrinter::EmitJumpTableEntry(const MachineJumpTableInfo *MJTI, assert(Value && "Unknown entry kind!"); unsigned EntrySize = MJTI->getEntrySize(*TM.getDataLayout()); - OutStreamer.EmitValue(Value, EntrySize, /*addrspace*/0); + OutStreamer.EmitValue(Value, EntrySize); } @@ -1320,19 +1326,19 @@ void AsmPrinter::EmitXXStructorList(const Constant *List, bool isCtor) { /// EmitInt8 - Emit a byte directive and value. /// void AsmPrinter::EmitInt8(int Value) const { - OutStreamer.EmitIntValue(Value, 1, 0/*addrspace*/); + OutStreamer.EmitIntValue(Value, 1); } /// EmitInt16 - Emit a short directive and value. /// void AsmPrinter::EmitInt16(int Value) const { - OutStreamer.EmitIntValue(Value, 2, 0/*addrspace*/); + OutStreamer.EmitIntValue(Value, 2); } /// EmitInt32 - Emit a long directive and value. /// void AsmPrinter::EmitInt32(int Value) const { - OutStreamer.EmitIntValue(Value, 4, 0/*addrspace*/); + OutStreamer.EmitIntValue(Value, 4); } /// EmitLabelDifference - Emit something like ".long Hi-Lo" where the size @@ -1347,14 +1353,14 @@ void AsmPrinter::EmitLabelDifference(const MCSymbol *Hi, const MCSymbol *Lo, OutContext); if (!MAI->hasSetDirective()) { - OutStreamer.EmitValue(Diff, Size, 0/*AddrSpace*/); + OutStreamer.EmitValue(Diff, Size); return; } // Otherwise, emit with .set (aka assignment). MCSymbol *SetLabel = GetTempSymbol("set", SetCounter++); OutStreamer.EmitAssignment(SetLabel, Diff); - OutStreamer.EmitSymbolValue(SetLabel, Size, 0/*AddrSpace*/); + OutStreamer.EmitSymbolValue(SetLabel, Size); } /// EmitLabelOffsetDifference - Emit something like ".long Hi+Offset-Lo" @@ -1378,12 +1384,12 @@ void AsmPrinter::EmitLabelOffsetDifference(const MCSymbol *Hi, uint64_t Offset, OutContext); if (!MAI->hasSetDirective()) - OutStreamer.EmitValue(Diff, 4, 0/*AddrSpace*/); + OutStreamer.EmitValue(Diff, 4); else { // Otherwise, emit with .set (aka assignment). MCSymbol *SetLabel = GetTempSymbol("set", SetCounter++); OutStreamer.EmitAssignment(SetLabel, Diff); - OutStreamer.EmitSymbolValue(SetLabel, 4, 0/*AddrSpace*/); + OutStreamer.EmitSymbolValue(SetLabel, 4); } } @@ -1401,7 +1407,7 @@ void AsmPrinter::EmitLabelPlusOffset(const MCSymbol *Label, uint64_t Offset, MCConstantExpr::Create(Offset, OutContext), OutContext); - OutStreamer.EmitValue(Expr, Size, 0/*AddrSpace*/); + OutStreamer.EmitValue(Expr, Size); } @@ -1472,19 +1478,14 @@ static const MCExpr *lowerConstant(const Constant *CV, AsmPrinter &AP) { case Instruction::GetElementPtr: { const DataLayout &TD = *AP.TM.getDataLayout(); // Generate a symbolic expression for the byte address - const Constant *PtrVal = CE->getOperand(0); - SmallVector<Value*, 8> IdxVec(CE->op_begin()+1, CE->op_end()); - int64_t Offset = TD.getIndexedOffset(PtrVal->getType(), IdxVec); + APInt OffsetAI(TD.getPointerSizeInBits(), 0); + cast<GEPOperator>(CE)->accumulateConstantOffset(TD, OffsetAI); const MCExpr *Base = lowerConstant(CE->getOperand(0), AP); - if (Offset == 0) + if (!OffsetAI) return Base; - // Truncate/sext the offset to the pointer size. - unsigned Width = TD.getPointerSizeInBits(); - if (Width < 64) - Offset = SignExtend64(Offset, Width); - + int64_t Offset = OffsetAI.getSExtValue(); return MCBinaryExpr::CreateAdd(Base, MCConstantExpr::Create(Offset, Ctx), Ctx); } @@ -1614,7 +1615,7 @@ static int isRepeatedByteSequence(const Value *V, TargetMachine &TM) { } return Byte; } - + if (const ConstantDataSequential *CDS = dyn_cast<ConstantDataSequential>(V)) return isRepeatedByteSequence(CDS); @@ -1623,7 +1624,7 @@ static int isRepeatedByteSequence(const Value *V, TargetMachine &TM) { static void emitGlobalConstantDataSequential(const ConstantDataSequential *CDS, unsigned AddrSpace,AsmPrinter &AP){ - + // See if we can aggregate this into a .fill, if so, emit it as such. int Value = isRepeatedByteSequence(CDS, AP.TM); if (Value != -1) { @@ -1632,7 +1633,7 @@ static void emitGlobalConstantDataSequential(const ConstantDataSequential *CDS, if (Bytes > 1) return AP.OutStreamer.EmitFill(Bytes, Value, AddrSpace); } - + // If this can be emitted with .ascii/.asciz, emit it as such. if (CDS->isString()) return AP.OutStreamer.EmitBytes(CDS->getAsString(), AddrSpace); @@ -1656,7 +1657,7 @@ static void emitGlobalConstantDataSequential(const ConstantDataSequential *CDS, float F; uint32_t I; }; - + F = CDS->getElementAsFloat(i); if (AP.isVerbose()) AP.OutStreamer.GetCommentOS() << "float " << F << '\n'; @@ -1669,7 +1670,7 @@ static void emitGlobalConstantDataSequential(const ConstantDataSequential *CDS, double F; uint64_t I; }; - + F = CDS->getElementAsDouble(i); if (AP.isVerbose()) AP.OutStreamer.GetCommentOS() << "double " << F << '\n'; @@ -1745,87 +1746,48 @@ static void emitGlobalConstantStruct(const ConstantStruct *CS, static void emitGlobalConstantFP(const ConstantFP *CFP, unsigned AddrSpace, AsmPrinter &AP) { - if (CFP->getType()->isHalfTy()) { - if (AP.isVerbose()) { - SmallString<10> Str; - CFP->getValueAPF().toString(Str); - AP.OutStreamer.GetCommentOS() << "half " << Str << '\n'; - } - uint64_t Val = CFP->getValueAPF().bitcastToAPInt().getZExtValue(); - AP.OutStreamer.EmitIntValue(Val, 2, AddrSpace); - return; - } - - if (CFP->getType()->isFloatTy()) { - if (AP.isVerbose()) { - float Val = CFP->getValueAPF().convertToFloat(); - uint64_t IntVal = CFP->getValueAPF().bitcastToAPInt().getZExtValue(); - AP.OutStreamer.GetCommentOS() << "float " << Val << '\n' - << " (" << format("0x%x", IntVal) << ")\n"; - } - uint64_t Val = CFP->getValueAPF().bitcastToAPInt().getZExtValue(); - AP.OutStreamer.EmitIntValue(Val, 4, AddrSpace); - return; - } + APInt API = CFP->getValueAPF().bitcastToAPInt(); - // FP Constants are printed as integer constants to avoid losing - // precision. - if (CFP->getType()->isDoubleTy()) { - if (AP.isVerbose()) { - double Val = CFP->getValueAPF().convertToDouble(); - uint64_t IntVal = CFP->getValueAPF().bitcastToAPInt().getZExtValue(); - AP.OutStreamer.GetCommentOS() << "double " << Val << '\n' - << " (" << format("0x%lx", IntVal) << ")\n"; - } + // First print a comment with what we think the original floating-point value + // should have been. + if (AP.isVerbose()) { + SmallString<8> StrVal; + CFP->getValueAPF().toString(StrVal); - uint64_t Val = CFP->getValueAPF().bitcastToAPInt().getZExtValue(); - AP.OutStreamer.EmitIntValue(Val, 8, AddrSpace); - return; + CFP->getType()->print(AP.OutStreamer.GetCommentOS()); + AP.OutStreamer.GetCommentOS() << ' ' << StrVal << '\n'; } - if (CFP->getType()->isX86_FP80Ty()) { - // all long double variants are printed as hex - // API needed to prevent premature destruction - APInt API = CFP->getValueAPF().bitcastToAPInt(); - const uint64_t *p = API.getRawData(); - if (AP.isVerbose()) { - // Convert to double so we can print the approximate val as a comment. - APFloat DoubleVal = CFP->getValueAPF(); - bool ignored; - DoubleVal.convert(APFloat::IEEEdouble, APFloat::rmNearestTiesToEven, - &ignored); - AP.OutStreamer.GetCommentOS() << "x86_fp80 ~= " - << DoubleVal.convertToDouble() << '\n'; - } + // Now iterate through the APInt chunks, emitting them in endian-correct + // order, possibly with a smaller chunk at beginning/end (e.g. for x87 80-bit + // floats). + unsigned NumBytes = API.getBitWidth() / 8; + unsigned TrailingBytes = NumBytes % sizeof(uint64_t); + const uint64_t *p = API.getRawData(); - if (AP.TM.getDataLayout()->isBigEndian()) { - AP.OutStreamer.EmitIntValue(p[1], 2, AddrSpace); - AP.OutStreamer.EmitIntValue(p[0], 8, AddrSpace); - } else { - AP.OutStreamer.EmitIntValue(p[0], 8, AddrSpace); - AP.OutStreamer.EmitIntValue(p[1], 2, AddrSpace); - } + // PPC's long double has odd notions of endianness compared to how LLVM + // handles it: p[0] goes first for *big* endian on PPC. + if (AP.TM.getDataLayout()->isBigEndian() != CFP->getType()->isPPC_FP128Ty()) { + int Chunk = API.getNumWords() - 1; - // Emit the tail padding for the long double. - const DataLayout &TD = *AP.TM.getDataLayout(); - AP.OutStreamer.EmitZeros(TD.getTypeAllocSize(CFP->getType()) - - TD.getTypeStoreSize(CFP->getType()), AddrSpace); - return; - } + if (TrailingBytes) + AP.OutStreamer.EmitIntValue(p[Chunk--], TrailingBytes, AddrSpace); - assert(CFP->getType()->isPPC_FP128Ty() && - "Floating point constant type not handled"); - // All long double variants are printed as hex - // API needed to prevent premature destruction. - APInt API = CFP->getValueAPF().bitcastToAPInt(); - const uint64_t *p = API.getRawData(); - if (AP.TM.getDataLayout()->isBigEndian()) { - AP.OutStreamer.EmitIntValue(p[0], 8, AddrSpace); - AP.OutStreamer.EmitIntValue(p[1], 8, AddrSpace); + for (; Chunk >= 0; --Chunk) + AP.OutStreamer.EmitIntValue(p[Chunk], sizeof(uint64_t), AddrSpace); } else { - AP.OutStreamer.EmitIntValue(p[1], 8, AddrSpace); - AP.OutStreamer.EmitIntValue(p[0], 8, AddrSpace); + unsigned Chunk; + for (Chunk = 0; Chunk < NumBytes / sizeof(uint64_t); ++Chunk) + AP.OutStreamer.EmitIntValue(p[Chunk], sizeof(uint64_t), AddrSpace); + + if (TrailingBytes) + AP.OutStreamer.EmitIntValue(p[Chunk], TrailingBytes, AddrSpace); } + + // Emit the tail padding for the long double. + const DataLayout &TD = *AP.TM.getDataLayout(); + AP.OutStreamer.EmitZeros(TD.getTypeAllocSize(CFP->getType()) - + TD.getTypeStoreSize(CFP->getType()), AddrSpace); } static void emitGlobalConstantLargeInt(const ConstantInt *CI, @@ -1878,7 +1840,7 @@ static void emitGlobalConstantImpl(const Constant *CV, unsigned AddrSpace, if (const ConstantDataSequential *CDS = dyn_cast<ConstantDataSequential>(CV)) return emitGlobalConstantDataSequential(CDS, AddrSpace, AP); - + if (const ConstantArray *CVA = dyn_cast<ConstantArray>(CV)) return emitGlobalConstantArray(CVA, AddrSpace, AP); @@ -1900,10 +1862,10 @@ static void emitGlobalConstantImpl(const Constant *CV, unsigned AddrSpace, return emitGlobalConstantImpl(New, AddrSpace, AP); } } - + if (const ConstantVector *V = dyn_cast<ConstantVector>(CV)) return emitGlobalConstantVector(V, AddrSpace, AP); - + // Otherwise, it must be a ConstantExpr. Lower it to an MCExpr, then emit it // thread the streamer with EmitValue. AP.OutStreamer.EmitValue(lowerConstant(CV, AP), Size, AddrSpace); diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp index d94e1fe..156acac 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp @@ -13,19 +13,19 @@ #define DEBUG_TYPE "asm-printer" #include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/MC/MachineLocation.h" +#include "llvm/ADT/Twine.h" +#include "llvm/IR/DataLayout.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" -#include "llvm/DataLayout.h" +#include "llvm/MC/MachineLocation.h" +#include "llvm/Support/Dwarf.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Support/Dwarf.h" -#include "llvm/Support/ErrorHandling.h" using namespace llvm; //===----------------------------------------------------------------------===// @@ -46,19 +46,19 @@ void AsmPrinter::EmitULEB128(unsigned Value, const char *Desc, if (isVerbose() && Desc) OutStreamer.AddComment(Desc); - OutStreamer.EmitULEB128IntValue(Value, 0/*addrspace*/, PadTo); + OutStreamer.EmitULEB128IntValue(Value, PadTo); } /// EmitCFAByte - Emit a .byte 42 directive for a DW_CFA_xxx value. void AsmPrinter::EmitCFAByte(unsigned Val) const { if (isVerbose()) { if (Val >= dwarf::DW_CFA_offset && Val < dwarf::DW_CFA_offset+64) - OutStreamer.AddComment("DW_CFA_offset + Reg (" + + OutStreamer.AddComment("DW_CFA_offset + Reg (" + Twine(Val-dwarf::DW_CFA_offset) + ")"); else OutStreamer.AddComment(dwarf::CallFrameString(Val)); } - OutStreamer.EmitIntValue(Val, 1, 0/*addrspace*/); + OutStreamer.EmitIntValue(Val, 1); } static const char *DecodeDWARFEncoding(unsigned Encoding) { @@ -83,7 +83,7 @@ static const char *DecodeDWARFEncoding(unsigned Encoding) { case dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel |dwarf::DW_EH_PE_sdata8: return "indirect pcrel sdata8"; } - + return "<unknown encoding>"; } @@ -101,15 +101,15 @@ void AsmPrinter::EmitEncodingByte(unsigned Val, const char *Desc) const { OutStreamer.AddComment(Twine("Encoding = ") + DecodeDWARFEncoding(Val)); } - - OutStreamer.EmitIntValue(Val, 1, 0/*addrspace*/); + + OutStreamer.EmitIntValue(Val, 1); } /// GetSizeOfEncodedValue - Return the size of the encoding in bytes. unsigned AsmPrinter::GetSizeOfEncodedValue(unsigned Encoding) const { if (Encoding == dwarf::DW_EH_PE_omit) return 0; - + switch (Encoding & 0x07) { default: llvm_unreachable("Invalid encoded value."); case dwarf::DW_EH_PE_absptr: return TM.getDataLayout()->getPointerSize(); @@ -119,20 +119,16 @@ unsigned AsmPrinter::GetSizeOfEncodedValue(unsigned Encoding) const { } } -void AsmPrinter::EmitReference(const MCSymbol *Sym, unsigned Encoding) const { - const TargetLoweringObjectFile &TLOF = getObjFileLowering(); - - const MCExpr *Exp = - TLOF.getExprForDwarfReference(Sym, Encoding, OutStreamer); - OutStreamer.EmitAbsValue(Exp, GetSizeOfEncodedValue(Encoding)); -} +void AsmPrinter::EmitTTypeReference(const GlobalValue *GV, + unsigned Encoding) const { + if (GV) { + const TargetLoweringObjectFile &TLOF = getObjFileLowering(); -void AsmPrinter::EmitReference(const GlobalValue *GV, unsigned Encoding)const{ - const TargetLoweringObjectFile &TLOF = getObjFileLowering(); - - const MCExpr *Exp = - TLOF.getExprForDwarfGlobalReference(GV, Mang, MMI, Encoding, OutStreamer); - OutStreamer.EmitValue(Exp, GetSizeOfEncodedValue(Encoding), /*addrspace*/0); + const MCExpr *Exp = + TLOF.getTTypeGlobalReference(GV, Mang, MMI, Encoding, OutStreamer); + OutStreamer.EmitValue(Exp, GetSizeOfEncodedValue(Encoding)); + } else + OutStreamer.EmitIntValue(0, GetSizeOfEncodedValue(Encoding)); } /// EmitSectionOffset - Emit the 4-byte offset of Label from the start of its @@ -149,22 +145,22 @@ void AsmPrinter::EmitSectionOffset(const MCSymbol *Label, OutStreamer.EmitCOFFSecRel32(Label); return; } - + // Get the section that we're referring to, based on SectionLabel. const MCSection &Section = SectionLabel->getSection(); - + // If Label has already been emitted, verify that it is in the same section as // section label for sanity. assert((!Label->isInSection() || &Label->getSection() == &Section) && "Section offset using wrong section base for label"); - + // If the section in question will end up with an address of 0 anyway, we can // just emit an absolute reference to save a relocation. if (Section.isBaseAddressKnownZero()) { - OutStreamer.EmitSymbolValue(Label, 4, 0/*AddrSpace*/); + OutStreamer.EmitSymbolValue(Label, 4); return; } - + // Otherwise, emit it as a label difference from the start of the section. EmitLabelDifference(Label, SectionLabel, 4); } diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp index 50f0fc3..abfa330 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp @@ -13,26 +13,26 @@ #define DEBUG_TYPE "asm-printer" #include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/Constants.h" -#include "llvm/InlineAsm.h" -#include "llvm/LLVMContext.h" -#include "llvm/Module.h" +#include "llvm/ADT/OwningPtr.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/Twine.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCTargetAsmParser.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/ADT/OwningPtr.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/Twine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; namespace { diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.cpp index 4d73b3c..57e0acd 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.cpp @@ -8,16 +8,16 @@ //===----------------------------------------------------------------------===// // // Data structures for DWARF info entries. -// +// //===----------------------------------------------------------------------===// #include "DIE.h" #include "llvm/ADT/Twine.h" #include "llvm/CodeGen/AsmPrinter.h" +#include "llvm/IR/DataLayout.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" -#include "llvm/DataLayout.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -112,6 +112,17 @@ DIE::~DIE() { delete Children[i]; } +/// Climb up the parent chain to get the compile unit DIE this DIE belongs to. +DIE *DIE::getCompileUnit() const{ + DIE *p = getParent(); + while (p) { + if (p->getTag() == dwarf::DW_TAG_compile_unit) + return p; + p = p->getParent(); + } + llvm_unreachable("We should not have orphaned DIEs."); +} + #ifndef NDEBUG void DIE::print(raw_ostream &O, unsigned IncIndent) { IndentCount += IncIndent; @@ -133,7 +144,7 @@ void DIE::print(raw_ostream &O, unsigned IncIndent) { O << "Size: " << Size << "\n"; } - const SmallVector<DIEAbbrevData, 8> &Data = Abbrev.getData(); + const SmallVectorImpl<DIEAbbrevData> &Data = Abbrev.getData(); IndentCount += 2; for (unsigned i = 0, N = Data.size(); i < N; ++i) { @@ -193,17 +204,20 @@ void DIEInteger::EmitValue(AsmPrinter *Asm, unsigned Form) const { case dwarf::DW_FORM_data1: Size = 1; break; case dwarf::DW_FORM_ref2: // Fall thru case dwarf::DW_FORM_data2: Size = 2; break; + case dwarf::DW_FORM_sec_offset: // Fall thru case dwarf::DW_FORM_ref4: // Fall thru case dwarf::DW_FORM_data4: Size = 4; break; case dwarf::DW_FORM_ref8: // Fall thru case dwarf::DW_FORM_data8: Size = 8; break; + case dwarf::DW_FORM_GNU_str_index: Asm->EmitULEB128(Integer); return; + case dwarf::DW_FORM_GNU_addr_index: Asm->EmitULEB128(Integer); return; case dwarf::DW_FORM_udata: Asm->EmitULEB128(Integer); return; case dwarf::DW_FORM_sdata: Asm->EmitSLEB128(Integer); return; case dwarf::DW_FORM_addr: Size = Asm->getDataLayout().getPointerSize(); break; default: llvm_unreachable("DIE Value form not supported yet"); } - Asm->OutStreamer.EmitIntValue(Integer, Size, 0/*addrspace*/); + Asm->OutStreamer.EmitIntValue(Integer, Size); } /// SizeOf - Determine size of integer value in bytes. @@ -216,10 +230,13 @@ unsigned DIEInteger::SizeOf(AsmPrinter *AP, unsigned Form) const { case dwarf::DW_FORM_data1: return sizeof(int8_t); case dwarf::DW_FORM_ref2: // Fall thru case dwarf::DW_FORM_data2: return sizeof(int16_t); + case dwarf::DW_FORM_sec_offset: // Fall thru case dwarf::DW_FORM_ref4: // Fall thru case dwarf::DW_FORM_data4: return sizeof(int32_t); case dwarf::DW_FORM_ref8: // Fall thru case dwarf::DW_FORM_data8: return sizeof(int64_t); + case dwarf::DW_FORM_GNU_str_index: return MCAsmInfo::getULEB128Size(Integer); + case dwarf::DW_FORM_GNU_addr_index: return MCAsmInfo::getULEB128Size(Integer); case dwarf::DW_FORM_udata: return MCAsmInfo::getULEB128Size(Integer); case dwarf::DW_FORM_sdata: return MCAsmInfo::getSLEB128Size(Integer); case dwarf::DW_FORM_addr: return AP->getDataLayout().getPointerSize(); @@ -241,13 +258,14 @@ void DIEInteger::print(raw_ostream &O) { /// EmitValue - Emit label value. /// void DIELabel::EmitValue(AsmPrinter *AP, unsigned Form) const { - AP->OutStreamer.EmitSymbolValue(Label, SizeOf(AP, Form), 0/*AddrSpace*/); + AP->OutStreamer.EmitSymbolValue(Label, SizeOf(AP, Form)); } /// SizeOf - Determine size of label value in bytes. /// unsigned DIELabel::SizeOf(AsmPrinter *AP, unsigned Form) const { if (Form == dwarf::DW_FORM_data4) return 4; + if (Form == dwarf::DW_FORM_sec_offset) return 4; if (Form == dwarf::DW_FORM_strp) return 4; return AP->getDataLayout().getPointerSize(); } @@ -306,7 +324,7 @@ void DIEEntry::print(raw_ostream &O) { /// unsigned DIEBlock::ComputeSize(AsmPrinter *AP) { if (!Size) { - const SmallVector<DIEAbbrevData, 8> &AbbrevData = Abbrev.getData(); + const SmallVectorImpl<DIEAbbrevData> &AbbrevData = Abbrev.getData(); for (unsigned i = 0, N = Values.size(); i < N; ++i) Size += Values[i]->SizeOf(AP, AbbrevData[i].getForm()); } @@ -325,7 +343,7 @@ void DIEBlock::EmitValue(AsmPrinter *Asm, unsigned Form) const { case dwarf::DW_FORM_block: Asm->EmitULEB128(Size); break; } - const SmallVector<DIEAbbrevData, 8> &AbbrevData = Abbrev.getData(); + const SmallVectorImpl<DIEAbbrevData> &AbbrevData = Abbrev.getData(); for (unsigned i = 0, N = Values.size(); i < N; ++i) Values[i]->EmitValue(Asm, AbbrevData[i].getForm()); } diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.h b/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.h index 28a96f3..c332aa2 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.h +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DIE.h @@ -8,7 +8,7 @@ //===----------------------------------------------------------------------===// // // Data structures for DWARF info entries. -// +// //===----------------------------------------------------------------------===// #ifndef CODEGEN_ASMPRINTER_DIE_H__ @@ -66,7 +66,7 @@ namespace llvm { /// Data - Raw data bytes for abbreviation. /// - SmallVector<DIEAbbrevData, 8> Data; + SmallVector<DIEAbbrevData, 12> Data; public: DIEAbbrev(uint16_t T, uint16_t C) : Tag(T), ChildrenFlag(C), Data() {} @@ -75,7 +75,7 @@ namespace llvm { uint16_t getTag() const { return Tag; } unsigned getNumber() const { return Number; } uint16_t getChildrenFlag() const { return ChildrenFlag; } - const SmallVector<DIEAbbrevData, 8> &getData() const { return Data; } + const SmallVectorImpl<DIEAbbrevData> &getData() const { return Data; } void setTag(uint16_t T) { Tag = T; } void setChildrenFlag(uint16_t CF) { ChildrenFlag = CF; } void setNumber(unsigned N) { Number = N; } @@ -108,7 +108,7 @@ namespace llvm { //===--------------------------------------------------------------------===// /// DIE - A structured debug information entry. Has an abbreviation which - /// describes it's organization. + /// describes its organization. class DIEValue; class DIE { @@ -131,9 +131,9 @@ namespace llvm { DIE *Parent; - /// Attributes values. + /// Attribute values. /// - SmallVector<DIEValue*, 32> Values; + SmallVector<DIEValue*, 12> Values; // Private data for print() mutable unsigned IndentCount; @@ -150,12 +150,15 @@ namespace llvm { unsigned getOffset() const { return Offset; } unsigned getSize() const { return Size; } const std::vector<DIE *> &getChildren() const { return Children; } - const SmallVector<DIEValue*, 32> &getValues() const { return Values; } + const SmallVectorImpl<DIEValue*> &getValues() const { return Values; } DIE *getParent() const { return Parent; } + /// Climb up the parent chain to get the compile unit DIE this DIE belongs + /// to. + DIE *getCompileUnit() const; void setTag(unsigned Tag) { Abbrev.setTag(Tag); } void setOffset(unsigned O) { Offset = O; } void setSize(unsigned S) { Size = S; } - + /// addValue - Add a value and attributes to a DIE. /// void addValue(unsigned Attribute, unsigned Form, DIEValue *Value) { @@ -232,9 +235,10 @@ namespace llvm { /// static unsigned BestForm(bool IsSigned, uint64_t Int) { if (IsSigned) { - if ((char)Int == (signed)Int) return dwarf::DW_FORM_data1; - if ((short)Int == (signed)Int) return dwarf::DW_FORM_data2; - if ((int)Int == (signed)Int) return dwarf::DW_FORM_data4; + const int64_t SignedInt = Int; + if ((char)Int == SignedInt) return dwarf::DW_FORM_data1; + if ((short)Int == SignedInt) return dwarf::DW_FORM_data2; + if ((int)Int == SignedInt) return dwarf::DW_FORM_data4; } else { if ((unsigned char)Int == Int) return dwarf::DW_FORM_data1; if ((unsigned short)Int == Int) return dwarf::DW_FORM_data2; diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp index 05e0f2f..f58ec9b 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp @@ -12,10 +12,10 @@ //===----------------------------------------------------------------------===// #include "DwarfAccelTable.h" -#include "DwarfDebug.h" #include "DIE.h" -#include "llvm/ADT/Twine.h" +#include "DwarfDebug.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Twine.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCStreamer.h" @@ -32,7 +32,7 @@ const char *DwarfAccelTable::Atom::AtomTypeString(enum AtomType AT) { case eAtomTypeTag: return "eAtomTypeTag"; case eAtomTypeNameFlags: return "eAtomTypeNameFlags"; case eAtomTypeTypeFlags: return "eAtomTypeTypeFlags"; - } + } llvm_unreachable("invalid AtomType!"); } @@ -155,7 +155,7 @@ void DwarfAccelTable::EmitHashes(AsmPrinter *Asm) { HE = Buckets[i].end(); HI != HE; ++HI) { Asm->OutStreamer.AddComment("Hash in Bucket " + Twine(i)); Asm->EmitInt32((*HI)->HashValue); - } + } } } @@ -173,7 +173,7 @@ void DwarfAccelTable::EmitOffsets(AsmPrinter *Asm, MCSymbol *SecBegin) { MCBinaryExpr::CreateSub(MCSymbolRefExpr::Create((*HI)->Sym, Context), MCSymbolRefExpr::Create(SecBegin, Context), Context); - Asm->OutStreamer.EmitValue(Sub, sizeof(uint32_t), 0); + Asm->OutStreamer.EmitValue(Sub, sizeof(uint32_t)); } } } @@ -181,7 +181,7 @@ void DwarfAccelTable::EmitOffsets(AsmPrinter *Asm, MCSymbol *SecBegin) { // Walk through the buckets and emit the full data for each element in // the bucket. For the string case emit the dies and the various offsets. // Terminate each HashData bucket with 0. -void DwarfAccelTable::EmitData(AsmPrinter *Asm, DwarfDebug *D) { +void DwarfAccelTable::EmitData(AsmPrinter *Asm, DwarfUnits *D) { uint64_t PrevHash = UINT64_MAX; for (size_t i = 0, e = Buckets.size(); i < e; ++i) { for (HashList::const_iterator HI = Buckets[i].begin(), @@ -190,7 +190,7 @@ void DwarfAccelTable::EmitData(AsmPrinter *Asm, DwarfDebug *D) { Asm->OutStreamer.EmitLabel((*HI)->Sym); Asm->OutStreamer.AddComment((*HI)->Str); Asm->EmitSectionOffset(D->getStringPoolEntry((*HI)->Str), - D->getStringPool()); + D->getStringPoolSym()); Asm->OutStreamer.AddComment("Num DIEs"); Asm->EmitInt32((*HI)->Data.size()); for (ArrayRef<HashDataContents*>::const_iterator @@ -215,7 +215,7 @@ void DwarfAccelTable::EmitData(AsmPrinter *Asm, DwarfDebug *D) { // Emit the entire data structure to the output file. void DwarfAccelTable::Emit(AsmPrinter *Asm, MCSymbol *SecBegin, - DwarfDebug *D) { + DwarfUnits *D) { // Emit the header. EmitHeader(Asm); @@ -258,7 +258,7 @@ void DwarfAccelTable::print(raw_ostream &O) { for (std::vector<HashData*>::const_iterator DI = Data.begin(), DE = Data.end(); DI != DE; ++DI) (*DI)->print(O); - + } #endif diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfAccelTable.h b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfAccelTable.h index 92d1bbe..9915bca 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfAccelTable.h +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfAccelTable.h @@ -14,18 +14,18 @@ #ifndef CODEGEN_ASMPRINTER_DWARFACCELTABLE_H__ #define CODEGEN_ASMPRINTER_DWARFACCELTABLE_H__ -#include "llvm/ADT/StringMap.h" +#include "DIE.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringMap.h" #include "llvm/MC/MCSymbol.h" -#include "llvm/Support/Dwarf.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Dwarf.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" #include "llvm/Support/FormattedStream.h" -#include "DIE.h" -#include <vector> #include <map> +#include <vector> // The dwarf accelerator tables are an indirect hash table optimized // for null lookup rather than access to known data. They are output into @@ -51,7 +51,7 @@ // section contains all of the 32-bit hash values in contiguous memory, and // the offsets contain the offset into the data area for the particular // hash. -// +// // For a lookup example, we could hash a function name and take it modulo the // number of buckets giving us our bucket. From there we take the bucket value // as an index into the hashes table and look at each successive hash as long @@ -63,8 +63,8 @@ namespace llvm { class AsmPrinter; class DIE; -class DwarfDebug; - +class DwarfUnits; + class DwarfAccelTable { enum HashFunctionType { @@ -81,7 +81,7 @@ class DwarfAccelTable { // Helper function to compute the number of buckets needed based on // the number of unique hashes. void ComputeBucketCount (void); - + struct TableHeader { uint32_t magic; // 'HASH' magic value to allow endian detection uint16_t version; // Version number. @@ -94,7 +94,7 @@ class DwarfAccelTable { // Also written to disk is the implementation specific header data. static const uint32_t MagicHash = 0x48415348; - + TableHeader (uint32_t data_len) : magic (MagicHash), version (1), hash_function (eHashFunctionDJB), bucket_count (0), hashes_count (0), header_data_len (data_len) @@ -123,7 +123,7 @@ public: // // uint32_t die_offset_base // uint32_t atom_count - // atom_count Atoms + // atom_count Atoms enum AtomType { eAtomTypeNULL = 0u, eAtomTypeDIEOffset = 1u, // DIE offset, check form for encoding @@ -138,12 +138,12 @@ public: enum TypeFlags { eTypeFlagClassMask = 0x0000000fu, - + // Always set for C++, only set for ObjC if this is the // @implementation for a class. eTypeFlagClassIsImplementation = ( 1u << 1 ) - }; - + }; + // Make these public so that they can be used as a general interface to // the class. struct Atom { @@ -245,7 +245,7 @@ private: void EmitBuckets(AsmPrinter *); void EmitHashes(AsmPrinter *); void EmitOffsets(AsmPrinter *, MCSymbol *); - void EmitData(AsmPrinter *, DwarfDebug *D); + void EmitData(AsmPrinter *, DwarfUnits *D); // Allocator for HashData and HashDataContents. BumpPtrAllocator Allocator; @@ -265,14 +265,14 @@ private: typedef std::vector<HashList> BucketList; BucketList Buckets; HashList Hashes; - + // Public Implementation public: DwarfAccelTable(ArrayRef<DwarfAccelTable::Atom>); ~DwarfAccelTable(); void AddName(StringRef, DIE*, char = 0); void FinalizeTable(AsmPrinter *, const char *); - void Emit(AsmPrinter *, MCSymbol *, DwarfDebug *); + void Emit(AsmPrinter *, MCSymbol *, DwarfUnits *); #ifndef NDEBUG void print(raw_ostream &O); void dump() { print(dbgs()); } diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp index 4fdd5ca..fec5ced 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp @@ -12,31 +12,31 @@ //===----------------------------------------------------------------------===// #include "DwarfException.h" -#include "llvm/Module.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/Twine.h" #include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/MC/MachineLocation.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/MC/MachineLocation.h" +#include "llvm/Support/Dwarf.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormattedStream.h" #include "llvm/Target/Mangler.h" -#include "llvm/DataLayout.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Support/Dwarf.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/Twine.h" using namespace llvm; DwarfCFIException::DwarfCFIException(AsmPrinter *A) @@ -122,8 +122,9 @@ void DwarfCFIException::BeginFunction(const MachineFunction *MF) { const MCSymbol *Sym = TLOF.getCFIPersonalitySymbol(Per, Asm->Mang, MMI); Asm->OutStreamer.EmitCFIPersonality(Sym, PerEncoding); - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("eh_func_begin", - Asm->getFunctionNumber())); + Asm->OutStreamer.EmitDebugLabel + (Asm->GetTempSymbol("eh_func_begin", + Asm->getFunctionNumber())); // Provide LSDA information. if (!shouldEmitLSDA) diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp index 2b07dda..f9b6f94 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp @@ -13,28 +13,29 @@ #define DEBUG_TYPE "dwarfdebug" -#include "DwarfAccelTable.h" #include "DwarfCompileUnit.h" +#include "DwarfAccelTable.h" #include "DwarfDebug.h" -#include "llvm/Constants.h" +#include "llvm/ADT/APFloat.h" #include "llvm/DIBuilder.h" -#include "llvm/GlobalVariable.h" -#include "llvm/Instructions.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Instructions.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Target/Mangler.h" -#include "llvm/DataLayout.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/ADT/APFloat.h" -#include "llvm/Support/ErrorHandling.h" using namespace llvm; /// CompileUnit - Compile unit constructor. -CompileUnit::CompileUnit(unsigned I, unsigned L, DIE *D, AsmPrinter *A, - DwarfDebug *DW) - : ID(I), Language(L), CUDie(D), Asm(A), DD(DW), IndexTyDie(0) { +CompileUnit::CompileUnit(unsigned UID, unsigned L, DIE *D, AsmPrinter *A, + DwarfDebug *DW, DwarfUnits *DWU) + : UniqueID(UID), Language(L), CUDie(D), Asm(A), DD(DW), DU(DWU), + IndexTyDie(0), DebugInfoOffset(0) { DIEIntegerOne = new (DIEValueAllocator) DIEInteger(1); } @@ -51,6 +52,50 @@ DIEEntry *CompileUnit::createDIEEntry(DIE *Entry) { return Value; } +/// getDefaultLowerBound - Return the default lower bound for an array. If the +/// DWARF version doesn't handle the language, return -1. +int64_t CompileUnit::getDefaultLowerBound() const { + switch (Language) { + default: + break; + + case dwarf::DW_LANG_C89: + case dwarf::DW_LANG_C99: + case dwarf::DW_LANG_C: + case dwarf::DW_LANG_C_plus_plus: + case dwarf::DW_LANG_ObjC: + case dwarf::DW_LANG_ObjC_plus_plus: + return 0; + + case dwarf::DW_LANG_Fortran77: + case dwarf::DW_LANG_Fortran90: + case dwarf::DW_LANG_Fortran95: + return 1; + + // The languages below have valid values only if the DWARF version >= 4. + case dwarf::DW_LANG_Java: + case dwarf::DW_LANG_Python: + case dwarf::DW_LANG_UPC: + case dwarf::DW_LANG_D: + if (dwarf::DWARF_VERSION >= 4) + return 0; + break; + + case dwarf::DW_LANG_Ada83: + case dwarf::DW_LANG_Ada95: + case dwarf::DW_LANG_Cobol74: + case dwarf::DW_LANG_Cobol85: + case dwarf::DW_LANG_Modula2: + case dwarf::DW_LANG_Pascal83: + case dwarf::DW_LANG_PLI: + if (dwarf::DWARF_VERSION >= 4) + return 1; + break; + } + + return -1; +} + /// addFlag - Add a flag that is true. void CompileUnit::addFlag(DIE *Die, unsigned Attribute) { if (!DD->useDarwinGDBCompat()) @@ -81,14 +126,37 @@ void CompileUnit::addSInt(DIE *Die, unsigned Attribute, /// addString - Add a string attribute data and value. We always emit a /// reference to the string pool instead of immediate strings so that DIEs have -/// more predictable sizes. +/// more predictable sizes. In the case of split dwarf we emit an index +/// into another table which gets us the static offset into the string +/// table. void CompileUnit::addString(DIE *Die, unsigned Attribute, StringRef String) { - MCSymbol *Symb = DD->getStringPoolEntry(String); + if (!DD->useSplitDwarf()) { + MCSymbol *Symb = DU->getStringPoolEntry(String); + DIEValue *Value; + if (Asm->needsRelocationsForDwarfStringPool()) + Value = new (DIEValueAllocator) DIELabel(Symb); + else { + MCSymbol *StringPool = DU->getStringPoolSym(); + Value = new (DIEValueAllocator) DIEDelta(Symb, StringPool); + } + Die->addValue(Attribute, dwarf::DW_FORM_strp, Value); + } else { + unsigned idx = DU->getStringPoolIndex(String); + DIEValue *Value = new (DIEValueAllocator) DIEInteger(idx); + Die->addValue(Attribute, dwarf::DW_FORM_GNU_str_index, Value); + } +} + +/// addLocalString - Add a string attribute data and value. This is guaranteed +/// to be in the local string pool instead of indirected. +void CompileUnit::addLocalString(DIE *Die, unsigned Attribute, + StringRef String) { + MCSymbol *Symb = DU->getStringPoolEntry(String); DIEValue *Value; if (Asm->needsRelocationsForDwarfStringPool()) Value = new (DIEValueAllocator) DIELabel(Symb); else { - MCSymbol *StringPool = DD->getStringPool(); + MCSymbol *StringPool = DU->getStringPoolSym(); Value = new (DIEValueAllocator) DIEDelta(Symb, StringPool); } Die->addValue(Attribute, dwarf::DW_FORM_strp, Value); @@ -102,6 +170,42 @@ void CompileUnit::addLabel(DIE *Die, unsigned Attribute, unsigned Form, Die->addValue(Attribute, Form, Value); } +/// addLabelAddress - Add a dwarf label attribute data and value using +/// DW_FORM_addr or DW_FORM_GNU_addr_index. +/// +void CompileUnit::addLabelAddress(DIE *Die, unsigned Attribute, + MCSymbol *Label) { + if (!DD->useSplitDwarf()) { + if (Label != NULL) { + DIEValue *Value = new (DIEValueAllocator) DIELabel(Label); + Die->addValue(Attribute, dwarf::DW_FORM_addr, Value); + } else { + DIEValue *Value = new (DIEValueAllocator) DIEInteger(0); + Die->addValue(Attribute, dwarf::DW_FORM_addr, Value); + } + } else { + unsigned idx = DU->getAddrPoolIndex(Label); + DIEValue *Value = new (DIEValueAllocator) DIEInteger(idx); + Die->addValue(Attribute, dwarf::DW_FORM_GNU_addr_index, Value); + } +} + +/// addOpAddress - Add a dwarf op address data and value using the +/// form given and an op of either DW_FORM_addr or DW_FORM_GNU_addr_index. +/// +void CompileUnit::addOpAddress(DIE *Die, MCSymbol *Sym) { + + if (!DD->useSplitDwarf()) { + addUInt(Die, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr); + addLabel(Die, 0, dwarf::DW_FORM_udata, Sym); + } else { + unsigned idx = DU->getAddrPoolIndex(Sym); + DIEValue *Value = new (DIEValueAllocator) DIEInteger(idx); + addUInt(Die, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_GNU_addr_index); + Die->addValue(0, dwarf::DW_FORM_GNU_addr_index, Value); + } +} + /// addDelta - Add a label delta attribute data and value. /// void CompileUnit::addDelta(DIE *Die, unsigned Attribute, unsigned Form, @@ -132,12 +236,13 @@ void CompileUnit::addSourceLine(DIE *Die, DIVariable V) { // Verify variable. if (!V.Verify()) return; - + unsigned Line = V.getLineNumber(); if (Line == 0) return; - unsigned FileID = DD->GetOrCreateSourceID(V.getContext().getFilename(), - V.getContext().getDirectory()); + unsigned FileID = DD->getOrCreateSourceID(V.getContext().getFilename(), + V.getContext().getDirectory(), + getUniqueID()); assert(FileID && "Invalid file id"); addUInt(Die, dwarf::DW_AT_decl_file, 0, FileID); addUInt(Die, dwarf::DW_AT_decl_line, 0, Line); @@ -153,7 +258,8 @@ void CompileUnit::addSourceLine(DIE *Die, DIGlobalVariable G) { unsigned Line = G.getLineNumber(); if (Line == 0) return; - unsigned FileID = DD->GetOrCreateSourceID(G.getFilename(), G.getDirectory()); + unsigned FileID = DD->getOrCreateSourceID(G.getFilename(), G.getDirectory(), + getUniqueID()); assert(FileID && "Invalid file id"); addUInt(Die, dwarf::DW_AT_decl_file, 0, FileID); addUInt(Die, dwarf::DW_AT_decl_line, 0, Line); @@ -171,8 +277,8 @@ void CompileUnit::addSourceLine(DIE *Die, DISubprogram SP) { if (Line == 0) return; - unsigned FileID = DD->GetOrCreateSourceID(SP.getFilename(), - SP.getDirectory()); + unsigned FileID = DD->getOrCreateSourceID(SP.getFilename(), + SP.getDirectory(), getUniqueID()); assert(FileID && "Invalid file id"); addUInt(Die, dwarf::DW_AT_decl_file, 0, FileID); addUInt(Die, dwarf::DW_AT_decl_line, 0, Line); @@ -188,8 +294,8 @@ void CompileUnit::addSourceLine(DIE *Die, DIType Ty) { unsigned Line = Ty.getLineNumber(); if (Line == 0) return; - unsigned FileID = DD->GetOrCreateSourceID(Ty.getFilename(), - Ty.getDirectory()); + unsigned FileID = DD->getOrCreateSourceID(Ty.getFilename(), + Ty.getDirectory(), getUniqueID()); assert(FileID && "Invalid file id"); addUInt(Die, dwarf::DW_AT_decl_file, 0, FileID); addUInt(Die, dwarf::DW_AT_decl_line, 0, Line); @@ -206,8 +312,8 @@ void CompileUnit::addSourceLine(DIE *Die, DIObjCProperty Ty) { if (Line == 0) return; DIFile File = Ty.getFile(); - unsigned FileID = DD->GetOrCreateSourceID(File.getFilename(), - File.getDirectory()); + unsigned FileID = DD->getOrCreateSourceID(File.getFilename(), + File.getDirectory(), getUniqueID()); assert(FileID && "Invalid file id"); addUInt(Die, dwarf::DW_AT_decl_file, 0, FileID); addUInt(Die, dwarf::DW_AT_decl_line, 0, Line); @@ -225,15 +331,16 @@ void CompileUnit::addSourceLine(DIE *Die, DINameSpace NS) { return; StringRef FN = NS.getFilename(); - unsigned FileID = DD->GetOrCreateSourceID(FN, NS.getDirectory()); + unsigned FileID = DD->getOrCreateSourceID(FN, NS.getDirectory(), + getUniqueID()); assert(FileID && "Invalid file id"); addUInt(Die, dwarf::DW_AT_decl_file, 0, FileID); addUInt(Die, dwarf::DW_AT_decl_line, 0, Line); } -/// addVariableAddress - Add DW_AT_location attribute for a +/// addVariableAddress - Add DW_AT_location attribute for a /// DbgVariable based on provided MachineLocation. -void CompileUnit::addVariableAddress(DbgVariable *&DV, DIE *Die, +void CompileUnit::addVariableAddress(DbgVariable *&DV, DIE *Die, MachineLocation Location) { if (DV->variableHasComplexAddress()) addComplexAddress(DV, Die, dwarf::DW_AT_location, Location); @@ -492,7 +599,7 @@ bool CompileUnit::addConstantValue(DIE *Die, const MachineOperand &MO, case 64: Form = dwarf::DW_FORM_data8; break; default: break; } - SignedConstant ? addSInt(Block, 0, Form, MO.getImm()) + SignedConstant ? addSInt(Block, 0, Form, MO.getImm()) : addUInt(Block, 0, Form, MO.getImm()); addBlock(Die, dwarf::DW_AT_const_value, 0, Block); @@ -524,10 +631,21 @@ bool CompileUnit::addConstantFPValue(DIE *Die, const MachineOperand &MO) { return true; } +/// addConstantFPValue - Add constant value entry in variable DIE. +bool CompileUnit::addConstantFPValue(DIE *Die, const ConstantFP *CFP) { + return addConstantValue(Die, CFP->getValueAPF().bitcastToAPInt(), false); +} + /// addConstantValue - Add constant value entry in variable DIE. bool CompileUnit::addConstantValue(DIE *Die, const ConstantInt *CI, bool Unsigned) { - unsigned CIBitWidth = CI->getBitWidth(); + return addConstantValue(Die, CI->getValue(), Unsigned); +} + +// addConstantValue - Add constant value entry in variable DIE. +bool CompileUnit::addConstantValue(DIE *Die, const APInt &Val, + bool Unsigned) { + unsigned CIBitWidth = Val.getBitWidth(); if (CIBitWidth <= 64) { unsigned form = 0; switch (CIBitWidth) { @@ -535,20 +653,19 @@ bool CompileUnit::addConstantValue(DIE *Die, const ConstantInt *CI, case 16: form = dwarf::DW_FORM_data2; break; case 32: form = dwarf::DW_FORM_data4; break; case 64: form = dwarf::DW_FORM_data8; break; - default: + default: form = Unsigned ? dwarf::DW_FORM_udata : dwarf::DW_FORM_sdata; } if (Unsigned) - addUInt(Die, dwarf::DW_AT_const_value, form, CI->getZExtValue()); + addUInt(Die, dwarf::DW_AT_const_value, form, Val.getZExtValue()); else - addSInt(Die, dwarf::DW_AT_const_value, form, CI->getSExtValue()); + addSInt(Die, dwarf::DW_AT_const_value, form, Val.getSExtValue()); return true; } DIEBlock *Block = new (DIEValueAllocator) DIEBlock(); // Get the raw data form of the large APInt. - const APInt Val = CI->getValue(); const uint64_t *Ptr64 = Val.getRawData(); int NumBytes = Val.getBitWidth() / 8; // 8 bits per byte. @@ -582,18 +699,21 @@ void CompileUnit::addTemplateParams(DIE &Buffer, DIArray TParams) { } } +/// getOrCreateContextDIE - Get context owner's DIE. +DIE *CompileUnit::getOrCreateContextDIE(DIDescriptor Context) { + if (Context.isType()) + return getOrCreateTypeDIE(DIType(Context)); + else if (Context.isNameSpace()) + return getOrCreateNameSpace(DINameSpace(Context)); + else if (Context.isSubprogram()) + return getOrCreateSubprogramDIE(DISubprogram(Context)); + else + return getDIE(Context); +} + /// addToContextOwner - Add Die into the list of its context owner's children. void CompileUnit::addToContextOwner(DIE *Die, DIDescriptor Context) { - if (Context.isType()) { - DIE *ContextDIE = getOrCreateTypeDIE(DIType(Context)); - ContextDIE->addChild(Die); - } else if (Context.isNameSpace()) { - DIE *ContextDIE = getOrCreateNameSpace(DINameSpace(Context)); - ContextDIE->addChild(Die); - } else if (Context.isSubprogram()) { - DIE *ContextDIE = getOrCreateSubprogramDIE(DISubprogram(Context)); - ContextDIE->addChild(Die); - } else if (DIE *ContextDIE = getDIE(Context)) + if (DIE *ContextDIE = getOrCreateContextDIE(Context)) ContextDIE->addChild(Die); else addDie(Die); @@ -635,7 +755,7 @@ DIE *CompileUnit::getOrCreateTypeDIE(const MDNode *TyNode) { DwarfAccelTable::eTypeFlagClassIsImplementation : 0; addAccelType(Ty.getName(), std::make_pair(TyDIE, Flags)); } - + addToContextOwner(TyDIE, Ty.getContext()); return TyDIE; } @@ -670,8 +790,8 @@ void CompileUnit::addType(DIE *Entity, DIType Ty, unsigned Attribute) { /// void CompileUnit::addGlobalType(DIType Ty) { DIDescriptor Context = Ty.getContext(); - if (Ty.isCompositeType() && !Ty.getName().empty() && !Ty.isForwardDecl() - && (!Context || Context.isCompileUnit() || Context.isFile() + if (Ty.isCompositeType() && !Ty.getName().empty() && !Ty.isForwardDecl() + && (!Context || Context.isCompileUnit() || Context.isFile() || Context.isNameSpace())) if (DIEEntry *Entry = getDIEEntry(Ty)) GlobalTypes[Ty.getName()] = Entry->getEntry(); @@ -739,6 +859,9 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DIDerivedType DTy) { if (Size && Tag != dwarf::DW_TAG_pointer_type) addUInt(&Buffer, dwarf::DW_AT_byte_size, 0, Size); + if (Tag == dwarf::DW_TAG_ptr_to_member_type) + addDIEEntry(&Buffer, dwarf::DW_AT_containing_type, dwarf::DW_FORM_ref4, + getOrCreateTypeDIE(DTy.getClassType())); // Add source line info if available and TyDesc is not a forward declaration. if (!DTy.isForwardDecl()) addSourceLine(&Buffer, DTy); @@ -754,7 +877,6 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { Buffer.setTag(Tag); switch (Tag) { - case dwarf::DW_TAG_vector_type: case dwarf::DW_TAG_array_type: constructArrayTypeDIE(Buffer, &CTy); break; @@ -794,6 +916,8 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { } else { DIE *Arg = new DIE(dwarf::DW_TAG_formal_parameter); addType(Arg, DIType(Ty)); + if (DIType(Ty).isArtificial()) + addFlag(Arg, dwarf::DW_AT_artificial); Buffer.addChild(Arg); } } @@ -830,27 +954,20 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { else if (SP.isPrivate()) addUInt(ElemDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_private); - else + else addUInt(ElemDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_public); if (SP.isExplicit()) addFlag(ElemDie, dwarf::DW_AT_explicit); - } - else if (Element.isVariable()) { - DIVariable DV(Element); - ElemDie = new DIE(dwarf::DW_TAG_variable); - addString(ElemDie, dwarf::DW_AT_name, DV.getName()); - addType(ElemDie, DV.getType()); - addFlag(ElemDie, dwarf::DW_AT_declaration); - addFlag(ElemDie, dwarf::DW_AT_external); - addSourceLine(ElemDie, DV); } else if (Element.isDerivedType()) { DIDerivedType DDTy(Element); if (DDTy.getTag() == dwarf::DW_TAG_friend) { ElemDie = new DIE(dwarf::DW_TAG_friend); addType(ElemDie, DDTy.getTypeDerivedFrom(), dwarf::DW_AT_friend); - } else - ElemDie = createMemberDIE(DIDerivedType(Element)); + } else if (DDTy.isStaticMember()) + ElemDie = createStaticMemberDIE(DDTy); + else + ElemDie = createMemberDIE(DDTy); } else if (Element.isObjCProperty()) { DIObjCProperty Property(Element); ElemDie = new DIE(Property.getTag()); @@ -878,7 +995,7 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { if (Property.isNonAtomicObjCProperty()) PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_nonatomic; if (PropertyAttributes) - addUInt(ElemDie, dwarf::DW_AT_APPLE_property_attribute, 0, + addUInt(ElemDie, dwarf::DW_AT_APPLE_property_attribute, 0, PropertyAttributes); DIEEntry *Entry = getDIEEntry(Element); @@ -951,7 +1068,7 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { } } -/// getOrCreateTemplateTypeParameterDIE - Find existing DIE or create new DIE +/// getOrCreateTemplateTypeParameterDIE - Find existing DIE or create new DIE /// for the given DITemplateTypeParameter. DIE * CompileUnit::getOrCreateTemplateTypeParameterDIE(DITemplateTypeParameter TP) { @@ -965,7 +1082,7 @@ CompileUnit::getOrCreateTemplateTypeParameterDIE(DITemplateTypeParameter TP) { return ParamDIE; } -/// getOrCreateTemplateValueParameterDIE - Find existing DIE or create new DIE +/// getOrCreateTemplateValueParameterDIE - Find existing DIE or create new DIE /// for the given DITemplateValueParameter. DIE * CompileUnit::getOrCreateTemplateValueParameterDIE(DITemplateValueParameter TPV){ @@ -977,7 +1094,7 @@ CompileUnit::getOrCreateTemplateValueParameterDIE(DITemplateValueParameter TPV){ addType(ParamDIE, TPV.getType()); if (!TPV.getName().empty()) addString(ParamDIE, dwarf::DW_AT_name, TPV.getName()); - addUInt(ParamDIE, dwarf::DW_AT_const_value, dwarf::DW_FORM_udata, + addUInt(ParamDIE, dwarf::DW_AT_const_value, dwarf::DW_FORM_udata, TPV.getValue()); return ParamDIE; } @@ -1095,7 +1212,7 @@ DIE *CompileUnit::getOrCreateSubprogramDIE(DISubprogram SP) { if (!SP.isDefinition()) { addFlag(SPDie, dwarf::DW_AT_declaration); - + // Add arguments. Do not add arguments for subprogram definition. They will // be handled while processing variables. DICompositeType SPTy = SP.getType(); @@ -1166,39 +1283,56 @@ void CompileUnit::createGlobalVariableDIE(const MDNode *N) { if (!GV.Verify()) return; - DIE *VariableDIE = new DIE(GV.getTag()); - // Add to map. - insertDIE(N, VariableDIE); - - // Add name. - addString(VariableDIE, dwarf::DW_AT_name, GV.getDisplayName()); - StringRef LinkageName = GV.getLinkageName(); - bool isGlobalVariable = GV.getGlobal() != NULL; - if (!LinkageName.empty() && isGlobalVariable) - addString(VariableDIE, dwarf::DW_AT_MIPS_linkage_name, - getRealLinkageName(LinkageName)); - // Add type. + DIDescriptor GVContext = GV.getContext(); DIType GTy = GV.getType(); - addType(VariableDIE, GTy); - // Add scoping info. - if (!GV.isLocalToUnit()) - addFlag(VariableDIE, dwarf::DW_AT_external); + // If this is a static data member definition, some attributes belong + // to the declaration DIE. + DIE *VariableDIE = NULL; + bool IsStaticMember = false; + DIDerivedType SDMDecl = GV.getStaticDataMemberDeclaration(); + if (SDMDecl.Verify()) { + assert(SDMDecl.isStaticMember() && "Expected static member decl"); + // We need the declaration DIE that is in the static member's class. + // But that class might not exist in the DWARF yet. + // Creating the class will create the static member decl DIE. + getOrCreateContextDIE(SDMDecl.getContext()); + VariableDIE = getDIE(SDMDecl); + assert(VariableDIE && "Static member decl has no context?"); + IsStaticMember = true; + } + + // If this is not a static data member definition, create the variable + // DIE and add the initial set of attributes to it. + if (!VariableDIE) { + VariableDIE = new DIE(GV.getTag()); + // Add to map. + insertDIE(N, VariableDIE); + + // Add name and type. + addString(VariableDIE, dwarf::DW_AT_name, GV.getDisplayName()); + addType(VariableDIE, GTy); + + // Add scoping info. + if (!GV.isLocalToUnit()) { + addFlag(VariableDIE, dwarf::DW_AT_external); + addGlobalName(GV.getName(), VariableDIE); + } + + // Add line number info. + addSourceLine(VariableDIE, GV); + // Add to context owner. + addToContextOwner(VariableDIE, GVContext); + } - // Add line number info. - addSourceLine(VariableDIE, GV); - // Add to context owner. - DIDescriptor GVContext = GV.getContext(); - addToContextOwner(VariableDIE, GVContext); // Add location. bool addToAccelTable = false; DIE *VariableSpecDIE = NULL; + bool isGlobalVariable = GV.getGlobal() != NULL; if (isGlobalVariable) { addToAccelTable = true; DIEBlock *Block = new (DIEValueAllocator) DIEBlock(); - addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr); - addLabel(Block, 0, dwarf::DW_FORM_udata, - Asm->Mang->getSymbol(GV.getGlobal())); + addOpAddress(Block, Asm->Mang->getSymbol(GV.getGlobal())); // Do not create specification DIE if context is either compile unit // or a subprogram. if (GVContext && GV.isDefinition() && !GVContext.isCompileUnit() && @@ -1208,25 +1342,44 @@ void CompileUnit::createGlobalVariableDIE(const MDNode *N) { addDIEEntry(VariableSpecDIE, dwarf::DW_AT_specification, dwarf::DW_FORM_ref4, VariableDIE); addBlock(VariableSpecDIE, dwarf::DW_AT_location, 0, Block); - addFlag(VariableDIE, dwarf::DW_AT_declaration); + // A static member's declaration is already flagged as such. + if (!SDMDecl.Verify()) + addFlag(VariableDIE, dwarf::DW_AT_declaration); addDie(VariableSpecDIE); } else { addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block); } - } else if (const ConstantInt *CI = - dyn_cast_or_null<ConstantInt>(GV.getConstant())) - addConstantValue(VariableDIE, CI, GTy.isUnsignedDIType()); - else if (const ConstantExpr *CE = getMergedGlobalExpr(N->getOperand(11))) { + // Add linkage name. + StringRef LinkageName = GV.getLinkageName(); + if (!LinkageName.empty()) { + // From DWARF4: DIEs to which DW_AT_linkage_name may apply include: + // TAG_common_block, TAG_constant, TAG_entry_point, TAG_subprogram and + // TAG_variable. + addString(IsStaticMember && VariableSpecDIE ? + VariableSpecDIE : VariableDIE, dwarf::DW_AT_MIPS_linkage_name, + getRealLinkageName(LinkageName)); + // In compatibility mode with older gdbs we put the linkage name on both + // the TAG_variable DIE and on the TAG_member DIE. + if (IsStaticMember && VariableSpecDIE && DD->useDarwinGDBCompat()) + addString(VariableDIE, dwarf::DW_AT_MIPS_linkage_name, + getRealLinkageName(LinkageName)); + } + } else if (const ConstantInt *CI = + dyn_cast_or_null<ConstantInt>(GV.getConstant())) { + // AT_const_value was added when the static member was created. To avoid + // emitting AT_const_value multiple times, we only add AT_const_value when + // it is not a static member. + if (!IsStaticMember) + addConstantValue(VariableDIE, CI, GTy.isUnsignedDIType()); + } else if (const ConstantExpr *CE = getMergedGlobalExpr(N->getOperand(11))) { addToAccelTable = true; // GV is a merged global. DIEBlock *Block = new (DIEValueAllocator) DIEBlock(); Value *Ptr = CE->getOperand(0); - addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr); - addLabel(Block, 0, dwarf::DW_FORM_udata, - Asm->Mang->getSymbol(cast<GlobalValue>(Ptr))); + addOpAddress(Block, Asm->Mang->getSymbol(cast<GlobalValue>(Ptr))); addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_constu); SmallVector<Value*, 3> Idx(CE->op_begin()+1, CE->op_end()); - addUInt(Block, 0, dwarf::DW_FORM_udata, + addUInt(Block, 0, dwarf::DW_FORM_udata, Asm->getDataLayout().getIndexedOffset(Ptr->getType(), Idx)); addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_plus); addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block); @@ -1250,22 +1403,25 @@ void CompileUnit::constructSubrangeDIE(DIE &Buffer, DISubrange SR, DIE *IndexTy) { DIE *DW_Subrange = new DIE(dwarf::DW_TAG_subrange_type); addDIEEntry(DW_Subrange, dwarf::DW_AT_type, dwarf::DW_FORM_ref4, IndexTy); - uint64_t L = SR.getLo(); - uint64_t H = SR.getHi(); - // The L value defines the lower bounds which is typically zero for C/C++. The - // H value is the upper bounds. Values are 64 bit. H - L + 1 is the size - // of the array. If L > H then do not emit DW_AT_lower_bound and - // DW_AT_upper_bound attributes. If L is zero and H is also zero then the - // array has one element and in such case do not emit lower bound. + // The LowerBound value defines the lower bounds which is typically zero for + // C/C++. The Count value is the number of elements. Values are 64 bit. If + // Count == -1 then the array is unbounded and we do not emit + // DW_AT_lower_bound and DW_AT_upper_bound attributes. If LowerBound == 0 and + // Count == 0, then the array has zero elements in which case we do not emit + // an upper bound. + int64_t LowerBound = SR.getLo(); + int64_t DefaultLowerBound = getDefaultLowerBound(); + int64_t Count = SR.getCount(); + + if (DefaultLowerBound == -1 || LowerBound != DefaultLowerBound) + addUInt(DW_Subrange, dwarf::DW_AT_lower_bound, 0, LowerBound); + + if (Count != -1 && Count != 0) + // FIXME: An unbounded array should reference the expression that defines + // the array. + addUInt(DW_Subrange, dwarf::DW_AT_upper_bound, 0, LowerBound + Count - 1); - if (L > H) { - Buffer.addChild(DW_Subrange); - return; - } - if (L) - addUInt(DW_Subrange, dwarf::DW_AT_lower_bound, 0, L); - addUInt(DW_Subrange, dwarf::DW_AT_upper_bound, 0, H); Buffer.addChild(DW_Subrange); } @@ -1273,7 +1429,7 @@ void CompileUnit::constructSubrangeDIE(DIE &Buffer, DISubrange SR, void CompileUnit::constructArrayTypeDIE(DIE &Buffer, DICompositeType *CTy) { Buffer.setTag(dwarf::DW_TAG_array_type); - if (CTy->getTag() == dwarf::DW_TAG_vector_type) + if (CTy->isVector()) addFlag(&Buffer, dwarf::DW_AT_GNU_vector); // Emit derived type. @@ -1281,10 +1437,13 @@ void CompileUnit::constructArrayTypeDIE(DIE &Buffer, DIArray Elements = CTy->getTypeArray(); // Get an anonymous type for index type. + // FIXME: This type should be passed down from the front end + // as different languages may have different sizes for indexes. DIE *IdxTy = getIndexTyDie(); if (!IdxTy) { // Construct an anonymous type for index type. IdxTy = new DIE(dwarf::DW_TAG_base_type); + addString(IdxTy, dwarf::DW_AT_name, "int"); addUInt(IdxTy, dwarf::DW_AT_byte_size, 0, sizeof(int32_t)); addUInt(IdxTy, dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, dwarf::DW_ATE_signed); @@ -1327,8 +1486,6 @@ void CompileUnit::constructContainingTypeDIEs() { /// constructVariableDIE - Construct a DIE for the given DbgVariable. DIE *CompileUnit::constructVariableDIE(DbgVariable *DV, bool isScopeAbstract) { StringRef Name = DV->getName(); - if (Name.empty()) - return NULL; // Translate tag to proper Dwarf tag. unsigned Tag = DV->getTag(); @@ -1376,20 +1533,20 @@ DIE *CompileUnit::constructVariableDIE(DbgVariable *DV, bool isScopeAbstract) { TRI->getFrameRegister(*Asm->MF) == RegOp.getReg()) { unsigned FrameReg = 0; const TargetFrameLowering *TFI = Asm->TM.getFrameLowering(); - int Offset = - TFI->getFrameIndexReference(*Asm->MF, - DVInsn->getOperand(1).getImm(), + int Offset = + TFI->getFrameIndexReference(*Asm->MF, + DVInsn->getOperand(1).getImm(), FrameReg); MachineLocation Location(FrameReg, Offset); addVariableAddress(DV, VariableDie, Location); - + } else if (RegOp.getReg()) - addVariableAddress(DV, VariableDie, + addVariableAddress(DV, VariableDie, MachineLocation(RegOp.getReg())); updated = true; } else if (DVInsn->getOperand(0).isImm()) - updated = + updated = addConstantValue(VariableDie, DVInsn->getOperand(0), DV->getType()); else if (DVInsn->getOperand(0).isFPImm()) @@ -1397,11 +1554,11 @@ DIE *CompileUnit::constructVariableDIE(DbgVariable *DV, bool isScopeAbstract) { addConstantFPValue(VariableDie, DVInsn->getOperand(0)); else if (DVInsn->getOperand(0).isCImm()) updated = - addConstantValue(VariableDie, + addConstantValue(VariableDie, DVInsn->getOperand(0).getCImm(), DV->getType().isUnsignedDIType()); } else { - addVariableAddress(DV, VariableDie, + addVariableAddress(DV, VariableDie, Asm->getDebugValueLocation(DVInsn)); updated = true; } @@ -1419,7 +1576,7 @@ DIE *CompileUnit::constructVariableDIE(DbgVariable *DV, bool isScopeAbstract) { if (FI != ~0) { unsigned FrameReg = 0; const TargetFrameLowering *TFI = Asm->TM.getFrameLowering(); - int Offset = + int Offset = TFI->getFrameIndexReference(*Asm->MF, FI, FrameReg); MachineLocation Location(FrameReg, Offset); addVariableAddress(DV, VariableDie, Location); @@ -1499,7 +1656,7 @@ DIE *CompileUnit::createMemberDIE(DIDerivedType DT) { addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_private); // Otherwise C++ member and base classes are considered public. - else + else addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_public); if (DT.isVirtual()) @@ -1509,35 +1666,46 @@ DIE *CompileUnit::createMemberDIE(DIDerivedType DT) { // Objective-C properties. if (MDNode *PNode = DT.getObjCProperty()) if (DIEEntry *PropertyDie = getDIEEntry(PNode)) - MemberDie->addValue(dwarf::DW_AT_APPLE_property, dwarf::DW_FORM_ref4, + MemberDie->addValue(dwarf::DW_AT_APPLE_property, dwarf::DW_FORM_ref4, PropertyDie); - // This is only for backward compatibility. - StringRef PropertyName = DT.getObjCPropertyName(); - if (!PropertyName.empty()) { - addString(MemberDie, dwarf::DW_AT_APPLE_property_name, PropertyName); - StringRef GetterName = DT.getObjCPropertyGetterName(); - if (!GetterName.empty()) - addString(MemberDie, dwarf::DW_AT_APPLE_property_getter, GetterName); - StringRef SetterName = DT.getObjCPropertySetterName(); - if (!SetterName.empty()) - addString(MemberDie, dwarf::DW_AT_APPLE_property_setter, SetterName); - unsigned PropertyAttributes = 0; - if (DT.isReadOnlyObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_readonly; - if (DT.isReadWriteObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_readwrite; - if (DT.isAssignObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_assign; - if (DT.isRetainObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_retain; - if (DT.isCopyObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_copy; - if (DT.isNonAtomicObjCProperty()) - PropertyAttributes |= dwarf::DW_APPLE_PROPERTY_nonatomic; - if (PropertyAttributes) - addUInt(MemberDie, dwarf::DW_AT_APPLE_property_attribute, 0, - PropertyAttributes); - } + if (DT.isArtificial()) + addFlag(MemberDie, dwarf::DW_AT_artificial); + return MemberDie; } + +/// createStaticMemberDIE - Create new DIE for C++ static member. +DIE *CompileUnit::createStaticMemberDIE(const DIDerivedType DT) { + if (!DT.Verify()) + return NULL; + + DIE *StaticMemberDIE = new DIE(DT.getTag()); + DIType Ty = DT.getTypeDerivedFrom(); + + addString(StaticMemberDIE, dwarf::DW_AT_name, DT.getName()); + addType(StaticMemberDIE, Ty); + addSourceLine(StaticMemberDIE, DT); + addFlag(StaticMemberDIE, dwarf::DW_AT_external); + addFlag(StaticMemberDIE, dwarf::DW_AT_declaration); + + // FIXME: We could omit private if the parent is a class_type, and + // public if the parent is something else. + if (DT.isProtected()) + addUInt(StaticMemberDIE, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, + dwarf::DW_ACCESS_protected); + else if (DT.isPrivate()) + addUInt(StaticMemberDIE, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, + dwarf::DW_ACCESS_private); + else + addUInt(StaticMemberDIE, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, + dwarf::DW_ACCESS_public); + + if (const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(DT.getConstant())) + addConstantValue(StaticMemberDIE, CI, Ty.isUnsignedDIType()); + if (const ConstantFP *CFP = dyn_cast_or_null<ConstantFP>(DT.getConstant())) + addConstantFPValue(StaticMemberDIE, CFP); + + insertDIE(DT, StaticMemberDIE); + return StaticMemberDIE; +} diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h index fad9b6e..2b180c6 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h @@ -15,26 +15,28 @@ #define CODEGEN_ASMPRINTER_DWARFCOMPILEUNIT_H #include "DIE.h" -#include "llvm/DebugInfo.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/StringMap.h" #include "llvm/ADT/OwningPtr.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/DebugInfo.h" namespace llvm { class DwarfDebug; +class DwarfUnits; class MachineLocation; class MachineOperand; class ConstantInt; +class ConstantFP; class DbgVariable; //===----------------------------------------------------------------------===// /// CompileUnit - This dwarf writer support class manages information associated /// with a source file. class CompileUnit { - /// ID - File identifier for source. + /// UniqueID - a numeric ID unique among all CUs in the module /// - unsigned ID; + unsigned UniqueID; /// Language - The DW_AT_language of the compile unit /// @@ -47,7 +49,9 @@ class CompileUnit { /// Asm - Target of Dwarf emission. AsmPrinter *Asm; + // Holders for some common dwarf information. DwarfDebug *DD; + DwarfUnits *DU; /// IndexTyDie - An anonymous type for index type. Owned by CUDie. DIE *IndexTyDie; @@ -60,6 +64,10 @@ class CompileUnit { /// descriptors to debug information entries using a DIEEntry proxy. DenseMap<const MDNode *, DIEEntry *> MDNodeToDIEEntryMap; + /// GlobalNames - A map of globally visible named entities for this unit. + /// + StringMap<DIE*> GlobalNames; + /// GlobalTypes - A map of globally visible types for this unit. /// StringMap<DIE*> GlobalTypes; @@ -79,14 +87,27 @@ class CompileUnit { /// corresponds to the MDNode mapped with the subprogram DIE. DenseMap<DIE *, const MDNode *> ContainingTypeMap; + /// Offset of the CUDie from beginning of debug info section. + unsigned DebugInfoOffset; + + /// getLowerBoundDefault - Return the default lower bound for an array. If the + /// DWARF version doesn't handle the language, return -1. + int64_t getDefaultLowerBound() const; + + /// getOrCreateContextDIE - Get context owner's DIE. + DIE *getOrCreateContextDIE(DIDescriptor Context); + public: - CompileUnit(unsigned I, unsigned L, DIE *D, AsmPrinter *A, DwarfDebug *DW); + CompileUnit(unsigned UID, unsigned L, DIE *D, AsmPrinter *A, DwarfDebug *DW, + DwarfUnits *); ~CompileUnit(); // Accessors. - unsigned getID() const { return ID; } + unsigned getUniqueID() const { return UniqueID; } unsigned getLanguage() const { return Language; } DIE* getCUDie() const { return CUDie.get(); } + unsigned getDebugInfoOffset() const { return DebugInfoOffset; } + const StringMap<DIE*> &getGlobalNames() const { return GlobalNames; } const StringMap<DIE*> &getGlobalTypes() const { return GlobalTypes; } const StringMap<std::vector<DIE*> > &getAccelNames() const { @@ -102,11 +123,16 @@ public: &getAccelTypes() const { return AccelTypes; } - + + void setDebugInfoOffset(unsigned DbgInfoOff) { DebugInfoOffset = DbgInfoOff; } /// hasContent - Return true if this compile unit has something to write out. /// bool hasContent() const { return !CUDie->getChildren().empty(); } + /// addGlobalName - Add a new global entity to the compile unit. + /// + void addGlobalName(StringRef Name, DIE *Die) { GlobalNames[Name] = Die; } + /// addGlobalType - Add a new global type to the compile unit. /// void addGlobalType(DIType Ty); @@ -129,12 +155,12 @@ public: std::vector<std::pair<DIE*, unsigned > > &DIEs = AccelTypes[Name]; DIEs.push_back(Die); } - + /// getDIE - Returns the debug information entry map slot for the /// specified debug variable. DIE *getDIE(const MDNode *N) { return MDNodeToDieMap.lookup(N); } - DIEBlock *getDIEBlock() { + DIEBlock *getDIEBlock() { return new (DIEValueAllocator) DIEBlock(); } @@ -174,11 +200,10 @@ public: void setIndexTyDie(DIE *D) { IndexTyDie = D; } -public: /// addFlag - Add a flag that is true to the DIE. void addFlag(DIE *Die, unsigned Attribute); - + /// addUInt - Add an unsigned integer attribute data and value. /// void addUInt(DIE *Die, unsigned Attribute, unsigned Form, uint64_t Integer); @@ -191,11 +216,25 @@ public: /// void addString(DIE *Die, unsigned Attribute, const StringRef Str); + /// addLocalString - Add a string attribute data and value. + /// + void addLocalString(DIE *Die, unsigned Attribute, const StringRef Str); + /// addLabel - Add a Dwarf label attribute data and value. /// void addLabel(DIE *Die, unsigned Attribute, unsigned Form, const MCSymbol *Label); + /// addLabelAddress - Add a dwarf label attribute data and value using + /// either DW_FORM_addr or DW_FORM_GNU_addr_index. + /// + void addLabelAddress(DIE *Die, unsigned Attribute, MCSymbol *Label); + + /// addOpAddress - Add a dwarf op address data and value using the + /// form given and an op of either DW_FORM_addr or DW_FORM_GNU_addr_index. + /// + void addOpAddress(DIE *Die, MCSymbol *Label); + /// addDelta - Add a label delta attribute data and value. /// void addDelta(DIE *Die, unsigned Attribute, unsigned Form, @@ -204,7 +243,7 @@ public: /// addDIEEntry - Add a DIE attribute data and value. /// void addDIEEntry(DIE *Die, unsigned Attribute, unsigned Form, DIE *Entry); - + /// addBlock - Add block data. /// void addBlock(DIE *Die, unsigned Attribute, unsigned Form, DIEBlock *Block); @@ -226,9 +265,11 @@ public: /// addConstantValue - Add constant value entry in variable DIE. bool addConstantValue(DIE *Die, const MachineOperand &MO, DIType Ty); bool addConstantValue(DIE *Die, const ConstantInt *CI, bool Unsigned); + bool addConstantValue(DIE *Die, const APInt &Val, bool Unsigned); /// addConstantFPValue - Add constant value entry in variable DIE. bool addConstantFPValue(DIE *Die, const MachineOperand &MO); + bool addConstantFPValue(DIE *Die, const ConstantFP *CFP); /// addTemplateParams - Add template parameters in buffer. void addTemplateParams(DIE &Buffer, DIArray TParams); @@ -257,7 +298,7 @@ public: void addBlockByrefAddress(DbgVariable *&DV, DIE *Die, unsigned Attribute, const MachineLocation &Location); - /// addVariableAddress - Add DW_AT_location attribute for a + /// addVariableAddress - Add DW_AT_location attribute for a /// DbgVariable based on provided MachineLocation. void addVariableAddress(DbgVariable *&DV, DIE *Die, MachineLocation Location); @@ -279,7 +320,7 @@ public: /// given DIType. DIE *getOrCreateTypeDIE(const MDNode *N); - /// getOrCreateTemplateTypeParameterDIE - Find existing DIE or create new DIE + /// getOrCreateTemplateTypeParameterDIE - Find existing DIE or create new DIE /// for the given DITemplateTypeParameter. DIE *getOrCreateTemplateTypeParameterDIE(DITemplateTypeParameter TP); @@ -312,7 +353,7 @@ public: void constructSubrangeDIE(DIE &Buffer, DISubrange SR, DIE *IndexTy); /// constructArrayTypeDIE - Construct array type DIE from DICompositeType. - void constructArrayTypeDIE(DIE &Buffer, + void constructArrayTypeDIE(DIE &Buffer, DICompositeType *CTy); /// constructEnumTypeDIE - Construct enum type DIE from DIEnumerator. @@ -328,6 +369,9 @@ public: /// createMemberDIE - Create new member DIE. DIE *createMemberDIE(DIDerivedType DT); + /// createStaticMemberDIE - Create new static data member DIE. + DIE *createStaticMemberDIE(DIDerivedType DT); + private: // DIEValueAllocator - All DIEValues are allocated through this allocator. diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index 367b523..d3cb4f9 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -16,34 +16,34 @@ #include "DIE.h" #include "DwarfAccelTable.h" #include "DwarfCompileUnit.h" -#include "llvm/Constants.h" -#include "llvm/DebugInfo.h" -#include "llvm/DIBuilder.h" -#include "llvm/Module.h" -#include "llvm/Instructions.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/Triple.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/DIBuilder.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetFrameLowering.h" -#include "llvm/Target/TargetLoweringObjectFile.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetOptions.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/Triple.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ValueHandle.h" #include "llvm/Support/FormattedStream.h" -#include "llvm/Support/Timer.h" #include "llvm/Support/Path.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/ValueHandle.h" +#include "llvm/Target/TargetFrameLowering.h" +#include "llvm/Target/TargetLoweringObjectFile.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; static cl::opt<bool> DisableDebugInfoPrinting("disable-debug-info-print", @@ -54,6 +54,10 @@ static cl::opt<bool> UnknownLocations("use-unknown-locations", cl::Hidden, cl::desc("Make an absence of debug location information explicit."), cl::init(false)); +static cl::opt<bool> GenerateDwarfPubNamesSection("generate-dwarf-pubnames", + cl::Hidden, cl::init(false), + cl::desc("Generate DWARF pubnames section")); + namespace { enum DefaultOnOff { Default, Enable, Disable @@ -78,6 +82,15 @@ static cl::opt<DefaultOnOff> DarwinGDBCompat("darwin-gdb-compat", cl::Hidden, clEnumValEnd), cl::init(Default)); +static cl::opt<DefaultOnOff> SplitDwarf("split-dwarf", cl::Hidden, + cl::desc("Output prototype dwarf split debug info."), + cl::values( + clEnumVal(Default, "Default for platform"), + clEnumVal(Enable, "Enabled"), + clEnumVal(Disable, "Disabled"), + clEnumValEnd), + cl::init(Default)); + namespace { const char *DWARFGroupName = "DWARF Emission"; const char *DbgTimerName = "DWARF Debug Writer"; @@ -85,8 +98,8 @@ namespace { //===----------------------------------------------------------------------===// -/// Configuration values for initial hash set sizes (log2). -/// +// Configuration values for initial hash set sizes (log2). +// static const unsigned InitAbbreviationsSetSize = 9; // log2(512) namespace llvm { @@ -102,19 +115,19 @@ DIType DbgVariable::getType() const { either the struct, or a pointer to the struct, as its type. This is necessary for various behind-the-scenes things the compiler needs to do with by-reference variables in blocks. - + However, as far as the original *programmer* is concerned, the variable should still have type 'SomeType', as originally declared. - + The following function dives into the __Block_byref_x_VarName struct to find the original type of the variable. This will be passed back to the code generating the type for the Debug Information Entry for the variable 'VarName'. 'VarName' will then have the original type 'SomeType' in its debug information. - + The original type 'SomeType' will be the type of the field named 'VarName' inside the __Block_byref_x_VarName struct. - + NOTE: In order for this to not completely fail on the debugger side, the Debug Information Entry for the variable VarName needs to have a DW_AT_location that tells the debugger how to unwind through @@ -122,15 +135,15 @@ DIType DbgVariable::getType() const { value of the variable. The function addBlockByrefType does this. */ DIType subType = Ty; unsigned tag = Ty.getTag(); - + if (tag == dwarf::DW_TAG_pointer_type) { DIDerivedType DTy = DIDerivedType(Ty); subType = DTy.getTypeDerivedFrom(); } - + DICompositeType blockStruct = DICompositeType(subType); DIArray Elements = blockStruct.getTypeArray(); - + for (unsigned i = 0, N = Elements.getNumElements(); i < N; ++i) { DIDescriptor Element = Elements.getElement(i); DIDerivedType DT = DIDerivedType(Element); @@ -146,45 +159,55 @@ DIType DbgVariable::getType() const { DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M) : Asm(A), MMI(Asm->MMI), FirstCU(0), AbbreviationsSet(InitAbbreviationsSetSize), - SourceIdMap(DIEValueAllocator), StringPool(DIEValueAllocator), - PrevLabel(NULL) { - NextStringPoolNumber = 0; + SourceIdMap(DIEValueAllocator), + PrevLabel(NULL), GlobalCUIndexCount(0), + InfoHolder(A, &AbbreviationsSet, &Abbreviations, "info_string", + DIEValueAllocator), + SkeletonAbbrevSet(InitAbbreviationsSetSize), + SkeletonHolder(A, &SkeletonAbbrevSet, &SkeletonAbbrevs, "skel_string", + DIEValueAllocator) { DwarfInfoSectionSym = DwarfAbbrevSectionSym = 0; DwarfStrSectionSym = TextSectionSym = 0; - DwarfDebugRangeSectionSym = DwarfDebugLocSectionSym = 0; + DwarfDebugRangeSectionSym = DwarfDebugLocSectionSym = DwarfLineSectionSym = 0; + DwarfAbbrevDWOSectionSym = DwarfStrDWOSectionSym = 0; FunctionBeginSym = FunctionEndSym = 0; // Turn on accelerator tables and older gdb compatibility // for Darwin. - bool isDarwin = Triple(M->getTargetTriple()).isOSDarwin(); + bool IsDarwin = Triple(M->getTargetTriple()).isOSDarwin(); if (DarwinGDBCompat == Default) { - if (isDarwin) - isDarwinGDBCompat = true; + if (IsDarwin) + IsDarwinGDBCompat = true; else - isDarwinGDBCompat = false; + IsDarwinGDBCompat = false; } else - isDarwinGDBCompat = DarwinGDBCompat == Enable ? true : false; + IsDarwinGDBCompat = DarwinGDBCompat == Enable ? true : false; if (DwarfAccelTables == Default) { - if (isDarwin) - hasDwarfAccelTables = true; + if (IsDarwin) + HasDwarfAccelTables = true; else - hasDwarfAccelTables = false; + HasDwarfAccelTables = false; } else - hasDwarfAccelTables = DwarfAccelTables == Enable ? true : false; + HasDwarfAccelTables = DwarfAccelTables == Enable ? true : false; + + if (SplitDwarf == Default) + HasSplitDwarf = false; + else + HasSplitDwarf = SplitDwarf == Enable ? true : false; { NamedRegionTimer T(DbgTimerName, DWARFGroupName, TimePassesIsEnabled); - beginModule(M); + beginModule(); } } DwarfDebug::~DwarfDebug() { } -/// EmitSectionSym - Switch to the specified MCSection and emit an assembler -/// temporary label to it if SymbolStem is specified. -static MCSymbol *EmitSectionSym(AsmPrinter *Asm, const MCSection *Section, +// Switch to the specified MCSection and emit an assembler +// temporary label to it if SymbolStem is specified. +static MCSymbol *emitSectionSym(AsmPrinter *Asm, const MCSection *Section, const char *SymbolStem = 0) { Asm->OutStreamer.SwitchSection(Section); if (!SymbolStem) return 0; @@ -194,44 +217,64 @@ static MCSymbol *EmitSectionSym(AsmPrinter *Asm, const MCSection *Section, return TmpSym; } -MCSymbol *DwarfDebug::getStringPool() { - return Asm->GetTempSymbol("section_str"); +MCSymbol *DwarfUnits::getStringPoolSym() { + return Asm->GetTempSymbol(StringPref); } -MCSymbol *DwarfDebug::getStringPoolEntry(StringRef Str) { - std::pair<MCSymbol*, unsigned> &Entry = StringPool[Str]; +MCSymbol *DwarfUnits::getStringPoolEntry(StringRef Str) { + std::pair<MCSymbol*, unsigned> &Entry = + StringPool.GetOrCreateValue(Str).getValue(); if (Entry.first) return Entry.first; Entry.second = NextStringPoolNumber++; - return Entry.first = Asm->GetTempSymbol("string", Entry.second); + return Entry.first = Asm->GetTempSymbol(StringPref, Entry.second); } -/// assignAbbrevNumber - Define a unique number for the abbreviation. -/// -void DwarfDebug::assignAbbrevNumber(DIEAbbrev &Abbrev) { +unsigned DwarfUnits::getStringPoolIndex(StringRef Str) { + std::pair<MCSymbol*, unsigned> &Entry = + StringPool.GetOrCreateValue(Str).getValue(); + if (Entry.first) return Entry.second; + + Entry.second = NextStringPoolNumber++; + Entry.first = Asm->GetTempSymbol(StringPref, Entry.second); + return Entry.second; +} + +unsigned DwarfUnits::getAddrPoolIndex(MCSymbol *Sym) { + std::pair<MCSymbol*, unsigned> &Entry = AddressPool[Sym]; + if (Entry.first) return Entry.second; + + Entry.second = NextAddrPoolNumber++; + Entry.first = Sym; + return Entry.second; +} + +// Define a unique number for the abbreviation. +// +void DwarfUnits::assignAbbrevNumber(DIEAbbrev &Abbrev) { // Profile the node so that we can make it unique. FoldingSetNodeID ID; Abbrev.Profile(ID); // Check the set for priors. - DIEAbbrev *InSet = AbbreviationsSet.GetOrInsertNode(&Abbrev); + DIEAbbrev *InSet = AbbreviationsSet->GetOrInsertNode(&Abbrev); // If it's newly added. if (InSet == &Abbrev) { // Add to abbreviation list. - Abbreviations.push_back(&Abbrev); + Abbreviations->push_back(&Abbrev); // Assign the vector position + 1 as its number. - Abbrev.setNumber(Abbreviations.size()); + Abbrev.setNumber(Abbreviations->size()); } else { // Assign existing abbreviation number. Abbrev.setNumber(InSet->getNumber()); } } -/// getRealLinkageName - If special LLVM prefix that is used to inform the asm -/// printer to not emit usual symbol prefix before the symbol name is used then -/// return linkage name after skipping this special LLVM prefix. +// If special LLVM prefix that is used to inform the asm +// printer to not emit usual symbol prefix before the symbol name is used then +// return linkage name after skipping this special LLVM prefix. static StringRef getRealLinkageName(StringRef LinkageName) { char One = '\1'; if (LinkageName.startswith(StringRef(&One, 1))) @@ -275,7 +318,7 @@ static StringRef getObjCMethodName(StringRef In) { static void addSubprogramNames(CompileUnit *TheCU, DISubprogram SP, DIE* Die) { if (!SP.isDefinition()) return; - + TheCU->addAccelName(SP.getName(), Die); // If the linkage name is different than the name, go ahead and output @@ -296,10 +339,9 @@ static void addSubprogramNames(CompileUnit *TheCU, DISubprogram SP, } } -/// updateSubprogramScopeDIE - Find DIE for the given subprogram and -/// attach appropriate DW_AT_low_pc and DW_AT_high_pc attributes. -/// If there are global variables in this scope then create and insert -/// DIEs for these variables. +// Find DIE for the given subprogram and attach appropriate DW_AT_low_pc +// and DW_AT_high_pc attributes. If there are global variables in this +// scope then create and insert DIEs for these variables. DIE *DwarfDebug::updateSubprogramScopeDIE(CompileUnit *SPCU, const MDNode *SPNode) { DIE *SPDie = SPCU->getDIE(SPNode); @@ -310,11 +352,16 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(CompileUnit *SPCU, // If we're updating an abstract DIE, then we will be adding the children and // object pointer later on. But what we don't want to do is process the // concrete DIE twice. - if (DIE *AbsSPDIE = AbstractSPDies.lookup(SPNode)) { + DIE *AbsSPDIE = AbstractSPDies.lookup(SPNode); + if (AbsSPDIE) { + bool InSameCU = (AbsSPDIE->getCompileUnit() == SPCU->getCUDie()); // Pick up abstract subprogram DIE. SPDie = new DIE(dwarf::DW_TAG_subprogram); + // If AbsSPDIE belongs to a different CU, use DW_FORM_ref_addr instead of + // DW_FORM_ref4. SPCU->addDIEEntry(SPDie, dwarf::DW_AT_abstract_origin, - dwarf::DW_FORM_ref4, AbsSPDIE); + InSameCU ? dwarf::DW_FORM_ref4 : dwarf::DW_FORM_ref_addr, + AbsSPDIE); SPCU->addDie(SPDie); } else { DISubprogram SPDecl = SP.getFunctionDeclaration(); @@ -347,17 +394,19 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(CompileUnit *SPCU, } DIE *SPDeclDie = SPDie; SPDie = new DIE(dwarf::DW_TAG_subprogram); - SPCU->addDIEEntry(SPDie, dwarf::DW_AT_specification, dwarf::DW_FORM_ref4, - SPDeclDie); + SPCU->addDIEEntry(SPDie, dwarf::DW_AT_specification, + dwarf::DW_FORM_ref4, SPDeclDie); SPCU->addDie(SPDie); } } } - SPCU->addLabel(SPDie, dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, - Asm->GetTempSymbol("func_begin", Asm->getFunctionNumber())); - SPCU->addLabel(SPDie, dwarf::DW_AT_high_pc, dwarf::DW_FORM_addr, - Asm->GetTempSymbol("func_end", Asm->getFunctionNumber())); + SPCU->addLabelAddress(SPDie, dwarf::DW_AT_low_pc, + Asm->GetTempSymbol("func_begin", + Asm->getFunctionNumber())); + SPCU->addLabelAddress(SPDie, dwarf::DW_AT_high_pc, + Asm->GetTempSymbol("func_end", + Asm->getFunctionNumber())); const TargetRegisterInfo *RI = Asm->TM.getRegisterInfo(); MachineLocation Location(RI->getFrameRegister(*Asm->MF)); SPCU->addAddress(SPDie, dwarf::DW_AT_frame_base, Location); @@ -365,13 +414,13 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(CompileUnit *SPCU, // Add name to the name table, we do this here because we're guaranteed // to have concrete versions of our DW_TAG_subprogram nodes. addSubprogramNames(SPCU, SP, SPDie); - + return SPDie; } -/// constructLexicalScope - Construct new DW_TAG_lexical_block -/// for this scope and attach DW_AT_low_pc/DW_AT_high_pc labels. -DIE *DwarfDebug::constructLexicalScopeDIE(CompileUnit *TheCU, +// Construct new DW_TAG_lexical_block for this scope and attach +// DW_AT_low_pc/DW_AT_high_pc labels. +DIE *DwarfDebug::constructLexicalScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) { DIE *ScopeDIE = new DIE(dwarf::DW_TAG_lexical_block); if (Scope->isAbstractScope()) @@ -387,7 +436,7 @@ DIE *DwarfDebug::constructLexicalScopeDIE(CompileUnit *TheCU, // .debug_range as a uint, size 4, for now. emitDIE will handle // DW_AT_ranges appropriately. TheCU->addUInt(ScopeDIE, dwarf::DW_AT_ranges, dwarf::DW_FORM_data4, - DebugRangeSymbols.size() + DebugRangeSymbols.size() * Asm->getDataLayout().getPointerSize()); for (SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(), RE = Ranges.end(); RI != RE; ++RI) { @@ -399,23 +448,22 @@ DIE *DwarfDebug::constructLexicalScopeDIE(CompileUnit *TheCU, return ScopeDIE; } - const MCSymbol *Start = getLabelBeforeInsn(RI->first); - const MCSymbol *End = getLabelAfterInsn(RI->second); + MCSymbol *Start = getLabelBeforeInsn(RI->first); + MCSymbol *End = getLabelAfterInsn(RI->second); if (End == 0) return 0; assert(Start->isDefined() && "Invalid starting label for an inlined scope!"); assert(End->isDefined() && "Invalid end label for an inlined scope!"); - TheCU->addLabel(ScopeDIE, dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, Start); - TheCU->addLabel(ScopeDIE, dwarf::DW_AT_high_pc, dwarf::DW_FORM_addr, End); + TheCU->addLabelAddress(ScopeDIE, dwarf::DW_AT_low_pc, Start); + TheCU->addLabelAddress(ScopeDIE, dwarf::DW_AT_high_pc, End); return ScopeDIE; } -/// constructInlinedScopeDIE - This scope represents inlined body of -/// a function. Construct DIE to represent this concrete inlined copy -/// of the function. +// This scope represents inlined body of a function. Construct DIE to +// represent this concrete inlined copy of the function. DIE *DwarfDebug::constructInlinedScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) { const SmallVector<InsnRange, 4> &Ranges = Scope->getRanges(); @@ -433,8 +481,8 @@ DIE *DwarfDebug::constructInlinedScopeDIE(CompileUnit *TheCU, } SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(); - const MCSymbol *StartLabel = getLabelBeforeInsn(RI->first); - const MCSymbol *EndLabel = getLabelAfterInsn(RI->second); + MCSymbol *StartLabel = getLabelBeforeInsn(RI->first); + MCSymbol *EndLabel = getLabelAfterInsn(RI->second); if (StartLabel == 0 || EndLabel == 0) { llvm_unreachable("Unexpected Start and End labels for an inlined scope!"); @@ -453,7 +501,7 @@ DIE *DwarfDebug::constructInlinedScopeDIE(CompileUnit *TheCU, // .debug_range as a uint, size 4, for now. emitDIE will handle // DW_AT_ranges appropriately. TheCU->addUInt(ScopeDIE, dwarf::DW_AT_ranges, dwarf::DW_FORM_data4, - DebugRangeSymbols.size() + DebugRangeSymbols.size() * Asm->getDataLayout().getPointerSize()); for (SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(), RE = Ranges.end(); RI != RE; ++RI) { @@ -463,10 +511,8 @@ DIE *DwarfDebug::constructInlinedScopeDIE(CompileUnit *TheCU, DebugRangeSymbols.push_back(NULL); DebugRangeSymbols.push_back(NULL); } else { - TheCU->addLabel(ScopeDIE, dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, - StartLabel); - TheCU->addLabel(ScopeDIE, dwarf::DW_AT_high_pc, dwarf::DW_FORM_addr, - EndLabel); + TheCU->addLabelAddress(ScopeDIE, dwarf::DW_AT_low_pc, StartLabel); + TheCU->addLabelAddress(ScopeDIE, dwarf::DW_AT_high_pc, EndLabel); } InlinedSubprogramDIEs.insert(OriginDIE); @@ -487,21 +533,28 @@ DIE *DwarfDebug::constructInlinedScopeDIE(CompileUnit *TheCU, DILocation DL(Scope->getInlinedAt()); TheCU->addUInt(ScopeDIE, dwarf::DW_AT_call_file, 0, - GetOrCreateSourceID(DL.getFilename(), DL.getDirectory())); + getOrCreateSourceID(DL.getFilename(), DL.getDirectory(), + TheCU->getUniqueID())); TheCU->addUInt(ScopeDIE, dwarf::DW_AT_call_line, 0, DL.getLineNumber()); // Add name to the name table, we do this here because we're guaranteed // to have concrete versions of our DW_TAG_inlined_subprogram nodes. addSubprogramNames(TheCU, InlinedSP, ScopeDIE); - + return ScopeDIE; } -/// constructScopeDIE - Construct a DIE for this scope. +// Construct a DIE for this scope. DIE *DwarfDebug::constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) { if (!Scope || !Scope->getScopeNode()) return NULL; + DIScope DS(Scope->getScopeNode()); + // Early return to avoid creating dangling variable|scope DIEs. + if (!Scope->getInlinedAt() && DS.isSubprogram() && Scope->isAbstractScope() && + !TheCU->getDIE(DS)) + return NULL; + SmallVector<DIE *, 8> Children; DIE *ObjectPointer = NULL; @@ -509,7 +562,7 @@ DIE *DwarfDebug::constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) { if (LScopes.isCurrentFunctionScope(Scope)) for (unsigned i = 0, N = CurrentFnArguments.size(); i < N; ++i) if (DbgVariable *ArgDV = CurrentFnArguments[i]) - if (DIE *Arg = + if (DIE *Arg = TheCU->constructVariableDIE(ArgDV, Scope->isAbstractScope())) { Children.push_back(Arg); if (ArgDV->isObjectPointer()) ObjectPointer = Arg; @@ -518,7 +571,7 @@ DIE *DwarfDebug::constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) { // Collect lexical scope children first. const SmallVector<DbgVariable *, 8> &Variables = ScopeVariables.lookup(Scope); for (unsigned i = 0, N = Variables.size(); i < N; ++i) - if (DIE *Variable = + if (DIE *Variable = TheCU->constructVariableDIE(Variables[i], Scope->isAbstractScope())) { Children.push_back(Variable); if (Variables[i]->isObjectPointer()) ObjectPointer = Variable; @@ -527,7 +580,6 @@ DIE *DwarfDebug::constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) { for (unsigned j = 0, M = Scopes.size(); j < M; ++j) if (DIE *Nested = constructScopeDIE(TheCU, Scopes[j])) Children.push_back(Nested); - DIScope DS(Scope->getScopeNode()); DIE *ScopeDIE = NULL; if (Scope->getInlinedAt()) ScopeDIE = constructInlinedScopeDIE(TheCU, Scope); @@ -548,7 +600,7 @@ DIE *DwarfDebug::constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) { return NULL; ScopeDIE = constructLexicalScopeDIE(TheCU, Scope); } - + if (!ScopeDIE) return NULL; // Add children @@ -566,24 +618,33 @@ DIE *DwarfDebug::constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) { return ScopeDIE; } -/// GetOrCreateSourceID - Look up the source id with the given directory and -/// source file names. If none currently exists, create a new id and insert it -/// in the SourceIds map. This can update DirectoryNames and SourceFileNames -/// maps as well. -unsigned DwarfDebug::GetOrCreateSourceID(StringRef FileName, - StringRef DirName) { +// Look up the source id with the given directory and source file names. +// If none currently exists, create a new id and insert it in the +// SourceIds map. This can update DirectoryNames and SourceFileNames maps +// as well. +unsigned DwarfDebug::getOrCreateSourceID(StringRef FileName, + StringRef DirName, unsigned CUID) { + // If we use .loc in assembly, we can't separate .file entries according to + // compile units. Thus all files will belong to the default compile unit. + if (Asm->TM.hasMCUseLoc() && + Asm->OutStreamer.getKind() == MCStreamer::SK_AsmStreamer) + CUID = 0; + // If FE did not provide a file name, then assume stdin. if (FileName.empty()) - return GetOrCreateSourceID("<stdin>", StringRef()); + return getOrCreateSourceID("<stdin>", StringRef(), CUID); // TODO: this might not belong here. See if we can factor this better. if (DirName == CompilationDir) DirName = ""; - unsigned SrcId = SourceIdMap.size()+1; + // FileIDCUMap stores the current ID for the given compile unit. + unsigned SrcId = FileIDCUMap[CUID] + 1; - // We look up the file/dir pair by concatenating them with a zero byte. + // We look up the CUID/file/dir by concatenating them with a zero byte. SmallString<128> NamePair; + NamePair += CUID; + NamePair += '\0'; NamePair += DirName; NamePair += '\0'; // Zero bytes are not allowed in paths. NamePair += FileName; @@ -592,37 +653,57 @@ unsigned DwarfDebug::GetOrCreateSourceID(StringRef FileName, if (Ent.getValue() != SrcId) return Ent.getValue(); + FileIDCUMap[CUID] = SrcId; // Print out a .file directive to specify files for .loc directives. - Asm->OutStreamer.EmitDwarfFileDirective(SrcId, DirName, FileName); + Asm->OutStreamer.EmitDwarfFileDirective(SrcId, DirName, FileName, CUID); return SrcId; } -/// constructCompileUnit - Create new CompileUnit for the given -/// metadata node with tag DW_TAG_compile_unit. +// Create new CompileUnit for the given metadata node with tag +// DW_TAG_compile_unit. CompileUnit *DwarfDebug::constructCompileUnit(const MDNode *N) { DICompileUnit DIUnit(N); StringRef FN = DIUnit.getFilename(); CompilationDir = DIUnit.getDirectory(); - unsigned ID = GetOrCreateSourceID(FN, CompilationDir); DIE *Die = new DIE(dwarf::DW_TAG_compile_unit); - CompileUnit *NewCU = new CompileUnit(ID, DIUnit.getLanguage(), Die, - Asm, this); + CompileUnit *NewCU = new CompileUnit(GlobalCUIndexCount++, + DIUnit.getLanguage(), Die, Asm, + this, &InfoHolder); + + FileIDCUMap[NewCU->getUniqueID()] = 0; + // Call this to emit a .file directive if it wasn't emitted for the source + // file this CU comes from yet. + getOrCreateSourceID(FN, CompilationDir, NewCU->getUniqueID()); + NewCU->addString(Die, dwarf::DW_AT_producer, DIUnit.getProducer()); NewCU->addUInt(Die, dwarf::DW_AT_language, dwarf::DW_FORM_data2, DIUnit.getLanguage()); NewCU->addString(Die, dwarf::DW_AT_name, FN); // 2.17.1 requires that we use DW_AT_low_pc for a single entry point - // into an entity. - NewCU->addUInt(Die, dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, 0); + // into an entity. We're using 0 (or a NULL label) for this. + NewCU->addLabelAddress(Die, dwarf::DW_AT_low_pc, NULL); + + // Define start line table label for each Compile Unit. + MCSymbol *LineTableStartSym = Asm->GetTempSymbol("line_table_start", + NewCU->getUniqueID()); + Asm->OutStreamer.getContext().setMCLineTableSymbol(LineTableStartSym, + NewCU->getUniqueID()); + // DW_AT_stmt_list is a offset of line number information for this // compile unit in debug_line section. + // The line table entries are not always emitted in assembly, so it + // is not okay to use line_table_start here. if (Asm->MAI->doesDwarfUseRelocationsAcrossSections()) NewCU->addLabel(Die, dwarf::DW_AT_stmt_list, dwarf::DW_FORM_data4, - Asm->GetTempSymbol("section_line")); - else + NewCU->getUniqueID() == 0 ? + Asm->GetTempSymbol("section_line") : LineTableStartSym); + else if (NewCU->getUniqueID() == 0) NewCU->addUInt(Die, dwarf::DW_AT_stmt_list, dwarf::DW_FORM_data4, 0); + else + NewCU->addDelta(Die, dwarf::DW_AT_stmt_list, dwarf::DW_FORM_data4, + LineTableStartSym, DwarfLineSectionSym); if (!CompilationDir.empty()) NewCU->addString(Die, dwarf::DW_AT_comp_dir, CompilationDir); @@ -632,19 +713,22 @@ CompileUnit *DwarfDebug::constructCompileUnit(const MDNode *N) { StringRef Flags = DIUnit.getFlags(); if (!Flags.empty()) NewCU->addString(Die, dwarf::DW_AT_APPLE_flags, Flags); - + if (unsigned RVer = DIUnit.getRunTimeVersion()) NewCU->addUInt(Die, dwarf::DW_AT_APPLE_major_runtime_vers, dwarf::DW_FORM_data1, RVer); if (!FirstCU) FirstCU = NewCU; + + InfoHolder.addUnit(NewCU); + CUMap.insert(std::make_pair(N, NewCU)); return NewCU; } -/// construct SubprogramDIE - Construct subprogram DIE. -void DwarfDebug::constructSubprogramDIE(CompileUnit *TheCU, +// Construct subprogram DIE. +void DwarfDebug::constructSubprogramDIE(CompileUnit *TheCU, const MDNode *N) { CompileUnit *&CURef = SPMap[N]; if (CURef) @@ -665,133 +749,83 @@ void DwarfDebug::constructSubprogramDIE(CompileUnit *TheCU, // Add to context owner. TheCU->addToContextOwner(SubprogramDie, SP.getContext()); - return; -} - -/// collectInfoFromNamedMDNodes - Collect debug info from named mdnodes such -/// as llvm.dbg.enum and llvm.dbg.ty -void DwarfDebug::collectInfoFromNamedMDNodes(Module *M) { - if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.sp")) - for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { - const MDNode *N = NMD->getOperand(i); - if (CompileUnit *CU = CUMap.lookup(DISubprogram(N).getCompileUnit())) - constructSubprogramDIE(CU, N); - } - - if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.gv")) - for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { - const MDNode *N = NMD->getOperand(i); - if (CompileUnit *CU = CUMap.lookup(DIGlobalVariable(N).getCompileUnit())) - CU->createGlobalVariableDIE(N); - } - - if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.enum")) - for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { - DIType Ty(NMD->getOperand(i)); - if (CompileUnit *CU = CUMap.lookup(Ty.getCompileUnit())) - CU->getOrCreateTypeDIE(Ty); - } - - if (NamedMDNode *NMD = M->getNamedMetadata("llvm.dbg.ty")) - for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { - DIType Ty(NMD->getOperand(i)); - if (CompileUnit *CU = CUMap.lookup(Ty.getCompileUnit())) - CU->getOrCreateTypeDIE(Ty); - } -} - -/// collectLegacyDebugInfo - Collect debug info using DebugInfoFinder. -/// FIXME - Remove this when dragon-egg and llvm-gcc switch to DIBuilder. -bool DwarfDebug::collectLegacyDebugInfo(Module *M) { - DebugInfoFinder DbgFinder; - DbgFinder.processModule(*M); - - bool HasDebugInfo = false; - // Scan all the compile-units to see if there are any marked as the main - // unit. If not, we do not generate debug info. - for (DebugInfoFinder::iterator I = DbgFinder.compile_unit_begin(), - E = DbgFinder.compile_unit_end(); I != E; ++I) { - if (DICompileUnit(*I).isMain()) { - HasDebugInfo = true; - break; - } - } - if (!HasDebugInfo) return false; - - // Create all the compile unit DIEs. - for (DebugInfoFinder::iterator I = DbgFinder.compile_unit_begin(), - E = DbgFinder.compile_unit_end(); I != E; ++I) - constructCompileUnit(*I); - - // Create DIEs for each global variable. - for (DebugInfoFinder::iterator I = DbgFinder.global_variable_begin(), - E = DbgFinder.global_variable_end(); I != E; ++I) { - const MDNode *N = *I; - if (CompileUnit *CU = CUMap.lookup(DIGlobalVariable(N).getCompileUnit())) - CU->createGlobalVariableDIE(N); - } - - // Create DIEs for each subprogram. - for (DebugInfoFinder::iterator I = DbgFinder.subprogram_begin(), - E = DbgFinder.subprogram_end(); I != E; ++I) { - const MDNode *N = *I; - if (CompileUnit *CU = CUMap.lookup(DISubprogram(N).getCompileUnit())) - constructSubprogramDIE(CU, N); - } - - return HasDebugInfo; + // Expose as global, if requested. + if (GenerateDwarfPubNamesSection) + TheCU->addGlobalName(SP.getName(), SubprogramDie); } -/// beginModule - Emit all Dwarf sections that should come prior to the -/// content. Create global DIEs and emit initial debug info sections. -/// This is invoked by the target AsmPrinter. -void DwarfDebug::beginModule(Module *M) { +// Emit all Dwarf sections that should come prior to the content. Create +// global DIEs and emit initial debug info sections. This is invoked by +// the target AsmPrinter. +void DwarfDebug::beginModule() { if (DisableDebugInfoPrinting) return; + const Module *M = MMI->getModule(); + // If module has named metadata anchors then use them, otherwise scan the // module using debug info finder to collect debug info. NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu"); - if (CU_Nodes) { - for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) { - DICompileUnit CUNode(CU_Nodes->getOperand(i)); - CompileUnit *CU = constructCompileUnit(CUNode); - DIArray GVs = CUNode.getGlobalVariables(); - for (unsigned i = 0, e = GVs.getNumElements(); i != e; ++i) - CU->createGlobalVariableDIE(GVs.getElement(i)); - DIArray SPs = CUNode.getSubprograms(); - for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i) - constructSubprogramDIE(CU, SPs.getElement(i)); - DIArray EnumTypes = CUNode.getEnumTypes(); - for (unsigned i = 0, e = EnumTypes.getNumElements(); i != e; ++i) - CU->getOrCreateTypeDIE(EnumTypes.getElement(i)); - DIArray RetainedTypes = CUNode.getRetainedTypes(); - for (unsigned i = 0, e = RetainedTypes.getNumElements(); i != e; ++i) - CU->getOrCreateTypeDIE(RetainedTypes.getElement(i)); - } - } else if (!collectLegacyDebugInfo(M)) + if (!CU_Nodes) return; - collectInfoFromNamedMDNodes(M); - + // Emit initial sections so we can reference labels later. + emitSectionLabels(); + + for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) { + DICompileUnit CUNode(CU_Nodes->getOperand(i)); + CompileUnit *CU = constructCompileUnit(CUNode); + DIArray GVs = CUNode.getGlobalVariables(); + for (unsigned i = 0, e = GVs.getNumElements(); i != e; ++i) + CU->createGlobalVariableDIE(GVs.getElement(i)); + DIArray SPs = CUNode.getSubprograms(); + for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i) + constructSubprogramDIE(CU, SPs.getElement(i)); + DIArray EnumTypes = CUNode.getEnumTypes(); + for (unsigned i = 0, e = EnumTypes.getNumElements(); i != e; ++i) + CU->getOrCreateTypeDIE(EnumTypes.getElement(i)); + DIArray RetainedTypes = CUNode.getRetainedTypes(); + for (unsigned i = 0, e = RetainedTypes.getNumElements(); i != e; ++i) + CU->getOrCreateTypeDIE(RetainedTypes.getElement(i)); + // If we're splitting the dwarf out now that we've got the entire + // CU then construct a skeleton CU based upon it. + if (useSplitDwarf()) { + // This should be a unique identifier when we want to build .dwp files. + CU->addUInt(CU->getCUDie(), dwarf::DW_AT_GNU_dwo_id, dwarf::DW_FORM_data8, 0); + // Now construct the skeleton CU associated. + constructSkeletonCU(CUNode); + } + } + // Tell MMI that we have debug info. MMI->setDebugInfoAvailability(true); - - // Emit initial sections. - EmitSectionLabels(); // Prime section data. SectionMap.insert(Asm->getObjFileLowering().getTextSection()); } -/// endModule - Emit all Dwarf sections that should come after the content. -/// -void DwarfDebug::endModule() { - if (!FirstCU) return; +// Attach DW_AT_inline attribute with inlined subprogram DIEs. +void DwarfDebug::computeInlinedDIEs() { + // Attach DW_AT_inline attribute with inlined subprogram DIEs. + for (SmallPtrSet<DIE *, 4>::iterator AI = InlinedSubprogramDIEs.begin(), + AE = InlinedSubprogramDIEs.end(); AI != AE; ++AI) { + DIE *ISP = *AI; + FirstCU->addUInt(ISP, dwarf::DW_AT_inline, 0, dwarf::DW_INL_inlined); + } + for (DenseMap<const MDNode *, DIE *>::iterator AI = AbstractSPDies.begin(), + AE = AbstractSPDies.end(); AI != AE; ++AI) { + DIE *ISP = AI->second; + if (InlinedSubprogramDIEs.count(ISP)) + continue; + FirstCU->addUInt(ISP, dwarf::DW_AT_inline, 0, dwarf::DW_INL_inlined); + } +} + +// Collect info for variables that were optimized out. +void DwarfDebug::collectDeadVariables() { const Module *M = MMI->getModule(); DenseMap<const MDNode *, LexicalScope *> DeadFnScopeMap; - // Collect info for variables that were optimized out. if (NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu")) { for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) { DICompileUnit TheCU(CU_Nodes->getOperand(i)); @@ -804,7 +838,7 @@ void DwarfDebug::endModule() { DIArray Variables = SP.getVariables(); if (Variables.getNumElements() == 0) continue; - LexicalScope *Scope = + LexicalScope *Scope = new LexicalScope(NULL, DIDescriptor(SP), NULL, false); DeadFnScopeMap[SP] = Scope; @@ -817,27 +851,22 @@ void DwarfDebug::endModule() { DIVariable DV(Variables.getElement(vi)); if (!DV.Verify()) continue; DbgVariable *NewVar = new DbgVariable(DV, NULL); - if (DIE *VariableDIE = + if (DIE *VariableDIE = SPCU->constructVariableDIE(NewVar, Scope->isAbstractScope())) ScopeDIE->addChild(VariableDIE); } } } } + DeleteContainerSeconds(DeadFnScopeMap); +} + +void DwarfDebug::finalizeModuleInfo() { + // Collect info for variables that were optimized out. + collectDeadVariables(); // Attach DW_AT_inline attribute with inlined subprogram DIEs. - for (SmallPtrSet<DIE *, 4>::iterator AI = InlinedSubprogramDIEs.begin(), - AE = InlinedSubprogramDIEs.end(); AI != AE; ++AI) { - DIE *ISP = *AI; - FirstCU->addUInt(ISP, dwarf::DW_AT_inline, 0, dwarf::DW_INL_inlined); - } - for (DenseMap<const MDNode *, DIE *>::iterator AI = AbstractSPDies.begin(), - AE = AbstractSPDies.end(); AI != AE; ++AI) { - DIE *ISP = AI->second; - if (InlinedSubprogramDIEs.count(ISP)) - continue; - FirstCU->addUInt(ISP, dwarf::DW_AT_inline, 0, dwarf::DW_INL_inlined); - } + computeInlinedDIEs(); // Emit DW_AT_containing_type attribute to connect types with their // vtable holding type. @@ -847,6 +876,13 @@ void DwarfDebug::endModule() { TheCU->constructContainingTypeDIEs(); } + // Compute DIE offsets and sizes. + InfoHolder.computeSizeAndOffsets(); + if (useSplitDwarf()) + SkeletonHolder.computeSizeAndOffsets(); +} + +void DwarfDebug::endSections() { // Standard sections final addresses. Asm->OutStreamer.SwitchSection(Asm->getObjFileLowering().getTextSection()); Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("text_end")); @@ -858,15 +894,79 @@ void DwarfDebug::endModule() { Asm->OutStreamer.SwitchSection(SectionMap[I]); Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("section_end", I+1)); } +} + +// Emit all Dwarf sections that should come after the content. +void DwarfDebug::endModule() { - // Compute DIE offsets and sizes. - computeSizeAndOffsets(); + if (!FirstCU) return; + + // End any existing sections. + // TODO: Does this need to happen? + endSections(); + + // Finalize the debug info for the module. + finalizeModuleInfo(); + + if (!useSplitDwarf()) { + // Emit all the DIEs into a debug info section. + emitDebugInfo(); + + // Corresponding abbreviations into a abbrev section. + emitAbbreviations(); - // Emit all the DIEs into a debug info section - emitDebugInfo(); + // Emit info into a debug loc section. + emitDebugLoc(); + + // Emit info into a debug aranges section. + emitDebugARanges(); + + // Emit info into a debug ranges section. + emitDebugRanges(); + + // Emit info into a debug macinfo section. + emitDebugMacInfo(); + + // Emit inline info. + // TODO: When we don't need the option anymore we + // can remove all of the code that this section + // depends upon. + if (useDarwinGDBCompat()) + emitDebugInlineInfo(); + } else { + // TODO: Fill this in for separated debug sections and separate + // out information into new sections. - // Corresponding abbreviations into a abbrev section. - emitAbbreviations(); + // Emit the debug info section and compile units. + emitDebugInfo(); + emitDebugInfoDWO(); + + // Corresponding abbreviations into a abbrev section. + emitAbbreviations(); + emitDebugAbbrevDWO(); + + // Emit info into a debug loc section. + emitDebugLoc(); + + // Emit info into a debug aranges section. + emitDebugARanges(); + + // Emit info into a debug ranges section. + emitDebugRanges(); + + // Emit info into a debug macinfo section. + emitDebugMacInfo(); + + // Emit DWO addresses. + InfoHolder.emitAddresses(Asm->getObjFileLowering().getDwarfAddrSection()); + + // Emit inline info. + // TODO: When we don't need the option anymore we + // can remove all of the code that this section + // depends upon. + if (useDarwinGDBCompat()) + emitDebugInlineInfo(); + } // Emit info into the dwarf accelerator table sections. if (useDwarfAccelTables()) { @@ -875,45 +975,37 @@ void DwarfDebug::endModule() { emitAccelNamespaces(); emitAccelTypes(); } - + + // Emit info into a debug pubnames section, if requested. + if (GenerateDwarfPubNamesSection) + emitDebugPubnames(); + // Emit info into a debug pubtypes section. // TODO: When we don't need the option anymore we can // remove all of the code that adds to the table. if (useDarwinGDBCompat()) emitDebugPubTypes(); - // Emit info into a debug loc section. - emitDebugLoc(); - - // Emit info into a debug aranges section. - EmitDebugARanges(); - - // Emit info into a debug ranges section. - emitDebugRanges(); - - // Emit info into a debug macinfo section. - emitDebugMacInfo(); - - // Emit inline info. - // TODO: When we don't need the option anymore we - // can remove all of the code that this section - // depends upon. - if (useDarwinGDBCompat()) - emitDebugInlineInfo(); - - // Emit info into a debug str section. + // Finally emit string information into a string table. emitDebugStr(); + if (useSplitDwarf()) + emitDebugStrDWO(); // clean up. - DeleteContainerSeconds(DeadFnScopeMap); SPMap.clear(); for (DenseMap<const MDNode *, CompileUnit *>::iterator I = CUMap.begin(), E = CUMap.end(); I != E; ++I) delete I->second; - FirstCU = NULL; // Reset for the next Module, if any. + + for (SmallVector<CompileUnit *, 1>::iterator I = SkeletonCUs.begin(), + E = SkeletonCUs.end(); I != E; ++I) + delete *I; + + // Reset these for the next Module if we have one. + FirstCU = NULL; } -/// findAbstractVariable - Find abstract variable, if any, associated with Var. +// Find abstract variable, if any, associated with Var. DbgVariable *DwarfDebug::findAbstractVariable(DIVariable &DV, DebugLoc ScopeLoc) { LLVMContext &Ctx = DV->getContext(); @@ -933,8 +1025,7 @@ DbgVariable *DwarfDebug::findAbstractVariable(DIVariable &DV, return AbsDbgVariable; } -/// addCurrentFnArgument - If Var is a current function argument then add -/// it to CurrentFnArguments list. +// If Var is a current function argument then add it to CurrentFnArguments list. bool DwarfDebug::addCurrentFnArgument(const MachineFunction *MF, DbgVariable *Var, LexicalScope *Scope) { if (!LScopes.isCurrentFunctionScope(Scope)) @@ -943,7 +1034,7 @@ bool DwarfDebug::addCurrentFnArgument(const MachineFunction *MF, if (DV.getTag() != dwarf::DW_TAG_arg_variable) return false; unsigned ArgNo = DV.getArgNumber(); - if (ArgNo == 0) + if (ArgNo == 0) return false; size_t Size = CurrentFnArguments.size(); @@ -957,8 +1048,7 @@ bool DwarfDebug::addCurrentFnArgument(const MachineFunction *MF, return true; } -/// collectVariableInfoFromMMITable - Collect variable information from -/// side table maintained by MMI. +// Collect variable information from side table maintained by MMI. void DwarfDebug::collectVariableInfoFromMMITable(const MachineFunction *MF, SmallPtrSet<const MDNode *, 16> &Processed) { @@ -987,8 +1077,8 @@ DwarfDebug::collectVariableInfoFromMMITable(const MachineFunction *MF, } } -/// isDbgValueInDefinedReg - Return true if debug value, encoded by -/// DBG_VALUE instruction, is in a defined reg. +// Return true if debug value, encoded by DBG_VALUE instruction, is in a +// defined reg. static bool isDbgValueInDefinedReg(const MachineInstr *MI) { assert(MI->isDebugValue() && "Invalid DBG_VALUE machine instruction!"); return MI->getNumOperands() == 3 && @@ -996,10 +1086,9 @@ static bool isDbgValueInDefinedReg(const MachineInstr *MI) { MI->getOperand(1).isImm() && MI->getOperand(1).getImm() == 0; } -/// getDebugLocEntry - Get .debug_loc entry for the instruction range starting -/// at MI. -static DotDebugLocEntry getDebugLocEntry(AsmPrinter *Asm, - const MCSymbol *FLabel, +// Get .debug_loc entry for the instruction range starting at MI. +static DotDebugLocEntry getDebugLocEntry(AsmPrinter *Asm, + const MCSymbol *FLabel, const MCSymbol *SLabel, const MachineInstr *MI) { const MDNode *Var = MI->getOperand(MI->getNumOperands() - 1).getMetadata(); @@ -1023,12 +1112,12 @@ static DotDebugLocEntry getDebugLocEntry(AsmPrinter *Asm, llvm_unreachable("Unexpected 3 operand DBG_VALUE instruction!"); } -/// collectVariableInfo - Find variables for each lexical scope. +// Find variables for each lexical scope. void DwarfDebug::collectVariableInfo(const MachineFunction *MF, SmallPtrSet<const MDNode *, 16> &Processed) { - /// collection info from MMI table. + // collection info from MMI table. collectVariableInfoFromMMITable(MF, Processed); for (SmallVectorImpl<const MDNode*>::const_iterator @@ -1050,16 +1139,10 @@ DwarfDebug::collectVariableInfo(const MachineFunction *MF, if (DV.getTag() == dwarf::DW_TAG_arg_variable && DISubprogram(DV.getContext()).describes(MF->getFunction())) Scope = LScopes.getCurrentFunctionScope(); - else { - if (DV.getVersion() <= LLVMDebugVersion9) - Scope = LScopes.findLexicalScope(MInsn->getDebugLoc()); - else { - if (MDNode *IA = DV.getInlinedAt()) - Scope = LScopes.findInlinedScope(DebugLoc::getFromDILocation(IA)); - else - Scope = LScopes.findLexicalScope(cast<MDNode>(DV->getOperand(1))); - } - } + else if (MDNode *IA = DV.getInlinedAt()) + Scope = LScopes.findInlinedScope(DebugLoc::getFromDILocation(IA)); + else + Scope = LScopes.findLexicalScope(cast<MDNode>(DV->getOperand(1))); // If variable scope is not found then skip this variable. if (!Scope) continue; @@ -1080,7 +1163,7 @@ DwarfDebug::collectVariableInfo(const MachineFunction *MF, continue; } - // handle multiple DBG_VALUE instructions describing one variable. + // Handle multiple DBG_VALUE instructions describing one variable. RegVar->setDotDebugLocOffset(DotDebugLocEntries.size()); for (SmallVectorImpl<const MachineInstr*>::const_iterator @@ -1103,7 +1186,7 @@ DwarfDebug::collectVariableInfo(const MachineFunction *MF, SLabel = FunctionEndSym; else { const MachineInstr *End = HI[1]; - DEBUG(dbgs() << "DotDebugLoc Pair:\n" + DEBUG(dbgs() << "DotDebugLoc Pair:\n" << "\t" << *Begin << "\t" << *End << "\n"); if (End->isDebugValue()) SLabel = getLabelBeforeInsn(End); @@ -1134,19 +1217,19 @@ DwarfDebug::collectVariableInfo(const MachineFunction *MF, } } -/// getLabelBeforeInsn - Return Label preceding the instruction. -const MCSymbol *DwarfDebug::getLabelBeforeInsn(const MachineInstr *MI) { +// Return Label preceding the instruction. +MCSymbol *DwarfDebug::getLabelBeforeInsn(const MachineInstr *MI) { MCSymbol *Label = LabelsBeforeInsn.lookup(MI); assert(Label && "Didn't insert label before instruction"); return Label; } -/// getLabelAfterInsn - Return Label immediately following the instruction. -const MCSymbol *DwarfDebug::getLabelAfterInsn(const MachineInstr *MI) { +// Return Label immediately following the instruction. +MCSymbol *DwarfDebug::getLabelAfterInsn(const MachineInstr *MI) { return LabelsAfterInsn.lookup(MI); } -/// beginInstruction - Process beginning of an instruction. +// Process beginning of an instruction. void DwarfDebug::beginInstruction(const MachineInstr *MI) { // Check if source location changes, but ignore DBG_VALUE locations. if (!MI->isDebugValue()) { @@ -1188,7 +1271,7 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) { I->second = PrevLabel; } -/// endInstruction - Process end of an instruction. +// Process end of an instruction. void DwarfDebug::endInstruction(const MachineInstr *MI) { // Don't create a new label after DBG_VALUE instructions. // They don't generate code. @@ -1214,11 +1297,10 @@ void DwarfDebug::endInstruction(const MachineInstr *MI) { I->second = PrevLabel; } -/// identifyScopeMarkers() - -/// Each LexicalScope has first instruction and last instruction to mark -/// beginning and end of a scope respectively. Create an inverse map that list -/// scopes starts (and ends) with an instruction. One instruction may start (or -/// end) multiple scopes. Ignore scopes that are not reachable. +// Each LexicalScope has first instruction and last instruction to mark +// beginning and end of a scope respectively. Create an inverse map that list +// scopes starts (and ends) with an instruction. One instruction may start (or +// end) multiple scopes. Ignore scopes that are not reachable. void DwarfDebug::identifyScopeMarkers() { SmallVector<LexicalScope *, 4> WorkList; WorkList.push_back(LScopes.getCurrentFunctionScope()); @@ -1247,15 +1329,15 @@ void DwarfDebug::identifyScopeMarkers() { } } -/// getScopeNode - Get MDNode for DebugLoc's scope. +// Get MDNode for DebugLoc's scope. static MDNode *getScopeNode(DebugLoc DL, const LLVMContext &Ctx) { if (MDNode *InlinedAt = DL.getInlinedAt(Ctx)) return getScopeNode(DebugLoc::getFromDILocation(InlinedAt), Ctx); return DL.getScope(Ctx); } -/// getFnDebugLoc - Walk up the scope chain of given debug loc and find -/// line number info for the function. +// Walk up the scope chain of given debug loc and find line number info +// for the function. static DebugLoc getFnDebugLoc(DebugLoc DL, const LLVMContext &Ctx) { const MDNode *Scope = getScopeNode(DL, Ctx); DISubprogram SP = getDISubprogram(Scope); @@ -1271,14 +1353,21 @@ static DebugLoc getFnDebugLoc(DebugLoc DL, const LLVMContext &Ctx) { return DebugLoc(); } -/// beginFunction - Gather pre-function debug information. Assumes being -/// emitted immediately after the function entry point. +// Gather pre-function debug information. Assumes being called immediately +// after the function entry point has been emitted. void DwarfDebug::beginFunction(const MachineFunction *MF) { if (!MMI->hasDebugInfo()) return; LScopes.initialize(*MF); if (LScopes.empty()) return; identifyScopeMarkers(); + // Set DwarfCompileUnitID in MCContext to the Compile Unit this function + // belongs to. + LexicalScope *FnScope = LScopes.getCurrentFunctionScope(); + CompileUnit *TheCU = SPMap.lookup(FnScope->getScopeNode()); + assert(TheCU && "Unable to find compile unit!"); + Asm->OutStreamer.getContext().setDwarfCompileUnitID(TheCU->getUniqueID()); + FunctionBeginSym = Asm->GetTempSymbol("func_begin", Asm->getFunctionNumber()); // Assumes in correct section after the entry point. @@ -1287,7 +1376,7 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { assert(UserVariables.empty() && DbgValues.empty() && "Maps weren't cleaned"); const TargetRegisterInfo *TRI = Asm->TM.getRegisterInfo(); - /// LiveUserVar - Map physreg numbers to the MDNode they contain. + // LiveUserVar - Map physreg numbers to the MDNode they contain. std::vector<const MDNode*> LiveUserVar(TRI->getNumRegs()); for (MachineFunction::const_iterator I = MF->begin(), E = MF->end(); @@ -1327,7 +1416,7 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { if (History.size() >= 2 && Prev->isIdenticalTo(History[History.size() - 2])) { DEBUG(dbgs() << "Coalescing identical DBG_VALUE entries:\n" - << "\t" << *Prev + << "\t" << *Prev << "\t" << *History[History.size() - 2] << "\n"); History.pop_back(); } @@ -1413,7 +1502,7 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { const MachineInstr *Prev = History.back(); if (Prev->isDebugValue() && isDbgValueInDefinedReg(Prev)) { const MachineBasicBlock *PrevMBB = Prev->getParent(); - MachineBasicBlock::const_iterator LastMI = + MachineBasicBlock::const_iterator LastMI = PrevMBB->getLastNonDebugInstr(); if (LastMI == PrevMBB->end()) // Drop DBG_VALUE for empty range. @@ -1442,7 +1531,9 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) { MF->getFunction()->getContext()); recordSourceLine(FnStartDL.getLine(), FnStartDL.getCol(), FnStartDL.getScope(MF->getFunction()->getContext()), - 0); + // We'd like to list the prologue as "not statements" but GDB behaves + // poorly if we do that. Revisit this with caution/GDB (7.5+) testing. + DWARF2_FLAG_IS_STMT); } } @@ -1452,8 +1543,7 @@ void DwarfDebug::addScopeVariable(LexicalScope *LS, DbgVariable *Var) { // Vars.push_back(Var); } -/// endFunction - Gather and emit post-function debug information. -/// +// Gather and emit post-function debug information. void DwarfDebug::endFunction(const MachineFunction *MF) { if (!MMI->hasDebugInfo() || LScopes.empty()) return; @@ -1462,10 +1552,12 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { Asm->getFunctionNumber()); // Assumes in correct section after the entry point. Asm->OutStreamer.EmitLabel(FunctionEndSym); - + // Set DwarfCompileUnitID in MCContext to default value. + Asm->OutStreamer.getContext().setDwarfCompileUnitID(0); + SmallPtrSet<const MDNode *, 16> ProcessedVars; collectVariableInfo(MF, ProcessedVars); - + LexicalScope *FnScope = LScopes.getCurrentFunctionScope(); CompileUnit *TheCU = SPMap.lookup(FnScope->getScopeNode()); assert(TheCU && "Unable to find compile unit!"); @@ -1495,9 +1587,9 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { if (ProcessedSPNodes.count(AScope->getScopeNode()) == 0) constructScopeDIE(TheCU, AScope); } - + DIE *CurFnDIE = constructScopeDIE(TheCU, FnScope); - + if (!MF->getTarget().Options.DisableFramePointerElim(*MF)) TheCU->addFlag(CurFnDIE, dwarf::DW_AT_APPLE_omit_frame_ptr); @@ -1518,9 +1610,8 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { PrevLabel = NULL; } -/// recordSourceLine - Register a source line with debug info. Returns the -/// unique label that was emitted and which provides correspondence to -/// the source line list. +// Register a source line with debug info. Returns the unique label that was +// emitted and which provides correspondence to the source line list. void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S, unsigned Flags) { StringRef Fn; @@ -1552,7 +1643,8 @@ void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S, } else llvm_unreachable("Unexpected scope info"); - Src = GetOrCreateSourceID(Fn, Dir); + Src = getOrCreateSourceID(Fn, Dir, + Asm->OutStreamer.getContext().getDwarfCompileUnitID()); } Asm->OutStreamer.EmitDwarfLocDirective(Src, Line, Col, Flags, 0, 0, Fn); } @@ -1561,10 +1653,9 @@ void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S, // Emit Methods //===----------------------------------------------------------------------===// -/// computeSizeAndOffset - Compute the size and offset of a DIE. -/// +// Compute the size and offset of a DIE. unsigned -DwarfDebug::computeSizeAndOffset(DIE *Die, unsigned Offset, bool Last) { +DwarfUnits::computeSizeAndOffset(DIE *Die, unsigned Offset) { // Get the children. const std::vector<DIE *> &Children = Die->getChildren(); @@ -1573,7 +1664,7 @@ DwarfDebug::computeSizeAndOffset(DIE *Die, unsigned Offset, bool Last) { // Get the abbreviation for this DIE. unsigned AbbrevNumber = Die->getAbbrevNumber(); - const DIEAbbrev *Abbrev = Abbreviations[AbbrevNumber - 1]; + const DIEAbbrev *Abbrev = Abbreviations->at(AbbrevNumber - 1); // Set DIE offset Die->setOffset(Offset); @@ -1581,8 +1672,8 @@ DwarfDebug::computeSizeAndOffset(DIE *Die, unsigned Offset, bool Last) { // Start the size with the size of abbreviation code. Offset += MCAsmInfo::getULEB128Size(AbbrevNumber); - const SmallVector<DIEValue*, 32> &Values = Die->getValues(); - const SmallVector<DIEAbbrevData, 8> &AbbrevData = Abbrev->getData(); + const SmallVectorImpl<DIEValue*> &Values = Die->getValues(); + const SmallVectorImpl<DIEAbbrevData> &AbbrevData = Abbrev->getData(); // Size the DIE attribute values. for (unsigned i = 0, N = Values.size(); i < N; ++i) @@ -1595,7 +1686,7 @@ DwarfDebug::computeSizeAndOffset(DIE *Die, unsigned Offset, bool Last) { "Children flag not set"); for (unsigned j = 0, M = Children.size(); j < M; ++j) - Offset = computeSizeAndOffset(Children[j], Offset, (j + 1) == M); + Offset = computeSizeAndOffset(Children[j], Offset); // End of children marker. Offset += sizeof(int8_t); @@ -1605,57 +1696,68 @@ DwarfDebug::computeSizeAndOffset(DIE *Die, unsigned Offset, bool Last) { return Offset; } -/// computeSizeAndOffsets - Compute the size and offset of all the DIEs. -/// -void DwarfDebug::computeSizeAndOffsets() { - for (DenseMap<const MDNode *, CompileUnit *>::iterator I = CUMap.begin(), - E = CUMap.end(); I != E; ++I) { - // Compute size of compile unit header. - unsigned Offset = +// Compute the size and offset of all the DIEs. +void DwarfUnits::computeSizeAndOffsets() { + // Offset from the beginning of debug info section. + unsigned AccuOffset = 0; + for (SmallVectorImpl<CompileUnit *>::iterator I = CUs.begin(), + E = CUs.end(); I != E; ++I) { + (*I)->setDebugInfoOffset(AccuOffset); + unsigned Offset = sizeof(int32_t) + // Length of Compilation Unit Info sizeof(int16_t) + // DWARF version number sizeof(int32_t) + // Offset Into Abbrev. Section sizeof(int8_t); // Pointer Size (in bytes) - computeSizeAndOffset(I->second->getCUDie(), Offset, true); + + unsigned EndOffset = computeSizeAndOffset((*I)->getCUDie(), Offset); + AccuOffset += EndOffset; } } -/// EmitSectionLabels - Emit initial Dwarf sections with a label at -/// the start of each one. -void DwarfDebug::EmitSectionLabels() { +// Emit initial Dwarf sections with a label at the start of each one. +void DwarfDebug::emitSectionLabels() { const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering(); // Dwarf sections base addresses. DwarfInfoSectionSym = - EmitSectionSym(Asm, TLOF.getDwarfInfoSection(), "section_info"); + emitSectionSym(Asm, TLOF.getDwarfInfoSection(), "section_info"); DwarfAbbrevSectionSym = - EmitSectionSym(Asm, TLOF.getDwarfAbbrevSection(), "section_abbrev"); - EmitSectionSym(Asm, TLOF.getDwarfARangesSection()); + emitSectionSym(Asm, TLOF.getDwarfAbbrevSection(), "section_abbrev"); + if (useSplitDwarf()) + DwarfAbbrevDWOSectionSym = + emitSectionSym(Asm, TLOF.getDwarfAbbrevDWOSection(), + "section_abbrev_dwo"); + emitSectionSym(Asm, TLOF.getDwarfARangesSection()); if (const MCSection *MacroInfo = TLOF.getDwarfMacroInfoSection()) - EmitSectionSym(Asm, MacroInfo); - - EmitSectionSym(Asm, TLOF.getDwarfLineSection(), "section_line"); - EmitSectionSym(Asm, TLOF.getDwarfLocSection()); - EmitSectionSym(Asm, TLOF.getDwarfPubTypesSection()); + emitSectionSym(Asm, MacroInfo); + + DwarfLineSectionSym = + emitSectionSym(Asm, TLOF.getDwarfLineSection(), "section_line"); + emitSectionSym(Asm, TLOF.getDwarfLocSection()); + if (GenerateDwarfPubNamesSection) + emitSectionSym(Asm, TLOF.getDwarfPubNamesSection()); + emitSectionSym(Asm, TLOF.getDwarfPubTypesSection()); DwarfStrSectionSym = - EmitSectionSym(Asm, TLOF.getDwarfStrSection(), "section_str"); - DwarfDebugRangeSectionSym = EmitSectionSym(Asm, TLOF.getDwarfRangesSection(), + emitSectionSym(Asm, TLOF.getDwarfStrSection(), "info_string"); + if (useSplitDwarf()) + DwarfStrDWOSectionSym = + emitSectionSym(Asm, TLOF.getDwarfStrDWOSection(), "skel_string"); + DwarfDebugRangeSectionSym = emitSectionSym(Asm, TLOF.getDwarfRangesSection(), "debug_range"); - DwarfDebugLocSectionSym = EmitSectionSym(Asm, TLOF.getDwarfLocSection(), + DwarfDebugLocSectionSym = emitSectionSym(Asm, TLOF.getDwarfLocSection(), "section_debug_loc"); - TextSectionSym = EmitSectionSym(Asm, TLOF.getTextSection(), "text_begin"); - EmitSectionSym(Asm, TLOF.getDataSection()); + TextSectionSym = emitSectionSym(Asm, TLOF.getTextSection(), "text_begin"); + emitSectionSym(Asm, TLOF.getDataSection()); } -/// emitDIE - Recursively emits a debug information entry. -/// -void DwarfDebug::emitDIE(DIE *Die) { +// Recursively emits a debug information entry. +void DwarfDebug::emitDIE(DIE *Die, std::vector<DIEAbbrev *> *Abbrevs) { // Get the abbreviation for this DIE. unsigned AbbrevNumber = Die->getAbbrevNumber(); - const DIEAbbrev *Abbrev = Abbreviations[AbbrevNumber - 1]; + const DIEAbbrev *Abbrev = Abbrevs->at(AbbrevNumber - 1); // Emit the code (index) for the abbreviation. if (Asm->isVerbose()) @@ -1665,8 +1767,8 @@ void DwarfDebug::emitDIE(DIE *Die) { dwarf::TagString(Abbrev->getTag())); Asm->EmitULEB128(AbbrevNumber); - const SmallVector<DIEValue*, 32> &Values = Die->getValues(); - const SmallVector<DIEAbbrevData, 8> &AbbrevData = Abbrev->getData(); + const SmallVectorImpl<DIEValue*> &Values = Die->getValues(); + const SmallVectorImpl<DIEAbbrevData> &AbbrevData = Abbrev->getData(); // Emit the DIE attribute values. for (unsigned i = 0, N = Values.size(); i < N; ++i) { @@ -1682,6 +1784,13 @@ void DwarfDebug::emitDIE(DIE *Die) { DIEEntry *E = cast<DIEEntry>(Values[i]); DIE *Origin = E->getEntry(); unsigned Addr = Origin->getOffset(); + if (Form == dwarf::DW_FORM_ref_addr) { + // For DW_FORM_ref_addr, output the offset from beginning of debug info + // section. Origin->getOffset() returns the offset from start of the + // compile unit. + DwarfUnits &Holder = useSplitDwarf() ? SkeletonHolder : InfoHolder; + Addr += Holder.getCUOffset(Origin->getCompileUnit()); + } Asm->EmitInt32(Addr); break; } @@ -1732,7 +1841,7 @@ void DwarfDebug::emitDIE(DIE *Die) { const std::vector<DIE *> &Children = Die->getChildren(); for (unsigned j = 0, M = Children.size(); j < M; ++j) - emitDIE(Children[j]); + emitDIE(Children[j], Abbrevs); if (Asm->isVerbose()) Asm->OutStreamer.AddComment("End Of Children Mark"); @@ -1740,20 +1849,22 @@ void DwarfDebug::emitDIE(DIE *Die) { } } -/// emitDebugInfo - Emit the debug info section. -/// -void DwarfDebug::emitDebugInfo() { - // Start debug info section. - Asm->OutStreamer.SwitchSection( - Asm->getObjFileLowering().getDwarfInfoSection()); - for (DenseMap<const MDNode *, CompileUnit *>::iterator I = CUMap.begin(), - E = CUMap.end(); I != E; ++I) { - CompileUnit *TheCU = I->second; +// Emit the various dwarf units to the unit section USection with +// the abbreviations going into ASection. +void DwarfUnits::emitUnits(DwarfDebug *DD, + const MCSection *USection, + const MCSection *ASection, + const MCSymbol *ASectionSym) { + Asm->OutStreamer.SwitchSection(USection); + for (SmallVectorImpl<CompileUnit *>::iterator I = CUs.begin(), + E = CUs.end(); I != E; ++I) { + CompileUnit *TheCU = *I; DIE *Die = TheCU->getCUDie(); // Emit the compile units header. - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("info_begin", - TheCU->getID())); + Asm->OutStreamer + .EmitLabel(Asm->GetTempSymbol(USection->getLabelBeginName(), + TheCU->getUniqueID())); // Emit size of content not including length itself unsigned ContentSize = Die->getSize() + @@ -1766,31 +1877,62 @@ void DwarfDebug::emitDebugInfo() { Asm->OutStreamer.AddComment("DWARF version number"); Asm->EmitInt16(dwarf::DWARF_VERSION); Asm->OutStreamer.AddComment("Offset Into Abbrev. Section"); - Asm->EmitSectionOffset(Asm->GetTempSymbol("abbrev_begin"), - DwarfAbbrevSectionSym); + Asm->EmitSectionOffset(Asm->GetTempSymbol(ASection->getLabelBeginName()), + ASectionSym); Asm->OutStreamer.AddComment("Address Size (in bytes)"); Asm->EmitInt8(Asm->getDataLayout().getPointerSize()); - emitDIE(Die); - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("info_end", TheCU->getID())); + DD->emitDIE(Die, Abbreviations); + Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol(USection->getLabelEndName(), + TheCU->getUniqueID())); } } -/// emitAbbreviations - Emit the abbreviation section. -/// -void DwarfDebug::emitAbbreviations() const { +/// For a given compile unit DIE, returns offset from beginning of debug info. +unsigned DwarfUnits::getCUOffset(DIE *Die) { + assert(Die->getTag() == dwarf::DW_TAG_compile_unit && + "Input DIE should be compile unit in getCUOffset."); + for (SmallVectorImpl<CompileUnit *>::iterator I = CUs.begin(), + E = CUs.end(); I != E; ++I) { + CompileUnit *TheCU = *I; + if (TheCU->getCUDie() == Die) + return TheCU->getDebugInfoOffset(); + } + llvm_unreachable("The compile unit DIE should belong to CUs in DwarfUnits."); +} + +// Emit the debug info section. +void DwarfDebug::emitDebugInfo() { + DwarfUnits &Holder = useSplitDwarf() ? SkeletonHolder : InfoHolder; + + Holder.emitUnits(this, Asm->getObjFileLowering().getDwarfInfoSection(), + Asm->getObjFileLowering().getDwarfAbbrevSection(), + DwarfAbbrevSectionSym); +} + +// Emit the abbreviation section. +void DwarfDebug::emitAbbreviations() { + if (!useSplitDwarf()) + emitAbbrevs(Asm->getObjFileLowering().getDwarfAbbrevSection(), + &Abbreviations); + else + emitSkeletonAbbrevs(Asm->getObjFileLowering().getDwarfAbbrevSection()); +} + +void DwarfDebug::emitAbbrevs(const MCSection *Section, + std::vector<DIEAbbrev *> *Abbrevs) { // Check to see if it is worth the effort. - if (!Abbreviations.empty()) { + if (!Abbrevs->empty()) { // Start the debug abbrev section. - Asm->OutStreamer.SwitchSection( - Asm->getObjFileLowering().getDwarfAbbrevSection()); + Asm->OutStreamer.SwitchSection(Section); - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("abbrev_begin")); + MCSymbol *Begin = Asm->GetTempSymbol(Section->getLabelBeginName()); + Asm->OutStreamer.EmitLabel(Begin); // For each abbrevation. - for (unsigned i = 0, N = Abbreviations.size(); i < N; ++i) { + for (unsigned i = 0, N = Abbrevs->size(); i < N; ++i) { // Get abbreviation data - const DIEAbbrev *Abbrev = Abbreviations[i]; + const DIEAbbrev *Abbrev = Abbrevs->at(i); // Emit the abbrevations code (base 1 index.) Asm->EmitULEB128(Abbrev->getNumber(), "Abbreviation Code"); @@ -1802,13 +1944,12 @@ void DwarfDebug::emitAbbreviations() const { // Mark end of abbreviations. Asm->EmitULEB128(0, "EOM(3)"); - Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("abbrev_end")); + MCSymbol *End = Asm->GetTempSymbol(Section->getLabelEndName()); + Asm->OutStreamer.EmitLabel(End); } } -/// emitEndOfLineMatrix - Emit the last address of the section and the end of -/// the line matrix. -/// +// Emit the last address of the section and the end of the line matrix. void DwarfDebug::emitEndOfLineMatrix(unsigned SectionEnd) { // Define last address of section. Asm->OutStreamer.AddComment("Extended Op"); @@ -1822,8 +1963,7 @@ void DwarfDebug::emitEndOfLineMatrix(unsigned SectionEnd) { Asm->OutStreamer.AddComment("Section end label"); Asm->OutStreamer.EmitSymbolValue(Asm->GetTempSymbol("section_end",SectionEnd), - Asm->getDataLayout().getPointerSize(), - 0/*AddrSpace*/); + Asm->getDataLayout().getPointerSize()); // Mark end of matrix. Asm->OutStreamer.AddComment("DW_LNE_end_sequence"); @@ -1832,8 +1972,7 @@ void DwarfDebug::emitEndOfLineMatrix(unsigned SectionEnd) { Asm->EmitInt8(1); } -/// emitAccelNames - Emit visible names into a hashed accelerator table -/// section. +// Emit visible names into a hashed accelerator table section. void DwarfDebug::emitAccelNames() { DwarfAccelTable AT(DwarfAccelTable::Atom(DwarfAccelTable::eAtomTypeDIEOffset, dwarf::DW_FORM_data4)); @@ -1858,11 +1997,11 @@ void DwarfDebug::emitAccelNames() { Asm->OutStreamer.EmitLabel(SectionBegin); // Emit the full data. - AT.Emit(Asm, SectionBegin, this); + AT.Emit(Asm, SectionBegin, &InfoHolder); } -/// emitAccelObjC - Emit objective C classes and categories into a hashed -/// accelerator table section. +// Emit objective C classes and categories into a hashed accelerator table +// section. void DwarfDebug::emitAccelObjC() { DwarfAccelTable AT(DwarfAccelTable::Atom(DwarfAccelTable::eAtomTypeDIEOffset, dwarf::DW_FORM_data4)); @@ -1887,11 +2026,10 @@ void DwarfDebug::emitAccelObjC() { Asm->OutStreamer.EmitLabel(SectionBegin); // Emit the full data. - AT.Emit(Asm, SectionBegin, this); + AT.Emit(Asm, SectionBegin, &InfoHolder); } -/// emitAccelNamespace - Emit namespace dies into a hashed accelerator -/// table. +// Emit namespace dies into a hashed accelerator table. void DwarfDebug::emitAccelNamespaces() { DwarfAccelTable AT(DwarfAccelTable::Atom(DwarfAccelTable::eAtomTypeDIEOffset, dwarf::DW_FORM_data4)); @@ -1916,10 +2054,10 @@ void DwarfDebug::emitAccelNamespaces() { Asm->OutStreamer.EmitLabel(SectionBegin); // Emit the full data. - AT.Emit(Asm, SectionBegin, this); + AT.Emit(Asm, SectionBegin, &InfoHolder); } -/// emitAccelTypes() - Emit type dies into a hashed accelerator table. +// Emit type dies into a hashed accelerator table. void DwarfDebug::emitAccelTypes() { std::vector<DwarfAccelTable::Atom> Atoms; Atoms.push_back(DwarfAccelTable::Atom(DwarfAccelTable::eAtomTypeDIEOffset, @@ -1951,7 +2089,62 @@ void DwarfDebug::emitAccelTypes() { Asm->OutStreamer.EmitLabel(SectionBegin); // Emit the full data. - AT.Emit(Asm, SectionBegin, this); + AT.Emit(Asm, SectionBegin, &InfoHolder); +} + +/// emitDebugPubnames - Emit visible names into a debug pubnames section. +/// +void DwarfDebug::emitDebugPubnames() { + const MCSection *ISec = Asm->getObjFileLowering().getDwarfInfoSection(); + + typedef DenseMap<const MDNode*, CompileUnit*> CUMapType; + for (CUMapType::iterator I = CUMap.begin(), E = CUMap.end(); I != E; ++I) { + CompileUnit *TheCU = I->second; + unsigned ID = TheCU->getUniqueID(); + + if (TheCU->getGlobalNames().empty()) + continue; + + // Start the dwarf pubnames section. + Asm->OutStreamer.SwitchSection( + Asm->getObjFileLowering().getDwarfPubNamesSection()); + + Asm->OutStreamer.AddComment("Length of Public Names Info"); + Asm->EmitLabelDifference(Asm->GetTempSymbol("pubnames_end", ID), + Asm->GetTempSymbol("pubnames_begin", ID), 4); + + Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubnames_begin", ID)); + + Asm->OutStreamer.AddComment("DWARF Version"); + Asm->EmitInt16(dwarf::DWARF_VERSION); + + Asm->OutStreamer.AddComment("Offset of Compilation Unit Info"); + Asm->EmitSectionOffset(Asm->GetTempSymbol(ISec->getLabelBeginName(), ID), + DwarfInfoSectionSym); + + Asm->OutStreamer.AddComment("Compilation Unit Length"); + Asm->EmitLabelDifference(Asm->GetTempSymbol(ISec->getLabelEndName(), ID), + Asm->GetTempSymbol(ISec->getLabelBeginName(), ID), + 4); + + const StringMap<DIE*> &Globals = TheCU->getGlobalNames(); + for (StringMap<DIE*>::const_iterator + GI = Globals.begin(), GE = Globals.end(); GI != GE; ++GI) { + const char *Name = GI->getKeyData(); + const DIE *Entity = GI->second; + + Asm->OutStreamer.AddComment("DIE offset"); + Asm->EmitInt32(Entity->getOffset()); + + if (Asm->isVerbose()) + Asm->OutStreamer.AddComment("External Name"); + Asm->OutStreamer.EmitBytes(StringRef(Name, strlen(Name)+1), 0); + } + + Asm->OutStreamer.AddComment("End Mark"); + Asm->EmitInt32(0); + Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubnames_end", ID)); + } } void DwarfDebug::emitDebugPubTypes() { @@ -1963,22 +2156,26 @@ void DwarfDebug::emitDebugPubTypes() { Asm->getObjFileLowering().getDwarfPubTypesSection()); Asm->OutStreamer.AddComment("Length of Public Types Info"); Asm->EmitLabelDifference( - Asm->GetTempSymbol("pubtypes_end", TheCU->getID()), - Asm->GetTempSymbol("pubtypes_begin", TheCU->getID()), 4); + Asm->GetTempSymbol("pubtypes_end", TheCU->getUniqueID()), + Asm->GetTempSymbol("pubtypes_begin", TheCU->getUniqueID()), 4); Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubtypes_begin", - TheCU->getID())); + TheCU->getUniqueID())); if (Asm->isVerbose()) Asm->OutStreamer.AddComment("DWARF Version"); Asm->EmitInt16(dwarf::DWARF_VERSION); Asm->OutStreamer.AddComment("Offset of Compilation Unit Info"); - Asm->EmitSectionOffset(Asm->GetTempSymbol("info_begin", TheCU->getID()), + const MCSection *ISec = Asm->getObjFileLowering().getDwarfInfoSection(); + Asm->EmitSectionOffset(Asm->GetTempSymbol(ISec->getLabelBeginName(), + TheCU->getUniqueID()), DwarfInfoSectionSym); Asm->OutStreamer.AddComment("Compilation Unit Length"); - Asm->EmitLabelDifference(Asm->GetTempSymbol("info_end", TheCU->getID()), - Asm->GetTempSymbol("info_begin", TheCU->getID()), + Asm->EmitLabelDifference(Asm->GetTempSymbol(ISec->getLabelEndName(), + TheCU->getUniqueID()), + Asm->GetTempSymbol(ISec->getLabelBeginName(), + TheCU->getUniqueID()), 4); const StringMap<DIE*> &Globals = TheCU->getGlobalTypes(); @@ -1992,33 +2189,34 @@ void DwarfDebug::emitDebugPubTypes() { if (Asm->isVerbose()) Asm->OutStreamer.AddComment("External Name"); // Emit the name with a terminating null byte. - Asm->OutStreamer.EmitBytes(StringRef(Name, GI->getKeyLength()+1), 0); + Asm->OutStreamer.EmitBytes(StringRef(Name, GI->getKeyLength()+1)); } Asm->OutStreamer.AddComment("End Mark"); Asm->EmitInt32(0); Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("pubtypes_end", - TheCU->getID())); + TheCU->getUniqueID())); } } -/// emitDebugStr - Emit visible names into a debug str section. -/// -void DwarfDebug::emitDebugStr() { - // Check to see if it is worth the effort. +// Emit strings into a string section. +void DwarfUnits::emitStrings(const MCSection *StrSection, + const MCSection *OffsetSection = NULL, + const MCSymbol *StrSecSym = NULL) { + if (StringPool.empty()) return; // Start the dwarf str section. - Asm->OutStreamer.SwitchSection( - Asm->getObjFileLowering().getDwarfStrSection()); + Asm->OutStreamer.SwitchSection(StrSection); // Get all of the string pool entries and put them in an array by their ID so // we can sort them. SmallVector<std::pair<unsigned, - StringMapEntry<std::pair<MCSymbol*, unsigned> >*>, 64> Entries; + StringMapEntry<std::pair<MCSymbol*, unsigned> >*>, 64> Entries; for (StringMap<std::pair<MCSymbol*, unsigned> >::iterator - I = StringPool.begin(), E = StringPool.end(); I != E; ++I) + I = StringPool.begin(), E = StringPool.end(); + I != E; ++I) Entries.push_back(std::make_pair(I->second.second, &*I)); array_pod_sort(Entries.begin(), Entries.end()); @@ -2029,18 +2227,65 @@ void DwarfDebug::emitDebugStr() { // Emit the string itself with a terminating null byte. Asm->OutStreamer.EmitBytes(StringRef(Entries[i].second->getKeyData(), - Entries[i].second->getKeyLength()+1), - 0/*addrspace*/); + Entries[i].second->getKeyLength()+1)); + } + + // If we've got an offset section go ahead and emit that now as well. + if (OffsetSection) { + Asm->OutStreamer.SwitchSection(OffsetSection); + unsigned offset = 0; + unsigned size = 4; // FIXME: DWARF64 is 8. + for (unsigned i = 0, e = Entries.size(); i != e; ++i) { + Asm->OutStreamer.EmitIntValue(offset, size); + offset += Entries[i].second->getKeyLength() + 1; + } } } -/// emitDebugLoc - Emit visible names into a debug loc section. -/// +// Emit strings into a string section. +void DwarfUnits::emitAddresses(const MCSection *AddrSection) { + + if (AddressPool.empty()) return; + + // Start the dwarf addr section. + Asm->OutStreamer.SwitchSection(AddrSection); + + // Get all of the string pool entries and put them in an array by their ID so + // we can sort them. + SmallVector<std::pair<unsigned, + std::pair<MCSymbol*, unsigned>* >, 64> Entries; + + for (DenseMap<MCSymbol*, std::pair<MCSymbol*, unsigned> >::iterator + I = AddressPool.begin(), E = AddressPool.end(); + I != E; ++I) + Entries.push_back(std::make_pair(I->second.second, &(I->second))); + + array_pod_sort(Entries.begin(), Entries.end()); + + for (unsigned i = 0, e = Entries.size(); i != e; ++i) { + // Emit a label for reference from debug information entries. + MCSymbol *Sym = Entries[i].second->first; + if (Sym) + Asm->EmitLabelReference(Entries[i].second->first, + Asm->getDataLayout().getPointerSize()); + else + Asm->OutStreamer.EmitIntValue(0, Asm->getDataLayout().getPointerSize()); + } + +} + +// Emit visible names into a debug str section. +void DwarfDebug::emitDebugStr() { + DwarfUnits &Holder = useSplitDwarf() ? SkeletonHolder : InfoHolder; + Holder.emitStrings(Asm->getObjFileLowering().getDwarfStrSection()); +} + +// Emit visible names into a debug loc section. void DwarfDebug::emitDebugLoc() { if (DotDebugLocEntries.empty()) return; - for (SmallVector<DotDebugLocEntry, 4>::iterator + for (SmallVectorImpl<DotDebugLocEntry>::iterator I = DotDebugLocEntries.begin(), E = DotDebugLocEntries.end(); I != E; ++I) { DotDebugLocEntry &Entry = *I; @@ -2054,18 +2299,18 @@ void DwarfDebug::emitDebugLoc() { unsigned char Size = Asm->getDataLayout().getPointerSize(); Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("debug_loc", 0)); unsigned index = 1; - for (SmallVector<DotDebugLocEntry, 4>::iterator + for (SmallVectorImpl<DotDebugLocEntry>::iterator I = DotDebugLocEntries.begin(), E = DotDebugLocEntries.end(); I != E; ++I, ++index) { DotDebugLocEntry &Entry = *I; if (Entry.isMerged()) continue; if (Entry.isEmpty()) { - Asm->OutStreamer.EmitIntValue(0, Size, /*addrspace*/0); - Asm->OutStreamer.EmitIntValue(0, Size, /*addrspace*/0); + Asm->OutStreamer.EmitIntValue(0, Size); + Asm->OutStreamer.EmitIntValue(0, Size); Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("debug_loc", index)); } else { - Asm->OutStreamer.EmitSymbolValue(Entry.Begin, Size, 0); - Asm->OutStreamer.EmitSymbolValue(Entry.End, Size, 0); + Asm->OutStreamer.EmitSymbolValue(Entry.Begin, Size); + Asm->OutStreamer.EmitSymbolValue(Entry.End, Size); DIVariable DV(Entry.Variable); Asm->OutStreamer.AddComment("Loc expr size"); MCSymbol *begin = Asm->OutStreamer.getContext().CreateTempSymbol(); @@ -2075,7 +2320,7 @@ void DwarfDebug::emitDebugLoc() { if (Entry.isInt()) { DIBasicType BTy(DV.getType()); if (BTy.Verify() && - (BTy.getEncoding() == dwarf::DW_ATE_signed + (BTy.getEncoding() == dwarf::DW_ATE_signed || BTy.getEncoding() == dwarf::DW_ATE_signed_char)) { Asm->OutStreamer.AddComment("DW_OP_consts"); Asm->EmitInt8(dwarf::DW_OP_consts); @@ -2086,7 +2331,7 @@ void DwarfDebug::emitDebugLoc() { Asm->EmitULEB128(Entry.getInt()); } } else if (Entry.isLocation()) { - if (!DV.hasComplexAddress()) + if (!DV.hasComplexAddress()) // Regular entry. Asm->EmitDwarfRegOp(Entry.Loc); else { @@ -2112,7 +2357,7 @@ void DwarfDebug::emitDebugLoc() { } else { Asm->EmitDwarfRegOp(Entry.Loc); } - + // Emit remaining complex address elements. for (; i < N; ++i) { uint64_t Element = DV.getAddrElement(i); @@ -2134,33 +2379,30 @@ void DwarfDebug::emitDebugLoc() { } } -/// EmitDebugARanges - Emit visible names into a debug aranges section. -/// -void DwarfDebug::EmitDebugARanges() { +// Emit visible names into a debug aranges section. +void DwarfDebug::emitDebugARanges() { // Start the dwarf aranges section. Asm->OutStreamer.SwitchSection( Asm->getObjFileLowering().getDwarfARangesSection()); } -/// emitDebugRanges - Emit visible names into a debug ranges section. -/// +// Emit visible names into a debug ranges section. void DwarfDebug::emitDebugRanges() { // Start the dwarf ranges section. Asm->OutStreamer.SwitchSection( Asm->getObjFileLowering().getDwarfRangesSection()); unsigned char Size = Asm->getDataLayout().getPointerSize(); - for (SmallVector<const MCSymbol *, 8>::iterator + for (SmallVectorImpl<const MCSymbol *>::iterator I = DebugRangeSymbols.begin(), E = DebugRangeSymbols.end(); I != E; ++I) { if (*I) - Asm->OutStreamer.EmitSymbolValue(const_cast<MCSymbol*>(*I), Size, 0); + Asm->OutStreamer.EmitSymbolValue(const_cast<MCSymbol*>(*I), Size); else - Asm->OutStreamer.EmitIntValue(0, Size, /*addrspace*/0); + Asm->OutStreamer.EmitIntValue(0, Size); } } -/// emitDebugMacInfo - Emit visible names into a debug macinfo section. -/// +// Emit visible names into a debug macinfo section. void DwarfDebug::emitDebugMacInfo() { if (const MCSection *LineInfo = Asm->getObjFileLowering().getDwarfMacroInfoSection()) { @@ -2169,24 +2411,24 @@ void DwarfDebug::emitDebugMacInfo() { } } -/// emitDebugInlineInfo - Emit inline info using following format. -/// Section Header: -/// 1. length of section -/// 2. Dwarf version number -/// 3. address size. -/// -/// Entries (one "entry" for each function that was inlined): -/// -/// 1. offset into __debug_str section for MIPS linkage name, if exists; -/// otherwise offset into __debug_str for regular function name. -/// 2. offset into __debug_str section for regular function name. -/// 3. an unsigned LEB128 number indicating the number of distinct inlining -/// instances for the function. -/// -/// The rest of the entry consists of a {die_offset, low_pc} pair for each -/// inlined instance; the die_offset points to the inlined_subroutine die in the -/// __debug_info section, and the low_pc is the starting address for the -/// inlining instance. +// Emit inline info using following format. +// Section Header: +// 1. length of section +// 2. Dwarf version number +// 3. address size. +// +// Entries (one "entry" for each function that was inlined): +// +// 1. offset into __debug_str section for MIPS linkage name, if exists; +// otherwise offset into __debug_str for regular function name. +// 2. offset into __debug_str section for regular function name. +// 3. an unsigned LEB128 number indicating the number of distinct inlining +// instances for the function. +// +// The rest of the entry consists of a {die_offset, low_pc} pair for each +// inlined instance; the die_offset points to the inlined_subroutine die in the +// __debug_info section, and the low_pc is the starting address for the +// inlining instance. void DwarfDebug::emitDebugInlineInfo() { if (!Asm->MAI->doesDwarfUseInlineInfoSection()) return; @@ -2208,38 +2450,121 @@ void DwarfDebug::emitDebugInlineInfo() { Asm->OutStreamer.AddComment("Address Size (in bytes)"); Asm->EmitInt8(Asm->getDataLayout().getPointerSize()); - for (SmallVector<const MDNode *, 4>::iterator I = InlinedSPNodes.begin(), + for (SmallVectorImpl<const MDNode *>::iterator I = InlinedSPNodes.begin(), E = InlinedSPNodes.end(); I != E; ++I) { const MDNode *Node = *I; DenseMap<const MDNode *, SmallVector<InlineInfoLabels, 4> >::iterator II = InlineInfo.find(Node); - SmallVector<InlineInfoLabels, 4> &Labels = II->second; + SmallVectorImpl<InlineInfoLabels> &Labels = II->second; DISubprogram SP(Node); StringRef LName = SP.getLinkageName(); StringRef Name = SP.getName(); Asm->OutStreamer.AddComment("MIPS linkage name"); if (LName.empty()) - Asm->EmitSectionOffset(getStringPoolEntry(Name), DwarfStrSectionSym); + Asm->EmitSectionOffset(InfoHolder.getStringPoolEntry(Name), + DwarfStrSectionSym); else - Asm->EmitSectionOffset(getStringPoolEntry(getRealLinkageName(LName)), + Asm->EmitSectionOffset(InfoHolder + .getStringPoolEntry(getRealLinkageName(LName)), DwarfStrSectionSym); Asm->OutStreamer.AddComment("Function name"); - Asm->EmitSectionOffset(getStringPoolEntry(Name), DwarfStrSectionSym); + Asm->EmitSectionOffset(InfoHolder.getStringPoolEntry(Name), + DwarfStrSectionSym); Asm->EmitULEB128(Labels.size(), "Inline count"); - for (SmallVector<InlineInfoLabels, 4>::iterator LI = Labels.begin(), + for (SmallVectorImpl<InlineInfoLabels>::iterator LI = Labels.begin(), LE = Labels.end(); LI != LE; ++LI) { if (Asm->isVerbose()) Asm->OutStreamer.AddComment("DIE offset"); Asm->EmitInt32(LI->second->getOffset()); if (Asm->isVerbose()) Asm->OutStreamer.AddComment("low_pc"); Asm->OutStreamer.EmitSymbolValue(LI->first, - Asm->getDataLayout().getPointerSize(),0); + Asm->getDataLayout().getPointerSize()); } } Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("debug_inlined_end", 1)); } + +// DWARF5 Experimental Separate Dwarf emitters. + +// This DIE has the following attributes: DW_AT_comp_dir, DW_AT_stmt_list, +// DW_AT_low_pc, DW_AT_high_pc, DW_AT_ranges, DW_AT_dwo_name, DW_AT_dwo_id, +// DW_AT_ranges_base, DW_AT_addr_base. If DW_AT_ranges is present, +// DW_AT_low_pc and DW_AT_high_pc are not used, and vice versa. +CompileUnit *DwarfDebug::constructSkeletonCU(const MDNode *N) { + DICompileUnit DIUnit(N); + CompilationDir = DIUnit.getDirectory(); + + DIE *Die = new DIE(dwarf::DW_TAG_compile_unit); + CompileUnit *NewCU = new CompileUnit(GlobalCUIndexCount++, + DIUnit.getLanguage(), Die, Asm, + this, &SkeletonHolder); + + NewCU->addLocalString(Die, dwarf::DW_AT_GNU_dwo_name, + DIUnit.getSplitDebugFilename()); + + // This should be a unique identifier when we want to build .dwp files. + NewCU->addUInt(Die, dwarf::DW_AT_GNU_dwo_id, dwarf::DW_FORM_data8, 0); + + // FIXME: The addr base should be relative for each compile unit, however, + // this one is going to be 0 anyhow. + NewCU->addUInt(Die, dwarf::DW_AT_GNU_addr_base, dwarf::DW_FORM_sec_offset, 0); + + // 2.17.1 requires that we use DW_AT_low_pc for a single entry point + // into an entity. We're using 0, or a NULL label for this. + NewCU->addUInt(Die, dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, 0); + + // DW_AT_stmt_list is a offset of line number information for this + // compile unit in debug_line section. + if (Asm->MAI->doesDwarfUseRelocationsAcrossSections()) + NewCU->addLabel(Die, dwarf::DW_AT_stmt_list, dwarf::DW_FORM_sec_offset, + DwarfLineSectionSym); + else + NewCU->addUInt(Die, dwarf::DW_AT_stmt_list, dwarf::DW_FORM_sec_offset, 0); + + if (!CompilationDir.empty()) + NewCU->addLocalString(Die, dwarf::DW_AT_comp_dir, CompilationDir); + + SkeletonHolder.addUnit(NewCU); + SkeletonCUs.push_back(NewCU); + + return NewCU; +} + +void DwarfDebug::emitSkeletonAbbrevs(const MCSection *Section) { + assert(useSplitDwarf() && "No split dwarf debug info?"); + emitAbbrevs(Section, &SkeletonAbbrevs); +} + +// Emit the .debug_info.dwo section for separated dwarf. This contains the +// compile units that would normally be in debug_info. +void DwarfDebug::emitDebugInfoDWO() { + assert(useSplitDwarf() && "No split dwarf debug info?"); + InfoHolder.emitUnits(this, Asm->getObjFileLowering().getDwarfInfoDWOSection(), + Asm->getObjFileLowering().getDwarfAbbrevDWOSection(), + DwarfAbbrevDWOSectionSym); +} + +// Emit the .debug_abbrev.dwo section for separated dwarf. This contains the +// abbreviations for the .debug_info.dwo section. +void DwarfDebug::emitDebugAbbrevDWO() { + assert(useSplitDwarf() && "No split dwarf?"); + emitAbbrevs(Asm->getObjFileLowering().getDwarfAbbrevDWOSection(), + &Abbreviations); +} + +// Emit the .debug_str.dwo section for separated dwarf. This contains the +// string section and is identical in format to traditional .debug_str +// sections. +void DwarfDebug::emitDebugStrDWO() { + assert(useSplitDwarf() && "No split dwarf?"); + const MCSection *OffSec = Asm->getObjFileLowering() + .getDwarfStrOffDWOSection(); + const MCSymbol *StrSym = DwarfStrSectionSym; + InfoHolder.emitStrings(Asm->getObjFileLowering().getDwarfStrDWOSection(), + OffSec, StrSym); +} diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h index 61d9a51..81e345e 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h @@ -15,15 +15,15 @@ #define CODEGEN_ASMPRINTER_DWARFDEBUG_H__ #include "DIE.h" -#include "llvm/DebugInfo.h" -#include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/CodeGen/LexicalScopes.h" -#include "llvm/MC/MachineLocation.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringMap.h" +#include "llvm/CodeGen/AsmPrinter.h" +#include "llvm/CodeGen/LexicalScopes.h" +#include "llvm/DebugInfo.h" +#include "llvm/MC/MachineLocation.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/DebugLoc.h" @@ -41,10 +41,10 @@ class DIEAbbrev; class DIE; class DIEBlock; class DIEEntry; +class DwarfDebug; //===----------------------------------------------------------------------===// -/// SrcLineInfo - This class is used to record source line correspondence. -/// +/// \brief This class is used to record source line correspondence. class SrcLineInfo { unsigned Line; // Source line number. unsigned Column; // Source column. @@ -61,8 +61,8 @@ public: MCSymbol *getLabel() const { return Label; } }; -/// DotDebugLocEntry - This struct describes location entries emitted in -/// .debug_loc section. +/// \brief This struct describes location entries emitted in the .debug_loc +/// section. typedef struct DotDebugLocEntry { const MCSymbol *Begin; const MCSymbol *End; @@ -83,25 +83,25 @@ typedef struct DotDebugLocEntry { const ConstantFP *CFP; const ConstantInt *CIP; } Constants; - DotDebugLocEntry() - : Begin(0), End(0), Variable(0), Merged(false), + DotDebugLocEntry() + : Begin(0), End(0), Variable(0), Merged(false), Constant(false) { Constants.Int = 0;} DotDebugLocEntry(const MCSymbol *B, const MCSymbol *E, MachineLocation &L, - const MDNode *V) - : Begin(B), End(E), Loc(L), Variable(V), Merged(false), + const MDNode *V) + : Begin(B), End(E), Loc(L), Variable(V), Merged(false), Constant(false) { Constants.Int = 0; EntryKind = E_Location; } DotDebugLocEntry(const MCSymbol *B, const MCSymbol *E, int64_t i) - : Begin(B), End(E), Variable(0), Merged(false), + : Begin(B), End(E), Variable(0), Merged(false), Constant(true) { Constants.Int = i; EntryKind = E_Integer; } DotDebugLocEntry(const MCSymbol *B, const MCSymbol *E, const ConstantFP *FPtr) - : Begin(B), End(E), Variable(0), Merged(false), + : Begin(B), End(E), Variable(0), Merged(false), Constant(true) { Constants.CFP = FPtr; EntryKind = E_ConstantFP; } DotDebugLocEntry(const MCSymbol *B, const MCSymbol *E, const ConstantInt *IPtr) - : Begin(B), End(E), Variable(0), Merged(false), + : Begin(B), End(E), Variable(0), Merged(false), Constant(true) { Constants.CIP = IPtr; EntryKind = E_ConstantInt; } - /// Empty entries are also used as a trigger to emit temp label. Such + /// \brief Empty entries are also used as a trigger to emit temp label. Such /// labels are referenced is used to find debug_loc offset for a given DIE. bool isEmpty() { return Begin == 0 && End == 0; } bool isMerged() { return Merged; } @@ -121,8 +121,7 @@ typedef struct DotDebugLocEntry { } DotDebugLocEntry; //===----------------------------------------------------------------------===// -/// DbgVariable - This class is used to track local variable information. -/// +/// \brief This class is used to track local variable information. class DbgVariable { DIVariable Var; // Variable Descriptor. DIE *TheDIE; // Variable DIE. @@ -132,7 +131,7 @@ class DbgVariable { int FrameIndex; public: // AbsVar may be NULL. - DbgVariable(DIVariable V, DbgVariable *AV) + DbgVariable(DIVariable V, DbgVariable *AV) : Var(V), TheDIE(0), DotDebugLocOffset(~0U), AbsVar(AV), MInsn(0), FrameIndex(~0) {} @@ -148,14 +147,14 @@ public: void setMInsn(const MachineInstr *M) { MInsn = M; } int getFrameIndex() const { return FrameIndex; } void setFrameIndex(int FI) { FrameIndex = FI; } - // Translate tag to proper Dwarf tag. - unsigned getTag() const { + // Translate tag to proper Dwarf tag. + unsigned getTag() const { if (Var.getTag() == dwarf::DW_TAG_arg_variable) return dwarf::DW_TAG_formal_parameter; - + return dwarf::DW_TAG_variable; } - /// isArtificial - Return true if DbgVariable is artificial. + /// \brief Return true if DbgVariable is artificial. bool isArtificial() const { if (Var.isArtificial()) return true; @@ -171,7 +170,7 @@ public: return true; return false; } - + bool variableHasComplexAddress() const { assert(Var.Verify() && "Invalid complex DbgVariable!"); return Var.hasComplexAddress(); @@ -180,7 +179,7 @@ public: assert(Var.Verify() && "Invalid complex DbgVariable!"); return Var.isBlockByrefVariable(); } - unsigned getNumAddrElements() const { + unsigned getNumAddrElements() const { assert(Var.Verify() && "Invalid complex DbgVariable!"); return Var.getNumAddrElements(); } @@ -190,108 +189,192 @@ public: DIType getType() const; }; + +// A String->Symbol mapping of strings used by indirect +// references. +typedef StringMap<std::pair<MCSymbol*, unsigned>, + BumpPtrAllocator&> StrPool; + +// A Symbol->pair<Symbol, unsigned> mapping of addresses used by indirect +// references. +typedef DenseMap<MCSymbol *, std::pair<MCSymbol *, unsigned> > AddrPool; + +/// \brief Collects and handles information specific to a particular +/// collection of units. +class DwarfUnits { + // Target of Dwarf emission, used for sizing of abbreviations. + AsmPrinter *Asm; + + // Used to uniquely define abbreviations. + FoldingSet<DIEAbbrev> *AbbreviationsSet; + + // A list of all the unique abbreviations in use. + std::vector<DIEAbbrev *> *Abbreviations; + + // A pointer to all units in the section. + SmallVector<CompileUnit *, 1> CUs; + + // Collection of strings for this unit and assorted symbols. + StrPool StringPool; + unsigned NextStringPoolNumber; + std::string StringPref; + + // Collection of addresses for this unit and assorted labels. + AddrPool AddressPool; + unsigned NextAddrPoolNumber; + +public: + DwarfUnits(AsmPrinter *AP, FoldingSet<DIEAbbrev> *AS, + std::vector<DIEAbbrev *> *A, const char *Pref, + BumpPtrAllocator &DA) : + Asm(AP), AbbreviationsSet(AS), Abbreviations(A), + StringPool(DA), NextStringPoolNumber(0), StringPref(Pref), + AddressPool(), NextAddrPoolNumber(0) {} + + /// \brief Compute the size and offset of a DIE given an incoming Offset. + unsigned computeSizeAndOffset(DIE *Die, unsigned Offset); + + /// \brief Compute the size and offset of all the DIEs. + void computeSizeAndOffsets(); + + /// \brief Define a unique number for the abbreviation. + void assignAbbrevNumber(DIEAbbrev &Abbrev); + + /// \brief Add a unit to the list of CUs. + void addUnit(CompileUnit *CU) { CUs.push_back(CU); } + + /// \brief Emit all of the units to the section listed with the given + /// abbreviation section. + void emitUnits(DwarfDebug *, const MCSection *, const MCSection *, + const MCSymbol *); + + /// \brief Emit all of the strings to the section given. + void emitStrings(const MCSection *, const MCSection *, const MCSymbol *); + + /// \brief Emit all of the addresses to the section given. + void emitAddresses(const MCSection *); + + /// \brief Returns the entry into the start of the pool. + MCSymbol *getStringPoolSym(); + + /// \brief Returns an entry into the string pool with the given + /// string text. + MCSymbol *getStringPoolEntry(StringRef Str); + + /// \brief Returns the index into the string pool with the given + /// string text. + unsigned getStringPoolIndex(StringRef Str); + + /// \brief Returns the string pool. + StrPool *getStringPool() { return &StringPool; } + + /// \brief Returns the index into the address pool with the given + /// label/symbol. + unsigned getAddrPoolIndex(MCSymbol *); + + /// \brief Returns the address pool. + AddrPool *getAddrPool() { return &AddressPool; } + + /// \brief for a given compile unit DIE, returns offset from beginning of + /// debug info. + unsigned getCUOffset(DIE *Die); +}; + +/// \brief Collects and handles dwarf debug information. class DwarfDebug { - /// Asm - Target of Dwarf emission. + // Target of Dwarf emission. AsmPrinter *Asm; - /// MMI - Collected machine module information. + // Collected machine module information. MachineModuleInfo *MMI; - /// DIEValueAllocator - All DIEValues are allocated through this allocator. + // All DIEValues are allocated through this allocator. BumpPtrAllocator DIEValueAllocator; //===--------------------------------------------------------------------===// - // Attributes used to construct specific Dwarf sections. + // Attribute used to construct specific Dwarf sections. // CompileUnit *FirstCU; - /// Maps MDNode with its corresponding CompileUnit. + // Maps MDNode with its corresponding CompileUnit. DenseMap <const MDNode *, CompileUnit *> CUMap; - /// Maps subprogram MDNode with its corresponding CompileUnit. + // Maps subprogram MDNode with its corresponding CompileUnit. DenseMap <const MDNode *, CompileUnit *> SPMap; - /// AbbreviationsSet - Used to uniquely define abbreviations. - /// + // Used to uniquely define abbreviations. FoldingSet<DIEAbbrev> AbbreviationsSet; - /// Abbreviations - A list of all the unique abbreviations in use. - /// + // A list of all the unique abbreviations in use. std::vector<DIEAbbrev *> Abbreviations; - /// SourceIdMap - Source id map, i.e. pair of source filename and directory, - /// separated by a zero byte, mapped to a unique id. + // Stores the current file ID for a given compile unit. + DenseMap <unsigned, unsigned> FileIDCUMap; + // Source id map, i.e. CUID, source filename and directory, + // separated by a zero byte, mapped to a unique id. StringMap<unsigned, BumpPtrAllocator&> SourceIdMap; - /// StringPool - A String->Symbol mapping of strings used by indirect - /// references. - StringMap<std::pair<MCSymbol*, unsigned>, BumpPtrAllocator&> StringPool; - unsigned NextStringPoolNumber; - - /// SectionMap - Provides a unique id per text section. - /// + // Provides a unique id per text section. SetVector<const MCSection*> SectionMap; - /// CurrentFnArguments - List of Arguments (DbgValues) for current function. + // List of Arguments (DbgValues) for current function. SmallVector<DbgVariable *, 8> CurrentFnArguments; LexicalScopes LScopes; - /// AbstractSPDies - Collection of abstract subprogram DIEs. + // Collection of abstract subprogram DIEs. DenseMap<const MDNode *, DIE *> AbstractSPDies; - /// ScopeVariables - Collection of dbg variables of a scope. + // Collection of dbg variables of a scope. DenseMap<LexicalScope *, SmallVector<DbgVariable *, 8> > ScopeVariables; - /// AbstractVariables - Collection of abstract variables. + // Collection of abstract variables. DenseMap<const MDNode *, DbgVariable *> AbstractVariables; - /// DotDebugLocEntries - Collection of DotDebugLocEntry. + // Collection of DotDebugLocEntry. SmallVector<DotDebugLocEntry, 4> DotDebugLocEntries; - /// InlinedSubprogramDIEs - Collection of subprogram DIEs that are marked - /// (at the end of the module) as DW_AT_inline. + // Collection of subprogram DIEs that are marked (at the end of the module) + // as DW_AT_inline. SmallPtrSet<DIE *, 4> InlinedSubprogramDIEs; - /// InlineInfo - Keep track of inlined functions and their location. This - /// information is used to populate the debug_inlined section. + // Keep track of inlined functions and their location. This + // information is used to populate the debug_inlined section. typedef std::pair<const MCSymbol *, DIE *> InlineInfoLabels; DenseMap<const MDNode *, SmallVector<InlineInfoLabels, 4> > InlineInfo; SmallVector<const MDNode *, 4> InlinedSPNodes; - // ProcessedSPNodes - This is a collection of subprogram MDNodes that - // are processed to create DIEs. + // This is a collection of subprogram MDNodes that are processed to + // create DIEs. SmallPtrSet<const MDNode *, 16> ProcessedSPNodes; - /// LabelsBeforeInsn - Maps instruction with label emitted before - /// instruction. + // Maps instruction with label emitted before instruction. DenseMap<const MachineInstr *, MCSymbol *> LabelsBeforeInsn; - /// LabelsAfterInsn - Maps instruction with label emitted after - /// instruction. + // Maps instruction with label emitted after instruction. DenseMap<const MachineInstr *, MCSymbol *> LabelsAfterInsn; - /// UserVariables - Every user variable mentioned by a DBG_VALUE instruction - /// in order of appearance. + // Every user variable mentioned by a DBG_VALUE instruction in order of + // appearance. SmallVector<const MDNode*, 8> UserVariables; - /// DbgValues - For each user variable, keep a list of DBG_VALUE - /// instructions in order. The list can also contain normal instructions that - /// clobber the previous DBG_VALUE. + // For each user variable, keep a list of DBG_VALUE instructions in order. + // The list can also contain normal instructions that clobber the previous + // DBG_VALUE. typedef DenseMap<const MDNode*, SmallVector<const MachineInstr*, 4> > DbgValueHistoryMap; DbgValueHistoryMap DbgValues; SmallVector<const MCSymbol *, 8> DebugRangeSymbols; - /// Previous instruction's location information. This is used to determine - /// label location to indicate scope boundries in dwarf debug info. + // Previous instruction's location information. This is used to determine + // label location to indicate scope boundries in dwarf debug info. DebugLoc PrevInstLoc; MCSymbol *PrevLabel; - /// PrologEndLoc - This location indicates end of function prologue and - /// beginning of function body. + // This location indicates end of function prologue and beginning of function + // body. DebugLoc PrologEndLoc; struct FunctionDebugFrameInfo { @@ -309,180 +392,208 @@ class DwarfDebug { // section offsets and are created by EmitSectionLabels. MCSymbol *DwarfInfoSectionSym, *DwarfAbbrevSectionSym; MCSymbol *DwarfStrSectionSym, *TextSectionSym, *DwarfDebugRangeSectionSym; - MCSymbol *DwarfDebugLocSectionSym; + MCSymbol *DwarfDebugLocSectionSym, *DwarfLineSectionSym; MCSymbol *FunctionBeginSym, *FunctionEndSym; + MCSymbol *DwarfAbbrevDWOSectionSym, *DwarfStrDWOSectionSym; // As an optimization, there is no need to emit an entry in the directory // table for the same directory as DW_at_comp_dir. StringRef CompilationDir; - // A holder for the DarwinGDBCompat flag so that the compile unit can use it. - bool isDarwinGDBCompat; - bool hasDwarfAccelTables; -private: + // Counter for assigning globally unique IDs for CUs. + unsigned GlobalCUIndexCount; - /// assignAbbrevNumber - Define a unique number for the abbreviation. - /// - void assignAbbrevNumber(DIEAbbrev &Abbrev); + // Holder for the file specific debug information. + DwarfUnits InfoHolder; + + // Holders for the various debug information flags that we might need to + // have exposed. See accessor functions below for description. + + // Whether or not we're emitting info for older versions of gdb on darwin. + bool IsDarwinGDBCompat; + + // DWARF5 Experimental Options + bool HasDwarfAccelTables; + bool HasSplitDwarf; + + // Separated Dwarf Variables + // In general these will all be for bits that are left in the + // original object file, rather than things that are meant + // to be in the .dwo sections. + + // The CUs left in the original object file for separated debug info. + SmallVector<CompileUnit *, 1> SkeletonCUs; + + // Used to uniquely define abbreviations for the skeleton emission. + FoldingSet<DIEAbbrev> SkeletonAbbrevSet; + + // A list of all the unique abbreviations in use. + std::vector<DIEAbbrev *> SkeletonAbbrevs; + + // Holder for the skeleton information. + DwarfUnits SkeletonHolder; + +private: void addScopeVariable(LexicalScope *LS, DbgVariable *Var); - /// findAbstractVariable - Find abstract variable associated with Var. + /// \brief Find abstract variable associated with Var. DbgVariable *findAbstractVariable(DIVariable &Var, DebugLoc Loc); - /// updateSubprogramScopeDIE - Find DIE for the given subprogram and - /// attach appropriate DW_AT_low_pc and DW_AT_high_pc attributes. - /// If there are global variables in this scope then create and insert - /// DIEs for these variables. + /// \brief Find DIE for the given subprogram and attach appropriate + /// DW_AT_low_pc and DW_AT_high_pc attributes. If there are global + /// variables in this scope then create and insert DIEs for these + /// variables. DIE *updateSubprogramScopeDIE(CompileUnit *SPCU, const MDNode *SPNode); - /// constructLexicalScope - Construct new DW_TAG_lexical_block - /// for this scope and attach DW_AT_low_pc/DW_AT_high_pc labels. + /// \brief Construct new DW_TAG_lexical_block for this scope and + /// attach DW_AT_low_pc/DW_AT_high_pc labels. DIE *constructLexicalScopeDIE(CompileUnit *TheCU, LexicalScope *Scope); - /// constructInlinedScopeDIE - This scope represents inlined body of - /// a function. Construct DIE to represent this concrete inlined copy - /// of the function. + /// \brief This scope represents inlined body of a function. Construct + /// DIE to represent this concrete inlined copy of the function. DIE *constructInlinedScopeDIE(CompileUnit *TheCU, LexicalScope *Scope); - /// constructScopeDIE - Construct a DIE for this scope. + /// \brief Construct a DIE for this scope. DIE *constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope); - /// EmitSectionLabels - Emit initial Dwarf sections with a label at - /// the start of each one. - void EmitSectionLabels(); + /// \brief Emit initial Dwarf sections with a label at the start of each one. + void emitSectionLabels(); - /// emitDIE - Recursively Emits a debug information entry. - /// - void emitDIE(DIE *Die); + /// \brief Compute the size and offset of a DIE given an incoming Offset. + unsigned computeSizeAndOffset(DIE *Die, unsigned Offset); - /// computeSizeAndOffset - Compute the size and offset of a DIE. - /// - unsigned computeSizeAndOffset(DIE *Die, unsigned Offset, bool Last); - - /// computeSizeAndOffsets - Compute the size and offset of all the DIEs. - /// + /// \brief Compute the size and offset of all the DIEs. void computeSizeAndOffsets(); - /// EmitDebugInfo - Emit the debug info section. - /// + /// \brief Attach DW_AT_inline attribute with inlined subprogram DIEs. + void computeInlinedDIEs(); + + /// \brief Collect info for variables that were optimized out. + void collectDeadVariables(); + + /// \brief Finish off debug information after all functions have been + /// processed. + void finalizeModuleInfo(); + + /// \brief Emit labels to close any remaining sections that have been left + /// open. + void endSections(); + + /// \brief Emit a set of abbreviations to the specific section. + void emitAbbrevs(const MCSection *, std::vector<DIEAbbrev*> *); + + /// \brief Emit the debug info section. void emitDebugInfo(); - /// emitAbbreviations - Emit the abbreviation section. - /// - void emitAbbreviations() const; + /// \brief Emit the abbreviation section. + void emitAbbreviations(); - /// emitEndOfLineMatrix - Emit the last address of the section and the end of + /// \brief Emit the last address of the section and the end of /// the line matrix. - /// void emitEndOfLineMatrix(unsigned SectionEnd); - /// emitAccelNames - Emit visible names into a hashed accelerator table - /// section. + /// \brief Emit visible names into a hashed accelerator table section. void emitAccelNames(); - - /// emitAccelObjC - Emit objective C classes and categories into a hashed + + /// \brief Emit objective C classes and categories into a hashed /// accelerator table section. void emitAccelObjC(); - /// emitAccelNamespace - Emit namespace dies into a hashed accelerator - /// table. + /// \brief Emit namespace dies into a hashed accelerator table. void emitAccelNamespaces(); - /// emitAccelTypes() - Emit type dies into a hashed accelerator table. - /// + /// \brief Emit type dies into a hashed accelerator table. void emitAccelTypes(); - - /// emitDebugPubTypes - Emit visible types into a debug pubtypes section. - /// + + /// \brief Emit visible names into a debug pubnames section. + void emitDebugPubnames(); + + /// \brief Emit visible types into a debug pubtypes section. void emitDebugPubTypes(); - /// emitDebugStr - Emit visible names into a debug str section. - /// + /// \brief Emit visible names into a debug str section. void emitDebugStr(); - /// emitDebugLoc - Emit visible names into a debug loc section. - /// + /// \brief Emit visible names into a debug loc section. void emitDebugLoc(); - /// EmitDebugARanges - Emit visible names into a debug aranges section. - /// - void EmitDebugARanges(); + /// \brief Emit visible names into a debug aranges section. + void emitDebugARanges(); - /// emitDebugRanges - Emit visible names into a debug ranges section. - /// + /// \brief Emit visible names into a debug ranges section. void emitDebugRanges(); - /// emitDebugMacInfo - Emit visible names into a debug macinfo section. - /// + /// \brief Emit visible names into a debug macinfo section. void emitDebugMacInfo(); - /// emitDebugInlineInfo - Emit inline info using following format. - /// Section Header: - /// 1. length of section - /// 2. Dwarf version number - /// 3. address size. - /// - /// Entries (one "entry" for each function that was inlined): - /// - /// 1. offset into __debug_str section for MIPS linkage name, if exists; - /// otherwise offset into __debug_str for regular function name. - /// 2. offset into __debug_str section for regular function name. - /// 3. an unsigned LEB128 number indicating the number of distinct inlining - /// instances for the function. - /// - /// The rest of the entry consists of a {die_offset, low_pc} pair for each - /// inlined instance; the die_offset points to the inlined_subroutine die in - /// the __debug_info section, and the low_pc is the starting address for the - /// inlining instance. + /// \brief Emit inline info using custom format. void emitDebugInlineInfo(); - /// constructCompileUnit - Create new CompileUnit for the given - /// metadata node with tag DW_TAG_compile_unit. + /// DWARF 5 Experimental Split Dwarf Emitters + + /// \brief Construct the split debug info compile unit for the debug info + /// section. + CompileUnit *constructSkeletonCU(const MDNode *); + + /// \brief Emit the local split abbreviations. + void emitSkeletonAbbrevs(const MCSection *); + + /// \brief Emit the debug info dwo section. + void emitDebugInfoDWO(); + + /// \brief Emit the debug abbrev dwo section. + void emitDebugAbbrevDWO(); + + /// \brief Emit the debug str dwo section. + void emitDebugStrDWO(); + + /// \brief Create new CompileUnit for the given metadata node with tag + /// DW_TAG_compile_unit. CompileUnit *constructCompileUnit(const MDNode *N); - /// construct SubprogramDIE - Construct subprogram DIE. + /// \brief Construct subprogram DIE. void constructSubprogramDIE(CompileUnit *TheCU, const MDNode *N); - /// recordSourceLine - Register a source line with debug info. Returns the - /// unique label that was emitted and which provides correspondence to - /// the source line list. + /// \brief Register a source line with debug info. Returns the unique + /// label that was emitted and which provides correspondence to the + /// source line list. void recordSourceLine(unsigned Line, unsigned Col, const MDNode *Scope, unsigned Flags); - - /// identifyScopeMarkers() - Indentify instructions that are marking the - /// beginning of or ending of a scope. + + /// \brief Indentify instructions that are marking the beginning of or + /// ending of a scope. void identifyScopeMarkers(); - /// addCurrentFnArgument - If Var is an current function argument that add - /// it in CurrentFnArguments list. + /// \brief If Var is an current function argument that add it in + /// CurrentFnArguments list. bool addCurrentFnArgument(const MachineFunction *MF, DbgVariable *Var, LexicalScope *Scope); - /// collectVariableInfo - Populate LexicalScope entries with variables' info. + /// \brief Populate LexicalScope entries with variables' info. void collectVariableInfo(const MachineFunction *, SmallPtrSet<const MDNode *, 16> &ProcessedVars); - - /// collectVariableInfoFromMMITable - Collect variable information from - /// side table maintained by MMI. + + /// \brief Collect variable information from the side table maintained + /// by MMI. void collectVariableInfoFromMMITable(const MachineFunction * MF, SmallPtrSet<const MDNode *, 16> &P); - /// requestLabelBeforeInsn - Ensure that a label will be emitted before MI. + /// \brief Ensure that a label will be emitted before MI. void requestLabelBeforeInsn(const MachineInstr *MI) { LabelsBeforeInsn.insert(std::make_pair(MI, (MCSymbol*)0)); } - /// getLabelBeforeInsn - Return Label preceding the instruction. - const MCSymbol *getLabelBeforeInsn(const MachineInstr *MI); + /// \brief Return Label preceding the instruction. + MCSymbol *getLabelBeforeInsn(const MachineInstr *MI); - /// requestLabelAfterInsn - Ensure that a label will be emitted after MI. + /// \brief Ensure that a label will be emitted after MI. void requestLabelAfterInsn(const MachineInstr *MI) { LabelsAfterInsn.insert(std::make_pair(MI, (MCSymbol*)0)); } - /// getLabelAfterInsn - Return Label immediately following the instruction. - const MCSymbol *getLabelAfterInsn(const MachineInstr *MI); + /// \brief Return Label immediately following the instruction. + MCSymbol *getLabelAfterInsn(const MachineInstr *MI); public: //===--------------------------------------------------------------------===// @@ -491,52 +602,47 @@ public: DwarfDebug(AsmPrinter *A, Module *M); ~DwarfDebug(); - /// collectInfoFromNamedMDNodes - Collect debug info from named mdnodes such - /// as llvm.dbg.enum and llvm.dbg.ty - void collectInfoFromNamedMDNodes(Module *M); - - /// collectLegacyDebugInfo - Collect debug info using DebugInfoFinder. - /// FIXME - Remove this when DragonEgg switches to DIBuilder. - bool collectLegacyDebugInfo(Module *M); - - /// beginModule - Emit all Dwarf sections that should come prior to the + /// \brief Emit all Dwarf sections that should come prior to the /// content. - void beginModule(Module *M); + void beginModule(); - /// endModule - Emit all Dwarf sections that should come after the content. - /// + /// \brief Emit all Dwarf sections that should come after the content. void endModule(); - /// beginFunction - Gather pre-function debug information. Assumes being - /// emitted immediately after the function entry point. + /// \brief Gather pre-function debug information. void beginFunction(const MachineFunction *MF); - /// endFunction - Gather and emit post-function debug information. - /// + /// \brief Gather and emit post-function debug information. void endFunction(const MachineFunction *MF); - /// beginInstruction - Process beginning of an instruction. + /// \brief Process beginning of an instruction. void beginInstruction(const MachineInstr *MI); - /// endInstruction - Prcess end of an instruction. + /// \brief Process end of an instruction. void endInstruction(const MachineInstr *MI); - /// GetOrCreateSourceID - Look up the source id with the given directory and - /// source file names. If none currently exists, create a new id and insert it - /// in the SourceIds map. - unsigned GetOrCreateSourceID(StringRef DirName, StringRef FullName); - - /// getStringPool - returns the entry into the start of the pool. - MCSymbol *getStringPool(); + /// \brief Look up the source id with the given directory and source file + /// names. If none currently exists, create a new id and insert it in the + /// SourceIds map. + unsigned getOrCreateSourceID(StringRef DirName, StringRef FullName, + unsigned CUID); - /// getStringPoolEntry - returns an entry into the string pool with the given - /// string text. - MCSymbol *getStringPoolEntry(StringRef Str); + /// \brief Recursively Emits a debug information entry. + void emitDIE(DIE *Die, std::vector<DIEAbbrev *> *Abbrevs); - /// useDarwinGDBCompat - returns whether or not to limit some of our debug + /// \brief Returns whether or not to limit some of our debug /// output to the limitations of darwin gdb. - bool useDarwinGDBCompat() { return isDarwinGDBCompat; } - bool useDwarfAccelTables() { return hasDwarfAccelTables; } + bool useDarwinGDBCompat() { return IsDarwinGDBCompat; } + + // Experimental DWARF5 features. + + /// \brief Returns whether or not to emit tables that dwarf consumers can + /// use to accelerate lookup. + bool useDwarfAccelTables() { return HasDwarfAccelTables; } + + /// \brief Returns whether or not to change the current debug info for the + /// split dwarf proposal support. + bool useSplitDwarf() { return HasSplitDwarf; } }; } // End of namespace llvm diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.cpp index 08fb6b3..7133458 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.cpp @@ -12,30 +12,29 @@ //===----------------------------------------------------------------------===// #include "DwarfException.h" -#include "llvm/Module.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/Twine.h" #include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/Support/Dwarf.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormattedStream.h" #include "llvm/Target/Mangler.h" -#include "llvm/DataLayout.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetLoweringObjectFile.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Support/Dwarf.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/Twine.h" using namespace llvm; DwarfException::DwarfException(AsmPrinter *A) @@ -608,7 +607,7 @@ void DwarfException::EmitExceptionTable() { if (!S.PadLabel) { if (VerboseAsm) Asm->OutStreamer.AddComment(" has no landing pad"); - Asm->OutStreamer.EmitIntValue(0, 4/*size*/, 0/*addrspace*/); + Asm->OutStreamer.EmitIntValue(0, 4/*size*/); } else { if (VerboseAsm) Asm->OutStreamer.AddComment(Twine(" jumps to ") + @@ -672,6 +671,18 @@ void DwarfException::EmitExceptionTable() { Asm->EmitSLEB128(Action.NextAction); } + EmitTypeInfos(TTypeEncoding); + + Asm->EmitAlignment(2); +} + +void DwarfException::EmitTypeInfos(unsigned TTypeEncoding) { + const std::vector<const GlobalVariable *> &TypeInfos = MMI->getTypeInfos(); + const std::vector<unsigned> &FilterIds = MMI->getFilterIds(); + + bool VerboseAsm = Asm->OutStreamer.isVerboseAsm(); + + int Entry = 0; // Emit the Catch TypeInfos. if (VerboseAsm && !TypeInfos.empty()) { Asm->OutStreamer.AddComment(">> Catch TypeInfos <<"); @@ -684,11 +695,7 @@ void DwarfException::EmitExceptionTable() { const GlobalVariable *GV = *I; if (VerboseAsm) Asm->OutStreamer.AddComment("TypeInfo " + Twine(Entry--)); - if (GV) - Asm->EmitReference(GV, TTypeEncoding); - else - Asm->OutStreamer.EmitIntValue(0,Asm->GetSizeOfEncodedValue(TTypeEncoding), - 0); + Asm->EmitTTypeReference(GV, TTypeEncoding); } // Emit the Exception Specifications. @@ -708,8 +715,6 @@ void DwarfException::EmitExceptionTable() { Asm->EmitULEB128(TypeID); } - - Asm->EmitAlignment(2); } /// EndModule - Emit all exception information that should come after the diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.h b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.h index fe9e493..74b1b13 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.h +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/DwarfException.h @@ -121,6 +121,8 @@ protected: /// catches in the function. This tables is reversed indexed base 1. void EmitExceptionTable(); + virtual void EmitTypeInfos(unsigned TTypeEncoding); + public: //===--------------------------------------------------------------------===// // Main entry points. @@ -175,6 +177,7 @@ public: }; class ARMException : public DwarfException { + void EmitTypeInfos(unsigned TTypeEncoding); public: //===--------------------------------------------------------------------===// // Main entry points. diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp new file mode 100644 index 0000000..a8fb66d --- /dev/null +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp @@ -0,0 +1,120 @@ +//===-- ErlangGCPrinter.cpp - Erlang/OTP frametable emitter -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the compiler plugin that is used in order to emit +// garbage collection information in a convenient layout for parsing and +// loading in the Erlang/OTP runtime. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/AsmPrinter.h" +#include "llvm/CodeGen/GCs.h" +#include "llvm/CodeGen/GCMetadataPrinter.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Metadata.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCSectionELF.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSymbol.h" +#include "llvm/Target/TargetLoweringObjectFile.h" +#include "llvm/Target/TargetMachine.h" + +using namespace llvm; + +namespace { + + class ErlangGCPrinter : public GCMetadataPrinter { + public: + void beginAssembly(AsmPrinter &AP); + void finishAssembly(AsmPrinter &AP); + }; + +} + +static GCMetadataPrinterRegistry::Add<ErlangGCPrinter> +X("erlang", "erlang-compatible garbage collector"); + +void llvm::linkErlangGCPrinter() { } + +void ErlangGCPrinter::beginAssembly(AsmPrinter &AP) { } + +void ErlangGCPrinter::finishAssembly(AsmPrinter &AP) { + MCStreamer &OS = AP.OutStreamer; + unsigned IntPtrSize = AP.TM.getDataLayout()->getPointerSize(); + + // Put this in a custom .note section. + AP.OutStreamer.SwitchSection(AP.getObjFileLowering().getContext() + .getELFSection(".note.gc", ELF::SHT_PROGBITS, 0, + SectionKind::getDataRel())); + + // For each function... + for (iterator FI = begin(), FE = end(); FI != FE; ++FI) { + GCFunctionInfo &MD = **FI; + + /** A compact GC layout. Emit this data structure: + * + * struct { + * int16_t PointCount; + * void *SafePointAddress[PointCount]; + * int16_t StackFrameSize; (in words) + * int16_t StackArity; + * int16_t LiveCount; + * int16_t LiveOffsets[LiveCount]; + * } __gcmap_<FUNCTIONNAME>; + **/ + + // Align to address width. + AP.EmitAlignment(IntPtrSize == 4 ? 2 : 3); + + // Emit PointCount. + OS.AddComment("safe point count"); + AP.EmitInt16(MD.size()); + + // And each safe point... + for (GCFunctionInfo::iterator PI = MD.begin(), PE = MD.end(); PI != PE; + ++PI) { + // Emit the address of the safe point. + OS.AddComment("safe point address"); + MCSymbol *Label = PI->Label; + AP.EmitLabelPlusOffset(Label/*Hi*/, 0/*Offset*/, 4/*Size*/); + } + + // Stack information never change in safe points! Only print info from the + // first call-site. + GCFunctionInfo::iterator PI = MD.begin(); + + // Emit the stack frame size. + OS.AddComment("stack frame size (in words)"); + AP.EmitInt16(MD.getFrameSize() / IntPtrSize); + + // Emit stack arity, i.e. the number of stacked arguments. + unsigned RegisteredArgs = IntPtrSize == 4 ? 5 : 6; + unsigned StackArity = MD.getFunction().arg_size() > RegisteredArgs ? + MD.getFunction().arg_size() - RegisteredArgs : 0; + OS.AddComment("stack arity"); + AP.EmitInt16(StackArity); + + // Emit the number of live roots in the function. + OS.AddComment("live root count"); + AP.EmitInt16(MD.live_size(PI)); + + // And for each live root... + for (GCFunctionInfo::live_iterator LI = MD.live_begin(PI), + LE = MD.live_end(PI); + LI != LE; ++LI) { + // Emit live root's offset within the stack frame. + OS.AddComment("stack index (offset / wordsize)"); + AP.EmitInt16(LI->StackOffset / IntPtrSize); + } + } +} diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp index f7c0119..98177c0 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp @@ -12,20 +12,20 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/GCs.h" +#include "llvm/ADT/SmallString.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/GCMetadataPrinter.h" -#include "llvm/Module.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" -#include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSymbol.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormattedStream.h" #include "llvm/Target/Mangler.h" -#include "llvm/DataLayout.h" #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FormattedStream.h" #include <cctype> using namespace llvm; @@ -100,7 +100,7 @@ void OcamlGCMetadataPrinter::finishAssembly(AsmPrinter &AP) { EmitCamlGlobal(getModule(), AP, "data_end"); // FIXME: Why does ocaml emit this?? - AP.OutStreamer.EmitIntValue(0, IntPtrSize, 0); + AP.OutStreamer.EmitIntValue(0, IntPtrSize); AP.OutStreamer.SwitchSection(AP.getObjFileLowering().getDataSection()); EmitCamlGlobal(getModule(), AP, "frametable"); @@ -145,7 +145,7 @@ void OcamlGCMetadataPrinter::finishAssembly(AsmPrinter &AP) { "Live root count "+Twine(LiveCount)+" >= 65536."); } - AP.OutStreamer.EmitSymbolValue(J->Label, IntPtrSize, 0); + AP.OutStreamer.EmitSymbolValue(J->Label, IntPtrSize); AP.EmitInt16(FrameSize); AP.EmitInt16(LiveCount); diff --git a/contrib/llvm/lib/CodeGen/AsmPrinter/Win64Exception.cpp b/contrib/llvm/lib/CodeGen/AsmPrinter/Win64Exception.cpp index 70742a8..1561012 100644 --- a/contrib/llvm/lib/CodeGen/AsmPrinter/Win64Exception.cpp +++ b/contrib/llvm/lib/CodeGen/AsmPrinter/Win64Exception.cpp @@ -12,30 +12,29 @@ //===----------------------------------------------------------------------===// #include "DwarfException.h" -#include "llvm/Module.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/Twine.h" #include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/Support/Dwarf.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormattedStream.h" #include "llvm/Target/Mangler.h" -#include "llvm/DataLayout.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetLoweringObjectFile.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Support/Dwarf.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/Twine.h" using namespace llvm; Win64Exception::Win64Exception(AsmPrinter *A) diff --git a/contrib/llvm/lib/CodeGen/BasicTargetTransformInfo.cpp b/contrib/llvm/lib/CodeGen/BasicTargetTransformInfo.cpp new file mode 100644 index 0000000..012ff8a --- /dev/null +++ b/contrib/llvm/lib/CodeGen/BasicTargetTransformInfo.cpp @@ -0,0 +1,466 @@ +//===- BasicTargetTransformInfo.cpp - Basic target-independent TTI impl ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file provides the implementation of a basic TargetTransformInfo pass +/// predicated on the target abstractions present in the target independent +/// code generator. It uses these (primarily TargetLowering) to model as much +/// of the TTI query interface as possible. It is included by most targets so +/// that they can specialize only a small subset of the query space. +/// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "basictti" +#include "llvm/CodeGen/Passes.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Target/TargetLowering.h" +#include <utility> + +using namespace llvm; + +namespace { + +class BasicTTI : public ImmutablePass, public TargetTransformInfo { + const TargetLoweringBase *TLI; + + /// Estimate the overhead of scalarizing an instruction. Insert and Extract + /// are set if the result needs to be inserted and/or extracted from vectors. + unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) const; + +public: + BasicTTI() : ImmutablePass(ID), TLI(0) { + llvm_unreachable("This pass cannot be directly constructed"); + } + + BasicTTI(const TargetLoweringBase *TLI) : ImmutablePass(ID), TLI(TLI) { + initializeBasicTTIPass(*PassRegistry::getPassRegistry()); + } + + virtual void initializePass() { + pushTTIStack(this); + } + + virtual void finalizePass() { + popTTIStack(); + } + + virtual void getAnalysisUsage(AnalysisUsage &AU) const { + TargetTransformInfo::getAnalysisUsage(AU); + } + + /// Pass identification. + static char ID; + + /// Provide necessary pointer adjustments for the two base classes. + virtual void *getAdjustedAnalysisPointer(const void *ID) { + if (ID == &TargetTransformInfo::ID) + return (TargetTransformInfo*)this; + return this; + } + + /// \name Scalar TTI Implementations + /// @{ + + virtual bool isLegalAddImmediate(int64_t imm) const; + virtual bool isLegalICmpImmediate(int64_t imm) const; + virtual bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, + int64_t BaseOffset, bool HasBaseReg, + int64_t Scale) const; + virtual bool isTruncateFree(Type *Ty1, Type *Ty2) const; + virtual bool isTypeLegal(Type *Ty) const; + virtual unsigned getJumpBufAlignment() const; + virtual unsigned getJumpBufSize() const; + virtual bool shouldBuildLookupTables() const; + + /// @} + + /// \name Vector TTI Implementations + /// @{ + + virtual unsigned getNumberOfRegisters(bool Vector) const; + virtual unsigned getMaximumUnrollFactor() const; + virtual unsigned getRegisterBitWidth(bool Vector) const; + virtual unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty, + OperandValueKind, + OperandValueKind) const; + virtual unsigned getShuffleCost(ShuffleKind Kind, Type *Tp, + int Index, Type *SubTp) const; + virtual unsigned getCastInstrCost(unsigned Opcode, Type *Dst, + Type *Src) const; + virtual unsigned getCFInstrCost(unsigned Opcode) const; + virtual unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy, + Type *CondTy) const; + virtual unsigned getVectorInstrCost(unsigned Opcode, Type *Val, + unsigned Index) const; + virtual unsigned getMemoryOpCost(unsigned Opcode, Type *Src, + unsigned Alignment, + unsigned AddressSpace) const; + virtual unsigned getIntrinsicInstrCost(Intrinsic::ID, Type *RetTy, + ArrayRef<Type*> Tys) const; + virtual unsigned getNumberOfParts(Type *Tp) const; + virtual unsigned getAddressComputationCost(Type *Ty) const; + + /// @} +}; + +} + +INITIALIZE_AG_PASS(BasicTTI, TargetTransformInfo, "basictti", + "Target independent code generator's TTI", true, true, false) +char BasicTTI::ID = 0; + +ImmutablePass * +llvm::createBasicTargetTransformInfoPass(const TargetLoweringBase *TLI) { + return new BasicTTI(TLI); +} + + +bool BasicTTI::isLegalAddImmediate(int64_t imm) const { + return TLI->isLegalAddImmediate(imm); +} + +bool BasicTTI::isLegalICmpImmediate(int64_t imm) const { + return TLI->isLegalICmpImmediate(imm); +} + +bool BasicTTI::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, + int64_t BaseOffset, bool HasBaseReg, + int64_t Scale) const { + TargetLoweringBase::AddrMode AM; + AM.BaseGV = BaseGV; + AM.BaseOffs = BaseOffset; + AM.HasBaseReg = HasBaseReg; + AM.Scale = Scale; + return TLI->isLegalAddressingMode(AM, Ty); +} + +bool BasicTTI::isTruncateFree(Type *Ty1, Type *Ty2) const { + return TLI->isTruncateFree(Ty1, Ty2); +} + +bool BasicTTI::isTypeLegal(Type *Ty) const { + EVT T = TLI->getValueType(Ty); + return TLI->isTypeLegal(T); +} + +unsigned BasicTTI::getJumpBufAlignment() const { + return TLI->getJumpBufAlignment(); +} + +unsigned BasicTTI::getJumpBufSize() const { + return TLI->getJumpBufSize(); +} + +bool BasicTTI::shouldBuildLookupTables() const { + return TLI->supportJumpTables() && + (TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) || + TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other)); +} + +//===----------------------------------------------------------------------===// +// +// Calls used by the vectorizers. +// +//===----------------------------------------------------------------------===// + +unsigned BasicTTI::getScalarizationOverhead(Type *Ty, bool Insert, + bool Extract) const { + assert (Ty->isVectorTy() && "Can only scalarize vectors"); + unsigned Cost = 0; + + for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) { + if (Insert) + Cost += TopTTI->getVectorInstrCost(Instruction::InsertElement, Ty, i); + if (Extract) + Cost += TopTTI->getVectorInstrCost(Instruction::ExtractElement, Ty, i); + } + + return Cost; +} + +unsigned BasicTTI::getNumberOfRegisters(bool Vector) const { + return 1; +} + +unsigned BasicTTI::getRegisterBitWidth(bool Vector) const { + return 32; +} + +unsigned BasicTTI::getMaximumUnrollFactor() const { + return 1; +} + +unsigned BasicTTI::getArithmeticInstrCost(unsigned Opcode, Type *Ty, + OperandValueKind, + OperandValueKind) const { + // Check if any of the operands are vector operands. + int ISD = TLI->InstructionOpcodeToISD(Opcode); + assert(ISD && "Invalid opcode"); + + std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Ty); + + if (TLI->isOperationLegalOrPromote(ISD, LT.second)) { + // The operation is legal. Assume it costs 1. + // If the type is split to multiple registers, assume that thre is some + // overhead to this. + // TODO: Once we have extract/insert subvector cost we need to use them. + if (LT.first > 1) + return LT.first * 2; + return LT.first * 1; + } + + if (!TLI->isOperationExpand(ISD, LT.second)) { + // If the operation is custom lowered then assume + // thare the code is twice as expensive. + return LT.first * 2; + } + + // Else, assume that we need to scalarize this op. + if (Ty->isVectorTy()) { + unsigned Num = Ty->getVectorNumElements(); + unsigned Cost = TopTTI->getArithmeticInstrCost(Opcode, Ty->getScalarType()); + // return the cost of multiple scalar invocation plus the cost of inserting + // and extracting the values. + return getScalarizationOverhead(Ty, true, true) + Num * Cost; + } + + // We don't know anything about this scalar instruction. + return 1; +} + +unsigned BasicTTI::getShuffleCost(ShuffleKind Kind, Type *Tp, int Index, + Type *SubTp) const { + return 1; +} + +unsigned BasicTTI::getCastInstrCost(unsigned Opcode, Type *Dst, + Type *Src) const { + int ISD = TLI->InstructionOpcodeToISD(Opcode); + assert(ISD && "Invalid opcode"); + + std::pair<unsigned, MVT> SrcLT = TLI->getTypeLegalizationCost(Src); + std::pair<unsigned, MVT> DstLT = TLI->getTypeLegalizationCost(Dst); + + // Check for NOOP conversions. + if (SrcLT.first == DstLT.first && + SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits()) { + + // Bitcast between types that are legalized to the same type are free. + if (Opcode == Instruction::BitCast || Opcode == Instruction::Trunc) + return 0; + } + + if (Opcode == Instruction::Trunc && + TLI->isTruncateFree(SrcLT.second, DstLT.second)) + return 0; + + if (Opcode == Instruction::ZExt && + TLI->isZExtFree(SrcLT.second, DstLT.second)) + return 0; + + // If the cast is marked as legal (or promote) then assume low cost. + if (TLI->isOperationLegalOrPromote(ISD, DstLT.second)) + return 1; + + // Handle scalar conversions. + if (!Src->isVectorTy() && !Dst->isVectorTy()) { + + // Scalar bitcasts are usually free. + if (Opcode == Instruction::BitCast) + return 0; + + // Just check the op cost. If the operation is legal then assume it costs 1. + if (!TLI->isOperationExpand(ISD, DstLT.second)) + return 1; + + // Assume that illegal scalar instruction are expensive. + return 4; + } + + // Check vector-to-vector casts. + if (Dst->isVectorTy() && Src->isVectorTy()) { + + // If the cast is between same-sized registers, then the check is simple. + if (SrcLT.first == DstLT.first && + SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits()) { + + // Assume that Zext is done using AND. + if (Opcode == Instruction::ZExt) + return 1; + + // Assume that sext is done using SHL and SRA. + if (Opcode == Instruction::SExt) + return 2; + + // Just check the op cost. If the operation is legal then assume it costs + // 1 and multiply by the type-legalization overhead. + if (!TLI->isOperationExpand(ISD, DstLT.second)) + return SrcLT.first * 1; + } + + // If we are converting vectors and the operation is illegal, or + // if the vectors are legalized to different types, estimate the + // scalarization costs. + unsigned Num = Dst->getVectorNumElements(); + unsigned Cost = TopTTI->getCastInstrCost(Opcode, Dst->getScalarType(), + Src->getScalarType()); + + // Return the cost of multiple scalar invocation plus the cost of + // inserting and extracting the values. + return getScalarizationOverhead(Dst, true, true) + Num * Cost; + } + + // We already handled vector-to-vector and scalar-to-scalar conversions. This + // is where we handle bitcast between vectors and scalars. We need to assume + // that the conversion is scalarized in one way or another. + if (Opcode == Instruction::BitCast) + // Illegal bitcasts are done by storing and loading from a stack slot. + return (Src->isVectorTy()? getScalarizationOverhead(Src, false, true):0) + + (Dst->isVectorTy()? getScalarizationOverhead(Dst, true, false):0); + + llvm_unreachable("Unhandled cast"); + } + +unsigned BasicTTI::getCFInstrCost(unsigned Opcode) const { + // Branches are assumed to be predicted. + return 0; +} + +unsigned BasicTTI::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, + Type *CondTy) const { + int ISD = TLI->InstructionOpcodeToISD(Opcode); + assert(ISD && "Invalid opcode"); + + // Selects on vectors are actually vector selects. + if (ISD == ISD::SELECT) { + assert(CondTy && "CondTy must exist"); + if (CondTy->isVectorTy()) + ISD = ISD::VSELECT; + } + + std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(ValTy); + + if (!TLI->isOperationExpand(ISD, LT.second)) { + // The operation is legal. Assume it costs 1. Multiply + // by the type-legalization overhead. + return LT.first * 1; + } + + // Otherwise, assume that the cast is scalarized. + if (ValTy->isVectorTy()) { + unsigned Num = ValTy->getVectorNumElements(); + if (CondTy) + CondTy = CondTy->getScalarType(); + unsigned Cost = TopTTI->getCmpSelInstrCost(Opcode, ValTy->getScalarType(), + CondTy); + + // Return the cost of multiple scalar invocation plus the cost of inserting + // and extracting the values. + return getScalarizationOverhead(ValTy, true, false) + Num * Cost; + } + + // Unknown scalar opcode. + return 1; +} + +unsigned BasicTTI::getVectorInstrCost(unsigned Opcode, Type *Val, + unsigned Index) const { + return 1; +} + +unsigned BasicTTI::getMemoryOpCost(unsigned Opcode, Type *Src, + unsigned Alignment, + unsigned AddressSpace) const { + assert(!Src->isVoidTy() && "Invalid type"); + std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Src); + + // Assume that all loads of legal types cost 1. + return LT.first; +} + +unsigned BasicTTI::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<Type *> Tys) const { + unsigned ISD = 0; + switch (IID) { + default: { + // Assume that we need to scalarize this intrinsic. + unsigned ScalarizationCost = 0; + unsigned ScalarCalls = 1; + if (RetTy->isVectorTy()) { + ScalarizationCost = getScalarizationOverhead(RetTy, true, false); + ScalarCalls = std::max(ScalarCalls, RetTy->getVectorNumElements()); + } + for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) { + if (Tys[i]->isVectorTy()) { + ScalarizationCost += getScalarizationOverhead(Tys[i], false, true); + ScalarCalls = std::max(ScalarCalls, RetTy->getVectorNumElements()); + } + } + + return ScalarCalls + ScalarizationCost; + } + // Look for intrinsics that can be lowered directly or turned into a scalar + // intrinsic call. + case Intrinsic::sqrt: ISD = ISD::FSQRT; break; + case Intrinsic::sin: ISD = ISD::FSIN; break; + case Intrinsic::cos: ISD = ISD::FCOS; break; + case Intrinsic::exp: ISD = ISD::FEXP; break; + case Intrinsic::exp2: ISD = ISD::FEXP2; break; + case Intrinsic::log: ISD = ISD::FLOG; break; + case Intrinsic::log10: ISD = ISD::FLOG10; break; + case Intrinsic::log2: ISD = ISD::FLOG2; break; + case Intrinsic::fabs: ISD = ISD::FABS; break; + case Intrinsic::floor: ISD = ISD::FFLOOR; break; + case Intrinsic::ceil: ISD = ISD::FCEIL; break; + case Intrinsic::trunc: ISD = ISD::FTRUNC; break; + case Intrinsic::rint: ISD = ISD::FRINT; break; + case Intrinsic::pow: ISD = ISD::FPOW; break; + case Intrinsic::fma: ISD = ISD::FMA; break; + case Intrinsic::fmuladd: ISD = ISD::FMA; break; // FIXME: mul + add? + } + + std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(RetTy); + + if (TLI->isOperationLegalOrPromote(ISD, LT.second)) { + // The operation is legal. Assume it costs 1. + // If the type is split to multiple registers, assume that thre is some + // overhead to this. + // TODO: Once we have extract/insert subvector cost we need to use them. + if (LT.first > 1) + return LT.first * 2; + return LT.first * 1; + } + + if (!TLI->isOperationExpand(ISD, LT.second)) { + // If the operation is custom lowered then assume + // thare the code is twice as expensive. + return LT.first * 2; + } + + // Else, assume that we need to scalarize this intrinsic. For math builtins + // this will emit a costly libcall, adding call overhead and spills. Make it + // very expensive. + if (RetTy->isVectorTy()) { + unsigned Num = RetTy->getVectorNumElements(); + unsigned Cost = TopTTI->getIntrinsicInstrCost(IID, RetTy->getScalarType(), + Tys); + return 10 * Cost * Num; + } + + // This is going to be turned into a library call, make it expensive. + return 10; +} + +unsigned BasicTTI::getNumberOfParts(Type *Tp) const { + std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Tp); + return LT.first; +} + +unsigned BasicTTI::getAddressComputationCost(Type *Ty) const { + return 0; +} diff --git a/contrib/llvm/lib/CodeGen/BranchFolding.cpp b/contrib/llvm/lib/CodeGen/BranchFolding.cpp index 6f4c5a2..f8cc3b3 100644 --- a/contrib/llvm/lib/CodeGen/BranchFolding.cpp +++ b/contrib/llvm/lib/CodeGen/BranchFolding.cpp @@ -18,24 +18,23 @@ #define DEBUG_TYPE "branchfolding" #include "BranchFolding.h" -#include "llvm/Function.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" +#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegisterScavenging.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/IR/Function.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SetVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <algorithm> using namespace llvm; @@ -571,8 +570,8 @@ static bool ProfitableToMerge(MachineBasicBlock *MBB1, // instructions that would be deleted in the merge. MachineFunction *MF = MBB1->getParent(); if (EffectiveTailLen >= 2 && - MF->getFunction()->getFnAttributes(). - hasAttribute(Attributes::OptimizeForSize) && + MF->getFunction()->getAttributes(). + hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize) && (I1 == MBB1->begin() || I2 == MBB2->begin())) return true; diff --git a/contrib/llvm/lib/CodeGen/CallingConvLower.cpp b/contrib/llvm/lib/CodeGen/CallingConvLower.cpp index 22b9140..f1d4ace 100644 --- a/contrib/llvm/lib/CodeGen/CallingConvLower.cpp +++ b/contrib/llvm/lib/CodeGen/CallingConvLower.cpp @@ -14,13 +14,13 @@ #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/IR/DataLayout.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; CCState::CCState(CallingConv::ID CC, bool isVarArg, MachineFunction &mf, @@ -74,7 +74,7 @@ CCState::AnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins, if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, *this)) { #ifndef NDEBUG dbgs() << "Formal argument #" << i << " has unhandled type " - << EVT(ArgVT).getEVTString(); + << EVT(ArgVT).getEVTString() << '\n'; #endif llvm_unreachable(0); } @@ -106,7 +106,7 @@ void CCState::AnalyzeReturn(const SmallVectorImpl<ISD::OutputArg> &Outs, if (Fn(i, VT, VT, CCValAssign::Full, ArgFlags, *this)) { #ifndef NDEBUG dbgs() << "Return operand #" << i << " has unhandled type " - << EVT(VT).getEVTString(); + << EVT(VT).getEVTString() << '\n'; #endif llvm_unreachable(0); } @@ -124,7 +124,7 @@ void CCState::AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs, if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, *this)) { #ifndef NDEBUG dbgs() << "Call operand #" << i << " has unhandled type " - << EVT(ArgVT).getEVTString(); + << EVT(ArgVT).getEVTString() << '\n'; #endif llvm_unreachable(0); } @@ -143,7 +143,7 @@ void CCState::AnalyzeCallOperands(SmallVectorImpl<MVT> &ArgVTs, if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, *this)) { #ifndef NDEBUG dbgs() << "Call operand #" << i << " has unhandled type " - << EVT(ArgVT).getEVTString(); + << EVT(ArgVT).getEVTString() << '\n'; #endif llvm_unreachable(0); } @@ -160,7 +160,7 @@ void CCState::AnalyzeCallResult(const SmallVectorImpl<ISD::InputArg> &Ins, if (Fn(i, VT, VT, CCValAssign::Full, Flags, *this)) { #ifndef NDEBUG dbgs() << "Call result #" << i << " has unhandled type " - << EVT(VT).getEVTString() << "\n"; + << EVT(VT).getEVTString() << '\n'; #endif llvm_unreachable(0); } @@ -173,7 +173,7 @@ void CCState::AnalyzeCallResult(MVT VT, CCAssignFn Fn) { if (Fn(0, VT, VT, CCValAssign::Full, ISD::ArgFlagsTy(), *this)) { #ifndef NDEBUG dbgs() << "Call result has unhandled type " - << EVT(VT).getEVTString(); + << EVT(VT).getEVTString() << '\n'; #endif llvm_unreachable(0); } diff --git a/contrib/llvm/lib/CodeGen/CodeGen.cpp b/contrib/llvm/lib/CodeGen/CodeGen.cpp index a53f6f8..35ec68d 100644 --- a/contrib/llvm/lib/CodeGen/CodeGen.cpp +++ b/contrib/llvm/lib/CodeGen/CodeGen.cpp @@ -19,9 +19,9 @@ using namespace llvm; /// initializeCodeGen - Initialize all passes linked into the CodeGen library. void llvm::initializeCodeGen(PassRegistry &Registry) { + initializeBasicTTIPass(Registry); initializeBranchFolderPassPass(Registry); initializeCalculateSpillWeightsPass(Registry); - initializeCodePlacementOptPass(Registry); initializeDeadMachineInstructionElimPass(Registry); initializeEarlyIfConverterPass(Registry); initializeExpandPostRAPass(Registry); diff --git a/contrib/llvm/lib/CodeGen/CodePlacementOpt.cpp b/contrib/llvm/lib/CodeGen/CodePlacementOpt.cpp deleted file mode 100644 index d8e06c3..0000000 --- a/contrib/llvm/lib/CodeGen/CodePlacementOpt.cpp +++ /dev/null @@ -1,422 +0,0 @@ -//===-- CodePlacementOpt.cpp - Code Placement pass. -----------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file implements the pass that optimizes code placement and aligns loop -// headers to target-specific alignment boundaries. -// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "code-placement" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/Debug.h" -#include "llvm/ADT/Statistic.h" -using namespace llvm; - -STATISTIC(NumLoopsAligned, "Number of loops aligned"); -STATISTIC(NumIntraElim, "Number of intra loop branches eliminated"); -STATISTIC(NumIntraMoved, "Number of intra loop branches moved"); - -namespace { - class CodePlacementOpt : public MachineFunctionPass { - const MachineLoopInfo *MLI; - const TargetInstrInfo *TII; - const TargetLowering *TLI; - - public: - static char ID; - CodePlacementOpt() : MachineFunctionPass(ID) {} - - virtual bool runOnMachineFunction(MachineFunction &MF); - - virtual void getAnalysisUsage(AnalysisUsage &AU) const { - AU.addRequired<MachineLoopInfo>(); - AU.addPreservedID(MachineDominatorsID); - MachineFunctionPass::getAnalysisUsage(AU); - } - - private: - bool HasFallthrough(MachineBasicBlock *MBB); - bool HasAnalyzableTerminator(MachineBasicBlock *MBB); - void Splice(MachineFunction &MF, - MachineFunction::iterator InsertPt, - MachineFunction::iterator Begin, - MachineFunction::iterator End); - bool EliminateUnconditionalJumpsToTop(MachineFunction &MF, - MachineLoop *L); - bool MoveDiscontiguousLoopBlocks(MachineFunction &MF, - MachineLoop *L); - bool OptimizeIntraLoopEdgesInLoopNest(MachineFunction &MF, MachineLoop *L); - bool OptimizeIntraLoopEdges(MachineFunction &MF); - bool AlignLoops(MachineFunction &MF); - bool AlignLoop(MachineFunction &MF, MachineLoop *L, unsigned Align); - }; - - char CodePlacementOpt::ID = 0; -} // end anonymous namespace - -char &llvm::CodePlacementOptID = CodePlacementOpt::ID; -INITIALIZE_PASS(CodePlacementOpt, "code-placement", - "Code Placement Optimizer", false, false) - -/// HasFallthrough - Test whether the given branch has a fallthrough, either as -/// a plain fallthrough or as a fallthrough case of a conditional branch. -/// -bool CodePlacementOpt::HasFallthrough(MachineBasicBlock *MBB) { - MachineBasicBlock *TBB = 0, *FBB = 0; - SmallVector<MachineOperand, 4> Cond; - if (TII->AnalyzeBranch(*MBB, TBB, FBB, Cond)) - return false; - // This conditional branch has no fallthrough. - if (FBB) - return false; - // An unconditional branch has no fallthrough. - if (Cond.empty() && TBB) - return false; - // It has a fallthrough. - return true; -} - -/// HasAnalyzableTerminator - Test whether AnalyzeBranch will succeed on MBB. -/// This is called before major changes are begun to test whether it will be -/// possible to complete the changes. -/// -/// Target-specific code is hereby encouraged to make AnalyzeBranch succeed -/// whenever possible. -/// -bool CodePlacementOpt::HasAnalyzableTerminator(MachineBasicBlock *MBB) { - // Conservatively ignore EH landing pads. - if (MBB->isLandingPad()) return false; - - // Aggressively handle return blocks and similar constructs. - if (MBB->succ_empty()) return true; - - // Ask the target's AnalyzeBranch if it can handle this block. - MachineBasicBlock *TBB = 0, *FBB = 0; - SmallVector<MachineOperand, 4> Cond; - // Make sure the terminator is understood. - if (TII->AnalyzeBranch(*MBB, TBB, FBB, Cond)) - return false; - // Ignore blocks which look like they might have EH-related control flow. - // AnalyzeBranch thinks it knows how to analyze such things, but it doesn't - // recognize the possibility of a control transfer through an unwind. - // Such blocks contain EH_LABEL instructions, however they may be in the - // middle of the block. Instead of searching for them, just check to see - // if the CFG disagrees with AnalyzeBranch. - if (1u + !Cond.empty() != MBB->succ_size()) - return false; - // Make sure we have the option of reversing the condition. - if (!Cond.empty() && TII->ReverseBranchCondition(Cond)) - return false; - return true; -} - -/// Splice - Move the sequence of instructions [Begin,End) to just before -/// InsertPt. Update branch instructions as needed to account for broken -/// fallthrough edges and to take advantage of newly exposed fallthrough -/// opportunities. -/// -void CodePlacementOpt::Splice(MachineFunction &MF, - MachineFunction::iterator InsertPt, - MachineFunction::iterator Begin, - MachineFunction::iterator End) { - assert(Begin != MF.begin() && End != MF.begin() && InsertPt != MF.begin() && - "Splice can't change the entry block!"); - MachineFunction::iterator OldBeginPrior = prior(Begin); - MachineFunction::iterator OldEndPrior = prior(End); - - MF.splice(InsertPt, Begin, End); - - prior(Begin)->updateTerminator(); - OldBeginPrior->updateTerminator(); - OldEndPrior->updateTerminator(); -} - -/// EliminateUnconditionalJumpsToTop - Move blocks which unconditionally jump -/// to the loop top to the top of the loop so that they have a fall through. -/// This can introduce a branch on entry to the loop, but it can eliminate a -/// branch within the loop. See the @simple case in -/// test/CodeGen/X86/loop_blocks.ll for an example of this. -bool CodePlacementOpt::EliminateUnconditionalJumpsToTop(MachineFunction &MF, - MachineLoop *L) { - bool Changed = false; - MachineBasicBlock *TopMBB = L->getTopBlock(); - - bool BotHasFallthrough = HasFallthrough(L->getBottomBlock()); - - if (TopMBB == MF.begin() || - HasAnalyzableTerminator(prior(MachineFunction::iterator(TopMBB)))) { - new_top: - for (MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin(), - PE = TopMBB->pred_end(); PI != PE; ++PI) { - MachineBasicBlock *Pred = *PI; - if (Pred == TopMBB) continue; - if (HasFallthrough(Pred)) continue; - if (!L->contains(Pred)) continue; - - // Verify that we can analyze all the loop entry edges before beginning - // any changes which will require us to be able to analyze them. - if (Pred == MF.begin()) - continue; - if (!HasAnalyzableTerminator(Pred)) - continue; - if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(Pred)))) - continue; - - // Move the block. - DEBUG(dbgs() << "CGP: Moving blocks starting at BB#" << Pred->getNumber() - << " to top of loop.\n"); - Changed = true; - - // Move it and all the blocks that can reach it via fallthrough edges - // exclusively, to keep existing fallthrough edges intact. - MachineFunction::iterator Begin = Pred; - MachineFunction::iterator End = llvm::next(Begin); - while (Begin != MF.begin()) { - MachineFunction::iterator Prior = prior(Begin); - if (Prior == MF.begin()) - break; - // Stop when a non-fallthrough edge is found. - if (!HasFallthrough(Prior)) - break; - // Stop if a block which could fall-through out of the loop is found. - if (Prior->isSuccessor(End)) - break; - // If we've reached the top, stop scanning. - if (Prior == MachineFunction::iterator(TopMBB)) { - // We know top currently has a fall through (because we just checked - // it) which would be lost if we do the transformation, so it isn't - // worthwhile to do the transformation unless it would expose a new - // fallthrough edge. - if (!Prior->isSuccessor(End)) - goto next_pred; - // Otherwise we can stop scanning and proceed to move the blocks. - break; - } - // If we hit a switch or something complicated, don't move anything - // for this predecessor. - if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(Prior)))) - break; - // Ok, the block prior to Begin will be moved along with the rest. - // Extend the range to include it. - Begin = Prior; - ++NumIntraMoved; - } - - // Move the blocks. - Splice(MF, TopMBB, Begin, End); - - // Update TopMBB. - TopMBB = L->getTopBlock(); - - // We have a new loop top. Iterate on it. We shouldn't have to do this - // too many times if BranchFolding has done a reasonable job. - goto new_top; - next_pred:; - } - } - - // If the loop previously didn't exit with a fall-through and it now does, - // we eliminated a branch. - if (Changed && - !BotHasFallthrough && - HasFallthrough(L->getBottomBlock())) { - ++NumIntraElim; - } - - return Changed; -} - -/// MoveDiscontiguousLoopBlocks - Move any loop blocks that are not in the -/// portion of the loop contiguous with the header. This usually makes the loop -/// contiguous, provided that AnalyzeBranch can handle all the relevant -/// branching. See the @cfg_islands case in test/CodeGen/X86/loop_blocks.ll -/// for an example of this. -bool CodePlacementOpt::MoveDiscontiguousLoopBlocks(MachineFunction &MF, - MachineLoop *L) { - bool Changed = false; - MachineBasicBlock *TopMBB = L->getTopBlock(); - MachineBasicBlock *BotMBB = L->getBottomBlock(); - - // Determine a position to move orphaned loop blocks to. If TopMBB is not - // entered via fallthrough and BotMBB is exited via fallthrough, prepend them - // to the top of the loop to avoid losing that fallthrough. Otherwise append - // them to the bottom, even if it previously had a fallthrough, on the theory - // that it's worth an extra branch to keep the loop contiguous. - MachineFunction::iterator InsertPt = - llvm::next(MachineFunction::iterator(BotMBB)); - bool InsertAtTop = false; - if (TopMBB != MF.begin() && - !HasFallthrough(prior(MachineFunction::iterator(TopMBB))) && - HasFallthrough(BotMBB)) { - InsertPt = TopMBB; - InsertAtTop = true; - } - - // Keep a record of which blocks are in the portion of the loop contiguous - // with the loop header. - SmallPtrSet<MachineBasicBlock *, 8> ContiguousBlocks; - for (MachineFunction::iterator I = TopMBB, - E = llvm::next(MachineFunction::iterator(BotMBB)); I != E; ++I) - ContiguousBlocks.insert(I); - - // Find non-contigous blocks and fix them. - if (InsertPt != MF.begin() && HasAnalyzableTerminator(prior(InsertPt))) - for (MachineLoop::block_iterator BI = L->block_begin(), BE = L->block_end(); - BI != BE; ++BI) { - MachineBasicBlock *BB = *BI; - - // Verify that we can analyze all the loop entry edges before beginning - // any changes which will require us to be able to analyze them. - if (!HasAnalyzableTerminator(BB)) - continue; - if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(BB)))) - continue; - - // If the layout predecessor is part of the loop, this block will be - // processed along with it. This keeps them in their relative order. - if (BB != MF.begin() && - L->contains(prior(MachineFunction::iterator(BB)))) - continue; - - // Check to see if this block is already contiguous with the main - // portion of the loop. - if (!ContiguousBlocks.insert(BB)) - continue; - - // Move the block. - DEBUG(dbgs() << "CGP: Moving blocks starting at BB#" << BB->getNumber() - << " to be contiguous with loop.\n"); - Changed = true; - - // Process this block and all loop blocks contiguous with it, to keep - // them in their relative order. - MachineFunction::iterator Begin = BB; - MachineFunction::iterator End = llvm::next(MachineFunction::iterator(BB)); - for (; End != MF.end(); ++End) { - if (!L->contains(End)) break; - if (!HasAnalyzableTerminator(End)) break; - ContiguousBlocks.insert(End); - ++NumIntraMoved; - } - - // If we're inserting at the bottom of the loop, and the code we're - // moving originally had fall-through successors, bring the sucessors - // up with the loop blocks to preserve the fall-through edges. - if (!InsertAtTop) - for (; End != MF.end(); ++End) { - if (L->contains(End)) break; - if (!HasAnalyzableTerminator(End)) break; - if (!HasFallthrough(prior(End))) break; - } - - // Move the blocks. This may invalidate TopMBB and/or BotMBB, but - // we don't need them anymore at this point. - Splice(MF, InsertPt, Begin, End); - } - - return Changed; -} - -/// OptimizeIntraLoopEdgesInLoopNest - Reposition loop blocks to minimize -/// intra-loop branching and to form contiguous loops. -/// -/// This code takes the approach of making minor changes to the existing -/// layout to fix specific loop-oriented problems. Also, it depends on -/// AnalyzeBranch, which can't understand complex control instructions. -/// -bool CodePlacementOpt::OptimizeIntraLoopEdgesInLoopNest(MachineFunction &MF, - MachineLoop *L) { - bool Changed = false; - - // Do optimization for nested loops. - for (MachineLoop::iterator I = L->begin(), E = L->end(); I != E; ++I) - Changed |= OptimizeIntraLoopEdgesInLoopNest(MF, *I); - - // Do optimization for this loop. - Changed |= EliminateUnconditionalJumpsToTop(MF, L); - Changed |= MoveDiscontiguousLoopBlocks(MF, L); - - return Changed; -} - -/// OptimizeIntraLoopEdges - Reposition loop blocks to minimize -/// intra-loop branching and to form contiguous loops. -/// -bool CodePlacementOpt::OptimizeIntraLoopEdges(MachineFunction &MF) { - bool Changed = false; - - if (!TLI->shouldOptimizeCodePlacement()) - return Changed; - - // Do optimization for each loop in the function. - for (MachineLoopInfo::iterator I = MLI->begin(), E = MLI->end(); - I != E; ++I) - if (!(*I)->getParentLoop()) - Changed |= OptimizeIntraLoopEdgesInLoopNest(MF, *I); - - return Changed; -} - -/// AlignLoops - Align loop headers to target preferred alignments. -/// -bool CodePlacementOpt::AlignLoops(MachineFunction &MF) { - const Function *F = MF.getFunction(); - if (F->getFnAttributes().hasAttribute(Attributes::OptimizeForSize)) - return false; - - unsigned Align = TLI->getPrefLoopAlignment(); - if (!Align) - return false; // Don't care about loop alignment. - - bool Changed = false; - - for (MachineLoopInfo::iterator I = MLI->begin(), E = MLI->end(); - I != E; ++I) - Changed |= AlignLoop(MF, *I, Align); - - return Changed; -} - -/// AlignLoop - Align loop headers to target preferred alignments. -/// -bool CodePlacementOpt::AlignLoop(MachineFunction &MF, MachineLoop *L, - unsigned Align) { - bool Changed = false; - - // Do alignment for nested loops. - for (MachineLoop::iterator I = L->begin(), E = L->end(); I != E; ++I) - Changed |= AlignLoop(MF, *I, Align); - - L->getTopBlock()->setAlignment(Align); - Changed = true; - ++NumLoopsAligned; - - return Changed; -} - -bool CodePlacementOpt::runOnMachineFunction(MachineFunction &MF) { - MLI = &getAnalysis<MachineLoopInfo>(); - if (MLI->empty()) - return false; // No loops. - - TLI = MF.getTarget().getTargetLowering(); - TII = MF.getTarget().getInstrInfo(); - - bool Changed = OptimizeIntraLoopEdges(MF); - - Changed |= AlignLoops(MF); - - return Changed; -} diff --git a/contrib/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp b/contrib/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp index 377b471..0eb74a4 100644 --- a/contrib/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp +++ b/contrib/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp @@ -17,12 +17,12 @@ #include "CriticalAntiDepBreaker.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; @@ -57,23 +57,7 @@ void CriticalAntiDepBreaker::StartBlock(MachineBasicBlock *BB) { bool IsReturnBlock = (BBSize != 0 && BB->back().isReturn()); - // Determine the live-out physregs for this block. - if (IsReturnBlock) { - // In a return block, examine the function live-out regs. - for (MachineRegisterInfo::liveout_iterator I = MRI.liveout_begin(), - E = MRI.liveout_end(); I != E; ++I) { - for (MCRegAliasIterator AI(*I, TRI, true); AI.isValid(); ++AI) { - unsigned Reg = *AI; - Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1); - KillIndices[Reg] = BBSize; - DefIndices[Reg] = ~0u; - } - } - } - - // In a non-return block, examine the live-in regs of all successors. - // Note a return block can have successors if the return instruction is - // predicated. + // Examine the live-in regs of all successors. for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), SE = BB->succ_end(); SI != SE; ++SI) for (MachineBasicBlock::livein_iterator I = (*SI)->livein_begin(), @@ -371,14 +355,15 @@ CriticalAntiDepBreaker::isNewRegClobberedByRefs(RegRefIter RegRefBegin, return false; } -unsigned -CriticalAntiDepBreaker::findSuitableFreeRegister(RegRefIter RegRefBegin, - RegRefIter RegRefEnd, - unsigned AntiDepReg, - unsigned LastNewReg, - const TargetRegisterClass *RC) +unsigned CriticalAntiDepBreaker:: +findSuitableFreeRegister(RegRefIter RegRefBegin, + RegRefIter RegRefEnd, + unsigned AntiDepReg, + unsigned LastNewReg, + const TargetRegisterClass *RC, + SmallVector<unsigned, 2> &Forbid) { - ArrayRef<unsigned> Order = RegClassInfo.getOrder(RC); + ArrayRef<MCPhysReg> Order = RegClassInfo.getOrder(RC); for (unsigned i = 0; i != Order.size(); ++i) { unsigned NewReg = Order[i]; // Don't replace a register with itself. @@ -401,6 +386,15 @@ CriticalAntiDepBreaker::findSuitableFreeRegister(RegRefIter RegRefBegin, Classes[NewReg] == reinterpret_cast<TargetRegisterClass *>(-1) || KillIndices[AntiDepReg] > DefIndices[NewReg]) continue; + // If NewReg overlaps any of the forbidden registers, we can't use it. + bool Forbidden = false; + for (SmallVector<unsigned, 2>::iterator it = Forbid.begin(), + ite = Forbid.end(); it != ite; ++it) + if (TRI->regsOverlap(NewReg, *it)) { + Forbidden = true; + break; + } + if (Forbidden) continue; return NewReg; } @@ -564,6 +558,8 @@ BreakAntiDependencies(const std::vector<SUnit>& SUnits, PrescanInstruction(MI); + SmallVector<unsigned, 2> ForbidRegs; + // If MI's defs have a special allocation requirement, don't allow // any def registers to be changed. Also assume all registers // defined in a call must not be changed (ABI). @@ -574,7 +570,9 @@ BreakAntiDependencies(const std::vector<SUnit>& SUnits, AntiDepReg = 0; else if (AntiDepReg) { // If this instruction has a use of AntiDepReg, breaking it - // is invalid. + // is invalid. If the instruction defines other registers, + // save a list of them so that we don't pick a new register + // that overlaps any of them. for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); if (!MO.isReg()) continue; @@ -584,6 +582,8 @@ BreakAntiDependencies(const std::vector<SUnit>& SUnits, AntiDepReg = 0; break; } + if (MO.isDef() && Reg != AntiDepReg) + ForbidRegs.push_back(Reg); } } @@ -606,7 +606,7 @@ BreakAntiDependencies(const std::vector<SUnit>& SUnits, if (unsigned NewReg = findSuitableFreeRegister(Range.first, Range.second, AntiDepReg, LastNewReg[AntiDepReg], - RC)) { + RC, ForbidRegs)) { DEBUG(dbgs() << "Breaking anti-dependence edge on " << TRI->getName(AntiDepReg) << " with " << RegRefs.count(AntiDepReg) << " references" diff --git a/contrib/llvm/lib/CodeGen/CriticalAntiDepBreaker.h b/contrib/llvm/lib/CodeGen/CriticalAntiDepBreaker.h index ad95c48..df13dd3 100644 --- a/contrib/llvm/lib/CodeGen/CriticalAntiDepBreaker.h +++ b/contrib/llvm/lib/CodeGen/CriticalAntiDepBreaker.h @@ -17,13 +17,13 @@ #define LLVM_CODEGEN_CRITICALANTIDEPBREAKER_H #include "AntiDepBreaker.h" +#include "llvm/ADT/BitVector.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/CodeGen/ScheduleDAG.h" -#include "llvm/ADT/BitVector.h" #include <map> namespace llvm { @@ -102,7 +102,8 @@ class TargetRegisterInfo; RegRefIter RegRefEnd, unsigned AntiDepReg, unsigned LastNewReg, - const TargetRegisterClass *RC); + const TargetRegisterClass *RC, + SmallVector<unsigned, 2> &Forbid); }; } diff --git a/contrib/llvm/lib/CodeGen/DFAPacketizer.cpp b/contrib/llvm/lib/CodeGen/DFAPacketizer.cpp index ff2f113..840a101 100644 --- a/contrib/llvm/lib/CodeGen/DFAPacketizer.cpp +++ b/contrib/llvm/lib/CodeGen/DFAPacketizer.cpp @@ -23,12 +23,12 @@ // //===----------------------------------------------------------------------===// -#include "llvm/CodeGen/ScheduleDAGInstrs.h" #include "llvm/CodeGen/DFAPacketizer.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBundle.h" -#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/CodeGen/ScheduleDAGInstrs.h" #include "llvm/MC/MCInstrItineraries.h" +#include "llvm/Target/TargetInstrInfo.h" using namespace llvm; DFAPacketizer::DFAPacketizer(const InstrItineraryData *I, const int (*SIT)[2], diff --git a/contrib/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp b/contrib/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp index 8964269..a54217f 100644 --- a/contrib/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp +++ b/contrib/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp @@ -13,14 +13,14 @@ #define DEBUG_TYPE "codegen-dce" #include "llvm/CodeGen/Passes.h" -#include "llvm/Pass.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/ADT/Statistic.h" using namespace llvm; STATISTIC(NumDeletes, "Number of dead instructions deleted"); @@ -99,15 +99,6 @@ bool DeadMachineInstructionElim::runOnMachineFunction(MachineFunction &MF) { // Start out assuming that reserved registers are live out of this block. LivePhysRegs = MRI->getReservedRegs(); - // Also add any explicit live-out physregs for this block. - if (!MBB->empty() && MBB->back().isReturn()) - for (MachineRegisterInfo::liveout_iterator LOI = MRI->liveout_begin(), - LOE = MRI->liveout_end(); LOI != LOE; ++LOI) { - unsigned Reg = *LOI; - if (TargetRegisterInfo::isPhysicalRegister(Reg)) - LivePhysRegs.set(Reg); - } - // Add live-ins from sucessors to LivePhysRegs. Normally, physregs are not // live across blocks, but some targets (x86) can have flags live out of a // block. diff --git a/contrib/llvm/lib/CodeGen/DwarfEHPrepare.cpp b/contrib/llvm/lib/CodeGen/DwarfEHPrepare.cpp index 7095624..f27ec77 100644 --- a/contrib/llvm/lib/CodeGen/DwarfEHPrepare.cpp +++ b/contrib/llvm/lib/CodeGen/DwarfEHPrepare.cpp @@ -13,15 +13,15 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "dwarfehprepare" -#include "llvm/Function.h" -#include "llvm/Instructions.h" -#include "llvm/IntrinsicInst.h" -#include "llvm/Module.h" -#include "llvm/Pass.h" +#include "llvm/CodeGen/Passes.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/Dominators.h" -#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" +#include "llvm/Pass.h" #include "llvm/Support/CallSite.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" @@ -33,7 +33,7 @@ STATISTIC(NumResumesLowered, "Number of resume calls lowered"); namespace { class DwarfEHPrepare : public FunctionPass { const TargetMachine *TM; - const TargetLowering *TLI; + const TargetLoweringBase *TLI; // RewindFunction - _Unwind_Resume or the target equivalent. Constant *RewindFunction; diff --git a/contrib/llvm/lib/CodeGen/EarlyIfConversion.cpp b/contrib/llvm/lib/CodeGen/EarlyIfConversion.cpp index d5d8404..5447df0 100644 --- a/contrib/llvm/lib/CodeGen/EarlyIfConversion.cpp +++ b/contrib/llvm/lib/CodeGen/EarlyIfConversion.cpp @@ -17,7 +17,6 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "early-ifcvt" -#include "MachineTraceMetrics.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SetVector.h" @@ -30,13 +29,14 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/MachineTraceMetrics.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetSubtargetInfo.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; @@ -459,7 +459,6 @@ void SSAIfConv::replacePHIInstrs() { for (unsigned i = 0, e = PHIs.size(); i != e; ++i) { PHIInfo &PI = PHIs[i]; DEBUG(dbgs() << "If-converting " << *PI.PHI); - assert(PI.PHI->getNumOperands() == 5 && "Unexpected PHI operands."); unsigned DstReg = PI.PHI->getOperand(0).getReg(); TII->insertSelect(*Head, FirstTerm, HeadDL, DstReg, Cond, PI.TReg, PI.FReg); DEBUG(dbgs() << " --> " << *llvm::prior(FirstTerm)); @@ -593,6 +592,7 @@ public: EarlyIfConverter() : MachineFunctionPass(ID) {} void getAnalysisUsage(AnalysisUsage &AU) const; bool runOnMachineFunction(MachineFunction &MF); + const char *getPassName() const { return "Early If-Conversion"; } private: bool tryConvertIf(MachineBasicBlock*); diff --git a/contrib/llvm/lib/CodeGen/ErlangGC.cpp b/contrib/llvm/lib/CodeGen/ErlangGC.cpp new file mode 100644 index 0000000..8a1e2d9 --- /dev/null +++ b/contrib/llvm/lib/CodeGen/ErlangGC.cpp @@ -0,0 +1,81 @@ +//===-- ErlangGC.cpp - Erlang/OTP GC strategy -------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the Erlang/OTP runtime-compatible garbage collector +// (e.g. defines safe points, root initialization etc.) +// +// The frametable emitter is in ErlangGCPrinter.cpp. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/GCs.h" +#include "llvm/CodeGen/GCStrategy.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCSymbol.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" + +using namespace llvm; + +namespace { + + class ErlangGC : public GCStrategy { + MCSymbol *InsertLabel(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MI, + DebugLoc DL) const; + public: + ErlangGC(); + bool findCustomSafePoints(GCFunctionInfo &FI, MachineFunction &MF); + }; + +} + +static GCRegistry::Add<ErlangGC> +X("erlang", "erlang-compatible garbage collector"); + +void llvm::linkErlangGC() { } + +ErlangGC::ErlangGC() { + InitRoots = false; + NeededSafePoints = 1 << GC::PostCall; + UsesMetadata = true; + CustomRoots = false; + CustomSafePoints = true; +} + +MCSymbol *ErlangGC::InsertLabel(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MI, + DebugLoc DL) const { + const TargetInstrInfo* TII = MBB.getParent()->getTarget().getInstrInfo(); + MCSymbol *Label = MBB.getParent()->getContext().CreateTempSymbol(); + BuildMI(MBB, MI, DL, TII->get(TargetOpcode::GC_LABEL)).addSym(Label); + return Label; +} + +bool ErlangGC::findCustomSafePoints(GCFunctionInfo &FI, MachineFunction &MF) { + for (MachineFunction::iterator BBI = MF.begin(), BBE = MF.end(); BBI != BBE; + ++BBI) + for (MachineBasicBlock::iterator MI = BBI->begin(), ME = BBI->end(); + MI != ME; ++MI) + + if (MI->getDesc().isCall()) { + + // Do not treat tail call sites as safe points. + if (MI->getDesc().isTerminator()) + continue; + + /* Code copied from VisitCallPoint(...) */ + MachineBasicBlock::iterator RAI = MI; ++RAI; + MCSymbol* Label = InsertLabel(*MI->getParent(), RAI, MI->getDebugLoc()); + FI.addSafePoint(GC::PostCall, Label, MI->getDebugLoc()); + } + + return false; +} diff --git a/contrib/llvm/lib/CodeGen/ExecutionDepsFix.cpp b/contrib/llvm/lib/CodeGen/ExecutionDepsFix.cpp index ed78f19..9b0e76f 100644 --- a/contrib/llvm/lib/CodeGen/ExecutionDepsFix.cpp +++ b/contrib/llvm/lib/CodeGen/ExecutionDepsFix.cpp @@ -21,15 +21,15 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "execution-fix" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/ADT/PostOrderIterator.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; /// A DomainValue is a bit like LiveIntervals' ValNo, but it also keeps track diff --git a/contrib/llvm/lib/CodeGen/ExpandISelPseudos.cpp b/contrib/llvm/lib/CodeGen/ExpandISelPseudos.cpp index 2c4a935..b2b6882 100644 --- a/contrib/llvm/lib/CodeGen/ExpandISelPseudos.cpp +++ b/contrib/llvm/lib/CodeGen/ExpandISelPseudos.cpp @@ -15,12 +15,12 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "expand-isel-pseudos" +#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/Passes.h" +#include "llvm/Support/Debug.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/Support/Debug.h" using namespace llvm; namespace { diff --git a/contrib/llvm/lib/CodeGen/ExpandPostRAPseudos.cpp b/contrib/llvm/lib/CodeGen/ExpandPostRAPseudos.cpp index ffe4b63..1611db8 100644 --- a/contrib/llvm/lib/CodeGen/ExpandPostRAPseudos.cpp +++ b/contrib/llvm/lib/CodeGen/ExpandPostRAPseudos.cpp @@ -18,11 +18,11 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; namespace { @@ -49,8 +49,6 @@ private: bool LowerSubregToReg(MachineInstr *MI); bool LowerCopy(MachineInstr *MI); - void TransferDeadFlag(MachineInstr *MI, unsigned DstReg, - const TargetRegisterInfo *TRI); void TransferImplicitDefs(MachineInstr *MI); }; } // end anonymous namespace @@ -61,21 +59,6 @@ char &llvm::ExpandPostRAPseudosID = ExpandPostRA::ID; INITIALIZE_PASS(ExpandPostRA, "postrapseudos", "Post-RA pseudo instruction expansion pass", false, false) -/// TransferDeadFlag - MI is a pseudo-instruction with DstReg dead, -/// and the lowered replacement instructions immediately precede it. -/// Mark the replacement instructions with the dead flag. -void -ExpandPostRA::TransferDeadFlag(MachineInstr *MI, unsigned DstReg, - const TargetRegisterInfo *TRI) { - for (MachineBasicBlock::iterator MII = - prior(MachineBasicBlock::iterator(MI)); ; --MII) { - if (MII->addRegisterDead(DstReg, TRI)) - break; - assert(MII != MI->getParent()->begin() && - "copyPhysReg output doesn't reference destination register!"); - } -} - /// TransferImplicitDefs - MI is a pseudo-instruction, and the lowered /// replacement instructions immediately precede it. Copy any implicit-def /// operands from MI to the replacement instruction. @@ -114,6 +97,12 @@ bool ExpandPostRA::LowerSubregToReg(MachineInstr *MI) { DEBUG(dbgs() << "subreg: CONVERTING: " << *MI); + if (MI->allDefsAreDead()) { + MI->setDesc(TII->get(TargetOpcode::KILL)); + DEBUG(dbgs() << "subreg: replaced by: " << *MI); + return true; + } + if (DstSubReg == InsReg) { // No need to insert an identify copy instruction. // Watch out for case like this: @@ -135,10 +124,6 @@ bool ExpandPostRA::LowerSubregToReg(MachineInstr *MI) { MachineBasicBlock::iterator CopyMI = MI; --CopyMI; CopyMI->addRegisterDefined(DstReg); - - // Transfer the kill/dead flags, if needed. - if (MI->getOperand(0).isDead()) - TransferDeadFlag(MI, DstSubReg, TRI); DEBUG(dbgs() << "subreg: " << *CopyMI); } @@ -148,6 +133,14 @@ bool ExpandPostRA::LowerSubregToReg(MachineInstr *MI) { } bool ExpandPostRA::LowerCopy(MachineInstr *MI) { + + if (MI->allDefsAreDead()) { + DEBUG(dbgs() << "dead copy: " << *MI); + MI->setDesc(TII->get(TargetOpcode::KILL)); + DEBUG(dbgs() << "replaced by: " << *MI); + return true; + } + MachineOperand &DstMO = MI->getOperand(0); MachineOperand &SrcMO = MI->getOperand(1); @@ -155,7 +148,7 @@ bool ExpandPostRA::LowerCopy(MachineInstr *MI) { DEBUG(dbgs() << "identity copy: " << *MI); // No need to insert an identity copy instruction, but replace with a KILL // if liveness is changed. - if (DstMO.isDead() || SrcMO.isUndef() || MI->getNumOperands() > 2) { + if (SrcMO.isUndef() || MI->getNumOperands() > 2) { // We must make sure the super-register gets killed. Replace the // instruction with KILL. MI->setDesc(TII->get(TargetOpcode::KILL)); @@ -171,8 +164,6 @@ bool ExpandPostRA::LowerCopy(MachineInstr *MI) { TII->copyPhysReg(*MI->getParent(), MI, MI->getDebugLoc(), DstMO.getReg(), SrcMO.getReg(), SrcMO.isKill()); - if (DstMO.isDead()) - TransferDeadFlag(MI, DstMO.getReg(), TRI); if (MI->getNumOperands() > 2) TransferImplicitDefs(MI); DEBUG({ diff --git a/contrib/llvm/lib/CodeGen/GCMetadata.cpp b/contrib/llvm/lib/CodeGen/GCMetadata.cpp index 1caf8c2..ef5247c 100644 --- a/contrib/llvm/lib/CodeGen/GCMetadata.cpp +++ b/contrib/llvm/lib/CodeGen/GCMetadata.cpp @@ -14,10 +14,10 @@ #include "llvm/CodeGen/GCMetadata.h" #include "llvm/CodeGen/GCStrategy.h" #include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/Pass.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/Function.h" +#include "llvm/IR/Function.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" @@ -37,21 +37,9 @@ namespace { void getAnalysisUsage(AnalysisUsage &AU) const; bool runOnFunction(Function &F); - }; - - class Deleter : public FunctionPass { - static char ID; - - public: - Deleter(); - - const char *getPassName() const; - void getAnalysisUsage(AnalysisUsage &AU) const; - - bool runOnFunction(Function &F); bool doFinalization(Module &M); }; - + } INITIALIZE_PASS(GCModuleInfo, "collector-metadata", @@ -182,32 +170,9 @@ bool Printer::runOnFunction(Function &F) { return false; } -// ----------------------------------------------------------------------------- - -char Deleter::ID = 0; - -FunctionPass *llvm::createGCInfoDeleter() { - return new Deleter(); -} - -Deleter::Deleter() : FunctionPass(ID) {} - -const char *Deleter::getPassName() const { - return "Delete Garbage Collector Information"; -} - -void Deleter::getAnalysisUsage(AnalysisUsage &AU) const { - AU.setPreservesAll(); - AU.addRequired<GCModuleInfo>(); -} - -bool Deleter::runOnFunction(Function &MF) { - return false; -} - -bool Deleter::doFinalization(Module &M) { +bool Printer::doFinalization(Module &M) { GCModuleInfo *GMI = getAnalysisIfAvailable<GCModuleInfo>(); - assert(GMI && "Deleter didn't require GCModuleInfo?!"); + assert(GMI && "Printer didn't require GCModuleInfo?!"); GMI->clear(); return false; } diff --git a/contrib/llvm/lib/CodeGen/GCStrategy.cpp b/contrib/llvm/lib/CodeGen/GCStrategy.cpp index f4755bb..1173d11 100644 --- a/contrib/llvm/lib/CodeGen/GCStrategy.cpp +++ b/contrib/llvm/lib/CodeGen/GCStrategy.cpp @@ -16,22 +16,22 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/GCStrategy.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/IntrinsicInst.h" -#include "llvm/Module.h" -#include "llvm/Analysis/Dominators.h" #include "llvm/Analysis/DominatorInternals.h" +#include "llvm/Analysis/Dominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/raw_ostream.h" using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/IfConversion.cpp b/contrib/llvm/lib/CodeGen/IfConversion.cpp index 31e36f0..9958d7d 100644 --- a/contrib/llvm/lib/CodeGen/IfConversion.cpp +++ b/contrib/llvm/lib/CodeGen/IfConversion.cpp @@ -12,24 +12,25 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "ifcvt" -#include "BranchFolding.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/MachineModuleInfo.h" +#include "BranchFolding.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/MC/MCInstrItineraries.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; // Hidden options for help debugging. @@ -150,7 +151,7 @@ namespace { /// basic block number. std::vector<BBInfo> BBAnalysis; - const TargetLowering *TLI; + const TargetLoweringBase *TLI; const TargetInstrInfo *TII; const TargetRegisterInfo *TRI; const InstrItineraryData *InstrItins; @@ -994,14 +995,13 @@ static void UpdatePredRedefs(MachineInstr *MI, SmallSet<unsigned,4> &Redefs, Redefs.erase(*SubRegs); } } + MachineInstrBuilder MIB(*MI->getParent()->getParent(), MI); for (unsigned i = 0, e = Defs.size(); i != e; ++i) { unsigned Reg = Defs[i]; if (!Redefs.insert(Reg)) { if (AddImpUse) // Treat predicated update as read + write. - MI->addOperand(MachineOperand::CreateReg(Reg, false/*IsDef*/, - true/*IsImp*/,false/*IsKill*/, - false/*IsDead*/,true/*IsUndef*/)); + MIB.addReg(Reg, RegState::Implicit | RegState::Undef); } else { for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs) Redefs.insert(*SubRegs); @@ -1557,7 +1557,7 @@ void IfConverter::MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI, bool AddEdges) { if (Succ == FallThrough) continue; FromBBI.BB->removeSuccessor(Succ); - if (AddEdges) + if (AddEdges && !ToBBI.BB->isSuccessor(Succ)) ToBBI.BB->addSuccessor(Succ); } diff --git a/contrib/llvm/lib/CodeGen/InlineSpiller.cpp b/contrib/llvm/lib/CodeGen/InlineSpiller.cpp index 37828a7..c6d1a18 100644 --- a/contrib/llvm/lib/CodeGen/InlineSpiller.cpp +++ b/contrib/llvm/lib/CodeGen/InlineSpiller.cpp @@ -14,7 +14,6 @@ #define DEBUG_TYPE "regalloc" #include "Spiller.h" -#include "VirtRegMap.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Analysis/AliasAnalysis.h" @@ -22,16 +21,17 @@ #include "llvm/CodeGen/LiveRangeEdit.h" #include "llvm/CodeGen/LiveStackAnalysis.h" #include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/InterferenceCache.cpp b/contrib/llvm/lib/CodeGen/InterferenceCache.cpp index 1541bf0..a8e711e 100644 --- a/contrib/llvm/lib/CodeGen/InterferenceCache.cpp +++ b/contrib/llvm/lib/CodeGen/InterferenceCache.cpp @@ -13,9 +13,9 @@ #define DEBUG_TYPE "regalloc" #include "InterferenceCache.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Support/ErrorHandling.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/InterferenceCache.h b/contrib/llvm/lib/CodeGen/InterferenceCache.h index 3c928a5..c02fb9a 100644 --- a/contrib/llvm/lib/CodeGen/InterferenceCache.h +++ b/contrib/llvm/lib/CodeGen/InterferenceCache.h @@ -15,7 +15,7 @@ #ifndef LLVM_CODEGEN_INTERFERENCECACHE #define LLVM_CODEGEN_INTERFERENCECACHE -#include "LiveIntervalUnion.h" +#include "llvm/CodeGen/LiveIntervalUnion.h" namespace llvm { diff --git a/contrib/llvm/lib/CodeGen/IntrinsicLowering.cpp b/contrib/llvm/lib/CodeGen/IntrinsicLowering.cpp index 6120ae56..07f0ccf 100644 --- a/contrib/llvm/lib/CodeGen/IntrinsicLowering.cpp +++ b/contrib/llvm/lib/CodeGen/IntrinsicLowering.cpp @@ -12,16 +12,16 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/IntrinsicLowering.h" -#include "llvm/Constants.h" -#include "llvm/DerivedTypes.h" -#include "llvm/IRBuilder.h" -#include "llvm/Module.h" -#include "llvm/Type.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" #include "llvm/Support/CallSite.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/DataLayout.h" using namespace llvm; template <class ArgIt> diff --git a/contrib/llvm/lib/CodeGen/LLVMTargetMachine.cpp b/contrib/llvm/lib/CodeGen/LLVMTargetMachine.cpp index 24daafa..1a09837 100644 --- a/contrib/llvm/lib/CodeGen/LLVMTargetMachine.cpp +++ b/contrib/llvm/lib/CodeGen/LLVMTargetMachine.cpp @@ -11,30 +11,30 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Transforms/Scalar.h" -#include "llvm/PassManager.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/ADT/OwningPtr.h" #include "llvm/Assembly/PrintModulePass.h" #include "llvm/CodeGen/AsmPrinter.h" -#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/MachineFunctionAnalysis.h" #include "llvm/CodeGen/MachineModuleInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetLoweringObjectFile.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" -#include "llvm/Target/TargetSubtargetInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/CodeGen/Passes.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSubtargetInfo.h" -#include "llvm/ADT/OwningPtr.h" +#include "llvm/PassManager.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/FormattedStream.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormattedStream.h" #include "llvm/Support/TargetRegistry.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetLoweringObjectFile.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/Transforms/Scalar.h" using namespace llvm; // Enable or disable FastISel. Both options are needed, because @@ -79,6 +79,10 @@ LLVMTargetMachine::LLVMTargetMachine(const Target &T, StringRef Triple, "and that InitializeAllTargetMCs() is being invoked!"); } +void LLVMTargetMachine::addAnalysisPasses(PassManagerBase &PM) { + PM.add(createBasicTargetTransformInfoPass(getTargetLowering())); +} + /// addPassesToX helper drives creation and initialization of TargetPassConfig. static MCContext *addPassesToGenerateCode(LLVMTargetMachine *TM, PassManagerBase &PM, @@ -96,6 +100,8 @@ static MCContext *addPassesToGenerateCode(LLVMTargetMachine *TM, PassConfig->addIRPasses(); + PassConfig->addCodeGenPrepare(); + PassConfig->addPassesToHandleExceptions(); PassConfig->addISelPrepare(); @@ -191,7 +197,8 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, // emission fails. MCCodeEmitter *MCE = getTarget().createMCCodeEmitter(*getInstrInfo(), MRI, STI, *Context); - MCAsmBackend *MAB = getTarget().createMCAsmBackend(getTargetTriple(), TargetCPU); + MCAsmBackend *MAB = getTarget().createMCAsmBackend(getTargetTriple(), + TargetCPU); if (MCE == 0 || MAB == 0) return true; @@ -199,7 +206,7 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, *Context, *MAB, Out, MCE, hasMCRelaxAll(), hasMCNoExecStack())); - AsmStreamer.get()->InitSections(); + AsmStreamer.get()->setAutoInitSections(true); break; } case CGFT_Null: @@ -219,7 +226,6 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, PM.add(Printer); - PM.add(createGCInfoDeleter()); return false; } @@ -238,7 +244,6 @@ bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM, return true; addCodeEmitter(PM, JCE); - PM.add(createGCInfoDeleter()); return false; // success! } diff --git a/contrib/llvm/lib/CodeGen/LexicalScopes.cpp b/contrib/llvm/lib/CodeGen/LexicalScopes.cpp index 6b6b9d0..8172154 100644 --- a/contrib/llvm/lib/CodeGen/LexicalScopes.cpp +++ b/contrib/llvm/lib/CodeGen/LexicalScopes.cpp @@ -16,10 +16,10 @@ #define DEBUG_TYPE "lexicalscopes" #include "llvm/CodeGen/LexicalScopes.h" -#include "llvm/DebugInfo.h" -#include "llvm/Function.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/Function.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FormattedStream.h" @@ -314,24 +314,22 @@ bool LexicalScopes::dominates(DebugLoc DL, MachineBasicBlock *MBB) { void LexicalScope::anchor() { } /// dump - Print data structures. -void LexicalScope::dump() const { +void LexicalScope::dump(unsigned Indent) const { #ifndef NDEBUG raw_ostream &err = dbgs(); - err.indent(IndentLevel); + err.indent(Indent); err << "DFSIn: " << DFSIn << " DFSOut: " << DFSOut << "\n"; const MDNode *N = Desc; + err.indent(Indent); N->dump(); if (AbstractScope) - err << "Abstract Scope\n"; + err << std::string(Indent, ' ') << "Abstract Scope\n"; - IndentLevel += 2; if (!Children.empty()) - err << "Children ...\n"; + err << std::string(Indent + 2, ' ') << "Children ...\n"; for (unsigned i = 0, e = Children.size(); i != e; ++i) if (Children[i] != this) - Children[i]->dump(); - - IndentLevel -= 2; + Children[i]->dump(Indent + 2); #endif } diff --git a/contrib/llvm/lib/CodeGen/LiveDebugVariables.cpp b/contrib/llvm/lib/CodeGen/LiveDebugVariables.cpp index defc127..0b117ac 100644 --- a/contrib/llvm/lib/CodeGen/LiveDebugVariables.cpp +++ b/contrib/llvm/lib/CodeGen/LiveDebugVariables.cpp @@ -21,11 +21,6 @@ #define DEBUG_TYPE "livedebug" #include "LiveDebugVariables.h" -#include "VirtRegMap.h" -#include "llvm/Constants.h" -#include "llvm/DebugInfo.h" -#include "llvm/Metadata.h" -#include "llvm/Value.h" #include "llvm/ADT/IntervalMap.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LexicalScopes.h" @@ -35,6 +30,11 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Value.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Target/TargetInstrInfo.h" @@ -247,10 +247,6 @@ public: LiveIntervals &LIS, MachineDominatorTree &MDT, UserValueScopes &UVS); - /// renameRegister - Update locations to rewrite OldReg as NewReg:SubIdx. - void renameRegister(unsigned OldReg, unsigned NewReg, unsigned SubIdx, - const TargetRegisterInfo *TRI); - /// splitRegister - Replace OldReg ranges with NewRegs ranges where NewRegs is /// live. Returns true if any changes were made. bool splitRegister(unsigned OldLocNo, ArrayRef<LiveInterval*> NewRegs); @@ -259,7 +255,7 @@ public: /// provided virtual register map. void rewriteLocations(VirtRegMap &VRM, const TargetRegisterInfo &TRI); - /// emitDebugVariables - Recreate DBG_VALUE instruction from data structures. + /// emitDebugValues - Recreate DBG_VALUE instruction from data structures. void emitDebugValues(VirtRegMap *VRM, LiveIntervals &LIS, const TargetInstrInfo &TRI); @@ -286,6 +282,11 @@ class LDVImpl { MachineDominatorTree *MDT; const TargetRegisterInfo *TRI; + /// Whether emitDebugValues is called. + bool EmitDone; + /// Whether the machine function is modified during the pass. + bool ModifiedMF; + /// userValues - All allocated UserValue instances. SmallVector<UserValue*, 8> userValues; @@ -320,27 +321,30 @@ class LDVImpl { void computeIntervals(); public: - LDVImpl(LiveDebugVariables *ps) : pass(*ps) {} + LDVImpl(LiveDebugVariables *ps) : pass(*ps), EmitDone(false), + ModifiedMF(false) {} bool runOnMachineFunction(MachineFunction &mf); - /// clear - Relase all memory. + /// clear - Release all memory. void clear() { DeleteContainerPointers(userValues); userValues.clear(); virtRegToEqClass.clear(); userVarMap.clear(); + // Make sure we call emitDebugValues if the machine function was modified. + assert((!ModifiedMF || EmitDone) && + "Dbg values are not emitted in LDV"); + EmitDone = false; + ModifiedMF = false; } /// mapVirtReg - Map virtual register to an equivalence class. void mapVirtReg(unsigned VirtReg, UserValue *EC); - /// renameRegister - Replace all references to OldReg with NewReg:SubIdx. - void renameRegister(unsigned OldReg, unsigned NewReg, unsigned SubIdx); - /// splitRegister - Replace all references to OldReg with NewRegs. void splitRegister(unsigned OldReg, ArrayRef<LiveInterval*> NewRegs); - /// emitDebugVariables - Recreate DBG_VALUE instruction from data structures. + /// emitDebugValues - Recreate DBG_VALUE instruction from data structures. void emitDebugValues(VirtRegMap *VRM); void print(raw_ostream&); @@ -693,6 +697,7 @@ bool LDVImpl::runOnMachineFunction(MachineFunction &mf) { computeIntervals(); DEBUG(print(dbgs())); LS.releaseMemory(); + ModifiedMF = Changed; return Changed; } @@ -714,45 +719,6 @@ LiveDebugVariables::~LiveDebugVariables() { delete static_cast<LDVImpl*>(pImpl); } -void UserValue:: -renameRegister(unsigned OldReg, unsigned NewReg, unsigned SubIdx, - const TargetRegisterInfo *TRI) { - for (unsigned i = locations.size(); i; --i) { - unsigned LocNo = i - 1; - MachineOperand &Loc = locations[LocNo]; - if (!Loc.isReg() || Loc.getReg() != OldReg) - continue; - if (TargetRegisterInfo::isPhysicalRegister(NewReg)) - Loc.substPhysReg(NewReg, *TRI); - else - Loc.substVirtReg(NewReg, SubIdx, *TRI); - coalesceLocation(LocNo); - } -} - -void LDVImpl:: -renameRegister(unsigned OldReg, unsigned NewReg, unsigned SubIdx) { - UserValue *UV = lookupVirtReg(OldReg); - if (!UV) - return; - - if (TargetRegisterInfo::isVirtualRegister(NewReg)) - mapVirtReg(NewReg, UV); - if (OldReg != NewReg) - virtRegToEqClass.erase(OldReg); - - do { - UV->renameRegister(OldReg, NewReg, SubIdx, TRI); - UV = UV->getNext(); - } while (UV); -} - -void LiveDebugVariables:: -renameRegister(unsigned OldReg, unsigned NewReg, unsigned SubIdx) { - if (pImpl) - static_cast<LDVImpl*>(pImpl)->renameRegister(OldReg, NewReg, SubIdx); -} - //===----------------------------------------------------------------------===// // Live Range Splitting //===----------------------------------------------------------------------===// @@ -1011,6 +977,7 @@ void LDVImpl::emitDebugValues(VirtRegMap *VRM) { userValues[i]->rewriteLocations(*VRM, *TRI); userValues[i]->emitDebugValues(VRM, *LIS, *TII); } + EmitDone = true; } void LiveDebugVariables::emitDebugValues(VirtRegMap *VRM) { diff --git a/contrib/llvm/lib/CodeGen/LiveInterval.cpp b/contrib/llvm/lib/CodeGen/LiveInterval.cpp index 8585cbb..dccd847 100644 --- a/contrib/llvm/lib/CodeGen/LiveInterval.cpp +++ b/contrib/llvm/lib/CodeGen/LiveInterval.cpp @@ -19,15 +19,15 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/LiveInterval.h" -#include "llvm/CodeGen/LiveIntervalAnalysis.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "RegisterCoalescer.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/CodeGen/LiveIntervalAnalysis.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "RegisterCoalescer.h" #include <algorithm> using namespace llvm; @@ -440,7 +440,7 @@ void LiveInterval::join(LiveInterval &Other, iterator OutIt = begin(); OutIt->valno = NewVNInfo[LHSValNoAssignments[OutIt->valno->id]]; - for (iterator I = next(OutIt), E = end(); I != E; ++I) { + for (iterator I = llvm::next(OutIt), E = end(); I != E; ++I) { VNInfo* nextValNo = NewVNInfo[LHSValNoAssignments[I->valno->id]]; assert(nextValNo != 0 && "Huh?"); @@ -464,10 +464,12 @@ void LiveInterval::join(LiveInterval &Other, ranges.erase(OutIt, end()); } - // Remember assignements because val# ids are changing. - SmallVector<unsigned, 16> OtherAssignments; + // Rewrite Other values before changing the VNInfo ids. + // This can leave Other in an invalid state because we're not coalescing + // touching segments that now have identical values. That's OK since Other is + // not supposed to be valid after calling join(); for (iterator I = Other.begin(), E = Other.end(); I != E; ++I) - OtherAssignments.push_back(RHSValNoAssignments[I->valno->id]); + I->valno = NewVNInfo[RHSValNoAssignments[I->valno->id]]; // Update val# info. Renumber them and make sure they all belong to this // LiveInterval now. Also remove dead val#'s. @@ -486,148 +488,9 @@ void LiveInterval::join(LiveInterval &Other, valnos.resize(NumNewVals); // shrinkify // Okay, now insert the RHS live ranges into the LHS. - unsigned RangeNo = 0; - for (iterator I = Other.begin(), E = Other.end(); I != E; ++I, ++RangeNo) { - // Map the valno in the other live range to the current live range. - I->valno = NewVNInfo[OtherAssignments[RangeNo]]; - assert(I->valno && "Adding a dead range?"); - } - mergeIntervalRanges(Other); - - verify(); -} - -/// \brief Helper function for merging in another LiveInterval's ranges. -/// -/// This is a helper routine implementing an efficient merge of another -/// LiveIntervals ranges into the current interval. -/// -/// \param LHSValNo If non-NULL, set as the new value number for every range -/// from RHS which is merged into the LHS. -/// \param RHSValNo If non-NULL, then only ranges in RHS whose original value -/// number maches this value number will be merged into LHS. -void LiveInterval::mergeIntervalRanges(const LiveInterval &RHS, - VNInfo *LHSValNo, - const VNInfo *RHSValNo) { - if (RHS.empty()) - return; - - // Ensure we're starting with a valid range. Note that we don't verify RHS - // because it may have had its value numbers adjusted in preparation for - // merging. - verify(); - - // The strategy for merging these efficiently is as follows: - // - // 1) Find the beginning of the impacted ranges in the LHS. - // 2) Create a new, merged sub-squence of ranges merging from the position in - // #1 until either LHS or RHS is exhausted. Any part of LHS between RHS - // entries being merged will be copied into this new range. - // 3) Replace the relevant section in LHS with these newly merged ranges. - // 4) Append any remaning ranges from RHS if LHS is exhausted in #2. - // - // We don't follow the typical in-place merge strategy for sorted ranges of - // appending the new ranges to the back and then using std::inplace_merge - // because one step of the merge can both mutate the original elements and - // remove elements from the original. Essentially, because the merge includes - // collapsing overlapping ranges, a more complex approach is required. - - // We do an initial binary search to optimize for a common pattern: a large - // LHS, and a very small RHS. - const_iterator RI = RHS.begin(), RE = RHS.end(); - iterator LE = end(), LI = std::upper_bound(begin(), LE, *RI); - - // Merge into NewRanges until one of the ranges is exhausted. - SmallVector<LiveRange, 4> NewRanges; - - // Keep track of where to begin the replacement. - iterator ReplaceI = LI; - - // If there are preceding ranges in the LHS, put the last one into NewRanges - // so we can optionally extend it. Adjust the replacement point accordingly. - if (LI != begin()) { - ReplaceI = llvm::prior(LI); - NewRanges.push_back(*ReplaceI); - } - - // Now loop over the mergable portions of both LHS and RHS, merging into - // NewRanges. - while (LI != LE && RI != RE) { - // Skip incoming ranges with the wrong value. - if (RHSValNo && RI->valno != RHSValNo) { - ++RI; - continue; - } - - // Select the first range. We pick the earliest start point, and then the - // largest range. - LiveRange R = *LI; - if (*RI < R) { - R = *RI; - ++RI; - if (LHSValNo) - R.valno = LHSValNo; - } else { - ++LI; - } - - if (NewRanges.empty()) { - NewRanges.push_back(R); - continue; - } - - LiveRange &LastR = NewRanges.back(); - if (R.valno == LastR.valno) { - // Try to merge this range into the last one. - if (R.start <= LastR.end) { - LastR.end = std::max(LastR.end, R.end); - continue; - } - } else { - // We can't merge ranges across a value number. - assert(R.start >= LastR.end && - "Cannot overlap two LiveRanges with differing ValID's"); - } - - // If all else fails, just append the range. - NewRanges.push_back(R); - } - assert(RI == RE || LI == LE); - - // Check for being able to merge into the trailing sequence of ranges on the LHS. - if (!NewRanges.empty()) - for (; LI != LE && (LI->valno == NewRanges.back().valno && - LI->start <= NewRanges.back().end); - ++LI) - NewRanges.back().end = std::max(NewRanges.back().end, LI->end); - - // Replace the ranges in the LHS with the newly merged ones. It would be - // really nice if there were a move-supporting 'replace' directly in - // SmallVector, but as there is not, we pay the price of copies to avoid - // wasted memory allocations. - SmallVectorImpl<LiveRange>::iterator NRI = NewRanges.begin(), - NRE = NewRanges.end(); - for (; ReplaceI != LI && NRI != NRE; ++ReplaceI, ++NRI) - *ReplaceI = *NRI; - if (NRI == NRE) - ranges.erase(ReplaceI, LI); - else - ranges.insert(LI, NRI, NRE); - - // And finally insert any trailing end of RHS (if we have one). - for (; RI != RE; ++RI) { - LiveRange R = *RI; - if (LHSValNo) - R.valno = LHSValNo; - if (!ranges.empty() && - ranges.back().valno == R.valno && R.start <= ranges.back().end) - ranges.back().end = std::max(ranges.back().end, R.end); - else - ranges.push_back(R); - } - - // Ensure we finished with a valid new sequence of ranges. - verify(); + LiveRangeUpdater Updater(this); + for (iterator I = Other.begin(), E = Other.end(); I != E; ++I) + Updater.add(*I); } /// MergeRangesInAsValue - Merge all of the intervals in RHS into this live @@ -636,7 +499,9 @@ void LiveInterval::mergeIntervalRanges(const LiveInterval &RHS, /// the overlapping LiveRanges have the specified value number. void LiveInterval::MergeRangesInAsValue(const LiveInterval &RHS, VNInfo *LHSValNo) { - mergeIntervalRanges(RHS, LHSValNo); + LiveRangeUpdater Updater(this); + for (const_iterator I = RHS.begin(), E = RHS.end(); I != E; ++I) + Updater.add(I->start, I->end, LHSValNo); } /// MergeValueInAsValue - Merge all of the live ranges of a specific val# @@ -647,7 +512,10 @@ void LiveInterval::MergeRangesInAsValue(const LiveInterval &RHS, void LiveInterval::MergeValueInAsValue(const LiveInterval &RHS, const VNInfo *RHSValNo, VNInfo *LHSValNo) { - mergeIntervalRanges(RHS, LHSValNo, RHSValNo); + LiveRangeUpdater Updater(this); + for (const_iterator I = RHS.begin(), E = RHS.end(); I != E; ++I) + if (I->valno == RHSValNo) + Updater.add(I->start, I->end, LHSValNo); } /// MergeValueNumberInto - This method is called when two value nubmers @@ -785,6 +653,206 @@ void LiveRange::print(raw_ostream &os) const { os << *this; } +//===----------------------------------------------------------------------===// +// LiveRangeUpdater class +//===----------------------------------------------------------------------===// +// +// The LiveRangeUpdater class always maintains these invariants: +// +// - When LastStart is invalid, Spills is empty and the iterators are invalid. +// This is the initial state, and the state created by flush(). +// In this state, isDirty() returns false. +// +// Otherwise, segments are kept in three separate areas: +// +// 1. [begin; WriteI) at the front of LI. +// 2. [ReadI; end) at the back of LI. +// 3. Spills. +// +// - LI.begin() <= WriteI <= ReadI <= LI.end(). +// - Segments in all three areas are fully ordered and coalesced. +// - Segments in area 1 precede and can't coalesce with segments in area 2. +// - Segments in Spills precede and can't coalesce with segments in area 2. +// - No coalescing is possible between segments in Spills and segments in area +// 1, and there are no overlapping segments. +// +// The segments in Spills are not ordered with respect to the segments in area +// 1. They need to be merged. +// +// When they exist, Spills.back().start <= LastStart, +// and WriteI[-1].start <= LastStart. + +void LiveRangeUpdater::print(raw_ostream &OS) const { + if (!isDirty()) { + if (LI) + OS << "Clean " << PrintReg(LI->reg) << " updater: " << *LI << '\n'; + else + OS << "Null updater.\n"; + return; + } + assert(LI && "Can't have null LI in dirty updater."); + OS << PrintReg(LI->reg) << " updater with gap = " << (ReadI - WriteI) + << ", last start = " << LastStart + << ":\n Area 1:"; + for (LiveInterval::const_iterator I = LI->begin(); I != WriteI; ++I) + OS << ' ' << *I; + OS << "\n Spills:"; + for (unsigned I = 0, E = Spills.size(); I != E; ++I) + OS << ' ' << Spills[I]; + OS << "\n Area 2:"; + for (LiveInterval::const_iterator I = ReadI, E = LI->end(); I != E; ++I) + OS << ' ' << *I; + OS << '\n'; +} + +void LiveRangeUpdater::dump() const +{ + print(errs()); +} + +// Determine if A and B should be coalesced. +static inline bool coalescable(const LiveRange &A, const LiveRange &B) { + assert(A.start <= B.start && "Unordered live ranges."); + if (A.end == B.start) + return A.valno == B.valno; + if (A.end < B.start) + return false; + assert(A.valno == B.valno && "Cannot overlap different values"); + return true; +} + +void LiveRangeUpdater::add(LiveRange Seg) { + assert(LI && "Cannot add to a null destination"); + + // Flush the state if Start moves backwards. + if (!LastStart.isValid() || LastStart > Seg.start) { + if (isDirty()) + flush(); + // This brings us to an uninitialized state. Reinitialize. + assert(Spills.empty() && "Leftover spilled segments"); + WriteI = ReadI = LI->begin(); + } + + // Remember start for next time. + LastStart = Seg.start; + + // Advance ReadI until it ends after Seg.start. + LiveInterval::iterator E = LI->end(); + if (ReadI != E && ReadI->end <= Seg.start) { + // First try to close the gap between WriteI and ReadI with spills. + if (ReadI != WriteI) + mergeSpills(); + // Then advance ReadI. + if (ReadI == WriteI) + ReadI = WriteI = LI->find(Seg.start); + else + while (ReadI != E && ReadI->end <= Seg.start) + *WriteI++ = *ReadI++; + } + + assert(ReadI == E || ReadI->end > Seg.start); + + // Check if the ReadI segment begins early. + if (ReadI != E && ReadI->start <= Seg.start) { + assert(ReadI->valno == Seg.valno && "Cannot overlap different values"); + // Bail if Seg is completely contained in ReadI. + if (ReadI->end >= Seg.end) + return; + // Coalesce into Seg. + Seg.start = ReadI->start; + ++ReadI; + } + + // Coalesce as much as possible from ReadI into Seg. + while (ReadI != E && coalescable(Seg, *ReadI)) { + Seg.end = std::max(Seg.end, ReadI->end); + ++ReadI; + } + + // Try coalescing Spills.back() into Seg. + if (!Spills.empty() && coalescable(Spills.back(), Seg)) { + Seg.start = Spills.back().start; + Seg.end = std::max(Spills.back().end, Seg.end); + Spills.pop_back(); + } + + // Try coalescing Seg into WriteI[-1]. + if (WriteI != LI->begin() && coalescable(WriteI[-1], Seg)) { + WriteI[-1].end = std::max(WriteI[-1].end, Seg.end); + return; + } + + // Seg doesn't coalesce with anything, and needs to be inserted somewhere. + if (WriteI != ReadI) { + *WriteI++ = Seg; + return; + } + + // Finally, append to LI or Spills. + if (WriteI == E) { + LI->ranges.push_back(Seg); + WriteI = ReadI = LI->ranges.end(); + } else + Spills.push_back(Seg); +} + +// Merge as many spilled segments as possible into the gap between WriteI +// and ReadI. Advance WriteI to reflect the inserted instructions. +void LiveRangeUpdater::mergeSpills() { + // Perform a backwards merge of Spills and [SpillI;WriteI). + size_t GapSize = ReadI - WriteI; + size_t NumMoved = std::min(Spills.size(), GapSize); + LiveInterval::iterator Src = WriteI; + LiveInterval::iterator Dst = Src + NumMoved; + LiveInterval::iterator SpillSrc = Spills.end(); + LiveInterval::iterator B = LI->begin(); + + // This is the new WriteI position after merging spills. + WriteI = Dst; + + // Now merge Src and Spills backwards. + while (Src != Dst) { + if (Src != B && Src[-1].start > SpillSrc[-1].start) + *--Dst = *--Src; + else + *--Dst = *--SpillSrc; + } + assert(NumMoved == size_t(Spills.end() - SpillSrc)); + Spills.erase(SpillSrc, Spills.end()); +} + +void LiveRangeUpdater::flush() { + if (!isDirty()) + return; + // Clear the dirty state. + LastStart = SlotIndex(); + + assert(LI && "Cannot add to a null destination"); + + // Nothing to merge? + if (Spills.empty()) { + LI->ranges.erase(WriteI, ReadI); + LI->verify(); + return; + } + + // Resize the WriteI - ReadI gap to match Spills. + size_t GapSize = ReadI - WriteI; + if (GapSize < Spills.size()) { + // The gap is too small. Make some room. + size_t WritePos = WriteI - LI->begin(); + LI->ranges.insert(ReadI, Spills.size() - GapSize, LiveRange()); + // This also invalidated ReadI, but it is recomputed below. + WriteI = LI->ranges.begin() + WritePos; + } else { + // Shrink the gap if necessary. + LI->ranges.erase(WriteI + Spills.size(), ReadI); + } + ReadI = WriteI + Spills.size(); + mergeSpills(); + LI->verify(); +} + unsigned ConnectedVNInfoEqClasses::Classify(const LiveInterval *LI) { // Create initial equivalence classes. EqClass.clear(); diff --git a/contrib/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp b/contrib/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp index 4e75d89..f1b8394 100644 --- a/contrib/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp +++ b/contrib/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp @@ -17,34 +17,29 @@ #define DEBUG_TYPE "regalloc" #include "llvm/CodeGen/LiveIntervalAnalysis.h" -#include "llvm/Value.h" +#include "LiveRangeCalc.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" +#include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/IR/Value.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/STLExtras.h" -#include "LiveRangeCalc.h" -#include "VirtRegMap.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <algorithm> -#include <limits> #include <cmath> +#include <limits> using namespace llvm; -// Switch to the new experimental algorithm for computing live intervals. -static cl::opt<bool> -NewLiveIntervals("new-live-intervals", cl::Hidden, - cl::desc("Use new algorithm forcomputing live intervals")); - char LiveIntervals::ID = 0; char &llvm::LiveIntervalsID = LiveIntervals::ID; INITIALIZE_PASS_BEGIN(LiveIntervals, "liveintervals", @@ -60,6 +55,9 @@ void LiveIntervals::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesCFG(); AU.addRequired<AliasAnalysis>(); AU.addPreserved<AliasAnalysis>(); + // LiveVariables isn't really required by this analysis, it is only required + // here to make sure it is live during TwoAddressInstructionPass and + // PHIElimination. This is temporary. AU.addRequired<LiveVariables>(); AU.addPreserved<LiveVariables>(); AU.addPreservedID(MachineLoopInfoID); @@ -105,7 +103,6 @@ bool LiveIntervals::runOnMachineFunction(MachineFunction &fn) { TRI = TM->getRegisterInfo(); TII = TM->getInstrInfo(); AA = &getAnalysis<AliasAnalysis>(); - LV = &getAnalysis<LiveVariables>(); Indexes = &getAnalysis<SlotIndexes>(); DomTree = &getAnalysis<MachineDominatorTree>(); if (!LRCalc) @@ -114,16 +111,8 @@ bool LiveIntervals::runOnMachineFunction(MachineFunction &fn) { // Allocate space for all virtual registers. VirtRegIntervals.resize(MRI->getNumVirtRegs()); - if (NewLiveIntervals) { - // This is the new way of computing live intervals. - // It is independent of LiveVariables, and it can run at any time. - computeVirtRegs(); - computeRegMasks(); - } else { - // This is the old way of computing live intervals. - // It depends on LiveVariables. - computeIntervals(); - } + computeVirtRegs(); + computeRegMasks(); computeLiveInRegUnits(); DEBUG(dump()); @@ -165,298 +154,6 @@ void LiveIntervals::dumpInstrs() const { } #endif -static -bool MultipleDefsBySameMI(const MachineInstr &MI, unsigned MOIdx) { - unsigned Reg = MI.getOperand(MOIdx).getReg(); - for (unsigned i = MOIdx+1, e = MI.getNumOperands(); i < e; ++i) { - const MachineOperand &MO = MI.getOperand(i); - if (!MO.isReg()) - continue; - if (MO.getReg() == Reg && MO.isDef()) { - assert(MI.getOperand(MOIdx).getSubReg() != MO.getSubReg() && - MI.getOperand(MOIdx).getSubReg() && - (MO.getSubReg() || MO.isImplicit())); - return true; - } - } - return false; -} - -/// isPartialRedef - Return true if the specified def at the specific index is -/// partially re-defining the specified live interval. A common case of this is -/// a definition of the sub-register. -bool LiveIntervals::isPartialRedef(SlotIndex MIIdx, MachineOperand &MO, - LiveInterval &interval) { - if (!MO.getSubReg() || MO.isEarlyClobber()) - return false; - - SlotIndex RedefIndex = MIIdx.getRegSlot(); - const LiveRange *OldLR = - interval.getLiveRangeContaining(RedefIndex.getRegSlot(true)); - MachineInstr *DefMI = getInstructionFromIndex(OldLR->valno->def); - if (DefMI != 0) { - return DefMI->findRegisterDefOperandIdx(interval.reg) != -1; - } - return false; -} - -void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, - MachineBasicBlock::iterator mi, - SlotIndex MIIdx, - MachineOperand& MO, - unsigned MOIdx, - LiveInterval &interval) { - DEBUG(dbgs() << "\t\tregister: " << PrintReg(interval.reg, TRI)); - - // Virtual registers may be defined multiple times (due to phi - // elimination and 2-addr elimination). Much of what we do only has to be - // done once for the vreg. We use an empty interval to detect the first - // time we see a vreg. - LiveVariables::VarInfo& vi = LV->getVarInfo(interval.reg); - if (interval.empty()) { - // Get the Idx of the defining instructions. - SlotIndex defIndex = MIIdx.getRegSlot(MO.isEarlyClobber()); - - // Make sure the first definition is not a partial redefinition. - assert(!MO.readsReg() && "First def cannot also read virtual register " - "missing <undef> flag?"); - - VNInfo *ValNo = interval.getNextValue(defIndex, VNInfoAllocator); - assert(ValNo->id == 0 && "First value in interval is not 0?"); - - // Loop over all of the blocks that the vreg is defined in. There are - // two cases we have to handle here. The most common case is a vreg - // whose lifetime is contained within a basic block. In this case there - // will be a single kill, in MBB, which comes after the definition. - if (vi.Kills.size() == 1 && vi.Kills[0]->getParent() == mbb) { - // FIXME: what about dead vars? - SlotIndex killIdx; - if (vi.Kills[0] != mi) - killIdx = getInstructionIndex(vi.Kills[0]).getRegSlot(); - else - killIdx = defIndex.getDeadSlot(); - - // If the kill happens after the definition, we have an intra-block - // live range. - if (killIdx > defIndex) { - assert(vi.AliveBlocks.empty() && - "Shouldn't be alive across any blocks!"); - LiveRange LR(defIndex, killIdx, ValNo); - interval.addRange(LR); - DEBUG(dbgs() << " +" << LR << "\n"); - return; - } - } - - // The other case we handle is when a virtual register lives to the end - // of the defining block, potentially live across some blocks, then is - // live into some number of blocks, but gets killed. Start by adding a - // range that goes from this definition to the end of the defining block. - LiveRange NewLR(defIndex, getMBBEndIdx(mbb), ValNo); - DEBUG(dbgs() << " +" << NewLR); - interval.addRange(NewLR); - - bool PHIJoin = LV->isPHIJoin(interval.reg); - - if (PHIJoin) { - // A phi join register is killed at the end of the MBB and revived as a - // new valno in the killing blocks. - assert(vi.AliveBlocks.empty() && "Phi join can't pass through blocks"); - DEBUG(dbgs() << " phi-join"); - } else { - // Iterate over all of the blocks that the variable is completely - // live in, adding [insrtIndex(begin), instrIndex(end)+4) to the - // live interval. - for (SparseBitVector<>::iterator I = vi.AliveBlocks.begin(), - E = vi.AliveBlocks.end(); I != E; ++I) { - MachineBasicBlock *aliveBlock = MF->getBlockNumbered(*I); - LiveRange LR(getMBBStartIdx(aliveBlock), getMBBEndIdx(aliveBlock), - ValNo); - interval.addRange(LR); - DEBUG(dbgs() << " +" << LR); - } - } - - // Finally, this virtual register is live from the start of any killing - // block to the 'use' slot of the killing instruction. - for (unsigned i = 0, e = vi.Kills.size(); i != e; ++i) { - MachineInstr *Kill = vi.Kills[i]; - SlotIndex Start = getMBBStartIdx(Kill->getParent()); - SlotIndex killIdx = getInstructionIndex(Kill).getRegSlot(); - - // Create interval with one of a NEW value number. Note that this value - // number isn't actually defined by an instruction, weird huh? :) - if (PHIJoin) { - assert(getInstructionFromIndex(Start) == 0 && - "PHI def index points at actual instruction."); - ValNo = interval.getNextValue(Start, VNInfoAllocator); - } - LiveRange LR(Start, killIdx, ValNo); - interval.addRange(LR); - DEBUG(dbgs() << " +" << LR); - } - - } else { - if (MultipleDefsBySameMI(*mi, MOIdx)) - // Multiple defs of the same virtual register by the same instruction. - // e.g. %reg1031:5<def>, %reg1031:6<def> = VLD1q16 %reg1024<kill>, ... - // This is likely due to elimination of REG_SEQUENCE instructions. Return - // here since there is nothing to do. - return; - - // If this is the second time we see a virtual register definition, it - // must be due to phi elimination or two addr elimination. If this is - // the result of two address elimination, then the vreg is one of the - // def-and-use register operand. - - // It may also be partial redef like this: - // 80 %reg1041:6<def> = VSHRNv4i16 %reg1034<kill>, 12, pred:14, pred:%reg0 - // 120 %reg1041:5<def> = VSHRNv4i16 %reg1039<kill>, 12, pred:14, pred:%reg0 - bool PartReDef = isPartialRedef(MIIdx, MO, interval); - if (PartReDef || mi->isRegTiedToUseOperand(MOIdx)) { - // If this is a two-address definition, then we have already processed - // the live range. The only problem is that we didn't realize there - // are actually two values in the live interval. Because of this we - // need to take the LiveRegion that defines this register and split it - // into two values. - SlotIndex RedefIndex = MIIdx.getRegSlot(MO.isEarlyClobber()); - - const LiveRange *OldLR = - interval.getLiveRangeContaining(RedefIndex.getRegSlot(true)); - VNInfo *OldValNo = OldLR->valno; - SlotIndex DefIndex = OldValNo->def.getRegSlot(); - - // Delete the previous value, which should be short and continuous, - // because the 2-addr copy must be in the same MBB as the redef. - interval.removeRange(DefIndex, RedefIndex); - - // The new value number (#1) is defined by the instruction we claimed - // defined value #0. - VNInfo *ValNo = interval.createValueCopy(OldValNo, VNInfoAllocator); - - // Value#0 is now defined by the 2-addr instruction. - OldValNo->def = RedefIndex; - - // Add the new live interval which replaces the range for the input copy. - LiveRange LR(DefIndex, RedefIndex, ValNo); - DEBUG(dbgs() << " replace range with " << LR); - interval.addRange(LR); - - // If this redefinition is dead, we need to add a dummy unit live - // range covering the def slot. - if (MO.isDead()) - interval.addRange(LiveRange(RedefIndex, RedefIndex.getDeadSlot(), - OldValNo)); - - DEBUG(dbgs() << " RESULT: " << interval); - } else if (LV->isPHIJoin(interval.reg)) { - // In the case of PHI elimination, each variable definition is only - // live until the end of the block. We've already taken care of the - // rest of the live range. - - SlotIndex defIndex = MIIdx.getRegSlot(); - if (MO.isEarlyClobber()) - defIndex = MIIdx.getRegSlot(true); - - VNInfo *ValNo = interval.getNextValue(defIndex, VNInfoAllocator); - - SlotIndex killIndex = getMBBEndIdx(mbb); - LiveRange LR(defIndex, killIndex, ValNo); - interval.addRange(LR); - DEBUG(dbgs() << " phi-join +" << LR); - } else { - llvm_unreachable("Multiply defined register"); - } - } - - DEBUG(dbgs() << '\n'); -} - -void LiveIntervals::handleRegisterDef(MachineBasicBlock *MBB, - MachineBasicBlock::iterator MI, - SlotIndex MIIdx, - MachineOperand& MO, - unsigned MOIdx) { - if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) - handleVirtualRegisterDef(MBB, MI, MIIdx, MO, MOIdx, - getOrCreateInterval(MO.getReg())); -} - -/// computeIntervals - computes the live intervals for virtual -/// registers. for some ordering of the machine instructions [1,N] a -/// live interval is an interval [i, j) where 1 <= i <= j < N for -/// which a variable is live -void LiveIntervals::computeIntervals() { - DEBUG(dbgs() << "********** COMPUTING LIVE INTERVALS **********\n" - << "********** Function: " << MF->getName() << '\n'); - - RegMaskBlocks.resize(MF->getNumBlockIDs()); - - SmallVector<unsigned, 8> UndefUses; - for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end(); - MBBI != E; ++MBBI) { - MachineBasicBlock *MBB = MBBI; - RegMaskBlocks[MBB->getNumber()].first = RegMaskSlots.size(); - - if (MBB->empty()) - continue; - - // Track the index of the current machine instr. - SlotIndex MIIndex = getMBBStartIdx(MBB); - DEBUG(dbgs() << "BB#" << MBB->getNumber() - << ":\t\t# derived from " << MBB->getName() << "\n"); - - // Skip over empty initial indices. - if (getInstructionFromIndex(MIIndex) == 0) - MIIndex = Indexes->getNextNonNullIndex(MIIndex); - - for (MachineBasicBlock::iterator MI = MBB->begin(), miEnd = MBB->end(); - MI != miEnd; ++MI) { - DEBUG(dbgs() << MIIndex << "\t" << *MI); - if (MI->isDebugValue()) - continue; - assert(Indexes->getInstructionFromIndex(MIIndex) == MI && - "Lost SlotIndex synchronization"); - - // Handle defs. - for (int i = MI->getNumOperands() - 1; i >= 0; --i) { - MachineOperand &MO = MI->getOperand(i); - - // Collect register masks. - if (MO.isRegMask()) { - RegMaskSlots.push_back(MIIndex.getRegSlot()); - RegMaskBits.push_back(MO.getRegMask()); - continue; - } - - if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg())) - continue; - - // handle register defs - build intervals - if (MO.isDef()) - handleRegisterDef(MBB, MI, MIIndex, MO, i); - else if (MO.isUndef()) - UndefUses.push_back(MO.getReg()); - } - - // Move to the next instr slot. - MIIndex = Indexes->getNextNonNullIndex(MIIndex); - } - - // Compute the number of register mask instructions in this block. - std::pair<unsigned, unsigned> &RMB = RegMaskBlocks[MBB->getNumber()]; - RMB.second = RegMaskSlots.size() - RMB.first; - } - - // Create empty intervals for registers defined by implicit_def's (except - // for those implicit_def that define values which are liveout of their - // blocks. - for (unsigned i = 0, e = UndefUses.size(); i != e; ++i) { - unsigned UndefReg = UndefUses[i]; - (void)getOrCreateInterval(UndefReg); - } -} - LiveInterval* LiveIntervals::createInterval(unsigned reg) { float Weight = TargetRegisterInfo::isPhysicalRegister(reg) ? HUGE_VALF : 0.0F; return new LiveInterval(reg, Weight); @@ -1275,9 +972,9 @@ private: // Return the last use of reg between NewIdx and OldIdx. SlotIndex findLastUseBefore(unsigned Reg) { - SlotIndex LastUse = NewIdx; if (TargetRegisterInfo::isVirtualRegister(Reg)) { + SlotIndex LastUse = NewIdx; for (MachineRegisterInfo::use_nodbg_iterator UI = MRI.use_nodbg_begin(Reg), UE = MRI.use_nodbg_end(); @@ -1287,26 +984,42 @@ private: if (InstSlot > LastUse && InstSlot < OldIdx) LastUse = InstSlot; } - } else { - MachineInstr* MI = LIS.getSlotIndexes()->getInstructionFromIndex(NewIdx); - MachineBasicBlock::iterator MII(MI); - ++MII; - MachineBasicBlock* MBB = MI->getParent(); - for (; MII != MBB->end() && LIS.getInstructionIndex(MII) < OldIdx; ++MII){ - for (MachineInstr::mop_iterator MOI = MII->operands_begin(), - MOE = MII->operands_end(); - MOI != MOE; ++MOI) { - const MachineOperand& mop = *MOI; - if (!mop.isReg() || mop.getReg() == 0 || - TargetRegisterInfo::isVirtualRegister(mop.getReg())) - continue; - - if (TRI.hasRegUnit(mop.getReg(), Reg)) - LastUse = LIS.getInstructionIndex(MII); - } - } + return LastUse; } - return LastUse; + + // This is a regunit interval, so scanning the use list could be very + // expensive. Scan upwards from OldIdx instead. + assert(NewIdx < OldIdx && "Expected upwards move"); + SlotIndexes *Indexes = LIS.getSlotIndexes(); + MachineBasicBlock *MBB = Indexes->getMBBFromIndex(NewIdx); + + // OldIdx may not correspond to an instruction any longer, so set MII to + // point to the next instruction after OldIdx, or MBB->end(). + MachineBasicBlock::iterator MII = MBB->end(); + if (MachineInstr *MI = Indexes->getInstructionFromIndex( + Indexes->getNextNonNullIndex(OldIdx))) + if (MI->getParent() == MBB) + MII = MI; + + MachineBasicBlock::iterator Begin = MBB->begin(); + while (MII != Begin) { + if ((--MII)->isDebugValue()) + continue; + SlotIndex Idx = Indexes->getInstructionIndex(MII); + + // Stop searching when NewIdx is reached. + if (!SlotIndex::isEarlierInstr(NewIdx, Idx)) + return NewIdx; + + // Check if MII uses Reg. + for (MIBundleOperands MO(MII); MO.isValid(); ++MO) + if (MO->isReg() && + TargetRegisterInfo::isPhysicalRegister(MO->getReg()) && + TRI.hasRegUnit(MO->getReg(), Reg)) + return Idx; + } + // Didn't reach NewIdx. It must be the first instruction in the block. + return NewIdx; } }; @@ -1331,3 +1044,129 @@ void LiveIntervals::handleMoveIntoBundle(MachineInstr* MI, HMEditor HME(*this, *MRI, *TRI, OldIndex, NewIndex, UpdateFlags); HME.updateAllRanges(MI); } + +void +LiveIntervals::repairIntervalsInRange(MachineBasicBlock *MBB, + MachineBasicBlock::iterator Begin, + MachineBasicBlock::iterator End, + ArrayRef<unsigned> OrigRegs) { + // Find anchor points, which are at the beginning/end of blocks or at + // instructions that already have indexes. + while (Begin != MBB->begin() && !Indexes->hasIndex(Begin)) + --Begin; + while (End != MBB->end() && !Indexes->hasIndex(End)) + ++End; + + SlotIndex endIdx; + if (End == MBB->end()) + endIdx = getMBBEndIdx(MBB).getPrevSlot(); + else + endIdx = getInstructionIndex(End); + + Indexes->repairIndexesInRange(MBB, Begin, End); + + for (MachineBasicBlock::iterator I = End; I != Begin;) { + --I; + MachineInstr *MI = I; + if (MI->isDebugValue()) + continue; + for (MachineInstr::const_mop_iterator MOI = MI->operands_begin(), + MOE = MI->operands_end(); MOI != MOE; ++MOI) { + if (MOI->isReg() && + TargetRegisterInfo::isVirtualRegister(MOI->getReg()) && + !hasInterval(MOI->getReg())) { + LiveInterval &LI = getOrCreateInterval(MOI->getReg()); + computeVirtRegInterval(&LI); + } + } + } + + for (unsigned i = 0, e = OrigRegs.size(); i != e; ++i) { + unsigned Reg = OrigRegs[i]; + if (!TargetRegisterInfo::isVirtualRegister(Reg)) + continue; + + LiveInterval &LI = getInterval(Reg); + // FIXME: Should we support undefs that gain defs? + if (!LI.hasAtLeastOneValue()) + continue; + + LiveInterval::iterator LII = LI.find(endIdx); + SlotIndex lastUseIdx; + if (LII != LI.end() && LII->start < endIdx) + lastUseIdx = LII->end; + else + --LII; + + for (MachineBasicBlock::iterator I = End; I != Begin;) { + --I; + MachineInstr *MI = I; + if (MI->isDebugValue()) + continue; + + SlotIndex instrIdx = getInstructionIndex(MI); + bool isStartValid = getInstructionFromIndex(LII->start); + bool isEndValid = getInstructionFromIndex(LII->end); + + // FIXME: This doesn't currently handle early-clobber or multiple removed + // defs inside of the region to repair. + for (MachineInstr::mop_iterator OI = MI->operands_begin(), + OE = MI->operands_end(); OI != OE; ++OI) { + const MachineOperand &MO = *OI; + if (!MO.isReg() || MO.getReg() != Reg) + continue; + + if (MO.isDef()) { + if (!isStartValid) { + if (LII->end.isDead()) { + SlotIndex prevStart; + if (LII != LI.begin()) + prevStart = llvm::prior(LII)->start; + + // FIXME: This could be more efficient if there was a removeRange + // method that returned an iterator. + LI.removeRange(*LII, true); + if (prevStart.isValid()) + LII = LI.find(prevStart); + else + LII = LI.begin(); + } else { + LII->start = instrIdx.getRegSlot(); + LII->valno->def = instrIdx.getRegSlot(); + if (MO.getSubReg() && !MO.isUndef()) + lastUseIdx = instrIdx.getRegSlot(); + else + lastUseIdx = SlotIndex(); + continue; + } + } + + if (!lastUseIdx.isValid()) { + VNInfo *VNI = LI.getNextValue(instrIdx.getRegSlot(), + VNInfoAllocator); + LiveRange LR(instrIdx.getRegSlot(), instrIdx.getDeadSlot(), VNI); + LII = LI.addRange(LR); + } else if (LII->start != instrIdx.getRegSlot()) { + VNInfo *VNI = LI.getNextValue(instrIdx.getRegSlot(), + VNInfoAllocator); + LiveRange LR(instrIdx.getRegSlot(), lastUseIdx, VNI); + LII = LI.addRange(LR); + } + + if (MO.getSubReg() && !MO.isUndef()) + lastUseIdx = instrIdx.getRegSlot(); + else + lastUseIdx = SlotIndex(); + } else if (MO.isUse()) { + // FIXME: This should probably be handled outside of this branch, + // either as part of the def case (for defs inside of the region) or + // after the loop over the region. + if (!isEndValid && !LII->end.isBlock()) + LII->end = instrIdx.getRegSlot(); + if (!lastUseIdx.isValid()) + lastUseIdx = instrIdx.getRegSlot(); + } + } + } + } +} diff --git a/contrib/llvm/lib/CodeGen/LiveIntervalUnion.cpp b/contrib/llvm/lib/CodeGen/LiveIntervalUnion.cpp index dadd02b..d5a81a3 100644 --- a/contrib/llvm/lib/CodeGen/LiveIntervalUnion.cpp +++ b/contrib/llvm/lib/CodeGen/LiveIntervalUnion.cpp @@ -14,13 +14,11 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "regalloc" -#include "LiveIntervalUnion.h" +#include "llvm/CodeGen/LiveIntervalUnion.h" #include "llvm/ADT/SparseBitVector.h" -#include "llvm/CodeGen/MachineLoopRanges.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetRegisterInfo.h" - #include <algorithm> using namespace llvm; @@ -182,33 +180,6 @@ collectInterferingVRegs(unsigned MaxInterferingRegs) { return InterferingVRegs.size(); } -bool LiveIntervalUnion::Query::checkLoopInterference(MachineLoopRange *Loop) { - // VirtReg is likely live throughout the loop, so start by checking LIU-Loop - // overlaps. - IntervalMapOverlaps<LiveIntervalUnion::Map, MachineLoopRange::Map> - Overlaps(LiveUnion->getMap(), Loop->getMap()); - if (!Overlaps.valid()) - return false; - - // The loop is overlapping an LIU assignment. Check VirtReg as well. - LiveInterval::iterator VRI = VirtReg->find(Overlaps.start()); - - for (;;) { - if (VRI == VirtReg->end()) - return false; - if (VRI->start < Overlaps.stop()) - return true; - - Overlaps.advanceTo(VRI->start); - if (!Overlaps.valid()) - return false; - if (Overlaps.start() < VRI->end) - return true; - - VRI = VirtReg->advanceTo(VRI, Overlaps.start()); - } -} - void LiveIntervalUnion::Array::init(LiveIntervalUnion::Allocator &Alloc, unsigned NSize) { // Reuse existing allocation. diff --git a/contrib/llvm/lib/CodeGen/LiveIntervalUnion.h b/contrib/llvm/lib/CodeGen/LiveIntervalUnion.h deleted file mode 100644 index 4d41fca..0000000 --- a/contrib/llvm/lib/CodeGen/LiveIntervalUnion.h +++ /dev/null @@ -1,210 +0,0 @@ -//===-- LiveIntervalUnion.h - Live interval union data struct --*- C++ -*--===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// LiveIntervalUnion is a union of live segments across multiple live virtual -// registers. This may be used during coalescing to represent a congruence -// class, or during register allocation to model liveness of a physical -// register. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_LIVEINTERVALUNION -#define LLVM_CODEGEN_LIVEINTERVALUNION - -#include "llvm/ADT/IntervalMap.h" -#include "llvm/CodeGen/LiveInterval.h" - -namespace llvm { - -class MachineLoopRange; -class TargetRegisterInfo; - -#ifndef NDEBUG -// forward declaration -template <unsigned Element> class SparseBitVector; -typedef SparseBitVector<128> LiveVirtRegBitSet; -#endif - -/// Compare a live virtual register segment to a LiveIntervalUnion segment. -inline bool -overlap(const LiveRange &VRSeg, - const IntervalMap<SlotIndex, LiveInterval*>::const_iterator &LUSeg) { - return VRSeg.start < LUSeg.stop() && LUSeg.start() < VRSeg.end; -} - -/// Union of live intervals that are strong candidates for coalescing into a -/// single register (either physical or virtual depending on the context). We -/// expect the constituent live intervals to be disjoint, although we may -/// eventually make exceptions to handle value-based interference. -class LiveIntervalUnion { - // A set of live virtual register segments that supports fast insertion, - // intersection, and removal. - // Mapping SlotIndex intervals to virtual register numbers. - typedef IntervalMap<SlotIndex, LiveInterval*> LiveSegments; - -public: - // SegmentIter can advance to the next segment ordered by starting position - // which may belong to a different live virtual register. We also must be able - // to reach the current segment's containing virtual register. - typedef LiveSegments::iterator SegmentIter; - - // LiveIntervalUnions share an external allocator. - typedef LiveSegments::Allocator Allocator; - - class Query; - -private: - unsigned Tag; // unique tag for current contents. - LiveSegments Segments; // union of virtual reg segments - -public: - explicit LiveIntervalUnion(Allocator &a) : Tag(0), Segments(a) {} - - // Iterate over all segments in the union of live virtual registers ordered - // by their starting position. - SegmentIter begin() { return Segments.begin(); } - SegmentIter end() { return Segments.end(); } - SegmentIter find(SlotIndex x) { return Segments.find(x); } - bool empty() const { return Segments.empty(); } - SlotIndex startIndex() const { return Segments.start(); } - - // Provide public access to the underlying map to allow overlap iteration. - typedef LiveSegments Map; - const Map &getMap() { return Segments; } - - /// getTag - Return an opaque tag representing the current state of the union. - unsigned getTag() const { return Tag; } - - /// changedSince - Return true if the union change since getTag returned tag. - bool changedSince(unsigned tag) const { return tag != Tag; } - - // Add a live virtual register to this union and merge its segments. - void unify(LiveInterval &VirtReg); - - // Remove a live virtual register's segments from this union. - void extract(LiveInterval &VirtReg); - - // Remove all inserted virtual registers. - void clear() { Segments.clear(); ++Tag; } - - // Print union, using TRI to translate register names - void print(raw_ostream &OS, const TargetRegisterInfo *TRI) const; - -#ifndef NDEBUG - // Verify the live intervals in this union and add them to the visited set. - void verify(LiveVirtRegBitSet& VisitedVRegs); -#endif - - /// Query interferences between a single live virtual register and a live - /// interval union. - class Query { - LiveIntervalUnion *LiveUnion; - LiveInterval *VirtReg; - LiveInterval::iterator VirtRegI; // current position in VirtReg - SegmentIter LiveUnionI; // current position in LiveUnion - SmallVector<LiveInterval*,4> InterferingVRegs; - bool CheckedFirstInterference; - bool SeenAllInterferences; - bool SeenUnspillableVReg; - unsigned Tag, UserTag; - - public: - Query(): LiveUnion(), VirtReg(), Tag(0), UserTag(0) {} - - Query(LiveInterval *VReg, LiveIntervalUnion *LIU): - LiveUnion(LIU), VirtReg(VReg), CheckedFirstInterference(false), - SeenAllInterferences(false), SeenUnspillableVReg(false) - {} - - void clear() { - LiveUnion = NULL; - VirtReg = NULL; - InterferingVRegs.clear(); - CheckedFirstInterference = false; - SeenAllInterferences = false; - SeenUnspillableVReg = false; - Tag = 0; - UserTag = 0; - } - - void init(unsigned UTag, LiveInterval *VReg, LiveIntervalUnion *LIU) { - assert(VReg && LIU && "Invalid arguments"); - if (UserTag == UTag && VirtReg == VReg && - LiveUnion == LIU && !LIU->changedSince(Tag)) { - // Retain cached results, e.g. firstInterference. - return; - } - clear(); - LiveUnion = LIU; - VirtReg = VReg; - Tag = LIU->getTag(); - UserTag = UTag; - } - - LiveInterval &virtReg() const { - assert(VirtReg && "uninitialized"); - return *VirtReg; - } - - // Does this live virtual register interfere with the union? - bool checkInterference() { return collectInterferingVRegs(1); } - - // Count the virtual registers in this union that interfere with this - // query's live virtual register, up to maxInterferingRegs. - unsigned collectInterferingVRegs(unsigned MaxInterferingRegs = UINT_MAX); - - // Was this virtual register visited during collectInterferingVRegs? - bool isSeenInterference(LiveInterval *VReg) const; - - // Did collectInterferingVRegs collect all interferences? - bool seenAllInterferences() const { return SeenAllInterferences; } - - // Did collectInterferingVRegs encounter an unspillable vreg? - bool seenUnspillableVReg() const { return SeenUnspillableVReg; } - - // Vector generated by collectInterferingVRegs. - const SmallVectorImpl<LiveInterval*> &interferingVRegs() const { - return InterferingVRegs; - } - - /// checkLoopInterference - Return true if there is interference overlapping - /// Loop. - bool checkLoopInterference(MachineLoopRange*); - - private: - Query(const Query&) LLVM_DELETED_FUNCTION; - void operator=(const Query&) LLVM_DELETED_FUNCTION; - }; - - // Array of LiveIntervalUnions. - class Array { - unsigned Size; - LiveIntervalUnion *LIUs; - public: - Array() : Size(0), LIUs(0) {} - ~Array() { clear(); } - - // Initialize the array to have Size entries. - // Reuse an existing allocation if the size matches. - void init(LiveIntervalUnion::Allocator&, unsigned Size); - - unsigned size() const { return Size; } - - void clear(); - - LiveIntervalUnion& operator[](unsigned idx) { - assert(idx < Size && "idx out of bounds"); - return LIUs[idx]; - } - }; -}; - -} // end namespace llvm - -#endif // !defined(LLVM_CODEGEN_LIVEINTERVALUNION) diff --git a/contrib/llvm/lib/CodeGen/LiveRangeCalc.cpp b/contrib/llvm/lib/CodeGen/LiveRangeCalc.cpp index c3ff4f1..dede490 100644 --- a/contrib/llvm/lib/CodeGen/LiveRangeCalc.cpp +++ b/contrib/llvm/lib/CodeGen/LiveRangeCalc.cpp @@ -18,10 +18,11 @@ using namespace llvm; -void LiveRangeCalc::reset(const MachineFunction *MF, +void LiveRangeCalc::reset(const MachineFunction *mf, SlotIndexes *SI, MachineDominatorTree *MDT, VNInfo::Allocator *VNIA) { + MF = mf; MRI = &MF->getRegInfo(); Indexes = SI; DomTree = MDT; @@ -104,28 +105,28 @@ void LiveRangeCalc::extendToUses(LiveInterval *LI, unsigned Reg) { // Transfer information from the LiveIn vector to the live ranges. -void LiveRangeCalc::updateLiveIns(VNInfo *OverrideVNI) { +void LiveRangeCalc::updateLiveIns() { + LiveRangeUpdater Updater; for (SmallVectorImpl<LiveInBlock>::iterator I = LiveIn.begin(), E = LiveIn.end(); I != E; ++I) { if (!I->DomNode) continue; MachineBasicBlock *MBB = I->DomNode->getBlock(); - - VNInfo *VNI = OverrideVNI ? OverrideVNI : I->Value; - assert(VNI && "No live-in value found"); - + assert(I->Value && "No live-in value found"); SlotIndex Start, End; tie(Start, End) = Indexes->getMBBRange(MBB); if (I->Kill.isValid()) - I->LI->addRange(LiveRange(Start, I->Kill, VNI)); + // Value is killed inside this block. + End = I->Kill; else { - I->LI->addRange(LiveRange(Start, End, VNI)); - // The value is live-through, update LiveOut as well. Defer the Domtree - // lookup until it is needed. + // The value is live-through, update LiveOut as well. + // Defer the Domtree lookup until it is needed. assert(Seen.test(MBB->getNumber())); - LiveOut[MBB] = LiveOutPair(VNI, (MachineDomTreeNode *)0); + LiveOut[MBB] = LiveOutPair(I->Value, (MachineDomTreeNode *)0); } + Updater.setDest(I->LI); + Updater.add(Start, End, I->Value); } LiveIn.clear(); } @@ -150,13 +151,11 @@ void LiveRangeCalc::extend(LiveInterval *LI, // multiple values, and we may need to create even more phi-defs to preserve // VNInfo SSA form. Perform a search for all predecessor blocks where we // know the dominating VNInfo. - VNInfo *VNI = findReachingDefs(LI, KillMBB, Kill, PhysReg); + if (findReachingDefs(LI, KillMBB, Kill, PhysReg)) + return; // When there were multiple different values, we may need new PHIs. - if (!VNI) - updateSSA(); - - updateLiveIns(VNI); + calculateValues(); } @@ -167,16 +166,18 @@ void LiveRangeCalc::calculateValues() { assert(Indexes && "Missing SlotIndexes"); assert(DomTree && "Missing dominator tree"); updateSSA(); - updateLiveIns(0); + updateLiveIns(); } -VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI, - MachineBasicBlock *KillMBB, - SlotIndex Kill, - unsigned PhysReg) { - // Blocks where LI should be live-in. - SmallVector<MachineBasicBlock*, 16> WorkList(1, KillMBB); +bool LiveRangeCalc::findReachingDefs(LiveInterval *LI, + MachineBasicBlock *KillMBB, + SlotIndex Kill, + unsigned PhysReg) { + unsigned KillMBBNum = KillMBB->getNumber(); + + // Block numbers where LI should be live-in. + SmallVector<unsigned, 16> WorkList(1, KillMBBNum); // Remember if we have seen more than one value. bool UniqueVNI = true; @@ -184,7 +185,7 @@ VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI, // Using Seen as a visited set, perform a BFS for all reaching defs. for (unsigned i = 0; i != WorkList.size(); ++i) { - MachineBasicBlock *MBB = WorkList[i]; + MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]); #ifndef NDEBUG if (MBB->pred_empty()) { @@ -231,25 +232,50 @@ VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI, // No, we need a live-in value for Pred as well if (Pred != KillMBB) - WorkList.push_back(Pred); + WorkList.push_back(Pred->getNumber()); else // Loopback to KillMBB, so value is really live through. Kill = SlotIndex(); } } - // Transfer WorkList to LiveInBlocks in reverse order. - // This ordering works best with updateSSA(). LiveIn.clear(); - LiveIn.reserve(WorkList.size()); - while(!WorkList.empty()) - addLiveInBlock(LI, DomTree->getNode(WorkList.pop_back_val())); - // The kill block may not be live-through. - assert(LiveIn.back().DomNode->getBlock() == KillMBB); - LiveIn.back().Kill = Kill; + // Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but + // neither require it. Skip the sorting overhead for small updates. + if (WorkList.size() > 4) + array_pod_sort(WorkList.begin(), WorkList.end()); + + // If a unique reaching def was found, blit in the live ranges immediately. + if (UniqueVNI) { + LiveRangeUpdater Updater(LI); + for (SmallVectorImpl<unsigned>::const_iterator + I = WorkList.begin(), E = WorkList.end(); I != E; ++I) { + SlotIndex Start, End; + tie(Start, End) = Indexes->getMBBRange(*I); + // Trim the live range in KillMBB. + if (*I == KillMBBNum && Kill.isValid()) + End = Kill; + else + LiveOut[MF->getBlockNumbered(*I)] = + LiveOutPair(TheVNI, (MachineDomTreeNode *)0); + Updater.add(Start, End, TheVNI); + } + return true; + } + + // Multiple values were found, so transfer the work list to the LiveIn array + // where UpdateSSA will use it as a work list. + LiveIn.reserve(WorkList.size()); + for (SmallVectorImpl<unsigned>::const_iterator + I = WorkList.begin(), E = WorkList.end(); I != E; ++I) { + MachineBasicBlock *MBB = MF->getBlockNumbered(*I); + addLiveInBlock(LI, DomTree->getNode(MBB)); + if (MBB == KillMBB) + LiveIn.back().Kill = Kill; + } - return UniqueVNI ? TheVNI : 0; + return false; } diff --git a/contrib/llvm/lib/CodeGen/LiveRangeCalc.h b/contrib/llvm/lib/CodeGen/LiveRangeCalc.h index 909829b..57cab7b 100644 --- a/contrib/llvm/lib/CodeGen/LiveRangeCalc.h +++ b/contrib/llvm/lib/CodeGen/LiveRangeCalc.h @@ -34,6 +34,7 @@ template <class NodeT> class DomTreeNodeBase; typedef DomTreeNodeBase<MachineBasicBlock> MachineDomTreeNode; class LiveRangeCalc { + const MachineFunction *MF; const MachineRegisterInfo *MRI; SlotIndexes *Indexes; MachineDominatorTree *DomTree; @@ -100,17 +101,20 @@ class LiveRangeCalc { /// used to add entries directly. SmallVector<LiveInBlock, 16> LiveIn; - /// findReachingDefs - Assuming that LI is live-in to KillMBB and killed at - /// Kill, search for values that can reach KillMBB. All blocks that need LI - /// to be live-in are added to LiveIn. If a unique reaching def is found, - /// its value is returned, if Kill is jointly dominated by multiple values, - /// NULL is returned. + /// Assuming that LI is live-in to KillMBB and killed at Kill, find the set + /// of defs that can reach it. + /// + /// If only one def can reach Kill, all paths from the def to kill are added + /// to LI, and the function returns true. + /// + /// If multiple values can reach Kill, the blocks that need LI to be live in + /// are added to the LiveIn array, and the function returns false. /// /// PhysReg, when set, is used to verify live-in lists on basic blocks. - VNInfo *findReachingDefs(LiveInterval *LI, - MachineBasicBlock *KillMBB, - SlotIndex Kill, - unsigned PhysReg); + bool findReachingDefs(LiveInterval *LI, + MachineBasicBlock *KillMBB, + SlotIndex Kill, + unsigned PhysReg); /// updateSSA - Compute the values that will be live in to all requested /// blocks in LiveIn. Create PHI-def values as required to preserve SSA form. @@ -119,12 +123,11 @@ class LiveRangeCalc { /// blocks. No values are read from the live ranges. void updateSSA(); - /// updateLiveIns - Add liveness as specified in the LiveIn vector, using VNI - /// as a wildcard value for LiveIn entries without a value. - void updateLiveIns(VNInfo *VNI); + /// Add liveness as specified in the LiveIn vector. + void updateLiveIns(); public: - LiveRangeCalc() : MRI(0), Indexes(0), DomTree(0), Alloc(0) {} + LiveRangeCalc() : MF(0), MRI(0), Indexes(0), DomTree(0), Alloc(0) {} //===--------------------------------------------------------------------===// // High-level interface. diff --git a/contrib/llvm/lib/CodeGen/LiveRangeEdit.cpp b/contrib/llvm/lib/CodeGen/LiveRangeEdit.cpp index f8fbc7d..7793e96 100644 --- a/contrib/llvm/lib/CodeGen/LiveRangeEdit.cpp +++ b/contrib/llvm/lib/CodeGen/LiveRangeEdit.cpp @@ -12,16 +12,16 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "regalloc" -#include "VirtRegMap.h" +#include "llvm/CodeGen/LiveRangeEdit.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/CalcSpillWeights.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" -#include "llvm/CodeGen/LiveRangeEdit.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" using namespace llvm; @@ -77,7 +77,7 @@ bool LiveRangeEdit::anyRematerializable(AliasAnalysis *aa) { /// OrigIdx are also available with the same value at UseIdx. bool LiveRangeEdit::allUsesAvailableAt(const MachineInstr *OrigMI, SlotIndex OrigIdx, - SlotIndex UseIdx) { + SlotIndex UseIdx) const { OrigIdx = OrigIdx.getRegSlot(true); UseIdx = UseIdx.getRegSlot(true); for (unsigned i = 0, e = OrigMI->getNumOperands(); i != e; ++i) { diff --git a/contrib/llvm/lib/CodeGen/LiveRegMatrix.cpp b/contrib/llvm/lib/CodeGen/LiveRegMatrix.cpp index 7f22478..0ef069f 100644 --- a/contrib/llvm/lib/CodeGen/LiveRegMatrix.cpp +++ b/contrib/llvm/lib/CodeGen/LiveRegMatrix.cpp @@ -12,16 +12,16 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "regalloc" -#include "LiveRegMatrix.h" +#include "llvm/CodeGen/LiveRegMatrix.h" #include "RegisterCoalescer.h" -#include "VirtRegMap.h" #include "llvm/ADT/Statistic.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/LiveRegMatrix.h b/contrib/llvm/lib/CodeGen/LiveRegMatrix.h deleted file mode 100644 index 8f22c24..0000000 --- a/contrib/llvm/lib/CodeGen/LiveRegMatrix.h +++ /dev/null @@ -1,148 +0,0 @@ -//===-- LiveRegMatrix.h - Track register interference ---------*- C++ -*---===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// The LiveRegMatrix analysis pass keeps track of virtual register interference -// along two dimensions: Slot indexes and register units. The matrix is used by -// register allocators to ensure that no interfering virtual registers get -// assigned to overlapping physical registers. -// -// Register units are defined in MCRegisterInfo.h, they represent the smallest -// unit of interference when dealing with overlapping physical registers. The -// LiveRegMatrix is represented as a LiveIntervalUnion per register unit. When -// a virtual register is assigned to a physical register, the live range for -// the virtual register is inserted into the LiveIntervalUnion for each regunit -// in the physreg. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_LIVEREGMATRIX_H -#define LLVM_CODEGEN_LIVEREGMATRIX_H - -#include "LiveIntervalUnion.h" -#include "llvm/ADT/BitVector.h" -#include "llvm/ADT/OwningPtr.h" -#include "llvm/CodeGen/MachineFunctionPass.h" - -namespace llvm { - -class LiveInterval; -class LiveIntervalAnalysis; -class MachineRegisterInfo; -class TargetRegisterInfo; -class VirtRegMap; - -class LiveRegMatrix : public MachineFunctionPass { - const TargetRegisterInfo *TRI; - MachineRegisterInfo *MRI; - LiveIntervals *LIS; - VirtRegMap *VRM; - - // UserTag changes whenever virtual registers have been modified. - unsigned UserTag; - - // The matrix is represented as a LiveIntervalUnion per register unit. - LiveIntervalUnion::Allocator LIUAlloc; - LiveIntervalUnion::Array Matrix; - - // Cached queries per register unit. - OwningArrayPtr<LiveIntervalUnion::Query> Queries; - - // Cached register mask interference info. - unsigned RegMaskTag; - unsigned RegMaskVirtReg; - BitVector RegMaskUsable; - - // MachineFunctionPass boilerplate. - virtual void getAnalysisUsage(AnalysisUsage&) const; - virtual bool runOnMachineFunction(MachineFunction&); - virtual void releaseMemory(); -public: - static char ID; - LiveRegMatrix(); - - //===--------------------------------------------------------------------===// - // High-level interface. - //===--------------------------------------------------------------------===// - // - // Check for interference before assigning virtual registers to physical - // registers. - // - - /// Invalidate cached interference queries after modifying virtual register - /// live ranges. Interference checks may return stale information unless - /// caches are invalidated. - void invalidateVirtRegs() { ++UserTag; } - - enum InterferenceKind { - /// No interference, go ahead and assign. - IK_Free = 0, - - /// Virtual register interference. There are interfering virtual registers - /// assigned to PhysReg or its aliases. This interference could be resolved - /// by unassigning those other virtual registers. - IK_VirtReg, - - /// Register unit interference. A fixed live range is in the way, typically - /// argument registers for a call. This can't be resolved by unassigning - /// other virtual registers. - IK_RegUnit, - - /// RegMask interference. The live range is crossing an instruction with a - /// regmask operand that doesn't preserve PhysReg. This typically means - /// VirtReg is live across a call, and PhysReg isn't call-preserved. - IK_RegMask - }; - - /// Check for interference before assigning VirtReg to PhysReg. - /// If this function returns IK_Free, it is legal to assign(VirtReg, PhysReg). - /// When there is more than one kind of interference, the InterferenceKind - /// with the highest enum value is returned. - InterferenceKind checkInterference(LiveInterval &VirtReg, unsigned PhysReg); - - /// Assign VirtReg to PhysReg. - /// This will mark VirtReg's live range as occupied in the LiveRegMatrix and - /// update VirtRegMap. The live range is expected to be available in PhysReg. - void assign(LiveInterval &VirtReg, unsigned PhysReg); - - /// Unassign VirtReg from its PhysReg. - /// Assuming that VirtReg was previously assigned to a PhysReg, this undoes - /// the assignment and updates VirtRegMap accordingly. - void unassign(LiveInterval &VirtReg); - - //===--------------------------------------------------------------------===// - // Low-level interface. - //===--------------------------------------------------------------------===// - // - // Provide access to the underlying LiveIntervalUnions. - // - - /// Check for regmask interference only. - /// Return true if VirtReg crosses a regmask operand that clobbers PhysReg. - /// If PhysReg is null, check if VirtReg crosses any regmask operands. - bool checkRegMaskInterference(LiveInterval &VirtReg, unsigned PhysReg = 0); - - /// Check for regunit interference only. - /// Return true if VirtReg overlaps a fixed assignment of one of PhysRegs's - /// register units. - bool checkRegUnitInterference(LiveInterval &VirtReg, unsigned PhysReg); - - /// Query a line of the assigned virtual register matrix directly. - /// Use MCRegUnitIterator to enumerate all regunits in the desired PhysReg. - /// This returns a reference to an internal Query data structure that is only - /// valid until the next query() call. - LiveIntervalUnion::Query &query(LiveInterval &VirtReg, unsigned RegUnit); - - /// Directly access the live interval unions per regunit. - /// This returns an array indexed by the regunit number. - LiveIntervalUnion *getLiveUnions() { return &Matrix[0]; } -}; - -} // end namespace llvm - -#endif // LLVM_CODEGEN_LIVEREGMATRIX_H diff --git a/contrib/llvm/lib/CodeGen/LiveStackAnalysis.cpp b/contrib/llvm/lib/CodeGen/LiveStackAnalysis.cpp index f0b522b..be11a8f 100644 --- a/contrib/llvm/lib/CodeGen/LiveStackAnalysis.cpp +++ b/contrib/llvm/lib/CodeGen/LiveStackAnalysis.cpp @@ -15,12 +15,12 @@ #define DEBUG_TYPE "livestacks" #include "llvm/CodeGen/LiveStackAnalysis.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/Statistic.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <limits> using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/LiveVariables.cpp b/contrib/llvm/lib/CodeGen/LiveVariables.cpp index 6ea933d..789eddc 100644 --- a/contrib/llvm/lib/CodeGen/LiveVariables.cpp +++ b/contrib/llvm/lib/CodeGen/LiveVariables.cpp @@ -27,17 +27,17 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/LiveVariables.h" +#include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/ADT/DepthFirstIterator.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/STLExtras.h" #include <algorithm> using namespace llvm; @@ -619,29 +619,6 @@ bool LiveVariables::runOnMachineFunction(MachineFunction &mf) { MBB); } - // Finally, if the last instruction in the block is a return, make sure to - // mark it as using all of the live-out values in the function. - // Things marked both call and return are tail calls; do not do this for - // them. The tail callee need not take the same registers as input - // that it produces as output, and there are dependencies for its input - // registers elsewhere. - if (!MBB->empty() && MBB->back().isReturn() - && !MBB->back().isCall()) { - MachineInstr *Ret = &MBB->back(); - - for (MachineRegisterInfo::liveout_iterator - I = MF->getRegInfo().liveout_begin(), - E = MF->getRegInfo().liveout_end(); I != E; ++I) { - assert(TargetRegisterInfo::isPhysicalRegister(*I) && - "Cannot have a live-out virtual register!"); - HandlePhysRegUse(*I, Ret); - - // Add live-out registers as implicit uses. - if (!Ret->readsRegister(*I)) - Ret->addOperand(MachineOperand::CreateReg(*I, false, true)); - } - } - // MachineCSE may CSE instructions which write to non-allocatable physical // registers across MBBs. Remember if any reserved register is liveout. SmallSet<unsigned, 4> LiveOuts; diff --git a/contrib/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp b/contrib/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp index fbc9e20..352ef94 100644 --- a/contrib/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp +++ b/contrib/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp @@ -15,26 +15,26 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "localstackalloc" -#include "llvm/Constants.h" -#include "llvm/DerivedTypes.h" -#include "llvm/Instructions.h" -#include "llvm/Intrinsics.h" -#include "llvm/LLVMContext.h" -#include "llvm/Module.h" -#include "llvm/Pass.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetFrameLowering.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/MachineBasicBlock.cpp b/contrib/llvm/lib/CodeGen/MachineBasicBlock.cpp index 18d021d..898e165 100644 --- a/contrib/llvm/lib/CodeGen/MachineBasicBlock.cpp +++ b/contrib/llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -12,24 +12,26 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/BasicBlock.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/Assembly/Writer.h" +#include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SlotIndexes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/DataLayout.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Assembly/Writer.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/Support/Debug.h" #include "llvm/Support/LeakDetector.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <algorithm> using namespace llvm; @@ -663,6 +665,13 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { << " -- BB#" << NMBB->getNumber() << " -- BB#" << Succ->getNumber() << '\n'); + LiveIntervals *LIS = P->getAnalysisIfAvailable<LiveIntervals>(); + SlotIndexes *Indexes = P->getAnalysisIfAvailable<SlotIndexes>(); + if (LIS) + LIS->insertMBBInMaps(NMBB); + else if (Indexes) + Indexes->insertMBBInMaps(NMBB); + // On some targets like Mips, branches may kill virtual registers. Make sure // that LiveVariables is properly updated after updateTerminator replaces the // terminators. @@ -689,14 +698,67 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { } } + SmallVector<unsigned, 4> UsedRegs; + if (LIS) { + for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); + I != E; ++I) { + MachineInstr *MI = I; + + for (MachineInstr::mop_iterator OI = MI->operands_begin(), + OE = MI->operands_end(); OI != OE; ++OI) { + if (!OI->isReg() || OI->getReg() == 0) + continue; + + unsigned Reg = OI->getReg(); + if (std::find(UsedRegs.begin(), UsedRegs.end(), Reg) == UsedRegs.end()) + UsedRegs.push_back(Reg); + } + } + } + ReplaceUsesOfBlockWith(Succ, NMBB); + + // If updateTerminator() removes instructions, we need to remove them from + // SlotIndexes. + SmallVector<MachineInstr*, 4> Terminators; + if (Indexes) { + for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); + I != E; ++I) + Terminators.push_back(I); + } + updateTerminator(); + if (Indexes) { + SmallVector<MachineInstr*, 4> NewTerminators; + for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); + I != E; ++I) + NewTerminators.push_back(I); + + for (SmallVectorImpl<MachineInstr*>::iterator I = Terminators.begin(), + E = Terminators.end(); I != E; ++I) { + if (std::find(NewTerminators.begin(), NewTerminators.end(), *I) == + NewTerminators.end()) + Indexes->removeMachineInstrFromMaps(*I); + } + } + // Insert unconditional "jump Succ" instruction in NMBB if necessary. NMBB->addSuccessor(Succ); if (!NMBB->isLayoutSuccessor(Succ)) { Cond.clear(); MF->getTarget().getInstrInfo()->InsertBranch(*NMBB, Succ, NULL, Cond, dl); + + if (Indexes) { + for (instr_iterator I = NMBB->instr_begin(), E = NMBB->instr_end(); + I != E; ++I) { + // Some instructions may have been moved to NMBB by updateTerminator(), + // so we first remove any instruction that already has an index. + if (Indexes->hasIndex(I)) + Indexes->removeMachineInstrFromMaps(I); + Indexes->insertMachineInstrInMaps(I); + } + } } // Fix PHI nodes in Succ so they refer to NMBB instead of this @@ -731,6 +793,67 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { LV->addNewBlock(NMBB, this, Succ); } + if (LIS) { + // After splitting the edge and updating SlotIndexes, live intervals may be + // in one of two situations, depending on whether this block was the last in + // the function. If the original block was the last in the function, all live + // intervals will end prior to the beginning of the new split block. If the + // original block was not at the end of the function, all live intervals will + // extend to the end of the new split block. + + bool isLastMBB = + llvm::next(MachineFunction::iterator(NMBB)) == getParent()->end(); + + SlotIndex StartIndex = Indexes->getMBBEndIdx(this); + SlotIndex PrevIndex = StartIndex.getPrevSlot(); + SlotIndex EndIndex = Indexes->getMBBEndIdx(NMBB); + + // Find the registers used from NMBB in PHIs in Succ. + SmallSet<unsigned, 8> PHISrcRegs; + for (MachineBasicBlock::instr_iterator + I = Succ->instr_begin(), E = Succ->instr_end(); + I != E && I->isPHI(); ++I) { + for (unsigned ni = 1, ne = I->getNumOperands(); ni != ne; ni += 2) { + if (I->getOperand(ni+1).getMBB() == NMBB) { + MachineOperand &MO = I->getOperand(ni); + unsigned Reg = MO.getReg(); + PHISrcRegs.insert(Reg); + if (MO.isUndef()) + continue; + + LiveInterval &LI = LIS->getInterval(Reg); + VNInfo *VNI = LI.getVNInfoAt(PrevIndex); + assert(VNI && "PHI sources should be live out of their predecessors."); + LI.addRange(LiveRange(StartIndex, EndIndex, VNI)); + } + } + } + + MachineRegisterInfo *MRI = &getParent()->getRegInfo(); + for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) { + unsigned Reg = TargetRegisterInfo::index2VirtReg(i); + if (PHISrcRegs.count(Reg) || !LIS->hasInterval(Reg)) + continue; + + LiveInterval &LI = LIS->getInterval(Reg); + if (!LI.liveAt(PrevIndex)) + continue; + + bool isLiveOut = LI.liveAt(LIS->getMBBStartIdx(Succ)); + if (isLiveOut && isLastMBB) { + VNInfo *VNI = LI.getVNInfoAt(PrevIndex); + assert(VNI && "LiveInterval should have VNInfo where it is live."); + LI.addRange(LiveRange(StartIndex, EndIndex, VNI)); + } else if (!isLiveOut && !isLastMBB) { + LI.removeRange(StartIndex, EndIndex); + } + } + + // Update all intervals for registers whose uses may have been modified by + // updateTerminator(). + LIS->repairIntervalsInRange(this, getFirstTerminator(), end(), UsedRegs); + } + if (MachineDominatorTree *MDT = P->getAnalysisIfAvailable<MachineDominatorTree>()) { // Update dominator information. @@ -788,40 +911,42 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { return NMBB; } -MachineBasicBlock::iterator -MachineBasicBlock::erase(MachineBasicBlock::iterator I) { - if (I->isBundle()) { - MachineBasicBlock::iterator E = llvm::next(I); - return Insts.erase(I.getInstrIterator(), E.getInstrIterator()); - } - - return Insts.erase(I.getInstrIterator()); +/// Prepare MI to be removed from its bundle. This fixes bundle flags on MI's +/// neighboring instructions so the bundle won't be broken by removing MI. +static void unbundleSingleMI(MachineInstr *MI) { + // Removing the first instruction in a bundle. + if (MI->isBundledWithSucc() && !MI->isBundledWithPred()) + MI->unbundleFromSucc(); + // Removing the last instruction in a bundle. + if (MI->isBundledWithPred() && !MI->isBundledWithSucc()) + MI->unbundleFromPred(); + // If MI is not bundled, or if it is internal to a bundle, the neighbor flags + // are already fine. } -MachineInstr *MachineBasicBlock::remove(MachineInstr *I) { - if (I->isBundle()) { - instr_iterator MII = llvm::next(I); - iterator E = end(); - while (MII != E && MII->isInsideBundle()) { - MachineInstr *MI = &*MII++; - Insts.remove(MI); - } - } +MachineBasicBlock::instr_iterator +MachineBasicBlock::erase(MachineBasicBlock::instr_iterator I) { + unbundleSingleMI(I); + return Insts.erase(I); +} - return Insts.remove(I); +MachineInstr *MachineBasicBlock::remove_instr(MachineInstr *MI) { + unbundleSingleMI(MI); + MI->clearFlag(MachineInstr::BundledPred); + MI->clearFlag(MachineInstr::BundledSucc); + return Insts.remove(MI); } -void MachineBasicBlock::splice(MachineBasicBlock::iterator where, - MachineBasicBlock *Other, - MachineBasicBlock::iterator From) { - if (From->isBundle()) { - MachineBasicBlock::iterator To = llvm::next(From); - Insts.splice(where.getInstrIterator(), Other->Insts, - From.getInstrIterator(), To.getInstrIterator()); - return; +MachineBasicBlock::instr_iterator +MachineBasicBlock::insert(instr_iterator I, MachineInstr *MI) { + assert(!MI->isBundledWithPred() && !MI->isBundledWithSucc() && + "Cannot insert instruction with bundle flags"); + // Set the bundle flags when inserting inside a bundle. + if (I != instr_end() && I->isBundledWithPred()) { + MI->setFlag(MachineInstr::BundledPred); + MI->setFlag(MachineInstr::BundledSucc); } - - Insts.splice(where.getInstrIterator(), Other->Insts, From.getInstrIterator()); + return Insts.insert(I, MI); } /// removeFromParent - This method unlinks 'this' from the containing function, @@ -982,7 +1107,6 @@ MachineBasicBlock::LivenessQueryResult MachineBasicBlock::computeRegisterLiveness(const TargetRegisterInfo *TRI, unsigned Reg, MachineInstr *MI, unsigned Neighborhood) { - unsigned N = Neighborhood; MachineBasicBlock *MBB = MI->getParent(); @@ -997,14 +1121,18 @@ MachineBasicBlock::computeRegisterLiveness(const TargetRegisterInfo *TRI, MachineOperandIteratorBase::PhysRegInfo Analysis = MIOperands(I).analyzePhysReg(Reg, TRI); - if (Analysis.Kills) + if (Analysis.Defines) + // Outputs happen after inputs so they take precedence if both are + // present. + return Analysis.DefinesDead ? LQR_Dead : LQR_Live; + + if (Analysis.Kills || Analysis.Clobbers) // Register killed, so isn't live. return LQR_Dead; - else if (Analysis.DefinesOverlap || Analysis.ReadsOverlap) + else if (Analysis.ReadsOverlap) // Defined or read without a previous kill - live. - return (Analysis.Defines || Analysis.Reads) ? - LQR_Live : LQR_OverlappingLive; + return Analysis.Reads ? LQR_Live : LQR_OverlappingLive; } while (I != MBB->begin() && --N > 0); } @@ -1036,7 +1164,7 @@ MachineBasicBlock::computeRegisterLiveness(const TargetRegisterInfo *TRI, return (Analysis.Reads) ? LQR_Live : LQR_OverlappingLive; - else if (Analysis.DefinesOverlap) + else if (Analysis.Clobbers || Analysis.Defines) // Defined (but not read) therefore cannot have been live. return LQR_Dead; } diff --git a/contrib/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/contrib/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp index a079d6e..070daf2 100644 --- a/contrib/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp +++ b/contrib/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp @@ -11,11 +11,11 @@ // //===----------------------------------------------------------------------===// -#include "llvm/InitializePasses.h" -#include "llvm/Analysis/BlockFrequencyImpl.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" -#include "llvm/CodeGen/Passes.h" +#include "llvm/Analysis/BlockFrequencyImpl.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/InitializePasses.h" using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/contrib/llvm/lib/CodeGen/MachineBlockPlacement.cpp index cd3f199..cd948e2 100644 --- a/contrib/llvm/lib/CodeGen/MachineBlockPlacement.cpp +++ b/contrib/llvm/lib/CodeGen/MachineBlockPlacement.cpp @@ -26,6 +26,11 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "block-placement2" +#include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" @@ -33,13 +38,8 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" -#include "llvm/CodeGen/Passes.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Debug.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" #include <algorithm> @@ -171,7 +171,7 @@ class MachineBlockPlacement : public MachineFunctionPass { const TargetInstrInfo *TII; /// \brief A handle to the target's lowering info. - const TargetLowering *TLI; + const TargetLoweringBase *TLI; /// \brief Allocator and owner of BlockChain structures. /// @@ -1013,8 +1013,8 @@ void MachineBlockPlacement::buildCFGChains(MachineFunction &F) { // exclusively on the loop info here so that we can align backedges in // unnatural CFGs and backedges that were introduced purely because of the // loop rotations done during this layout pass. - if (F.getFunction()->getFnAttributes(). - hasAttribute(Attributes::OptimizeForSize)) + if (F.getFunction()->getAttributes(). + hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize)) return; unsigned Align = TLI->getPrefLoopAlignment(); if (!Align) @@ -1061,7 +1061,7 @@ void MachineBlockPlacement::buildCFGChains(MachineFunction &F) { } // Align this block if the layout predecessor's edge into this block is - // cold relative to the block. When this is true, othe predecessors make up + // cold relative to the block. When this is true, other predecessors make up // all of the hot entries into the block and thus alignment is likely to be // important. BranchProbability LayoutProb = MBPI->getEdgeProbability(LayoutPred, *BI); diff --git a/contrib/llvm/lib/CodeGen/MachineBranchProbabilityInfo.cpp b/contrib/llvm/lib/CodeGen/MachineBranchProbabilityInfo.cpp index 4479211..ae70912 100644 --- a/contrib/llvm/lib/CodeGen/MachineBranchProbabilityInfo.cpp +++ b/contrib/llvm/lib/CodeGen/MachineBranchProbabilityInfo.cpp @@ -11,9 +11,9 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Instructions.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/IR/Instructions.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/contrib/llvm/lib/CodeGen/MachineCSE.cpp b/contrib/llvm/lib/CodeGen/MachineCSE.cpp index dbc41de..61d8d38 100644 --- a/contrib/llvm/lib/CodeGen/MachineCSE.cpp +++ b/contrib/llvm/lib/CodeGen/MachineCSE.cpp @@ -15,17 +15,17 @@ #define DEBUG_TYPE "machine-cse" #include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Target/TargetInstrInfo.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/ScopedHashTable.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/RecyclingAllocator.h" +#include "llvm/Target/TargetInstrInfo.h" using namespace llvm; STATISTIC(NumCoalesces, "Number of copies coalesced"); @@ -126,8 +126,6 @@ bool MachineCSE::PerformTrivialCoalescing(MachineInstr *MI, // deleted. continue; MachineInstr *DefMI = MRI->getVRegDef(Reg); - if (DefMI->getParent() != MBB) - continue; if (!DefMI->isCopy()) continue; unsigned SrcReg = DefMI->getOperand(1).getReg(); diff --git a/contrib/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/contrib/llvm/lib/CodeGen/MachineCopyPropagation.cpp index 4a79328..dc8a224 100644 --- a/contrib/llvm/lib/CodeGen/MachineCopyPropagation.cpp +++ b/contrib/llvm/lib/CodeGen/MachineCopyPropagation.cpp @@ -13,19 +13,19 @@ #define DEBUG_TYPE "codegen-cp" #include "llvm/CodeGen/Passes.h" -#include "llvm/Pass.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/BitVector.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; STATISTIC(NumDeletes, "Number of dead copies deleted"); @@ -33,6 +33,7 @@ STATISTIC(NumDeletes, "Number of dead copies deleted"); namespace { class MachineCopyPropagation : public MachineFunctionPass { const TargetRegisterInfo *TRI; + const TargetInstrInfo *TII; MachineRegisterInfo *MRI; public: @@ -51,6 +52,7 @@ namespace { SourceMap &SrcMap, DenseMap<unsigned, MachineInstr*> &AvailCopyMap); bool CopyPropagateBlock(MachineBasicBlock &MBB); + void removeCopy(MachineInstr *MI); }; } char MachineCopyPropagation::ID = 0; @@ -124,6 +126,16 @@ static bool isNopCopy(MachineInstr *CopyMI, unsigned Def, unsigned Src, return false; } +// Remove MI from the function because it has been determined it is dead. +// Turn it into a noop KILL instruction if it has super-register liveness +// adjustments. +void MachineCopyPropagation::removeCopy(MachineInstr *MI) { + if (MI->getNumOperands() == 2) + MI->eraseFromParent(); + else + MI->setDesc(TII->get(TargetOpcode::KILL)); +} + bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) { SmallSetVector<MachineInstr*, 8> MaybeDeadCopies; // Candidates for deletion DenseMap<unsigned, MachineInstr*> AvailCopyMap; // Def -> available copies map @@ -169,7 +181,7 @@ bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) { for (MachineBasicBlock::iterator I = CopyMI, E = MI; I != E; ++I) I->clearRegisterKills(Def, TRI); - MI->eraseFromParent(); + removeCopy(MI); Changed = true; ++NumDeletes; continue; @@ -262,7 +274,7 @@ bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) { unsigned Reg = (*DI)->getOperand(0).getReg(); if (MRI->isReserved(Reg) || !MaskMO.clobbersPhysReg(Reg)) continue; - (*DI)->eraseFromParent(); + removeCopy(*DI); Changed = true; ++NumDeletes; } @@ -298,7 +310,7 @@ bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) { DI = MaybeDeadCopies.begin(), DE = MaybeDeadCopies.end(); DI != DE; ++DI) { if (!MRI->isReserved((*DI)->getOperand(0).getReg())) { - (*DI)->eraseFromParent(); + removeCopy(*DI); Changed = true; ++NumDeletes; } @@ -312,6 +324,7 @@ bool MachineCopyPropagation::runOnMachineFunction(MachineFunction &MF) { bool Changed = false; TRI = MF.getTarget().getRegisterInfo(); + TII = MF.getTarget().getInstrInfo(); MRI = &MF.getRegInfo(); for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) diff --git a/contrib/llvm/lib/CodeGen/MachineFunction.cpp b/contrib/llvm/lib/CodeGen/MachineFunction.cpp index 91d5211..04321f3 100644 --- a/contrib/llvm/lib/CodeGen/MachineFunction.cpp +++ b/contrib/llvm/lib/CodeGen/MachineFunction.cpp @@ -14,28 +14,28 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/DebugInfo.h" -#include "llvm/Function.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/Analysis/ConstantFolding.h" #include "llvm/CodeGen/MachineConstantPool.h" -#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Function.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" -#include "llvm/Analysis/ConstantFolding.h" #include "llvm/Support/Debug.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetFrameLowering.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/Support/GraphWriter.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetFrameLowering.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; //===----------------------------------------------------------------------===// @@ -58,14 +58,17 @@ MachineFunction::MachineFunction(const Function *F, const TargetMachine &TM, else RegInfo = 0; MFInfo = 0; - FrameInfo = new (Allocator) MachineFrameInfo(*TM.getFrameLowering()); - if (Fn->getFnAttributes().hasAttribute(Attributes::StackAlignment)) + FrameInfo = new (Allocator) MachineFrameInfo(*TM.getFrameLowering(), + TM.Options.RealignStack); + if (Fn->getAttributes().hasAttribute(AttributeSet::FunctionIndex, + Attribute::StackAlignment)) FrameInfo->ensureMaxAlignment(Fn->getAttributes(). - getFnAttributes().getStackAlignment()); + getStackAlignment(AttributeSet::FunctionIndex)); ConstantPool = new (Allocator) MachineConstantPool(TM.getDataLayout()); Alignment = TM.getTargetLowering()->getMinFunctionAlignment(); // FIXME: Shouldn't use pref alignment if explicit alignment is set on Fn. - if (!Fn->getFnAttributes().hasAttribute(Attributes::OptimizeForSize)) + if (!Fn->getAttributes().hasAttribute(AttributeSet::FunctionIndex, + Attribute::OptimizeForSize)) Alignment = std::max(Alignment, TM.getTargetLowering()->getPrefFunctionAlignment()); FunctionNumber = FunctionNum; @@ -73,8 +76,15 @@ MachineFunction::MachineFunction(const Function *F, const TargetMachine &TM, } MachineFunction::~MachineFunction() { - BasicBlocks.clear(); + // Don't call destructors on MachineInstr and MachineOperand. All of their + // memory comes from the BumpPtrAllocator which is about to be purged. + // + // Do call MachineBasicBlock destructors, it contains std::vectors. + for (iterator I = begin(), E = end(); I != E; I = BasicBlocks.erase(I)) + I->Insts.clearAndLeakNodesUnsafely(); + InstructionRecycler.clear(Allocator); + OperandRecycler.clear(Allocator); BasicBlockRecycler.clear(Allocator); if (RegInfo) { RegInfo->~MachineRegisterInfo(); @@ -157,7 +167,7 @@ MachineInstr * MachineFunction::CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImp) { return new (InstructionRecycler.Allocate<MachineInstr>(Allocator)) - MachineInstr(MCID, DL, NoImp); + MachineInstr(*this, MCID, DL, NoImp); } /// CloneMachineInstr - Create a new MachineInstr which is a copy of the @@ -172,9 +182,17 @@ MachineFunction::CloneMachineInstr(const MachineInstr *Orig) { /// DeleteMachineInstr - Delete the given MachineInstr. /// +/// This function also serves as the MachineInstr destructor - the real +/// ~MachineInstr() destructor must be empty. void MachineFunction::DeleteMachineInstr(MachineInstr *MI) { - MI->~MachineInstr(); + // Strip it for parts. The operand array and the MI object itself are + // independently recyclable. + if (MI->Operands) + deallocateOperandArray(MI->CapOperands, MI->Operands); + // Don't call ~MachineInstr() which must be trivial anyway because + // ~MachineFunction drops whole lists of MachineInstrs wihout calling their + // destructors. InstructionRecycler.Deallocate(Allocator, MI); } @@ -328,13 +346,6 @@ void MachineFunction::print(raw_ostream &OS, SlotIndexes *Indexes) const { } OS << '\n'; } - if (RegInfo && !RegInfo->liveout_empty()) { - OS << "Function Live Outs:"; - for (MachineRegisterInfo::liveout_iterator - I = RegInfo->liveout_begin(), E = RegInfo->liveout_end(); I != E; ++I) - OS << ' ' << PrintReg(*I, TRI); - OS << '\n'; - } for (const_iterator BB = begin(), E = end(); BB != E; ++BB) { OS << '\n'; @@ -445,6 +456,70 @@ MCSymbol *MachineFunction::getPICBaseSymbol() const { // MachineFrameInfo implementation //===----------------------------------------------------------------------===// +/// ensureMaxAlignment - Make sure the function is at least Align bytes +/// aligned. +void MachineFrameInfo::ensureMaxAlignment(unsigned Align) { + if (!TFI.isStackRealignable() || !RealignOption) + assert(Align <= TFI.getStackAlignment() && + "For targets without stack realignment, Align is out of limit!"); + if (MaxAlignment < Align) MaxAlignment = Align; +} + +/// clampStackAlignment - Clamp the alignment if requested and emit a warning. +static inline unsigned clampStackAlignment(bool ShouldClamp, unsigned Align, + unsigned StackAlign) { + if (!ShouldClamp || Align <= StackAlign) + return Align; + DEBUG(dbgs() << "Warning: requested alignment " << Align + << " exceeds the stack alignment " << StackAlign + << " when stack realignment is off" << '\n'); + return StackAlign; +} + +/// CreateStackObject - Create a new statically sized stack object, returning +/// a nonnegative identifier to represent it. +/// +int MachineFrameInfo::CreateStackObject(uint64_t Size, unsigned Alignment, + bool isSS, bool MayNeedSP, const AllocaInst *Alloca) { + assert(Size != 0 && "Cannot allocate zero size stack objects!"); + Alignment = clampStackAlignment(!TFI.isStackRealignable() || !RealignOption, + Alignment, TFI.getStackAlignment()); + Objects.push_back(StackObject(Size, Alignment, 0, false, isSS, MayNeedSP, + Alloca)); + int Index = (int)Objects.size() - NumFixedObjects - 1; + assert(Index >= 0 && "Bad frame index!"); + ensureMaxAlignment(Alignment); + return Index; +} + +/// CreateSpillStackObject - Create a new statically sized stack object that +/// represents a spill slot, returning a nonnegative identifier to represent +/// it. +/// +int MachineFrameInfo::CreateSpillStackObject(uint64_t Size, + unsigned Alignment) { + Alignment = clampStackAlignment(!TFI.isStackRealignable() || !RealignOption, + Alignment, TFI.getStackAlignment()); + CreateStackObject(Size, Alignment, true, false); + int Index = (int)Objects.size() - NumFixedObjects - 1; + ensureMaxAlignment(Alignment); + return Index; +} + +/// CreateVariableSizedObject - Notify the MachineFrameInfo object that a +/// variable sized object has been created. This must be created whenever a +/// variable sized object is created, whether or not the index returned is +/// actually used. +/// +int MachineFrameInfo::CreateVariableSizedObject(unsigned Alignment) { + HasVarSizedObjects = true; + Alignment = clampStackAlignment(!TFI.isStackRealignable() || !RealignOption, + Alignment, TFI.getStackAlignment()); + Objects.push_back(StackObject(0, Alignment, 0, false, false, true, 0)); + ensureMaxAlignment(Alignment); + return (int)Objects.size()-NumFixedObjects-1; +} + /// CreateFixedObject - Create a new object at a fixed location on the stack. /// All fixed objects should be created before other objects are created for /// efficiency. By default, fixed objects are immutable. This returns an @@ -459,6 +534,8 @@ int MachineFrameInfo::CreateFixedObject(uint64_t Size, int64_t SPOffset, // object is 16-byte aligned. unsigned StackAlign = TFI.getStackAlignment(); unsigned Align = MinAlign(SPOffset, StackAlign); + Align = clampStackAlignment(!TFI.isStackRealignable() || !RealignOption, + Align, TFI.getStackAlignment()); Objects.insert(Objects.begin(), StackObject(Size, Align, SPOffset, Immutable, /*isSS*/ false, /*NeedSP*/ false, @@ -497,6 +574,54 @@ MachineFrameInfo::getPristineRegs(const MachineBasicBlock *MBB) const { return BV; } +unsigned MachineFrameInfo::estimateStackSize(const MachineFunction &MF) const { + const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering(); + const TargetRegisterInfo *RegInfo = MF.getTarget().getRegisterInfo(); + unsigned MaxAlign = getMaxAlignment(); + int Offset = 0; + + // This code is very, very similar to PEI::calculateFrameObjectOffsets(). + // It really should be refactored to share code. Until then, changes + // should keep in mind that there's tight coupling between the two. + + for (int i = getObjectIndexBegin(); i != 0; ++i) { + int FixedOff = -getObjectOffset(i); + if (FixedOff > Offset) Offset = FixedOff; + } + for (unsigned i = 0, e = getObjectIndexEnd(); i != e; ++i) { + if (isDeadObjectIndex(i)) + continue; + Offset += getObjectSize(i); + unsigned Align = getObjectAlignment(i); + // Adjust to alignment boundary + Offset = (Offset+Align-1)/Align*Align; + + MaxAlign = std::max(Align, MaxAlign); + } + + if (adjustsStack() && TFI->hasReservedCallFrame(MF)) + Offset += getMaxCallFrameSize(); + + // Round up the size to a multiple of the alignment. If the function has + // any calls or alloca's, align to the target's StackAlignment value to + // ensure that the callee's frame or the alloca data is suitably aligned; + // otherwise, for leaf functions, align to the TransientStackAlignment + // value. + unsigned StackAlign; + if (adjustsStack() || hasVarSizedObjects() || + (RegInfo->needsStackRealignment(MF) && getObjectIndexEnd() != 0)) + StackAlign = TFI->getStackAlignment(); + else + StackAlign = TFI->getTransientStackAlignment(); + + // If the frame pointer is eliminated, all frame offsets will be relative to + // SP not FP. Align to MaxAlign so this works. + StackAlign = std::max(StackAlign, MaxAlign); + unsigned AlignMask = StackAlign - 1; + Offset = (Offset + AlignMask) & ~uint64_t(AlignMask); + + return (unsigned)Offset; +} void MachineFrameInfo::print(const MachineFunction &MF, raw_ostream &OS) const{ if (Objects.empty()) return; diff --git a/contrib/llvm/lib/CodeGen/MachineFunctionPass.cpp b/contrib/llvm/lib/CodeGen/MachineFunctionPass.cpp index e5a4912..674cc80 100644 --- a/contrib/llvm/lib/CodeGen/MachineFunctionPass.cpp +++ b/contrib/llvm/lib/CodeGen/MachineFunctionPass.cpp @@ -11,7 +11,7 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Function.h" +#include "llvm/IR/Function.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/MachineFunctionAnalysis.h" #include "llvm/CodeGen/MachineFunctionPass.h" diff --git a/contrib/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp b/contrib/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp index ed94efb..fa9c821 100644 --- a/contrib/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp +++ b/contrib/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp @@ -12,11 +12,11 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/SlotIndexes.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/MachineInstr.cpp b/contrib/llvm/lib/CodeGen/MachineInstr.cpp index ce8d520..32d0668 100644 --- a/contrib/llvm/lib/CodeGen/MachineInstr.cpp +++ b/contrib/llvm/lib/CodeGen/MachineInstr.cpp @@ -12,15 +12,9 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/MachineInstr.h" -#include "llvm/Constants.h" -#include "llvm/DebugInfo.h" -#include "llvm/Function.h" -#include "llvm/InlineAsm.h" -#include "llvm/LLVMContext.h" -#include "llvm/Metadata.h" -#include "llvm/Module.h" -#include "llvm/Type.h" -#include "llvm/Value.h" +#include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Assembly/Writer.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" @@ -28,19 +22,24 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/MC/MCSymbol.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/LeakDetector.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/FoldingSet.h" -#include "llvm/ADT/Hashing.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; //===----------------------------------------------------------------------===// @@ -144,7 +143,7 @@ void MachineOperand::ChangeToRegister(unsigned Reg, bool isDef, bool isImp, // Change this to a register and set the reg#. OpKind = MO_Register; SmallContents.RegNo = Reg; - SubReg = 0; + SubReg_TargetFlags = 0; IsDef = isDef; IsImp = isImp; IsKill = isKill; @@ -518,89 +517,50 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) { // MachineInstr Implementation //===----------------------------------------------------------------------===// -/// MachineInstr ctor - This constructor creates a dummy MachineInstr with -/// MCID NULL and no operands. -MachineInstr::MachineInstr() - : MCID(0), Flags(0), AsmPrinterFlags(0), - NumMemRefs(0), MemRefs(0), - Parent(0) { - // Make sure that we get added to a machine basicblock - LeakDetector::addGarbageObject(this); -} - -void MachineInstr::addImplicitDefUseOperands() { +void MachineInstr::addImplicitDefUseOperands(MachineFunction &MF) { if (MCID->ImplicitDefs) for (const uint16_t *ImpDefs = MCID->getImplicitDefs(); *ImpDefs; ++ImpDefs) - addOperand(MachineOperand::CreateReg(*ImpDefs, true, true)); + addOperand(MF, MachineOperand::CreateReg(*ImpDefs, true, true)); if (MCID->ImplicitUses) for (const uint16_t *ImpUses = MCID->getImplicitUses(); *ImpUses; ++ImpUses) - addOperand(MachineOperand::CreateReg(*ImpUses, false, true)); + addOperand(MF, MachineOperand::CreateReg(*ImpUses, false, true)); } /// MachineInstr ctor - This constructor creates a MachineInstr and adds the /// implicit operands. It reserves space for the number of operands specified by /// the MCInstrDesc. -MachineInstr::MachineInstr(const MCInstrDesc &tid, const DebugLoc dl, - bool NoImp) - : MCID(&tid), Flags(0), AsmPrinterFlags(0), - NumMemRefs(0), MemRefs(0), Parent(0), debugLoc(dl) { - unsigned NumImplicitOps = 0; - if (!NoImp) - NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); - Operands.reserve(NumImplicitOps + MCID->getNumOperands()); +MachineInstr::MachineInstr(MachineFunction &MF, const MCInstrDesc &tid, + const DebugLoc dl, bool NoImp) + : MCID(&tid), Parent(0), Operands(0), NumOperands(0), + Flags(0), AsmPrinterFlags(0), + NumMemRefs(0), MemRefs(0), debugLoc(dl) { + // Reserve space for the expected number of operands. + if (unsigned NumOps = MCID->getNumOperands() + + MCID->getNumImplicitDefs() + MCID->getNumImplicitUses()) { + CapOperands = OperandCapacity::get(NumOps); + Operands = MF.allocateOperandArray(CapOperands); + } + if (!NoImp) - addImplicitDefUseOperands(); - // Make sure that we get added to a machine basicblock - LeakDetector::addGarbageObject(this); -} - -/// MachineInstr ctor - Work exactly the same as the ctor two above, except -/// that the MachineInstr is created and added to the end of the specified -/// basic block. -MachineInstr::MachineInstr(MachineBasicBlock *MBB, const DebugLoc dl, - const MCInstrDesc &tid) - : MCID(&tid), Flags(0), AsmPrinterFlags(0), - NumMemRefs(0), MemRefs(0), Parent(0), debugLoc(dl) { - assert(MBB && "Cannot use inserting ctor with null basic block!"); - unsigned NumImplicitOps = - MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); - Operands.reserve(NumImplicitOps + MCID->getNumOperands()); - addImplicitDefUseOperands(); - // Make sure that we get added to a machine basicblock - LeakDetector::addGarbageObject(this); - MBB->push_back(this); // Add instruction to end of basic block! + addImplicitDefUseOperands(MF); } /// MachineInstr ctor - Copies MachineInstr arg exactly /// MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI) - : MCID(&MI.getDesc()), Flags(0), AsmPrinterFlags(0), + : MCID(&MI.getDesc()), Parent(0), Operands(0), NumOperands(0), + Flags(0), AsmPrinterFlags(0), NumMemRefs(MI.NumMemRefs), MemRefs(MI.MemRefs), - Parent(0), debugLoc(MI.getDebugLoc()) { - Operands.reserve(MI.getNumOperands()); + debugLoc(MI.getDebugLoc()) { + CapOperands = OperandCapacity::get(MI.getNumOperands()); + Operands = MF.allocateOperandArray(CapOperands); - // Add operands + // Copy operands. for (unsigned i = 0; i != MI.getNumOperands(); ++i) - addOperand(MI.getOperand(i)); + addOperand(MF, MI.getOperand(i)); - // Copy all the flags. - Flags = MI.Flags; - - // Set parent to null. - Parent = 0; - - LeakDetector::addGarbageObject(this); -} - -MachineInstr::~MachineInstr() { - LeakDetector::removeGarbageObject(this); -#ifndef NDEBUG - for (unsigned i = 0, e = Operands.size(); i != e; ++i) { - assert(Operands[i].ParentMI == this && "ParentMI mismatch!"); - assert((!Operands[i].isReg() || !Operands[i].isOnRegUseList()) && - "Reg operand def/use list corrupted"); - } -#endif + // Copy all the sensible flags. + setFlags(MI.Flags); } /// getRegInfo - If this instruction is embedded into a MachineFunction, @@ -616,7 +576,7 @@ MachineRegisterInfo *MachineInstr::getRegInfo() { /// this instruction from their respective use lists. This requires that the /// operands already be on their use lists. void MachineInstr::RemoveRegOperandsFromUseLists(MachineRegisterInfo &MRI) { - for (unsigned i = 0, e = Operands.size(); i != e; ++i) + for (unsigned i = 0, e = getNumOperands(); i != e; ++i) if (Operands[i].isReg()) MRI.removeRegOperandFromUseList(&Operands[i]); } @@ -625,40 +585,65 @@ void MachineInstr::RemoveRegOperandsFromUseLists(MachineRegisterInfo &MRI) { /// this instruction from their respective use lists. This requires that the /// operands not be on their use lists yet. void MachineInstr::AddRegOperandsToUseLists(MachineRegisterInfo &MRI) { - for (unsigned i = 0, e = Operands.size(); i != e; ++i) + for (unsigned i = 0, e = getNumOperands(); i != e; ++i) if (Operands[i].isReg()) MRI.addRegOperandToUseList(&Operands[i]); } +void MachineInstr::addOperand(const MachineOperand &Op) { + MachineBasicBlock *MBB = getParent(); + assert(MBB && "Use MachineInstrBuilder to add operands to dangling instrs"); + MachineFunction *MF = MBB->getParent(); + assert(MF && "Use MachineInstrBuilder to add operands to dangling instrs"); + addOperand(*MF, Op); +} + +/// Move NumOps MachineOperands from Src to Dst, with support for overlapping +/// ranges. If MRI is non-null also update use-def chains. +static void moveOperands(MachineOperand *Dst, MachineOperand *Src, + unsigned NumOps, MachineRegisterInfo *MRI) { + if (MRI) + return MRI->moveOperands(Dst, Src, NumOps); + + // Here it would be convenient to call memmove, so that isn't allowed because + // MachineOperand has a constructor and so isn't a POD type. + if (Dst < Src) + for (unsigned i = 0; i != NumOps; ++i) + new (Dst + i) MachineOperand(Src[i]); + else + for (unsigned i = NumOps; i ; --i) + new (Dst + i - 1) MachineOperand(Src[i - 1]); +} + /// addOperand - Add the specified operand to the instruction. If it is an /// implicit operand, it is added to the end of the operand list. If it is /// an explicit operand it is added at the end of the explicit operand list /// (before the first implicit operand). -void MachineInstr::addOperand(const MachineOperand &Op) { +void MachineInstr::addOperand(MachineFunction &MF, const MachineOperand &Op) { assert(MCID && "Cannot add operands before providing an instr descriptor"); - bool isImpReg = Op.isReg() && Op.isImplicit(); - MachineRegisterInfo *RegInfo = getRegInfo(); - // If the Operands backing store is reallocated, all register operands must - // be removed and re-added to RegInfo. It is storing pointers to operands. - bool Reallocate = RegInfo && - !Operands.empty() && Operands.size() == Operands.capacity(); + // Check if we're adding one of our existing operands. + if (&Op >= Operands && &Op < Operands + NumOperands) { + // This is unusual: MI->addOperand(MI->getOperand(i)). + // If adding Op requires reallocating or moving existing operands around, + // the Op reference could go stale. Support it by copying Op. + MachineOperand CopyOp(Op); + return addOperand(MF, CopyOp); + } // Find the insert location for the new operand. Implicit registers go at - // the end, everything goes before the implicit regs. - unsigned OpNo = Operands.size(); - - // Remove all the implicit operands from RegInfo if they need to be shifted. + // the end, everything else goes before the implicit regs. + // // FIXME: Allow mixed explicit and implicit operands on inline asm. // InstrEmitter::EmitSpecialNode() is marking inline asm clobbers as // implicit-defs, but they must not be moved around. See the FIXME in // InstrEmitter.cpp. + unsigned OpNo = getNumOperands(); + bool isImpReg = Op.isReg() && Op.isImplicit(); if (!isImpReg && !isInlineAsm()) { while (OpNo && Operands[OpNo-1].isReg() && Operands[OpNo-1].isImplicit()) { --OpNo; assert(!Operands[OpNo].isTied() && "Cannot move tied operands"); - if (RegInfo) - RegInfo->removeRegOperandFromUseList(&Operands[OpNo]); } } @@ -669,55 +654,56 @@ void MachineInstr::addOperand(const MachineOperand &Op) { OpNo < MCID->getNumOperands()) && "Trying to add an operand to a machine instr that is already done!"); - // All operands from OpNo have been removed from RegInfo. If the Operands - // backing store needs to be reallocated, we also need to remove any other - // register operands. - if (Reallocate) - for (unsigned i = 0; i != OpNo; ++i) - if (Operands[i].isReg()) - RegInfo->removeRegOperandFromUseList(&Operands[i]); - - // Insert the new operand at OpNo. - Operands.insert(Operands.begin() + OpNo, Op); - Operands[OpNo].ParentMI = this; - - // The Operands backing store has now been reallocated, so we can re-add the - // operands before OpNo. - if (Reallocate) - for (unsigned i = 0; i != OpNo; ++i) - if (Operands[i].isReg()) - RegInfo->addRegOperandToUseList(&Operands[i]); - - // When adding a register operand, tell RegInfo about it. - if (Operands[OpNo].isReg()) { + MachineRegisterInfo *MRI = getRegInfo(); + + // Determine if the Operands array needs to be reallocated. + // Save the old capacity and operand array. + OperandCapacity OldCap = CapOperands; + MachineOperand *OldOperands = Operands; + if (!OldOperands || OldCap.getSize() == getNumOperands()) { + CapOperands = OldOperands ? OldCap.getNext() : OldCap.get(1); + Operands = MF.allocateOperandArray(CapOperands); + // Move the operands before the insertion point. + if (OpNo) + moveOperands(Operands, OldOperands, OpNo, MRI); + } + + // Move the operands following the insertion point. + if (OpNo != NumOperands) + moveOperands(Operands + OpNo + 1, OldOperands + OpNo, NumOperands - OpNo, + MRI); + ++NumOperands; + + // Deallocate the old operand array. + if (OldOperands != Operands && OldOperands) + MF.deallocateOperandArray(OldCap, OldOperands); + + // Copy Op into place. It still needs to be inserted into the MRI use lists. + MachineOperand *NewMO = new (Operands + OpNo) MachineOperand(Op); + NewMO->ParentMI = this; + + // When adding a register operand, tell MRI about it. + if (NewMO->isReg()) { // Ensure isOnRegUseList() returns false, regardless of Op's status. - Operands[OpNo].Contents.Reg.Prev = 0; + NewMO->Contents.Reg.Prev = 0; // Ignore existing ties. This is not a property that can be copied. - Operands[OpNo].TiedTo = 0; - // Add the new operand to RegInfo. - if (RegInfo) - RegInfo->addRegOperandToUseList(&Operands[OpNo]); + NewMO->TiedTo = 0; + // Add the new operand to MRI, but only for instructions in an MBB. + if (MRI) + MRI->addRegOperandToUseList(NewMO); // The MCID operand information isn't accurate until we start adding // explicit operands. The implicit operands are added first, then the // explicits are inserted before them. if (!isImpReg) { // Tie uses to defs as indicated in MCInstrDesc. - if (Operands[OpNo].isUse()) { + if (NewMO->isUse()) { int DefIdx = MCID->getOperandConstraint(OpNo, MCOI::TIED_TO); if (DefIdx != -1) tieOperands(DefIdx, OpNo); } // If the register operand is flagged as early, mark the operand as such. if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1) - Operands[OpNo].setIsEarlyClobber(true); - } - } - - // Re-add all the implicit ops. - if (RegInfo) { - for (unsigned i = OpNo + 1, e = Operands.size(); i != e; ++i) { - assert(Operands[i].isReg() && "Should only be an implicit reg!"); - RegInfo->addRegOperandToUseList(&Operands[i]); + NewMO->setIsEarlyClobber(true); } } } @@ -726,45 +712,27 @@ void MachineInstr::addOperand(const MachineOperand &Op) { /// fewer operand than it started with. /// void MachineInstr::RemoveOperand(unsigned OpNo) { - assert(OpNo < Operands.size() && "Invalid operand number"); + assert(OpNo < getNumOperands() && "Invalid operand number"); untieRegOperand(OpNo); - MachineRegisterInfo *RegInfo = getRegInfo(); - - // Special case removing the last one. - if (OpNo == Operands.size()-1) { - // If needed, remove from the reg def/use list. - if (RegInfo && Operands.back().isReg() && Operands.back().isOnRegUseList()) - RegInfo->removeRegOperandFromUseList(&Operands.back()); - - Operands.pop_back(); - return; - } - - // Otherwise, we are removing an interior operand. If we have reginfo to - // update, remove all operands that will be shifted down from their reg lists, - // move everything down, then re-add them. - if (RegInfo) { - for (unsigned i = OpNo, e = Operands.size(); i != e; ++i) { - if (Operands[i].isReg()) - RegInfo->removeRegOperandFromUseList(&Operands[i]); - } - } #ifndef NDEBUG // Moving tied operands would break the ties. - for (unsigned i = OpNo + 1, e = Operands.size(); i != e; ++i) + for (unsigned i = OpNo + 1, e = getNumOperands(); i != e; ++i) if (Operands[i].isReg()) assert(!Operands[i].isTied() && "Cannot move tied operands"); #endif - Operands.erase(Operands.begin()+OpNo); + MachineRegisterInfo *MRI = getRegInfo(); + if (MRI && Operands[OpNo].isReg()) + MRI->removeRegOperandFromUseList(Operands + OpNo); - if (RegInfo) { - for (unsigned i = OpNo, e = Operands.size(); i != e; ++i) { - if (Operands[i].isReg()) - RegInfo->addRegOperandToUseList(&Operands[i]); - } - } + // Don't call the MachineOperand destructor. A lot of this code depends on + // MachineOperand having a trivial destructor anyway, and adding a call here + // wouldn't make it 'destructor-correct'. + + if (unsigned N = NumOperands - 1 - OpNo) + moveOperands(Operands + OpNo, Operands + OpNo + 1, N, MRI); + --NumOperands; } /// addMemOperand - Add a MachineMemOperand to the machine instruction. @@ -773,33 +741,30 @@ void MachineInstr::RemoveOperand(unsigned OpNo) { void MachineInstr::addMemOperand(MachineFunction &MF, MachineMemOperand *MO) { mmo_iterator OldMemRefs = MemRefs; - uint16_t OldNumMemRefs = NumMemRefs; + unsigned OldNumMemRefs = NumMemRefs; - uint16_t NewNum = NumMemRefs + 1; + unsigned NewNum = NumMemRefs + 1; mmo_iterator NewMemRefs = MF.allocateMemRefsArray(NewNum); std::copy(OldMemRefs, OldMemRefs + OldNumMemRefs, NewMemRefs); NewMemRefs[NewNum - 1] = MO; - - MemRefs = NewMemRefs; - NumMemRefs = NewNum; + setMemRefs(NewMemRefs, NewMemRefs + NewNum); } bool MachineInstr::hasPropertyInBundle(unsigned Mask, QueryType Type) const { - const MachineBasicBlock *MBB = getParent(); - MachineBasicBlock::const_instr_iterator MII = *this; ++MII; - while (MII != MBB->end() && MII->isInsideBundle()) { + assert(!isBundledWithPred() && "Must be called on bundle header"); + for (MachineBasicBlock::const_instr_iterator MII = this;; ++MII) { if (MII->getDesc().getFlags() & Mask) { if (Type == AnyInBundle) return true; } else { - if (Type == AllInBundle) + if (Type == AllInBundle && !MII->isBundle()) return false; } - ++MII; + // This was the last instruction in the bundle. + if (!MII->isBundledWithSucc()) + return Type == AllInBundle; } - - return Type == AllInBundle; } bool MachineInstr::isIdenticalTo(const MachineInstr *Other, @@ -865,46 +830,25 @@ bool MachineInstr::isIdenticalTo(const MachineInstr *Other, return true; } -/// removeFromParent - This method unlinks 'this' from the containing basic -/// block, and returns it, but does not delete it. MachineInstr *MachineInstr::removeFromParent() { assert(getParent() && "Not embedded in a basic block!"); - - // If it's a bundle then remove the MIs inside the bundle as well. - if (isBundle()) { - MachineBasicBlock *MBB = getParent(); - MachineBasicBlock::instr_iterator MII = *this; ++MII; - MachineBasicBlock::instr_iterator E = MBB->instr_end(); - while (MII != E && MII->isInsideBundle()) { - MachineInstr *MI = &*MII; - ++MII; - MBB->remove(MI); - } - } - getParent()->remove(this); - return this; + return getParent()->remove(this); } +MachineInstr *MachineInstr::removeFromBundle() { + assert(getParent() && "Not embedded in a basic block!"); + return getParent()->remove_instr(this); +} -/// eraseFromParent - This method unlinks 'this' from the containing basic -/// block, and deletes it. void MachineInstr::eraseFromParent() { assert(getParent() && "Not embedded in a basic block!"); - // If it's a bundle then remove the MIs inside the bundle as well. - if (isBundle()) { - MachineBasicBlock *MBB = getParent(); - MachineBasicBlock::instr_iterator MII = *this; ++MII; - MachineBasicBlock::instr_iterator E = MBB->instr_end(); - while (MII != E && MII->isInsideBundle()) { - MachineInstr *MI = &*MII; - ++MII; - MBB->erase(MI); - } - } - // Erase the individual instruction, which may itself be inside a bundle. - getParent()->erase_instr(this); + getParent()->erase(this); } +void MachineInstr::eraseFromBundle() { + assert(getParent() && "Not embedded in a basic block!"); + getParent()->erase_instr(this); +} /// getNumExplicitOperands - Returns the number of non-implicit operands. /// @@ -921,14 +865,40 @@ unsigned MachineInstr::getNumExplicitOperands() const { return NumOperands; } -/// isBundled - Return true if this instruction part of a bundle. This is true -/// if either itself or its following instruction is marked "InsideBundle". -bool MachineInstr::isBundled() const { - if (isInsideBundle()) - return true; - MachineBasicBlock::const_instr_iterator nextMI = this; - ++nextMI; - return nextMI != Parent->instr_end() && nextMI->isInsideBundle(); +void MachineInstr::bundleWithPred() { + assert(!isBundledWithPred() && "MI is already bundled with its predecessor"); + setFlag(BundledPred); + MachineBasicBlock::instr_iterator Pred = this; + --Pred; + assert(!Pred->isBundledWithSucc() && "Inconsistent bundle flags"); + Pred->setFlag(BundledSucc); +} + +void MachineInstr::bundleWithSucc() { + assert(!isBundledWithSucc() && "MI is already bundled with its successor"); + setFlag(BundledSucc); + MachineBasicBlock::instr_iterator Succ = this; + ++Succ; + assert(!Succ->isBundledWithPred() && "Inconsistent bundle flags"); + Succ->setFlag(BundledPred); +} + +void MachineInstr::unbundleFromPred() { + assert(isBundledWithPred() && "MI isn't bundled with its predecessor"); + clearFlag(BundledPred); + MachineBasicBlock::instr_iterator Pred = this; + --Pred; + assert(Pred->isBundledWithSucc() && "Inconsistent bundle flags"); + Pred->clearFlag(BundledSucc); +} + +void MachineInstr::unbundleFromSucc() { + assert(isBundledWithSucc() && "MI isn't bundled with its successor"); + clearFlag(BundledSucc); + MachineBasicBlock::instr_iterator Succ = this; + ++Succ; + assert(Succ->isBundledWithPred() && "Inconsistent bundle flags"); + Succ->clearFlag(BundledPred); } bool MachineInstr::isStackAligningInlineAsm() const { @@ -1011,18 +981,13 @@ MachineInstr::getRegClassConstraint(unsigned OpIdx, return NULL; } -/// getBundleSize - Return the number of instructions inside the MI bundle. +/// Return the number of instructions inside the MI bundle, not counting the +/// header instruction. unsigned MachineInstr::getBundleSize() const { - assert(isBundle() && "Expecting a bundle"); - - const MachineBasicBlock *MBB = getParent(); - MachineBasicBlock::const_instr_iterator I = *this, E = MBB->instr_end(); + MachineBasicBlock::const_instr_iterator I = this; unsigned Size = 0; - while ((++I != E) && I->isInsideBundle()) { - ++Size; - } - assert(Size > 1 && "Malformed bundle"); - + while (I->isBundledWithSucc()) + ++Size, ++I; return Size; } @@ -1231,41 +1196,6 @@ void MachineInstr::clearKillInfo() { } } -/// copyKillDeadInfo - Copies kill / dead operand properties from MI. -/// -void MachineInstr::copyKillDeadInfo(const MachineInstr *MI) { - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - const MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || (!MO.isKill() && !MO.isDead())) - continue; - for (unsigned j = 0, ee = getNumOperands(); j != ee; ++j) { - MachineOperand &MOp = getOperand(j); - if (!MOp.isIdenticalTo(MO)) - continue; - if (MO.isKill()) - MOp.setIsKill(); - else - MOp.setIsDead(); - break; - } - } -} - -/// copyPredicates - Copies predicate operand(s) from MI. -void MachineInstr::copyPredicates(const MachineInstr *MI) { - assert(!isBundle() && "MachineInstr::copyPredicates() can't handle bundles"); - - const MCInstrDesc &MCID = MI->getDesc(); - if (!MCID.isPredicable()) - return; - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - if (MCID.OpInfo[i].isPredicate()) { - // Predicated operands must be last operands. - addOperand(MI->getOperand(i)); - } - } -} - void MachineInstr::substituteRegister(unsigned FromReg, unsigned ToReg, unsigned SubIdx, @@ -1460,12 +1390,13 @@ bool MachineInstr::allDefsAreDead() const { /// copyImplicitOps - Copy implicit register operands from specified /// instruction to this instruction. -void MachineInstr::copyImplicitOps(const MachineInstr *MI) { +void MachineInstr::copyImplicitOps(MachineFunction &MF, + const MachineInstr *MI) { for (unsigned i = MI->getDesc().getNumOperands(), e = MI->getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI->getOperand(i); if (MO.isReg() && MO.isImplicit()) - addOperand(MO); + addOperand(MF, MO); } } @@ -1497,7 +1428,8 @@ static void printDebugLoc(DebugLoc DL, const MachineFunction *MF, } } -void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { +void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM, + bool SkipOpers) const { // We can be a bit tidier if we know the TargetMachine and/or MachineFunction. const MachineFunction *MF = 0; const MachineRegisterInfo *MRI = 0; @@ -1534,6 +1466,9 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { else OS << "UNKNOWN"; + if (SkipOpers) + return; + // Print the rest of the operands. bool OmittedAnyCallClobbers = false; bool FirstOp = true; @@ -1545,10 +1480,14 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { OS << " "; getOperand(InlineAsm::MIOp_AsmString).print(OS, TM); - // Print HasSideEffects, IsAlignStack + // Print HasSideEffects, MayLoad, MayStore, IsAlignStack unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); if (ExtraInfo & InlineAsm::Extra_HasSideEffects) OS << " [sideeffect]"; + if (ExtraInfo & InlineAsm::Extra_MayLoad) + OS << " [mayload]"; + if (ExtraInfo & InlineAsm::Extra_MayStore) + OS << " [maystore]"; if (ExtraInfo & InlineAsm::Extra_IsAlignStack) OS << " [alignstack]"; if (getInlineAsmDialect() == InlineAsm::AD_ATT) @@ -1576,12 +1515,12 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { unsigned Reg = MO.getReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) { const MachineRegisterInfo &MRI = MF->getRegInfo(); - if (MRI.use_empty(Reg) && !MRI.isLiveOut(Reg)) { + if (MRI.use_empty(Reg)) { bool HasAliasLive = false; for (MCRegAliasIterator AI(Reg, TM->getRegisterInfo(), true); AI.isValid(); ++AI) { unsigned AliasReg = *AI; - if (!MRI.use_empty(AliasReg) || MRI.isLiveOut(AliasReg)) { + if (!MRI.use_empty(AliasReg)) { HasAliasLive = true; break; } @@ -1653,7 +1592,8 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { } bool HaveSemi = false; - if (Flags) { + const unsigned PrintableFlags = FrameSetup; + if (Flags & PrintableFlags) { if (!HaveSemi) OS << ";"; HaveSemi = true; OS << " flags: "; diff --git a/contrib/llvm/lib/CodeGen/MachineInstrBundle.cpp b/contrib/llvm/lib/CodeGen/MachineInstrBundle.cpp index 1f7fbfc..77bcd1d 100644 --- a/contrib/llvm/lib/CodeGen/MachineInstrBundle.cpp +++ b/contrib/llvm/lib/CodeGen/MachineInstrBundle.cpp @@ -8,14 +8,14 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/MachineInstrBundle.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallVector.h" using namespace llvm; namespace { @@ -47,8 +47,8 @@ bool UnpackMachineBundles::runOnMachineFunction(MachineFunction &MF) { // Remove BUNDLE instruction and the InsideBundle flags from bundled // instructions. if (MI->isBundle()) { - while (++MII != MIE && MII->isInsideBundle()) { - MII->setIsInsideBundle(false); + while (++MII != MIE && MII->isBundledWithPred()) { + MII->unbundleFromPred(); for (unsigned i = 0, e = MII->getNumOperands(); i != e; ++i) { MachineOperand &MO = MII->getOperand(i); if (MO.isReg() && MO.isInternalRead()) @@ -101,13 +101,15 @@ void llvm::finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI) { assert(FirstMI != LastMI && "Empty bundle?"); + MIBundleBuilder Bundle(MBB, FirstMI, LastMI); const TargetMachine &TM = MBB.getParent()->getTarget(); const TargetInstrInfo *TII = TM.getInstrInfo(); const TargetRegisterInfo *TRI = TM.getRegisterInfo(); - MachineInstrBuilder MIB = BuildMI(MBB, FirstMI, FirstMI->getDebugLoc(), + MachineInstrBuilder MIB = BuildMI(*MBB.getParent(), FirstMI->getDebugLoc(), TII->get(TargetOpcode::BUNDLE)); + Bundle.prepend(MIB); SmallVector<unsigned, 32> LocalDefs; SmallSet<unsigned, 32> LocalDefSet; @@ -177,7 +179,6 @@ void llvm::finalizeBundle(MachineBasicBlock &MBB, } } - FirstMI->setIsInsideBundle(); Defs.clear(); } @@ -223,14 +224,13 @@ bool llvm::finalizeBundles(MachineFunction &MF) { bool Changed = false; for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { MachineBasicBlock &MBB = *I; - MachineBasicBlock::instr_iterator MII = MBB.instr_begin(); - assert(!MII->isInsideBundle() && - "First instr cannot be inside bundle before finalization!"); - MachineBasicBlock::instr_iterator MIE = MBB.instr_end(); if (MII == MIE) continue; + assert(!MII->isInsideBundle() && + "First instr cannot be inside bundle before finalization!"); + for (++MII; MII != MIE; ) { if (!MII->isInsideBundle()) ++MII; @@ -281,7 +281,7 @@ MachineOperandIteratorBase::PhysRegInfo MachineOperandIteratorBase::analyzePhysReg(unsigned Reg, const TargetRegisterInfo *TRI) { bool AllDefsDead = true; - PhysRegInfo PRI = {false, false, false, false, false, false, false}; + PhysRegInfo PRI = {false, false, false, false, false, false}; assert(TargetRegisterInfo::isPhysicalRegister(Reg) && "analyzePhysReg not given a physical register!"); @@ -305,7 +305,9 @@ MachineOperandIteratorBase::analyzePhysReg(unsigned Reg, // Reg or a super-reg is read, and perhaps killed also. PRI.Reads = true; PRI.Kills = MO.isKill(); - } if (IsRegOrOverlapping && MO.readsReg()) { + } + + if (IsRegOrOverlapping && MO.readsReg()) { PRI.ReadsOverlap = true;// Reg or an overlapping register is read. } diff --git a/contrib/llvm/lib/CodeGen/MachineLICM.cpp b/contrib/llvm/lib/CodeGen/MachineLICM.cpp index 169443e..ed3ed4d 100644 --- a/contrib/llvm/lib/CodeGen/MachineLICM.cpp +++ b/contrib/llvm/lib/CodeGen/MachineLICM.cpp @@ -22,6 +22,10 @@ #define DEBUG_TYPE "machine-licm" #include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineLoopInfo.h" @@ -29,17 +33,13 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/MC/MCInstrItineraries.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/Statistic.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; static cl::opt<bool> @@ -62,7 +62,7 @@ namespace { class MachineLICM : public MachineFunctionPass { const TargetMachine *TM; const TargetInstrInfo *TII; - const TargetLowering *TLI; + const TargetLoweringBase *TLI; const TargetRegisterInfo *TRI; const MachineFrameInfo *MFI; MachineRegisterInfo *MRI; @@ -780,7 +780,7 @@ MachineLICM::getRegisterClassIDAndCost(const MachineInstr *MI, unsigned Reg, unsigned OpIdx, unsigned &RCId, unsigned &RCCost) const { const TargetRegisterClass *RC = MRI->getRegClass(Reg); - EVT VT = *RC->vt_begin(); + MVT VT = *RC->vt_begin(); if (VT == MVT::Untyped) { RCId = RC->getID(); RCCost = 1; diff --git a/contrib/llvm/lib/CodeGen/MachineLoopInfo.cpp b/contrib/llvm/lib/CodeGen/MachineLoopInfo.cpp index 27afeec..4e2cfdc 100644 --- a/contrib/llvm/lib/CodeGen/MachineLoopInfo.cpp +++ b/contrib/llvm/lib/CodeGen/MachineLoopInfo.cpp @@ -15,9 +15,9 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/Analysis/LoopInfoImpl.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/Analysis/LoopInfoImpl.h" #include "llvm/Support/Debug.h" using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/MachineLoopRanges.cpp b/contrib/llvm/lib/CodeGen/MachineLoopRanges.cpp deleted file mode 100644 index 17fe67f..0000000 --- a/contrib/llvm/lib/CodeGen/MachineLoopRanges.cpp +++ /dev/null @@ -1,116 +0,0 @@ -//===- MachineLoopRanges.cpp - Ranges of machine loops --------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file provides the implementation of the MachineLoopRanges analysis. -// -//===----------------------------------------------------------------------===// - -#include "llvm/CodeGen/MachineLoopRanges.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/Passes.h" - -using namespace llvm; - -char MachineLoopRanges::ID = 0; -INITIALIZE_PASS_BEGIN(MachineLoopRanges, "machine-loop-ranges", - "Machine Loop Ranges", true, true) -INITIALIZE_PASS_DEPENDENCY(SlotIndexes) -INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) -INITIALIZE_PASS_END(MachineLoopRanges, "machine-loop-ranges", - "Machine Loop Ranges", true, true) - -char &llvm::MachineLoopRangesID = MachineLoopRanges::ID; - -void MachineLoopRanges::getAnalysisUsage(AnalysisUsage &AU) const { - AU.setPreservesAll(); - AU.addRequiredTransitive<SlotIndexes>(); - AU.addRequiredTransitive<MachineLoopInfo>(); - MachineFunctionPass::getAnalysisUsage(AU); -} - -/// runOnMachineFunction - Don't do much, loop ranges are computed on demand. -bool MachineLoopRanges::runOnMachineFunction(MachineFunction &) { - releaseMemory(); - Indexes = &getAnalysis<SlotIndexes>(); - return false; -} - -void MachineLoopRanges::releaseMemory() { - DeleteContainerSeconds(Cache); - Cache.clear(); -} - -MachineLoopRange *MachineLoopRanges::getLoopRange(const MachineLoop *Loop) { - MachineLoopRange *&Range = Cache[Loop]; - if (!Range) - Range = new MachineLoopRange(Loop, Allocator, *Indexes); - return Range; -} - -/// Create a MachineLoopRange, only accessible to MachineLoopRanges. -MachineLoopRange::MachineLoopRange(const MachineLoop *loop, - MachineLoopRange::Allocator &alloc, - SlotIndexes &Indexes) - : Loop(loop), Intervals(alloc), Area(0) { - // Compute loop coverage. - for (MachineLoop::block_iterator I = Loop->block_begin(), - E = Loop->block_end(); I != E; ++I) { - const std::pair<SlotIndex, SlotIndex> &Range = Indexes.getMBBRange(*I); - Intervals.insert(Range.first, Range.second, 1u); - Area += Range.first.distance(Range.second); - } -} - -/// overlaps - Return true if this loop overlaps the given range of machine -/// instructions. -bool MachineLoopRange::overlaps(SlotIndex Start, SlotIndex Stop) { - Map::const_iterator I = Intervals.find(Start); - return I.valid() && Stop > I.start(); -} - -unsigned MachineLoopRange::getNumber() const { - return Loop->getHeader()->getNumber(); -} - -/// byNumber - Comparator for array_pod_sort that sorts a list of -/// MachineLoopRange pointers by number. -int MachineLoopRange::byNumber(const void *pa, const void *pb) { - const MachineLoopRange *a = *static_cast<MachineLoopRange *const *>(pa); - const MachineLoopRange *b = *static_cast<MachineLoopRange *const *>(pb); - unsigned na = a->getNumber(); - unsigned nb = b->getNumber(); - if (na < nb) - return -1; - if (na > nb) - return 1; - return 0; -} - -/// byAreaDesc - Comparator for array_pod_sort that sorts a list of -/// MachineLoopRange pointers by: -/// 1. Descending area. -/// 2. Ascending number. -int MachineLoopRange::byAreaDesc(const void *pa, const void *pb) { - const MachineLoopRange *a = *static_cast<MachineLoopRange *const *>(pa); - const MachineLoopRange *b = *static_cast<MachineLoopRange *const *>(pb); - if (a->getArea() != b->getArea()) - return a->getArea() > b->getArea() ? -1 : 1; - return byNumber(pa, pb); -} - -void MachineLoopRange::print(raw_ostream &OS) const { - OS << "Loop#" << getNumber() << " ="; - for (Map::const_iterator I = Intervals.begin(); I.valid(); ++I) - OS << " [" << I.start() << ';' << I.stop() << ')'; -} - -raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineLoopRange &MLR) { - MLR.print(OS); - return OS; -} diff --git a/contrib/llvm/lib/CodeGen/MachineModuleInfo.cpp b/contrib/llvm/lib/CodeGen/MachineModuleInfo.cpp index 005bf78..0ea9ae0 100644 --- a/contrib/llvm/lib/CodeGen/MachineModuleInfo.cpp +++ b/contrib/llvm/lib/CodeGen/MachineModuleInfo.cpp @@ -8,18 +8,17 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/MachineModuleInfo.h" - -#include "llvm/Constants.h" -#include "llvm/DerivedTypes.h" -#include "llvm/GlobalVariable.h" -#include "llvm/Module.h" +#include "llvm/ADT/PointerUnion.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/Passes.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCSymbol.h" -#include "llvm/ADT/PointerUnion.h" #include "llvm/Support/Dwarf.h" #include "llvm/Support/ErrorHandling.h" using namespace llvm; @@ -254,15 +253,8 @@ void MMIAddrLabelMapCallbackPtr::allUsesReplacedWith(Value *V2) { MachineModuleInfo::MachineModuleInfo(const MCAsmInfo &MAI, const MCRegisterInfo &MRI, const MCObjectFileInfo *MOFI) - : ImmutablePass(ID), Context(MAI, MRI, MOFI), - ObjFileMMI(0), CompactUnwindEncoding(0), CurCallSite(0), CallsEHReturn(0), - CallsUnwindInit(0), DbgInfoAvailable(false), - UsesVAFloatArgument(false) { + : ImmutablePass(ID), Context(MAI, MRI, MOFI, 0, false) { initializeMachineModuleInfoPass(*PassRegistry::getPassRegistry()); - // Always emit some info, by default "no personality" info. - Personalities.push_back(NULL); - AddrLabelSymbols = 0; - TheModule = 0; } MachineModuleInfo::MachineModuleInfo() @@ -274,26 +266,36 @@ MachineModuleInfo::MachineModuleInfo() } MachineModuleInfo::~MachineModuleInfo() { - delete ObjFileMMI; +} - // FIXME: Why isn't doFinalization being called?? - //assert(AddrLabelSymbols == 0 && "doFinalization not called"); - delete AddrLabelSymbols; +bool MachineModuleInfo::doInitialization(Module &M) { + + ObjFileMMI = 0; + CompactUnwindEncoding = 0; + CurCallSite = 0; + CallsEHReturn = 0; + CallsUnwindInit = 0; + DbgInfoAvailable = UsesVAFloatArgument = false; + // Always emit some info, by default "no personality" info. + Personalities.push_back(NULL); AddrLabelSymbols = 0; -} + TheModule = 0; -/// doInitialization - Initialize the state for a new module. -/// -bool MachineModuleInfo::doInitialization() { - assert(AddrLabelSymbols == 0 && "Improperly initialized"); return false; } -/// doFinalization - Tear down the state after completion of a module. -/// -bool MachineModuleInfo::doFinalization() { +bool MachineModuleInfo::doFinalization(Module &M) { + + Personalities.clear(); + delete AddrLabelSymbols; AddrLabelSymbols = 0; + + Context.reset(); + + delete ObjFileMMI; + ObjFileMMI = 0; + return false; } diff --git a/contrib/llvm/lib/CodeGen/MachineRegisterInfo.cpp b/contrib/llvm/lib/CodeGen/MachineRegisterInfo.cpp index 95d7a7d..1af00e8 100644 --- a/contrib/llvm/lib/CodeGen/MachineRegisterInfo.cpp +++ b/contrib/llvm/lib/CodeGen/MachineRegisterInfo.cpp @@ -30,12 +30,6 @@ MachineRegisterInfo::MachineRegisterInfo(const TargetRegisterInfo &TRI) } MachineRegisterInfo::~MachineRegisterInfo() { -#ifndef NDEBUG - clearVirtRegs(); - for (unsigned i = 0, e = TRI->getNumRegs(); i != e; ++i) - assert(!PhysRegUseDefLists[i] && - "PhysRegUseDefLists has entries after all instructions are deleted"); -#endif delete [] PhysRegUseDefLists; } @@ -43,6 +37,7 @@ MachineRegisterInfo::~MachineRegisterInfo() { /// void MachineRegisterInfo::setRegClass(unsigned Reg, const TargetRegisterClass *RC) { + assert(RC && RC->isAllocatable() && "Invalid RC for virtual register"); VRegInfo[Reg].first = RC; } @@ -180,6 +175,55 @@ void MachineRegisterInfo::removeRegOperandFromUseList(MachineOperand *MO) { MO->Contents.Reg.Next = 0; } +/// Move NumOps operands from Src to Dst, updating use-def lists as needed. +/// +/// The Dst range is assumed to be uninitialized memory. (Or it may contain +/// operands that won't be destroyed, which is OK because the MO destructor is +/// trivial anyway). +/// +/// The Src and Dst ranges may overlap. +void MachineRegisterInfo::moveOperands(MachineOperand *Dst, + MachineOperand *Src, + unsigned NumOps) { + assert(Src != Dst && NumOps && "Noop moveOperands"); + + // Copy backwards if Dst is within the Src range. + int Stride = 1; + if (Dst >= Src && Dst < Src + NumOps) { + Stride = -1; + Dst += NumOps - 1; + Src += NumOps - 1; + } + + // Copy one operand at a time. + do { + new (Dst) MachineOperand(*Src); + + // Dst takes Src's place in the use-def chain. + if (Src->isReg()) { + MachineOperand *&Head = getRegUseDefListHead(Src->getReg()); + MachineOperand *Prev = Src->Contents.Reg.Prev; + MachineOperand *Next = Src->Contents.Reg.Next; + assert(Head && "List empty, but operand is chained"); + assert(Prev && "Operand was not on use-def list"); + + // Prev links are circular, next link is NULL instead of looping back to + // Head. + if (Src == Head) + Head = Dst; + else + Prev->Contents.Reg.Next = Dst; + + // Update Prev pointer. This also works when Src was pointing to itself + // in a 1-element list. In that case Head == Dst. + (Next ? Next : Head)->Contents.Reg.Prev = Dst; + } + + Dst += Stride; + Src += Stride; + } while (--NumOps); +} + /// replaceRegWith - Replace all instances of FromReg with ToReg in the /// machine function. This is like llvm-level X->replaceAllUsesWith(Y), /// except that it also changes any definitions of the register as well. @@ -240,13 +284,6 @@ bool MachineRegisterInfo::isLiveIn(unsigned Reg) const { return false; } -bool MachineRegisterInfo::isLiveOut(unsigned Reg) const { - for (liveout_iterator I = liveout_begin(), E = liveout_end(); I != E; ++I) - if (*I == Reg) - return true; - return false; -} - /// getLiveInPhysReg - If VReg is a live-in virtual register, return the /// corresponding live-in physical register. unsigned MachineRegisterInfo::getLiveInPhysReg(unsigned VReg) const { diff --git a/contrib/llvm/lib/CodeGen/MachineSSAUpdater.cpp b/contrib/llvm/lib/CodeGen/MachineSSAUpdater.cpp index 076547a..bb6aad7 100644 --- a/contrib/llvm/lib/CodeGen/MachineSSAUpdater.cpp +++ b/contrib/llvm/lib/CodeGen/MachineSSAUpdater.cpp @@ -13,19 +13,19 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/MachineSSAUpdater.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/Support/AlignOf.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Transforms/Utils/SSAUpdaterImpl.h" using namespace llvm; @@ -109,7 +109,7 @@ unsigned LookForIdenticalPHI(MachineBasicBlock *BB, /// a value of the given register class at the start of the specified basic /// block. It returns the virtual register defined by the instruction. static -MachineInstr *InsertNewDef(unsigned Opcode, +MachineInstrBuilder InsertNewDef(unsigned Opcode, MachineBasicBlock *BB, MachineBasicBlock::iterator I, const TargetRegisterClass *RC, MachineRegisterInfo *MRI, @@ -183,13 +183,12 @@ unsigned MachineSSAUpdater::GetValueInMiddleOfBlock(MachineBasicBlock *BB) { // Otherwise, we do need a PHI: insert one now. MachineBasicBlock::iterator Loc = BB->empty() ? BB->end() : BB->begin(); - MachineInstr *InsertedPHI = InsertNewDef(TargetOpcode::PHI, BB, - Loc, VRC, MRI, TII); + MachineInstrBuilder InsertedPHI = InsertNewDef(TargetOpcode::PHI, BB, + Loc, VRC, MRI, TII); // Fill in all the predecessors of the PHI. - MachineInstrBuilder MIB(InsertedPHI); for (unsigned i = 0, e = PredValues.size(); i != e; ++i) - MIB.addReg(PredValues[i].second).addMBB(PredValues[i].first); + InsertedPHI.addReg(PredValues[i].second).addMBB(PredValues[i].first); // See if the PHI node can be merged to a single value. This can happen in // loop cases when we get a PHI of itself and one other value. @@ -316,8 +315,7 @@ public: /// the specified predecessor block. static void AddPHIOperand(MachineInstr *PHI, unsigned Val, MachineBasicBlock *Pred) { - PHI->addOperand(MachineOperand::CreateReg(Val, false)); - PHI->addOperand(MachineOperand::CreateMBB(Pred)); + MachineInstrBuilder(*Pred->getParent(), PHI).addReg(Val).addMBB(Pred); } /// InstrIsPHI - Check if an instruction is a PHI. diff --git a/contrib/llvm/lib/CodeGen/MachineScheduler.cpp b/contrib/llvm/lib/CodeGen/MachineScheduler.cpp index a4817d0..5bd2349 100644 --- a/contrib/llvm/lib/CodeGen/MachineScheduler.cpp +++ b/contrib/llvm/lib/CodeGen/MachineScheduler.cpp @@ -14,20 +14,22 @@ #define DEBUG_TYPE "misched" -#include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/MachineScheduler.h" +#include "llvm/ADT/OwningPtr.h" +#include "llvm/ADT/PriorityQueue.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/CodeGen/LiveIntervalAnalysis.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegisterClassInfo.h" -#include "llvm/CodeGen/ScheduleDAGILP.h" +#include "llvm/CodeGen/ScheduleDFS.h" #include "llvm/CodeGen/ScheduleHazardRecognizer.h" -#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/GraphWriter.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/OwningPtr.h" -#include "llvm/ADT/PriorityQueue.h" - #include <queue> using namespace llvm; @@ -49,14 +51,19 @@ static cl::opt<unsigned> MISchedCutoff("misched-cutoff", cl::Hidden, static bool ViewMISchedDAGs = false; #endif // NDEBUG -// Threshold to very roughly model an out-of-order processor's instruction -// buffers. If the actual value of this threshold matters much in practice, then -// it can be specified by the machine model. For now, it's an experimental -// tuning knob to determine when and if it matters. -static cl::opt<unsigned> ILPWindow("ilp-window", cl::Hidden, - cl::desc("Allow expected latency to exceed the critical path by N cycles " - "before attempting to balance ILP"), - cl::init(10U)); +// Experimental heuristics +static cl::opt<bool> EnableLoadCluster("misched-cluster", cl::Hidden, + cl::desc("Enable load clustering."), cl::init(true)); + +// Experimental heuristics +static cl::opt<bool> EnableMacroFusion("misched-fusion", cl::Hidden, + cl::desc("Enable scheduling for macro fusion."), cl::init(true)); + +static cl::opt<bool> VerifyScheduling("verify-misched", cl::Hidden, + cl::desc("Verify machine instrs before and after machine scheduling")); + +// DAG subtrees must have at least this many nodes. +static const unsigned MinSubtreeSize = 8; //===----------------------------------------------------------------------===// // Machine Instruction Scheduling Pass and Registry @@ -195,6 +202,10 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { LIS = &getAnalysis<LiveIntervals>(); const TargetInstrInfo *TII = MF->getTarget().getInstrInfo(); + if (VerifyScheduling) { + DEBUG(LIS->print(dbgs())); + MF->verify(this, "Before machine scheduling."); + } RegClassInfo->runOnMachineFunction(*MF); // Select the scheduler, or set the default. @@ -261,7 +272,8 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { } DEBUG(dbgs() << "********** MI Scheduling **********\n"); DEBUG(dbgs() << MF->getName() - << ":BB#" << MBB->getNumber() << "\n From: " << *I << " To: "; + << ":BB#" << MBB->getNumber() << " " << MBB->getName() + << "\n From: " << *I << " To: "; if (RegionEnd != MBB->end()) dbgs() << *RegionEnd; else dbgs() << "End"; dbgs() << " Remaining: " << RemainingInstrs << "\n"); @@ -282,6 +294,8 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { } Scheduler->finalizeSchedule(); DEBUG(LIS->print(dbgs())); + if (VerifyScheduling) + MF->verify(this, "After machine scheduling."); return true; } @@ -291,7 +305,7 @@ void MachineScheduler::print(raw_ostream &O, const Module* m) const { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void ReadyQueue::dump() { - dbgs() << Name << ": "; + dbgs() << " " << Name << ": "; for (unsigned i = 0, e = Queue.size(); i < e; ++i) dbgs() << Queue[i]->NodeNum << " "; dbgs() << "\n"; @@ -303,6 +317,25 @@ void ReadyQueue::dump() { // preservation. //===----------------------------------------------------------------------===// +ScheduleDAGMI::~ScheduleDAGMI() { + delete DFSResult; + DeleteContainerPointers(Mutations); + delete SchedImpl; +} + +bool ScheduleDAGMI::addEdge(SUnit *SuccSU, const SDep &PredDep) { + if (SuccSU != &ExitSU) { + // Do not use WillCreateCycle, it assumes SD scheduling. + // If Pred is reachable from Succ, then the edge creates a cycle. + if (Topo.IsReachable(PredDep.getSUnit(), SuccSU)) + return false; + Topo.AddPred(SuccSU, PredDep.getSUnit()); + } + SuccSU->addPred(PredDep, /*Required=*/!PredDep.isArtificial()); + // Return true regardless of whether a new edge needed to be inserted. + return true; +} + /// ReleaseSucc - Decrement the NumPredsLeft count of a successor. When /// NumPredsLeft reaches zero, release the successor node. /// @@ -310,6 +343,12 @@ void ReadyQueue::dump() { void ScheduleDAGMI::releaseSucc(SUnit *SU, SDep *SuccEdge) { SUnit *SuccSU = SuccEdge->getSUnit(); + if (SuccEdge->isWeak()) { + --SuccSU->WeakPredsLeft; + if (SuccEdge->isCluster()) + NextClusterSucc = SuccSU; + return; + } #ifndef NDEBUG if (SuccSU->NumPredsLeft == 0) { dbgs() << "*** Scheduling failed! ***\n"; @@ -338,6 +377,12 @@ void ScheduleDAGMI::releaseSuccessors(SUnit *SU) { void ScheduleDAGMI::releasePred(SUnit *SU, SDep *PredEdge) { SUnit *PredSU = PredEdge->getSUnit(); + if (PredEdge->isWeak()) { + --PredSU->WeakSuccsLeft; + if (PredEdge->isCluster()) + NextClusterPred = PredSU; + return; + } #ifndef NDEBUG if (PredSU->NumSuccsLeft == 0) { dbgs() << "*** Scheduling failed! ***\n"; @@ -433,7 +478,8 @@ void ScheduleDAGMI::initRegPressure() { // Cache the list of excess pressure sets in this region. This will also track // the max pressure in the scheduled code for these sets. RegionCriticalPSets.clear(); - std::vector<unsigned> RegionPressure = RPTracker.getPressure().MaxSetPressure; + const std::vector<unsigned> &RegionPressure = + RPTracker.getPressure().MaxSetPressure; for (unsigned i = 0, e = RegionPressure.size(); i < e; ++i) { unsigned Limit = TRI->getRegPressureSetLimit(i); DEBUG(dbgs() << TRI->getRegPressureSetName(i) @@ -452,7 +498,7 @@ void ScheduleDAGMI::initRegPressure() { // FIXME: When the pressure tracker deals in pressure differences then we won't // iterate over all RegionCriticalPSets[i]. void ScheduleDAGMI:: -updateScheduledPressure(std::vector<unsigned> NewMaxPressure) { +updateScheduledPressure(const std::vector<unsigned> &NewMaxPressure) { for (unsigned i = 0, e = RegionCriticalPSets.size(); i < e; ++i) { unsigned ID = RegionCriticalPSets[i].PSetID; int &MaxUnits = RegionCriticalPSets[i].UnitIncrease; @@ -474,14 +520,23 @@ updateScheduledPressure(std::vector<unsigned> NewMaxPressure) { void ScheduleDAGMI::schedule() { buildDAGWithRegPressure(); + Topo.InitDAGTopologicalSorting(); + postprocessDAG(); + SmallVector<SUnit*, 8> TopRoots, BotRoots; + findRootsAndBiasEdges(TopRoots, BotRoots); + + // Initialize the strategy before modifying the DAG. + // This may initialize a DFSResult to be used for queue priority. + SchedImpl->initialize(this); + DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su) SUnits[su].dumpAll(this)); - if (ViewMISchedDAGs) viewGraph(); - initQueues(); + // Initialize ready queues now that the DAG and priority data are finalized. + initQueues(TopRoots, BotRoots); bool IsTopNode = false; while (SUnit *SU = SchedImpl->pickNode(IsTopNode)) { @@ -498,7 +553,7 @@ void ScheduleDAGMI::schedule() { placeDebugValues(); DEBUG({ - unsigned BBNum = top()->getParent()->getNumber(); + unsigned BBNum = begin()->getParent()->getNumber(); dbgs() << "*** Final schedule for BB#" << BBNum << " ***\n"; dumpSchedule(); dbgs() << '\n'; @@ -516,7 +571,6 @@ void ScheduleDAGMI::buildDAGWithRegPressure() { // Build the DAG, and compute current register pressure. buildSchedGraph(AA, &RPTracker); - if (ViewMISchedDAGs) viewGraph(); // Initialize top/bottom trackers after computing region pressure. initRegPressure(); @@ -529,42 +583,67 @@ void ScheduleDAGMI::postprocessDAG() { } } -// Release all DAG roots for scheduling. -void ScheduleDAGMI::releaseRoots() { - SmallVector<SUnit*, 16> BotRoots; +void ScheduleDAGMI::computeDFSResult() { + if (!DFSResult) + DFSResult = new SchedDFSResult(/*BottomU*/true, MinSubtreeSize); + DFSResult->clear(); + ScheduledTrees.clear(); + DFSResult->resize(SUnits.size()); + DFSResult->compute(SUnits); + ScheduledTrees.resize(DFSResult->getNumSubtrees()); +} +void ScheduleDAGMI::findRootsAndBiasEdges(SmallVectorImpl<SUnit*> &TopRoots, + SmallVectorImpl<SUnit*> &BotRoots) { for (std::vector<SUnit>::iterator I = SUnits.begin(), E = SUnits.end(); I != E; ++I) { + SUnit *SU = &(*I); + assert(!SU->isBoundaryNode() && "Boundary node should not be in SUnits"); + + // Order predecessors so DFSResult follows the critical path. + SU->biasCriticalPath(); + // A SUnit is ready to top schedule if it has no predecessors. - if (I->Preds.empty()) - SchedImpl->releaseTopNode(&(*I)); + if (!I->NumPredsLeft) + TopRoots.push_back(SU); // A SUnit is ready to bottom schedule if it has no successors. - if (I->Succs.empty()) - BotRoots.push_back(&(*I)); + if (!I->NumSuccsLeft) + BotRoots.push_back(SU); } - // Release bottom roots in reverse order so the higher priority nodes appear - // first. This is more natural and slightly more efficient. - for (SmallVectorImpl<SUnit*>::const_reverse_iterator - I = BotRoots.rbegin(), E = BotRoots.rend(); I != E; ++I) - SchedImpl->releaseBottomNode(*I); + ExitSU.biasCriticalPath(); } /// Identify DAG roots and setup scheduler queues. -void ScheduleDAGMI::initQueues() { +void ScheduleDAGMI::initQueues(ArrayRef<SUnit*> TopRoots, + ArrayRef<SUnit*> BotRoots) { + NextClusterSucc = NULL; + NextClusterPred = NULL; - // Initialize the strategy before modifying the DAG. - SchedImpl->initialize(this); + // Release all DAG roots for scheduling, not including EntrySU/ExitSU. + // + // Nodes with unreleased weak edges can still be roots. + // Release top roots in forward order. + for (SmallVectorImpl<SUnit*>::const_iterator + I = TopRoots.begin(), E = TopRoots.end(); I != E; ++I) { + SchedImpl->releaseTopNode(*I); + } + // Release bottom roots in reverse order so the higher priority nodes appear + // first. This is more natural and slightly more efficient. + for (SmallVectorImpl<SUnit*>::const_reverse_iterator + I = BotRoots.rbegin(), E = BotRoots.rend(); I != E; ++I) { + SchedImpl->releaseBottomNode(*I); + } - // Release edges from the special Entry node or to the special Exit node. releaseSuccessors(&EntrySU); releasePredecessors(&ExitSU); - // Release all DAG roots for scheduling. - releaseRoots(); - SchedImpl->registerRoots(); + // Advance past initial DebugValues. + assert(TopRPTracker.getPos() == RegionBegin && "bad initial Top tracker"); CurrentTop = nextIfDebug(RegionBegin, RegionEnd); + TopRPTracker.setPos(CurrentTop); + CurrentBottom = RegionEnd; } @@ -618,6 +697,15 @@ void ScheduleDAGMI::updateQueues(SUnit *SU, bool IsTopNode) { SU->isScheduled = true; + if (DFSResult) { + unsigned SubtreeID = DFSResult->getSubtreeID(SU); + if (!ScheduledTrees.test(SubtreeID)) { + ScheduledTrees.set(SubtreeID); + DFSResult->scheduleTree(SubtreeID); + SchedImpl->scheduleTree(SubtreeID); + } + } + // Notify the scheduling strategy after updating the DAG. SchedImpl->schedNode(SU, IsTopNode); } @@ -635,6 +723,8 @@ void ScheduleDAGMI::placeDebugValues() { std::pair<MachineInstr *, MachineInstr *> P = *prior(DI); MachineInstr *DbgValue = P.first; MachineBasicBlock::iterator OrigPrevMI = P.second; + if (&*RegionBegin == DbgValue) + ++RegionBegin; BB->splice(++OrigPrevMI, BB, DbgValue); if (OrigPrevMI == llvm::prior(RegionEnd)) RegionEnd = DbgValue; @@ -655,6 +745,166 @@ void ScheduleDAGMI::dumpSchedule() const { #endif //===----------------------------------------------------------------------===// +// LoadClusterMutation - DAG post-processing to cluster loads. +//===----------------------------------------------------------------------===// + +namespace { +/// \brief Post-process the DAG to create cluster edges between neighboring +/// loads. +class LoadClusterMutation : public ScheduleDAGMutation { + struct LoadInfo { + SUnit *SU; + unsigned BaseReg; + unsigned Offset; + LoadInfo(SUnit *su, unsigned reg, unsigned ofs) + : SU(su), BaseReg(reg), Offset(ofs) {} + }; + static bool LoadInfoLess(const LoadClusterMutation::LoadInfo &LHS, + const LoadClusterMutation::LoadInfo &RHS); + + const TargetInstrInfo *TII; + const TargetRegisterInfo *TRI; +public: + LoadClusterMutation(const TargetInstrInfo *tii, + const TargetRegisterInfo *tri) + : TII(tii), TRI(tri) {} + + virtual void apply(ScheduleDAGMI *DAG); +protected: + void clusterNeighboringLoads(ArrayRef<SUnit*> Loads, ScheduleDAGMI *DAG); +}; +} // anonymous + +bool LoadClusterMutation::LoadInfoLess( + const LoadClusterMutation::LoadInfo &LHS, + const LoadClusterMutation::LoadInfo &RHS) { + if (LHS.BaseReg != RHS.BaseReg) + return LHS.BaseReg < RHS.BaseReg; + return LHS.Offset < RHS.Offset; +} + +void LoadClusterMutation::clusterNeighboringLoads(ArrayRef<SUnit*> Loads, + ScheduleDAGMI *DAG) { + SmallVector<LoadClusterMutation::LoadInfo,32> LoadRecords; + for (unsigned Idx = 0, End = Loads.size(); Idx != End; ++Idx) { + SUnit *SU = Loads[Idx]; + unsigned BaseReg; + unsigned Offset; + if (TII->getLdStBaseRegImmOfs(SU->getInstr(), BaseReg, Offset, TRI)) + LoadRecords.push_back(LoadInfo(SU, BaseReg, Offset)); + } + if (LoadRecords.size() < 2) + return; + std::sort(LoadRecords.begin(), LoadRecords.end(), LoadInfoLess); + unsigned ClusterLength = 1; + for (unsigned Idx = 0, End = LoadRecords.size(); Idx < (End - 1); ++Idx) { + if (LoadRecords[Idx].BaseReg != LoadRecords[Idx+1].BaseReg) { + ClusterLength = 1; + continue; + } + + SUnit *SUa = LoadRecords[Idx].SU; + SUnit *SUb = LoadRecords[Idx+1].SU; + if (TII->shouldClusterLoads(SUa->getInstr(), SUb->getInstr(), ClusterLength) + && DAG->addEdge(SUb, SDep(SUa, SDep::Cluster))) { + + DEBUG(dbgs() << "Cluster loads SU(" << SUa->NodeNum << ") - SU(" + << SUb->NodeNum << ")\n"); + // Copy successor edges from SUa to SUb. Interleaving computation + // dependent on SUa can prevent load combining due to register reuse. + // Predecessor edges do not need to be copied from SUb to SUa since nearby + // loads should have effectively the same inputs. + for (SUnit::const_succ_iterator + SI = SUa->Succs.begin(), SE = SUa->Succs.end(); SI != SE; ++SI) { + if (SI->getSUnit() == SUb) + continue; + DEBUG(dbgs() << " Copy Succ SU(" << SI->getSUnit()->NodeNum << ")\n"); + DAG->addEdge(SI->getSUnit(), SDep(SUb, SDep::Artificial)); + } + ++ClusterLength; + } + else + ClusterLength = 1; + } +} + +/// \brief Callback from DAG postProcessing to create cluster edges for loads. +void LoadClusterMutation::apply(ScheduleDAGMI *DAG) { + // Map DAG NodeNum to store chain ID. + DenseMap<unsigned, unsigned> StoreChainIDs; + // Map each store chain to a set of dependent loads. + SmallVector<SmallVector<SUnit*,4>, 32> StoreChainDependents; + for (unsigned Idx = 0, End = DAG->SUnits.size(); Idx != End; ++Idx) { + SUnit *SU = &DAG->SUnits[Idx]; + if (!SU->getInstr()->mayLoad()) + continue; + unsigned ChainPredID = DAG->SUnits.size(); + for (SUnit::const_pred_iterator + PI = SU->Preds.begin(), PE = SU->Preds.end(); PI != PE; ++PI) { + if (PI->isCtrl()) { + ChainPredID = PI->getSUnit()->NodeNum; + break; + } + } + // Check if this chain-like pred has been seen + // before. ChainPredID==MaxNodeID for loads at the top of the schedule. + unsigned NumChains = StoreChainDependents.size(); + std::pair<DenseMap<unsigned, unsigned>::iterator, bool> Result = + StoreChainIDs.insert(std::make_pair(ChainPredID, NumChains)); + if (Result.second) + StoreChainDependents.resize(NumChains + 1); + StoreChainDependents[Result.first->second].push_back(SU); + } + // Iterate over the store chains. + for (unsigned Idx = 0, End = StoreChainDependents.size(); Idx != End; ++Idx) + clusterNeighboringLoads(StoreChainDependents[Idx], DAG); +} + +//===----------------------------------------------------------------------===// +// MacroFusion - DAG post-processing to encourage fusion of macro ops. +//===----------------------------------------------------------------------===// + +namespace { +/// \brief Post-process the DAG to create cluster edges between instructions +/// that may be fused by the processor into a single operation. +class MacroFusion : public ScheduleDAGMutation { + const TargetInstrInfo *TII; +public: + MacroFusion(const TargetInstrInfo *tii): TII(tii) {} + + virtual void apply(ScheduleDAGMI *DAG); +}; +} // anonymous + +/// \brief Callback from DAG postProcessing to create cluster edges to encourage +/// fused operations. +void MacroFusion::apply(ScheduleDAGMI *DAG) { + // For now, assume targets can only fuse with the branch. + MachineInstr *Branch = DAG->ExitSU.getInstr(); + if (!Branch) + return; + + for (unsigned Idx = DAG->SUnits.size(); Idx > 0;) { + SUnit *SU = &DAG->SUnits[--Idx]; + if (!TII->shouldScheduleAdjacent(SU->getInstr(), Branch)) + continue; + + // Create a single weak edge from SU to ExitSU. The only effect is to cause + // bottom-up scheduling to heavily prioritize the clustered SU. There is no + // need to copy predecessor edges from ExitSU to SU, since top-down + // scheduling cannot prioritize ExitSU anyway. To defer top-down scheduling + // of SU, we could create an artificial edge from the deepest root, but it + // hasn't been needed yet. + bool Success = DAG->addEdge(&DAG->ExitSU, SDep(SU, SDep::Cluster)); + (void)Success; + assert(Success && "No DAG nodes should be reachable from ExitSU"); + + DEBUG(dbgs() << "Macro Fuse SU(" << SU->NodeNum << ")\n"); + break; + } +} + +//===----------------------------------------------------------------------===// // ConvergingScheduler - Implementation of the standard MachineSchedStrategy. //===----------------------------------------------------------------------===// @@ -666,9 +916,10 @@ public: /// Represent the type of SchedCandidate found within a single queue. /// pickNodeBidirectional depends on these listed by decreasing priority. enum CandReason { - NoCand, SingleExcess, SingleCritical, ResourceReduce, ResourceDemand, - BotHeightReduce, BotPathReduce, TopDepthReduce, TopPathReduce, - SingleMax, MultiPressure, NextDefUse, NodeOrder}; + NoCand, SingleExcess, SingleCritical, Cluster, + ResourceReduce, ResourceDemand, BotHeightReduce, BotPathReduce, + TopDepthReduce, TopPathReduce, SingleMax, MultiPressure, NextDefUse, + NodeOrder}; #ifndef NDEBUG static const char *getReasonStr(ConvergingScheduler::CandReason Reason); @@ -748,23 +999,26 @@ public: unsigned CritResIdx; // Number of micro-ops left to schedule. unsigned RemainingMicroOps; - // Is the unscheduled zone resource limited. - bool IsResourceLimited; - - unsigned MaxRemainingCount; void reset() { CriticalPath = 0; RemainingCounts.clear(); CritResIdx = 0; RemainingMicroOps = 0; - IsResourceLimited = false; - MaxRemainingCount = 0; } SchedRemainder() { reset(); } void init(ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel); + + unsigned getMaxRemainingCount(const TargetSchedModel *SchedModel) const { + if (!SchedModel->hasInstrSchedModel()) + return 0; + + return std::max( + RemainingMicroOps * SchedModel->getMicroOpFactor(), + RemainingCounts[CritResIdx]); + } }; /// Each Scheduling boundary is associated with ready queues. It tracks the @@ -805,15 +1059,15 @@ public: unsigned ExpectedCount; - // Policy flag: attempt to find ILP until expected latency is covered. - bool ShouldIncreaseILP; - #ifndef NDEBUG // Remember the greatest min operand latency. unsigned MaxMinLatency; #endif void reset() { + // A new HazardRec is created for each DAG and owned by SchedBoundary. + delete HazardRec; + Available.clear(); Pending.clear(); CheckPending = false; @@ -828,7 +1082,6 @@ public: CritResIdx = 0; IsResourceLimited = false; ExpectedCount = 0; - ShouldIncreaseILP = false; #ifndef NDEBUG MaxMinLatency = 0; #endif @@ -840,7 +1093,8 @@ public: /// PendingFlag set. SchedBoundary(unsigned ID, const Twine &Name): DAG(0), SchedModel(0), Rem(0), Available(ID, Name+".A"), - Pending(ID << ConvergingScheduler::LogMaxQID, Name+".P") { + Pending(ID << ConvergingScheduler::LogMaxQID, Name+".P"), + HazardRec(0) { reset(); } @@ -856,7 +1110,7 @@ public: unsigned getUnscheduledLatency(SUnit *SU) const { if (isTop()) return SU->getHeight(); - return SU->getDepth(); + return SU->getDepth() + SU->Latency; } unsigned getCriticalCount() const { @@ -865,7 +1119,7 @@ public: bool checkHazard(SUnit *SU); - void checkILPPolicy(); + void setLatencyPolicy(CandPolicy &Policy); void releaseNode(SUnit *SU, unsigned ReadyCycle); @@ -938,7 +1192,7 @@ protected: SchedCandidate &Candidate); #ifndef NDEBUG - void traceCandidate(const SchedCandidate &Cand, const SchedBoundary &Zone); + void traceCandidate(const SchedCandidate &Cand); #endif }; } // namespace @@ -961,6 +1215,13 @@ init(ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel) { RemainingCounts[PIdx] += (Factor * PI->Cycles); } } + for (unsigned PIdx = 0, PEnd = SchedModel->getNumProcResourceKinds(); + PIdx != PEnd; ++PIdx) { + if ((int)(RemainingCounts[PIdx] - RemainingCounts[CritResIdx]) + >= (int)SchedModel->getLatencyFactor()) { + CritResIdx = PIdx; + } + } } void ConvergingScheduler::SchedBoundary:: @@ -977,6 +1238,7 @@ void ConvergingScheduler::initialize(ScheduleDAGMI *dag) { DAG = dag; SchedModel = DAG->getSchedModel(); TRI = DAG->TRI; + Rem.init(DAG, SchedModel); Top.init(DAG, SchedModel, &Rem); Bot.init(DAG, SchedModel, &Rem); @@ -998,7 +1260,7 @@ void ConvergingScheduler::releaseTopNode(SUnit *SU) { if (SU->isScheduled) return; - for (SUnit::succ_iterator I = SU->Preds.begin(), E = SU->Preds.end(); + for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); I != E; ++I) { unsigned PredReadyCycle = I->getSUnit()->TopReadyCycle; unsigned MinLatency = I->getMinLatency(); @@ -1019,6 +1281,8 @@ void ConvergingScheduler::releaseBottomNode(SUnit *SU) { for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); I != E; ++I) { + if (I->isWeak()) + continue; unsigned SuccReadyCycle = I->getSUnit()->BotReadyCycle; unsigned MinLatency = I->getMinLatency(); #ifndef NDEBUG @@ -1067,12 +1331,28 @@ bool ConvergingScheduler::SchedBoundary::checkHazard(SUnit *SU) { return false; } -/// If expected latency is covered, disable ILP policy. -void ConvergingScheduler::SchedBoundary::checkILPPolicy() { - if (ShouldIncreaseILP - && (IsResourceLimited || ExpectedLatency <= CurrCycle)) { - ShouldIncreaseILP = false; - DEBUG(dbgs() << "Disable ILP: " << Available.getName() << '\n'); +/// Compute the remaining latency to determine whether ILP should be increased. +void ConvergingScheduler::SchedBoundary::setLatencyPolicy(CandPolicy &Policy) { + // FIXME: compile time. In all, we visit four queues here one we should only + // need to visit the one that was last popped if we cache the result. + unsigned RemLatency = 0; + for (ReadyQueue::iterator I = Available.begin(), E = Available.end(); + I != E; ++I) { + unsigned L = getUnscheduledLatency(*I); + if (L > RemLatency) + RemLatency = L; + } + for (ReadyQueue::iterator I = Pending.begin(), E = Pending.end(); + I != E; ++I) { + unsigned L = getUnscheduledLatency(*I); + if (L > RemLatency) + RemLatency = L; + } + unsigned CriticalPathLimit = Rem->CriticalPath + SchedModel->getILPWindow(); + if (RemLatency + ExpectedLatency >= CriticalPathLimit + && RemLatency > Rem->getMaxRemainingCount(SchedModel)) { + Policy.ReduceLatency = true; + DEBUG(dbgs() << "Increase ILP: " << Available.getName() << '\n'); } } @@ -1091,15 +1371,6 @@ void ConvergingScheduler::SchedBoundary::releaseNode(SUnit *SU, // Record this node as an immediate dependent of the scheduled node. NextSUs.insert(SU); - - // If CriticalPath has been computed, then check if the unscheduled nodes - // exceed the ILP window. Before registerRoots, CriticalPath==0. - if (Rem->CriticalPath && (ExpectedLatency + getUnscheduledLatency(SU) - > Rem->CriticalPath + ILPWindow)) { - ShouldIncreaseILP = true; - DEBUG(dbgs() << "Increase ILP: " << Available.getName() << " " - << ExpectedLatency << " + " << getUnscheduledLatency(SU) << '\n'); - } } /// Move the boundary of scheduled code by one cycle. @@ -1130,8 +1401,8 @@ void ConvergingScheduler::SchedBoundary::bumpCycle() { CheckPending = true; IsResourceLimited = getCriticalCount() > std::max(ExpectedLatency, CurrCycle); - DEBUG(dbgs() << " *** " << Available.getName() << " cycle " - << CurrCycle << '\n'); + DEBUG(dbgs() << " " << Available.getName() + << " Cycle: " << CurrCycle << '\n'); } /// Add the given processor resource to this scheduled zone. @@ -1147,9 +1418,6 @@ void ConvergingScheduler::SchedBoundary::countResource(unsigned PIdx, assert(Rem->RemainingCounts[PIdx] >= Count && "resource double counted"); Rem->RemainingCounts[PIdx] -= Count; - // Reset MaxRemainingCount for sanity. - Rem->MaxRemainingCount = 0; - // Check if this resource exceeds the current critical resource by a full // cycle. If so, it becomes the critical resource. if ((int)(ResourceCounts[PIdx] - ResourceCounts[CritResIdx]) @@ -1281,9 +1549,7 @@ SUnit *ConvergingScheduler::SchedBoundary::pickOnlyChoice() { /// resources. /// /// If the CriticalZone is latency limited, don't force a policy for the -/// candidates here. Instead, When releasing each candidate, releaseNode -/// compares the region's critical path to the candidate's height or depth and -/// the scheduled zone's expected latency then sets ShouldIncreaseILP. +/// candidates here. Instead, setLatencyPolicy sets ReduceLatency if needed. void ConvergingScheduler::balanceZones( ConvergingScheduler::SchedBoundary &CriticalZone, ConvergingScheduler::SchedCandidate &CriticalCand, @@ -1292,6 +1558,7 @@ void ConvergingScheduler::balanceZones( if (!CriticalZone.IsResourceLimited) return; + assert(SchedModel->hasInstrSchedModel() && "required schedmodel"); SchedRemainder *Rem = CriticalZone.Rem; @@ -1299,7 +1566,7 @@ void ConvergingScheduler::balanceZones( // remainder, try to reduce it. unsigned RemainingCritCount = Rem->RemainingCounts[CriticalZone.CritResIdx]; - if ((int)(Rem->MaxRemainingCount - RemainingCritCount) + if ((int)(Rem->getMaxRemainingCount(SchedModel) - RemainingCritCount) > (int)SchedModel->getLatencyFactor()) { CriticalCand.Policy.ReduceResIdx = CriticalZone.CritResIdx; DEBUG(dbgs() << "Balance " << CriticalZone.Available.getName() << " reduce " @@ -1325,12 +1592,9 @@ void ConvergingScheduler::checkResourceLimits( ConvergingScheduler::SchedCandidate &TopCand, ConvergingScheduler::SchedCandidate &BotCand) { - Bot.checkILPPolicy(); - Top.checkILPPolicy(); - if (Bot.ShouldIncreaseILP) - BotCand.Policy.ReduceLatency = true; - if (Top.ShouldIncreaseILP) - TopCand.Policy.ReduceLatency = true; + // Set ReduceLatency to true if needed. + Bot.setLatencyPolicy(BotCand.Policy); + Top.setLatencyPolicy(TopCand.Policy); // Handle resource-limited regions. if (Top.IsResourceLimited && Bot.IsResourceLimited @@ -1365,9 +1629,6 @@ void ConvergingScheduler::checkResourceLimits( // The critical resource is different in each zone, so request balancing. // Compute the cost of each zone. - Rem.MaxRemainingCount = std::max( - Rem.RemainingMicroOps * SchedModel->getMicroOpFactor(), - Rem.RemainingCounts[Rem.CritResIdx]); Top.ExpectedCount = std::max(Top.ExpectedLatency, Top.CurrCycle); Top.ExpectedCount = std::max( Top.getCriticalCount(), @@ -1399,7 +1660,7 @@ initResourceDelta(const ScheduleDAGMI *DAG, } /// Return true if this heuristic determines order. -static bool tryLess(unsigned TryVal, unsigned CandVal, +static bool tryLess(int TryVal, int CandVal, ConvergingScheduler::SchedCandidate &TryCand, ConvergingScheduler::SchedCandidate &Cand, ConvergingScheduler::CandReason Reason) { @@ -1414,7 +1675,8 @@ static bool tryLess(unsigned TryVal, unsigned CandVal, } return false; } -static bool tryGreater(unsigned TryVal, unsigned CandVal, + +static bool tryGreater(int TryVal, int CandVal, ConvergingScheduler::SchedCandidate &TryCand, ConvergingScheduler::SchedCandidate &Cand, ConvergingScheduler::CandReason Reason) { @@ -1430,6 +1692,10 @@ static bool tryGreater(unsigned TryVal, unsigned CandVal, return false; } +static unsigned getWeakLeft(const SUnit *SU, bool isTop) { + return (isTop) ? SU->WeakPredsLeft : SU->WeakSuccsLeft; +} + /// Apply a set of heursitics to a new candidate. Heuristics are currently /// hierarchical. This may be more efficient than a graduated cost model because /// we don't need to evaluate all aspects of the model for each node in the @@ -1472,6 +1738,26 @@ void ConvergingScheduler::tryCandidate(SchedCandidate &Cand, if (Cand.Reason == SingleCritical) Cand.Reason = MultiPressure; + // Keep clustered nodes together to encourage downstream peephole + // optimizations which may reduce resource requirements. + // + // This is a best effort to set things up for a post-RA pass. Optimizations + // like generating loads of multiple registers should ideally be done within + // the scheduler pass by combining the loads during DAG postprocessing. + const SUnit *NextClusterSU = + Zone.isTop() ? DAG->getNextClusterSucc() : DAG->getNextClusterPred(); + if (tryGreater(TryCand.SU == NextClusterSU, Cand.SU == NextClusterSU, + TryCand, Cand, Cluster)) + return; + // Currently, weak edges are for clustering, so we hard-code that reason. + // However, deferring the current TryCand will not change Cand's reason. + CandReason OrigReason = Cand.Reason; + if (tryLess(getWeakLeft(TryCand.SU, Zone.isTop()), + getWeakLeft(Cand.SU, Zone.isTop()), + TryCand, Cand, Cluster)) { + Cand.Reason = OrigReason; + return; + } // Avoid critical resource consumption and balance the schedule. TryCand.initResourceDelta(DAG, SchedModel); if (tryLess(TryCand.ResDelta.CritResources, Cand.ResDelta.CritResources, @@ -1518,15 +1804,10 @@ void ConvergingScheduler::tryCandidate(SchedCandidate &Cand, // Prefer immediate defs/users of the last scheduled instruction. This is a // nice pressure avoidance strategy that also conserves the processor's // register renaming resources and keeps the machine code readable. - if (Zone.NextSUs.count(TryCand.SU) && !Zone.NextSUs.count(Cand.SU)) { - TryCand.Reason = NextDefUse; - return; - } - if (!Zone.NextSUs.count(TryCand.SU) && Zone.NextSUs.count(Cand.SU)) { - if (Cand.Reason > NextDefUse) - Cand.Reason = NextDefUse; + if (tryGreater(Zone.NextSUs.count(TryCand.SU), Zone.NextSUs.count(Cand.SU), + TryCand, Cand, NextDefUse)) return; - } + // Fall through to original instruction order. if ((Zone.isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum) || (!Zone.isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) { @@ -1572,6 +1853,7 @@ const char *ConvergingScheduler::getReasonStr( case NoCand: return "NOCAND "; case SingleExcess: return "REG-EXCESS"; case SingleCritical: return "REG-CRIT "; + case Cluster: return "CLUSTER "; case SingleMax: return "REG-MAX "; case MultiPressure: return "REG-MULTI "; case ResourceReduce: return "RES-REDUCE"; @@ -1586,9 +1868,7 @@ const char *ConvergingScheduler::getReasonStr( llvm_unreachable("Unknown reason!"); } -void ConvergingScheduler::traceCandidate(const SchedCandidate &Cand, - const SchedBoundary &Zone) { - const char *Label = getReasonStr(Cand.Reason); +void ConvergingScheduler::traceCandidate(const SchedCandidate &Cand) { PressureElement P; unsigned ResIdx = 0; unsigned Latency = 0; @@ -1623,21 +1903,21 @@ void ConvergingScheduler::traceCandidate(const SchedCandidate &Cand, Latency = Cand.SU->getDepth(); break; } - dbgs() << Label << " " << Zone.Available.getName() << " "; + dbgs() << " SU(" << Cand.SU->NodeNum << ") " << getReasonStr(Cand.Reason); if (P.isValid()) - dbgs() << TRI->getRegPressureSetName(P.PSetID) << ":" << P.UnitIncrease - << " "; + dbgs() << " " << TRI->getRegPressureSetName(P.PSetID) + << ":" << P.UnitIncrease << " "; else - dbgs() << " "; + dbgs() << " "; if (ResIdx) - dbgs() << SchedModel->getProcResource(ResIdx)->Name << " "; + dbgs() << " " << SchedModel->getProcResource(ResIdx)->Name << " "; else - dbgs() << " "; + dbgs() << " "; if (Latency) - dbgs() << Latency << " cycles "; + dbgs() << " " << Latency << " cycles "; else - dbgs() << " "; - Cand.SU->dump(DAG); + dbgs() << " "; + dbgs() << '\n'; } #endif @@ -1666,15 +1946,14 @@ void ConvergingScheduler::pickNodeFromQueue(SchedBoundary &Zone, if (TryCand.ResDelta == SchedResourceDelta()) TryCand.initResourceDelta(DAG, SchedModel); Cand.setBest(TryCand); - DEBUG(traceCandidate(Cand, Zone)); + DEBUG(traceCandidate(Cand)); } - TryCand.SU = *I; } } static void tracePick(const ConvergingScheduler::SchedCandidate &Cand, bool IsTop) { - DEBUG(dbgs() << "Pick " << (IsTop ? "top" : "bot") + DEBUG(dbgs() << "Pick " << (IsTop ? "Top" : "Bot") << " SU(" << Cand.SU->NodeNum << ") " << ConvergingScheduler::getReasonStr(Cand.Reason) << '\n'); } @@ -1786,10 +2065,7 @@ SUnit *ConvergingScheduler::pickNode(bool &IsTopNode) { if (SU->isBottomReady()) Bot.removeReady(SU); - DEBUG(dbgs() << "*** " << (IsTopNode ? "Top" : "Bottom") - << " Scheduling Instruction in cycle " - << (IsTopNode ? Top.CurrCycle : Bot.CurrCycle) << '\n'; - SU->dump(DAG)); + DEBUG(dbgs() << "Scheduling " << *SU->getInstr()); return SU; } @@ -1812,7 +2088,13 @@ void ConvergingScheduler::schedNode(SUnit *SU, bool IsTopNode) { static ScheduleDAGInstrs *createConvergingSched(MachineSchedContext *C) { assert((!ForceTopDown || !ForceBottomUp) && "-misched-topdown incompatible with -misched-bottomup"); - return new ScheduleDAGMI(C, new ConvergingScheduler()); + ScheduleDAGMI *DAG = new ScheduleDAGMI(C, new ConvergingScheduler()); + // Register DAG post-processors. + if (EnableLoadCluster) + DAG->addMutation(new LoadClusterMutation(DAG->TII, DAG->TRI)); + if (EnableMacroFusion) + DAG->addMutation(new MacroFusion(DAG->TII)); + return DAG; } static MachineSchedRegistry ConvergingSchedRegistry("converge", "Standard converging scheduler.", @@ -1825,58 +2107,97 @@ ConvergingSchedRegistry("converge", "Standard converging scheduler.", namespace { /// \brief Order nodes by the ILP metric. struct ILPOrder { - ScheduleDAGILP *ILP; + const SchedDFSResult *DFSResult; + const BitVector *ScheduledTrees; bool MaximizeILP; - ILPOrder(ScheduleDAGILP *ilp, bool MaxILP): ILP(ilp), MaximizeILP(MaxILP) {} + ILPOrder(bool MaxILP): DFSResult(0), ScheduledTrees(0), MaximizeILP(MaxILP) {} /// \brief Apply a less-than relation on node priority. + /// + /// (Return true if A comes after B in the Q.) bool operator()(const SUnit *A, const SUnit *B) const { - // Return true if A comes after B in the Q. + unsigned SchedTreeA = DFSResult->getSubtreeID(A); + unsigned SchedTreeB = DFSResult->getSubtreeID(B); + if (SchedTreeA != SchedTreeB) { + // Unscheduled trees have lower priority. + if (ScheduledTrees->test(SchedTreeA) != ScheduledTrees->test(SchedTreeB)) + return ScheduledTrees->test(SchedTreeB); + + // Trees with shallower connections have have lower priority. + if (DFSResult->getSubtreeLevel(SchedTreeA) + != DFSResult->getSubtreeLevel(SchedTreeB)) { + return DFSResult->getSubtreeLevel(SchedTreeA) + < DFSResult->getSubtreeLevel(SchedTreeB); + } + } if (MaximizeILP) - return ILP->getILP(A) < ILP->getILP(B); + return DFSResult->getILP(A) < DFSResult->getILP(B); else - return ILP->getILP(A) > ILP->getILP(B); + return DFSResult->getILP(A) > DFSResult->getILP(B); } }; /// \brief Schedule based on the ILP metric. class ILPScheduler : public MachineSchedStrategy { - ScheduleDAGILP ILP; + /// In case all subtrees are eventually connected to a common root through + /// data dependence (e.g. reduction), place an upper limit on their size. + /// + /// FIXME: A subtree limit is generally good, but in the situation commented + /// above, where multiple similar subtrees feed a common root, we should + /// only split at a point where the resulting subtrees will be balanced. + /// (a motivating test case must be found). + static const unsigned SubtreeLimit = 16; + + ScheduleDAGMI *DAG; ILPOrder Cmp; std::vector<SUnit*> ReadyQ; public: - ILPScheduler(bool MaximizeILP) - : ILP(/*BottomUp=*/true), Cmp(&ILP, MaximizeILP) {} + ILPScheduler(bool MaximizeILP): DAG(0), Cmp(MaximizeILP) {} - virtual void initialize(ScheduleDAGMI *DAG) { + virtual void initialize(ScheduleDAGMI *dag) { + DAG = dag; + DAG->computeDFSResult(); + Cmp.DFSResult = DAG->getDFSResult(); + Cmp.ScheduledTrees = &DAG->getScheduledTrees(); ReadyQ.clear(); - ILP.resize(DAG->SUnits.size()); } virtual void registerRoots() { - for (std::vector<SUnit*>::const_iterator - I = ReadyQ.begin(), E = ReadyQ.end(); I != E; ++I) { - ILP.computeILP(*I); - } + // Restore the heap in ReadyQ with the updated DFS results. + std::make_heap(ReadyQ.begin(), ReadyQ.end(), Cmp); } /// Implement MachineSchedStrategy interface. /// ----------------------------------------- + /// Callback to select the highest priority node from the ready Q. virtual SUnit *pickNode(bool &IsTopNode) { if (ReadyQ.empty()) return NULL; - pop_heap(ReadyQ.begin(), ReadyQ.end(), Cmp); + std::pop_heap(ReadyQ.begin(), ReadyQ.end(), Cmp); SUnit *SU = ReadyQ.back(); ReadyQ.pop_back(); IsTopNode = false; - DEBUG(dbgs() << "*** Scheduling " << *SU->getInstr() - << " ILP: " << ILP.getILP(SU) << '\n'); + DEBUG(dbgs() << "*** Scheduling " << "SU(" << SU->NodeNum << "): " + << *SU->getInstr() + << " ILP: " << DAG->getDFSResult()->getILP(SU) + << " Tree: " << DAG->getDFSResult()->getSubtreeID(SU) << " @" + << DAG->getDFSResult()->getSubtreeLevel( + DAG->getDFSResult()->getSubtreeID(SU)) << '\n'); return SU; } - virtual void schedNode(SUnit *, bool) {} + /// \brief Scheduler callback to notify that a new subtree is scheduled. + virtual void scheduleTree(unsigned SubtreeID) { + std::make_heap(ReadyQ.begin(), ReadyQ.end(), Cmp); + } + + /// Callback after a node is scheduled. Mark a newly scheduled tree, notify + /// DFSResults, and resort the priority Q. + virtual void schedNode(SUnit *SU, bool IsTopNode) { + assert(!IsTopNode && "SchedDFSResult needs bottom-up"); + } virtual void releaseTopNode(SUnit *) { /*only called for top roots*/ } @@ -1986,3 +2307,90 @@ static MachineSchedRegistry ShufflerRegistry( "shuffle", "Shuffle machine instructions alternating directions", createInstructionShuffler); #endif // !NDEBUG + +//===----------------------------------------------------------------------===// +// GraphWriter support for ScheduleDAGMI. +//===----------------------------------------------------------------------===// + +#ifndef NDEBUG +namespace llvm { + +template<> struct GraphTraits< + ScheduleDAGMI*> : public GraphTraits<ScheduleDAG*> {}; + +template<> +struct DOTGraphTraits<ScheduleDAGMI*> : public DefaultDOTGraphTraits { + + DOTGraphTraits (bool isSimple=false) : DefaultDOTGraphTraits(isSimple) {} + + static std::string getGraphName(const ScheduleDAG *G) { + return G->MF.getName(); + } + + static bool renderGraphFromBottomUp() { + return true; + } + + static bool isNodeHidden(const SUnit *Node) { + return (Node->NumPreds > 10 || Node->NumSuccs > 10); + } + + static bool hasNodeAddressLabel(const SUnit *Node, + const ScheduleDAG *Graph) { + return false; + } + + /// If you want to override the dot attributes printed for a particular + /// edge, override this method. + static std::string getEdgeAttributes(const SUnit *Node, + SUnitIterator EI, + const ScheduleDAG *Graph) { + if (EI.isArtificialDep()) + return "color=cyan,style=dashed"; + if (EI.isCtrlDep()) + return "color=blue,style=dashed"; + return ""; + } + + static std::string getNodeLabel(const SUnit *SU, const ScheduleDAG *G) { + std::string Str; + raw_string_ostream SS(Str); + SS << "SU(" << SU->NodeNum << ')'; + return SS.str(); + } + static std::string getNodeDescription(const SUnit *SU, const ScheduleDAG *G) { + return G->getGraphNodeLabel(SU); + } + + static std::string getNodeAttributes(const SUnit *N, + const ScheduleDAG *Graph) { + std::string Str("shape=Mrecord"); + const SchedDFSResult *DFS = + static_cast<const ScheduleDAGMI*>(Graph)->getDFSResult(); + if (DFS) { + Str += ",style=filled,fillcolor=\"#"; + Str += DOT::getColorString(DFS->getSubtreeID(N)); + Str += '"'; + } + return Str; + } +}; +} // namespace llvm +#endif // NDEBUG + +/// viewGraph - Pop up a ghostview window with the reachable parts of the DAG +/// rendered using 'dot'. +/// +void ScheduleDAGMI::viewGraph(const Twine &Name, const Twine &Title) { +#ifndef NDEBUG + ViewGraph(this, Name, false, Title); +#else + errs() << "ScheduleDAGMI::viewGraph is only available in debug builds on " + << "systems with Graphviz or gv!\n"; +#endif // NDEBUG +} + +/// Out-of-line implementation with no arguments is handy for gdb. +void ScheduleDAGMI::viewGraph() { + viewGraph(getDAGName(), "Scheduling-Units Graph for " + getDAGName()); +} diff --git a/contrib/llvm/lib/CodeGen/MachineSink.cpp b/contrib/llvm/lib/CodeGen/MachineSink.cpp index b117f8c..4dafbe5 100644 --- a/contrib/llvm/lib/CodeGen/MachineSink.cpp +++ b/contrib/llvm/lib/CodeGen/MachineSink.cpp @@ -18,18 +18,18 @@ #define DEBUG_TYPE "machine-sink" #include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; static cl::opt<bool> diff --git a/contrib/llvm/lib/CodeGen/MachineTraceMetrics.cpp b/contrib/llvm/lib/CodeGen/MachineTraceMetrics.cpp index 9686b04..49d8c4e 100644 --- a/contrib/llvm/lib/CodeGen/MachineTraceMetrics.cpp +++ b/contrib/llvm/lib/CodeGen/MachineTraceMetrics.cpp @@ -8,20 +8,21 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "machine-trace-metrics" -#include "MachineTraceMetrics.h" +#include "llvm/CodeGen/MachineTraceMetrics.h" +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/SparseSet.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/PostOrderIterator.h" -#include "llvm/ADT/SparseSet.h" using namespace llvm; @@ -57,6 +58,8 @@ bool MachineTraceMetrics::runOnMachineFunction(MachineFunction &Func) { MF->getTarget().getSubtarget<TargetSubtargetInfo>(); SchedModel.init(*ST.getSchedModel(), &ST, TII); BlockInfo.resize(MF->getNumBlockIDs()); + ProcResourceCycles.resize(MF->getNumBlockIDs() * + SchedModel.getNumProcResourceKinds()); return false; } @@ -85,9 +88,13 @@ MachineTraceMetrics::getResources(const MachineBasicBlock *MBB) { return FBI; // Compute resource usage in the block. - // FIXME: Compute per-functional unit counts. FBI->HasCalls = false; unsigned InstrCount = 0; + + // Add up per-processor resource cycles as well. + unsigned PRKinds = SchedModel.getNumProcResourceKinds(); + SmallVector<unsigned, 32> PRCycles(PRKinds); + for (MachineBasicBlock::const_iterator I = MBB->begin(), E = MBB->end(); I != E; ++I) { const MachineInstr *MI = I; @@ -96,11 +103,43 @@ MachineTraceMetrics::getResources(const MachineBasicBlock *MBB) { ++InstrCount; if (MI->isCall()) FBI->HasCalls = true; + + // Count processor resources used. + if (!SchedModel.hasInstrSchedModel()) + continue; + const MCSchedClassDesc *SC = SchedModel.resolveSchedClass(MI); + if (!SC->isValid()) + continue; + + for (TargetSchedModel::ProcResIter + PI = SchedModel.getWriteProcResBegin(SC), + PE = SchedModel.getWriteProcResEnd(SC); PI != PE; ++PI) { + assert(PI->ProcResourceIdx < PRKinds && "Bad processor resource kind"); + PRCycles[PI->ProcResourceIdx] += PI->Cycles; + } } FBI->InstrCount = InstrCount; + + // Scale the resource cycles so they are comparable. + unsigned PROffset = MBB->getNumber() * PRKinds; + for (unsigned K = 0; K != PRKinds; ++K) + ProcResourceCycles[PROffset + K] = + PRCycles[K] * SchedModel.getResourceFactor(K); + return FBI; } +ArrayRef<unsigned> +MachineTraceMetrics::getProcResourceCycles(unsigned MBBNum) const { + assert(BlockInfo[MBBNum].hasResources() && + "getResources() must be called before getProcResourceCycles()"); + unsigned PRKinds = SchedModel.getNumProcResourceKinds(); + assert((MBBNum+1) * PRKinds <= ProcResourceCycles.size()); + return ArrayRef<unsigned>(ProcResourceCycles.data() + MBBNum * PRKinds, + PRKinds); +} + + //===----------------------------------------------------------------------===// // Ensemble utility functions //===----------------------------------------------------------------------===// @@ -108,6 +147,9 @@ MachineTraceMetrics::getResources(const MachineBasicBlock *MBB) { MachineTraceMetrics::Ensemble::Ensemble(MachineTraceMetrics *ct) : MTM(*ct) { BlockInfo.resize(MTM.BlockInfo.size()); + unsigned PRKinds = MTM.SchedModel.getNumProcResourceKinds(); + ProcResourceDepths.resize(MTM.BlockInfo.size() * PRKinds); + ProcResourceHeights.resize(MTM.BlockInfo.size() * PRKinds); } // Virtual destructor serves as an anchor. @@ -123,21 +165,32 @@ MachineTraceMetrics::Ensemble::getLoopFor(const MachineBasicBlock *MBB) const { void MachineTraceMetrics::Ensemble:: computeDepthResources(const MachineBasicBlock *MBB) { TraceBlockInfo *TBI = &BlockInfo[MBB->getNumber()]; + unsigned PRKinds = MTM.SchedModel.getNumProcResourceKinds(); + unsigned PROffset = MBB->getNumber() * PRKinds; // Compute resources from trace above. The top block is simple. if (!TBI->Pred) { TBI->InstrDepth = 0; TBI->Head = MBB->getNumber(); + std::fill(ProcResourceDepths.begin() + PROffset, + ProcResourceDepths.begin() + PROffset + PRKinds, 0); return; } // Compute from the block above. A post-order traversal ensures the // predecessor is always computed first. - TraceBlockInfo *PredTBI = &BlockInfo[TBI->Pred->getNumber()]; + unsigned PredNum = TBI->Pred->getNumber(); + TraceBlockInfo *PredTBI = &BlockInfo[PredNum]; assert(PredTBI->hasValidDepth() && "Trace above has not been computed yet"); const FixedBlockInfo *PredFBI = MTM.getResources(TBI->Pred); TBI->InstrDepth = PredTBI->InstrDepth + PredFBI->InstrCount; TBI->Head = PredTBI->Head; + + // Compute per-resource depths. + ArrayRef<unsigned> PredPRDepths = getProcResourceDepths(PredNum); + ArrayRef<unsigned> PredPRCycles = MTM.getProcResourceCycles(PredNum); + for (unsigned K = 0; K != PRKinds; ++K) + ProcResourceDepths[PROffset + K] = PredPRDepths[K] + PredPRCycles[K]; } // Update resource-related information in the TraceBlockInfo for MBB. @@ -145,22 +198,33 @@ computeDepthResources(const MachineBasicBlock *MBB) { void MachineTraceMetrics::Ensemble:: computeHeightResources(const MachineBasicBlock *MBB) { TraceBlockInfo *TBI = &BlockInfo[MBB->getNumber()]; + unsigned PRKinds = MTM.SchedModel.getNumProcResourceKinds(); + unsigned PROffset = MBB->getNumber() * PRKinds; // Compute resources for the current block. TBI->InstrHeight = MTM.getResources(MBB)->InstrCount; + ArrayRef<unsigned> PRCycles = MTM.getProcResourceCycles(MBB->getNumber()); // The trace tail is done. if (!TBI->Succ) { TBI->Tail = MBB->getNumber(); + std::copy(PRCycles.begin(), PRCycles.end(), + ProcResourceHeights.begin() + PROffset); return; } // Compute from the block below. A post-order traversal ensures the // predecessor is always computed first. - TraceBlockInfo *SuccTBI = &BlockInfo[TBI->Succ->getNumber()]; + unsigned SuccNum = TBI->Succ->getNumber(); + TraceBlockInfo *SuccTBI = &BlockInfo[SuccNum]; assert(SuccTBI->hasValidHeight() && "Trace below has not been computed yet"); TBI->InstrHeight += SuccTBI->InstrHeight; TBI->Tail = SuccTBI->Tail; + + // Compute per-resource heights. + ArrayRef<unsigned> SuccPRHeights = getProcResourceHeights(SuccNum); + for (unsigned K = 0; K != PRKinds; ++K) + ProcResourceHeights[PROffset + K] = SuccPRHeights[K] + PRCycles[K]; } // Check if depth resources for MBB are valid and return the TBI. @@ -181,6 +245,35 @@ getHeightResources(const MachineBasicBlock *MBB) const { return TBI->hasValidHeight() ? TBI : 0; } +/// Get an array of processor resource depths for MBB. Indexed by processor +/// resource kind, this array contains the scaled processor resources consumed +/// by all blocks preceding MBB in its trace. It does not include instructions +/// in MBB. +/// +/// Compare TraceBlockInfo::InstrDepth. +ArrayRef<unsigned> +MachineTraceMetrics::Ensemble:: +getProcResourceDepths(unsigned MBBNum) const { + unsigned PRKinds = MTM.SchedModel.getNumProcResourceKinds(); + assert((MBBNum+1) * PRKinds <= ProcResourceDepths.size()); + return ArrayRef<unsigned>(ProcResourceDepths.data() + MBBNum * PRKinds, + PRKinds); +} + +/// Get an array of processor resource heights for MBB. Indexed by processor +/// resource kind, this array contains the scaled processor resources consumed +/// by this block and all blocks following it in its trace. +/// +/// Compare TraceBlockInfo::InstrHeight. +ArrayRef<unsigned> +MachineTraceMetrics::Ensemble:: +getProcResourceHeights(unsigned MBBNum) const { + unsigned PRKinds = MTM.SchedModel.getNumProcResourceKinds(); + assert((MBBNum+1) * PRKinds <= ProcResourceHeights.size()); + return ArrayRef<unsigned>(ProcResourceHeights.data() + MBBNum * PRKinds, + PRKinds); +} + //===----------------------------------------------------------------------===// // Trace Selection Strategies //===----------------------------------------------------------------------===// @@ -677,7 +770,7 @@ computeCrossBlockCriticalPath(const TraceBlockInfo &TBI) { const MachineInstr *DefMI = MTM.MRI->getVRegDef(LIR.Reg); // Ignore dependencies outside the current trace. const TraceBlockInfo &DefTBI = BlockInfo[DefMI->getParent()->getNumber()]; - if (!DefTBI.isEarlierInSameTrace(TBI)) + if (!DefTBI.isUsefulDominator(TBI)) continue; unsigned Len = LIR.Height + Cycles[DefMI].Depth; MaxLen = std::max(MaxLen, Len); @@ -713,11 +806,24 @@ computeInstrDepths(const MachineBasicBlock *MBB) { SmallVector<DataDep, 8> Deps; while (!Stack.empty()) { MBB = Stack.pop_back_val(); - DEBUG(dbgs() << "Depths for BB#" << MBB->getNumber() << ":\n"); + DEBUG(dbgs() << "\nDepths for BB#" << MBB->getNumber() << ":\n"); TraceBlockInfo &TBI = BlockInfo[MBB->getNumber()]; TBI.HasValidInstrDepths = true; TBI.CriticalPath = 0; + // Print out resource depths here as well. + DEBUG({ + dbgs() << format("%7u Instructions\n", TBI.InstrDepth); + ArrayRef<unsigned> PRDepths = getProcResourceDepths(MBB->getNumber()); + for (unsigned K = 0; K != PRDepths.size(); ++K) + if (PRDepths[K]) { + unsigned Factor = MTM.SchedModel.getResourceFactor(K); + dbgs() << format("%6uc @ ", MTM.getCycles(PRDepths[K])) + << MTM.SchedModel.getProcResource(K)->Name << " (" + << PRDepths[K]/Factor << " ops x" << Factor << ")\n"; + } + }); + // Also compute the critical path length through MBB when possible. if (TBI.HasValidInstrHeights) TBI.CriticalPath = computeCrossBlockCriticalPath(TBI); @@ -740,7 +846,7 @@ computeInstrDepths(const MachineBasicBlock *MBB) { const TraceBlockInfo&DepTBI = BlockInfo[Dep.DefMI->getParent()->getNumber()]; // Ignore dependencies from outside the current trace. - if (!DepTBI.isEarlierInSameTrace(TBI)) + if (!DepTBI.isUsefulDominator(TBI)) continue; assert(DepTBI.HasValidInstrDepths && "Inconsistent dependency"); unsigned DepCycle = Cycles.lookup(Dep.DefMI).Depth; @@ -928,6 +1034,18 @@ computeInstrHeights(const MachineBasicBlock *MBB) { TBI.HasValidInstrHeights = true; TBI.CriticalPath = 0; + DEBUG({ + dbgs() << format("%7u Instructions\n", TBI.InstrHeight); + ArrayRef<unsigned> PRHeights = getProcResourceHeights(MBB->getNumber()); + for (unsigned K = 0; K != PRHeights.size(); ++K) + if (PRHeights[K]) { + unsigned Factor = MTM.SchedModel.getResourceFactor(K); + dbgs() << format("%6uc @ ", MTM.getCycles(PRHeights[K])) + << MTM.SchedModel.getProcResource(K)->Name << " (" + << PRHeights[K]/Factor << " ops x" << Factor << ")\n"; + } + }); + // Get dependencies from PHIs in the trace successor. const MachineBasicBlock *Succ = TBI.Succ; // If MBB is the last block in the trace, and it has a back-edge to the @@ -1058,27 +1176,52 @@ MachineTraceMetrics::Trace::getPHIDepth(const MachineInstr *PHI) const { } unsigned MachineTraceMetrics::Trace::getResourceDepth(bool Bottom) const { - // For now, we compute the resource depth from instruction count / issue - // width. Eventually, we should compute resource depth per functional unit - // and return the max. + // Find the limiting processor resource. + // Numbers have been pre-scaled to be comparable. + unsigned PRMax = 0; + ArrayRef<unsigned> PRDepths = TE.getProcResourceDepths(getBlockNum()); + if (Bottom) { + ArrayRef<unsigned> PRCycles = TE.MTM.getProcResourceCycles(getBlockNum()); + for (unsigned K = 0; K != PRDepths.size(); ++K) + PRMax = std::max(PRMax, PRDepths[K] + PRCycles[K]); + } else { + for (unsigned K = 0; K != PRDepths.size(); ++K) + PRMax = std::max(PRMax, PRDepths[K]); + } + // Convert to cycle count. + PRMax = TE.MTM.getCycles(PRMax); + unsigned Instrs = TBI.InstrDepth; if (Bottom) Instrs += TE.MTM.BlockInfo[getBlockNum()].InstrCount; if (unsigned IW = TE.MTM.SchedModel.getIssueWidth()) Instrs /= IW; // Assume issue width 1 without a schedule model. - return Instrs; + return std::max(Instrs, PRMax); } unsigned MachineTraceMetrics::Trace:: getResourceLength(ArrayRef<const MachineBasicBlock*> Extrablocks) const { + // Add up resources above and below the center block. + ArrayRef<unsigned> PRDepths = TE.getProcResourceDepths(getBlockNum()); + ArrayRef<unsigned> PRHeights = TE.getProcResourceHeights(getBlockNum()); + unsigned PRMax = 0; + for (unsigned K = 0; K != PRDepths.size(); ++K) { + unsigned PRCycles = PRDepths[K] + PRHeights[K]; + for (unsigned I = 0; I != Extrablocks.size(); ++I) + PRCycles += TE.MTM.getProcResourceCycles(Extrablocks[I]->getNumber())[K]; + PRMax = std::max(PRMax, PRCycles); + } + // Convert to cycle count. + PRMax = TE.MTM.getCycles(PRMax); + unsigned Instrs = TBI.InstrDepth + TBI.InstrHeight; for (unsigned i = 0, e = Extrablocks.size(); i != e; ++i) Instrs += TE.MTM.getResources(Extrablocks[i])->InstrCount; if (unsigned IW = TE.MTM.SchedModel.getIssueWidth()) Instrs /= IW; // Assume issue width 1 without a schedule model. - return Instrs; + return std::max(Instrs, PRMax); } void MachineTraceMetrics::Ensemble::print(raw_ostream &OS) const { diff --git a/contrib/llvm/lib/CodeGen/MachineTraceMetrics.h b/contrib/llvm/lib/CodeGen/MachineTraceMetrics.h deleted file mode 100644 index 460730b..0000000 --- a/contrib/llvm/lib/CodeGen/MachineTraceMetrics.h +++ /dev/null @@ -1,350 +0,0 @@ -//===- lib/CodeGen/MachineTraceMetrics.h - Super-scalar metrics -*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines the interface for the MachineTraceMetrics analysis pass -// that estimates CPU resource usage and critical data dependency paths through -// preferred traces. This is useful for super-scalar CPUs where execution speed -// can be limited both by data dependencies and by limited execution resources. -// -// Out-of-order CPUs will often be executing instructions from multiple basic -// blocks at the same time. This makes it difficult to estimate the resource -// usage accurately in a single basic block. Resources can be estimated better -// by looking at a trace through the current basic block. -// -// For every block, the MachineTraceMetrics pass will pick a preferred trace -// that passes through the block. The trace is chosen based on loop structure, -// branch probabilities, and resource usage. The intention is to pick likely -// traces that would be the most affected by code transformations. -// -// It is expensive to compute a full arbitrary trace for every block, so to -// save some computations, traces are chosen to be convergent. This means that -// if the traces through basic blocks A and B ever cross when moving away from -// A and B, they never diverge again. This applies in both directions - If the -// traces meet above A and B, they won't diverge when going further back. -// -// Traces tend to align with loops. The trace through a block in an inner loop -// will begin at the loop entry block and end at a back edge. If there are -// nested loops, the trace may begin and end at those instead. -// -// For each trace, we compute the critical path length, which is the number of -// cycles required to execute the trace when execution is limited by data -// dependencies only. We also compute the resource height, which is the number -// of cycles required to execute all instructions in the trace when ignoring -// data dependencies. -// -// Every instruction in the current block has a slack - the number of cycles -// execution of the instruction can be delayed without extending the critical -// path. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_MACHINE_TRACE_METRICS_H -#define LLVM_CODEGEN_MACHINE_TRACE_METRICS_H - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/TargetSchedule.h" - -namespace llvm { - -class InstrItineraryData; -class MachineBasicBlock; -class MachineInstr; -class MachineLoop; -class MachineLoopInfo; -class MachineRegisterInfo; -class TargetInstrInfo; -class TargetRegisterInfo; -class raw_ostream; - -class MachineTraceMetrics : public MachineFunctionPass { - const MachineFunction *MF; - const TargetInstrInfo *TII; - const TargetRegisterInfo *TRI; - const MachineRegisterInfo *MRI; - const MachineLoopInfo *Loops; - TargetSchedModel SchedModel; - -public: - class Ensemble; - class Trace; - static char ID; - MachineTraceMetrics(); - void getAnalysisUsage(AnalysisUsage&) const; - bool runOnMachineFunction(MachineFunction&); - void releaseMemory(); - void verifyAnalysis() const; - - friend class Ensemble; - friend class Trace; - - /// Per-basic block information that doesn't depend on the trace through the - /// block. - struct FixedBlockInfo { - /// The number of non-trivial instructions in the block. - /// Doesn't count PHI and COPY instructions that are likely to be removed. - unsigned InstrCount; - - /// True when the block contains calls. - bool HasCalls; - - FixedBlockInfo() : InstrCount(~0u), HasCalls(false) {} - - /// Returns true when resource information for this block has been computed. - bool hasResources() const { return InstrCount != ~0u; } - - /// Invalidate resource information. - void invalidate() { InstrCount = ~0u; } - }; - - /// Get the fixed resource information about MBB. Compute it on demand. - const FixedBlockInfo *getResources(const MachineBasicBlock*); - - /// A virtual register or regunit required by a basic block or its trace - /// successors. - struct LiveInReg { - /// The virtual register required, or a register unit. - unsigned Reg; - - /// For virtual registers: Minimum height of the defining instruction. - /// For regunits: Height of the highest user in the trace. - unsigned Height; - - LiveInReg(unsigned Reg, unsigned Height = 0) : Reg(Reg), Height(Height) {} - }; - - /// Per-basic block information that relates to a specific trace through the - /// block. Convergent traces means that only one of these is required per - /// block in a trace ensemble. - struct TraceBlockInfo { - /// Trace predecessor, or NULL for the first block in the trace. - /// Valid when hasValidDepth(). - const MachineBasicBlock *Pred; - - /// Trace successor, or NULL for the last block in the trace. - /// Valid when hasValidHeight(). - const MachineBasicBlock *Succ; - - /// The block number of the head of the trace. (When hasValidDepth()). - unsigned Head; - - /// The block number of the tail of the trace. (When hasValidHeight()). - unsigned Tail; - - /// Accumulated number of instructions in the trace above this block. - /// Does not include instructions in this block. - unsigned InstrDepth; - - /// Accumulated number of instructions in the trace below this block. - /// Includes instructions in this block. - unsigned InstrHeight; - - TraceBlockInfo() : - Pred(0), Succ(0), - InstrDepth(~0u), InstrHeight(~0u), - HasValidInstrDepths(false), HasValidInstrHeights(false) {} - - /// Returns true if the depth resources have been computed from the trace - /// above this block. - bool hasValidDepth() const { return InstrDepth != ~0u; } - - /// Returns true if the height resources have been computed from the trace - /// below this block. - bool hasValidHeight() const { return InstrHeight != ~0u; } - - /// Invalidate depth resources when some block above this one has changed. - void invalidateDepth() { InstrDepth = ~0u; HasValidInstrDepths = false; } - - /// Invalidate height resources when a block below this one has changed. - void invalidateHeight() { InstrHeight = ~0u; HasValidInstrHeights = false; } - - /// Determine if this block belongs to the same trace as TBI and comes - /// before it in the trace. - /// Also returns true when TBI == this. - bool isEarlierInSameTrace(const TraceBlockInfo &TBI) const { - return hasValidDepth() && TBI.hasValidDepth() && - Head == TBI.Head && InstrDepth <= TBI.InstrDepth; - } - - // Data-dependency-related information. Per-instruction depth and height - // are computed from data dependencies in the current trace, using - // itinerary data. - - /// Instruction depths have been computed. This implies hasValidDepth(). - bool HasValidInstrDepths; - - /// Instruction heights have been computed. This implies hasValidHeight(). - bool HasValidInstrHeights; - - /// Critical path length. This is the number of cycles in the longest data - /// dependency chain through the trace. This is only valid when both - /// HasValidInstrDepths and HasValidInstrHeights are set. - unsigned CriticalPath; - - /// Live-in registers. These registers are defined above the current block - /// and used by this block or a block below it. - /// This does not include PHI uses in the current block, but it does - /// include PHI uses in deeper blocks. - SmallVector<LiveInReg, 4> LiveIns; - - void print(raw_ostream&) const; - }; - - /// InstrCycles represents the cycle height and depth of an instruction in a - /// trace. - struct InstrCycles { - /// Earliest issue cycle as determined by data dependencies and instruction - /// latencies from the beginning of the trace. Data dependencies from - /// before the trace are not included. - unsigned Depth; - - /// Minimum number of cycles from this instruction is issued to the of the - /// trace, as determined by data dependencies and instruction latencies. - unsigned Height; - }; - - /// A trace represents a plausible sequence of executed basic blocks that - /// passes through the current basic block one. The Trace class serves as a - /// handle to internal cached data structures. - class Trace { - Ensemble &TE; - TraceBlockInfo &TBI; - - unsigned getBlockNum() const { return &TBI - &TE.BlockInfo[0]; } - - public: - explicit Trace(Ensemble &te, TraceBlockInfo &tbi) : TE(te), TBI(tbi) {} - void print(raw_ostream&) const; - - /// Compute the total number of instructions in the trace. - unsigned getInstrCount() const { - return TBI.InstrDepth + TBI.InstrHeight; - } - - /// Return the resource depth of the top/bottom of the trace center block. - /// This is the number of cycles required to execute all instructions from - /// the trace head to the trace center block. The resource depth only - /// considers execution resources, it ignores data dependencies. - /// When Bottom is set, instructions in the trace center block are included. - unsigned getResourceDepth(bool Bottom) const; - - /// Return the resource length of the trace. This is the number of cycles - /// required to execute the instructions in the trace if they were all - /// independent, exposing the maximum instruction-level parallelism. - /// - /// Any blocks in Extrablocks are included as if they were part of the - /// trace. - unsigned getResourceLength(ArrayRef<const MachineBasicBlock*> Extrablocks = - ArrayRef<const MachineBasicBlock*>()) const; - - /// Return the length of the (data dependency) critical path through the - /// trace. - unsigned getCriticalPath() const { return TBI.CriticalPath; } - - /// Return the depth and height of MI. The depth is only valid for - /// instructions in or above the trace center block. The height is only - /// valid for instructions in or below the trace center block. - InstrCycles getInstrCycles(const MachineInstr *MI) const { - return TE.Cycles.lookup(MI); - } - - /// Return the slack of MI. This is the number of cycles MI can be delayed - /// before the critical path becomes longer. - /// MI must be an instruction in the trace center block. - unsigned getInstrSlack(const MachineInstr *MI) const; - - /// Return the Depth of a PHI instruction in a trace center block successor. - /// The PHI does not have to be part of the trace. - unsigned getPHIDepth(const MachineInstr *PHI) const; - }; - - /// A trace ensemble is a collection of traces selected using the same - /// strategy, for example 'minimum resource height'. There is one trace for - /// every block in the function. - class Ensemble { - SmallVector<TraceBlockInfo, 4> BlockInfo; - DenseMap<const MachineInstr*, InstrCycles> Cycles; - friend class Trace; - - void computeTrace(const MachineBasicBlock*); - void computeDepthResources(const MachineBasicBlock*); - void computeHeightResources(const MachineBasicBlock*); - unsigned computeCrossBlockCriticalPath(const TraceBlockInfo&); - void computeInstrDepths(const MachineBasicBlock*); - void computeInstrHeights(const MachineBasicBlock*); - void addLiveIns(const MachineInstr *DefMI, unsigned DefOp, - ArrayRef<const MachineBasicBlock*> Trace); - - protected: - MachineTraceMetrics &MTM; - virtual const MachineBasicBlock *pickTracePred(const MachineBasicBlock*) =0; - virtual const MachineBasicBlock *pickTraceSucc(const MachineBasicBlock*) =0; - explicit Ensemble(MachineTraceMetrics*); - const MachineLoop *getLoopFor(const MachineBasicBlock*) const; - const TraceBlockInfo *getDepthResources(const MachineBasicBlock*) const; - const TraceBlockInfo *getHeightResources(const MachineBasicBlock*) const; - - public: - virtual ~Ensemble(); - virtual const char *getName() const =0; - void print(raw_ostream&) const; - void invalidate(const MachineBasicBlock *MBB); - void verify() const; - - /// Get the trace that passes through MBB. - /// The trace is computed on demand. - Trace getTrace(const MachineBasicBlock *MBB); - }; - - /// Strategies for selecting traces. - enum Strategy { - /// Select the trace through a block that has the fewest instructions. - TS_MinInstrCount, - - TS_NumStrategies - }; - - /// Get the trace ensemble representing the given trace selection strategy. - /// The returned Ensemble object is owned by the MachineTraceMetrics analysis, - /// and valid for the lifetime of the analysis pass. - Ensemble *getEnsemble(Strategy); - - /// Invalidate cached information about MBB. This must be called *before* MBB - /// is erased, or the CFG is otherwise changed. - /// - /// This invalidates per-block information about resource usage for MBB only, - /// and it invalidates per-trace information for any trace that passes - /// through MBB. - /// - /// Call Ensemble::getTrace() again to update any trace handles. - void invalidate(const MachineBasicBlock *MBB); - -private: - // One entry per basic block, indexed by block number. - SmallVector<FixedBlockInfo, 4> BlockInfo; - - // One ensemble per strategy. - Ensemble* Ensembles[TS_NumStrategies]; -}; - -inline raw_ostream &operator<<(raw_ostream &OS, - const MachineTraceMetrics::Trace &Tr) { - Tr.print(OS); - return OS; -} - -inline raw_ostream &operator<<(raw_ostream &OS, - const MachineTraceMetrics::Ensemble &En) { - En.print(OS); - return OS; -} -} // end namespace llvm - -#endif diff --git a/contrib/llvm/lib/CodeGen/MachineVerifier.cpp b/contrib/llvm/lib/CodeGen/MachineVerifier.cpp index 69a3ae8..4b12300 100644 --- a/contrib/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/contrib/llvm/lib/CodeGen/MachineVerifier.cpp @@ -23,28 +23,28 @@ // the verifier errors. //===----------------------------------------------------------------------===// -#include "llvm/BasicBlock.h" -#include "llvm/InlineAsm.h" -#include "llvm/Instructions.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SetOperations.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" -#include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/LiveStackAnalysis.h" -#include "llvm/CodeGen/MachineInstrBundle.h" -#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/Instructions.h" #include "llvm/MC/MCAsmInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/SetOperations.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; namespace { @@ -307,6 +307,9 @@ bool MachineVerifier::runOnMachineFunction(MachineFunction &MF) { visitMachineBasicBlockBefore(MFI); // Keep track of the current bundle header. const MachineInstr *CurBundle = 0; + // Do we expect the next instruction to be part of the same bundle? + bool InBundle = false; + for (MachineBasicBlock::const_instr_iterator MBBI = MFI->instr_begin(), MBBE = MFI->instr_end(); MBBI != MBBE; ++MBBI) { if (MBBI->getParent() != MFI) { @@ -314,6 +317,15 @@ bool MachineVerifier::runOnMachineFunction(MachineFunction &MF) { *OS << "Instruction: " << *MBBI; continue; } + + // Check for consistent bundle flags. + if (InBundle && !MBBI->isBundledWithPred()) + report("Missing BundledPred flag, " + "BundledSucc was set on predecessor", MBBI); + if (!InBundle && MBBI->isBundledWithPred()) + report("BundledPred flag is set, " + "but BundledSucc not set on predecessor", MBBI); + // Is this a bundle header? if (!MBBI->isInsideBundle()) { if (CurBundle) @@ -326,9 +338,14 @@ bool MachineVerifier::runOnMachineFunction(MachineFunction &MF) { for (unsigned I = 0, E = MBBI->getNumOperands(); I != E; ++I) visitMachineOperand(&MBBI->getOperand(I), I); visitMachineInstrAfter(MBBI); + + // Was this the last bundled instruction? + InBundle = MBBI->isBundledWithSucc(); } if (CurBundle) visitMachineBundleAfter(CurBundle); + if (InBundle) + report("BundledSucc flag set on last instruction in block", &MFI->back()); visitMachineBasicBlockAfter(MFI); } visitMachineFunctionAfter(); @@ -580,7 +597,7 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) { ++MBBI; if (MBBI == MF->end()) { report("MBB conditionally falls through out of function!", MBB); - } if (MBB->succ_size() == 1) { + } else if (MBB->succ_size() == 1) { // A conditional branch with only one successor is weird, but allowed. if (&*MBBI != TBB) report("MBB exits via conditional branch/fall-through but only has " diff --git a/contrib/llvm/lib/CodeGen/OptimizePHIs.cpp b/contrib/llvm/lib/CodeGen/OptimizePHIs.cpp index 6da313e..3982612 100644 --- a/contrib/llvm/lib/CodeGen/OptimizePHIs.cpp +++ b/contrib/llvm/lib/CodeGen/OptimizePHIs.cpp @@ -14,13 +14,13 @@ #define DEBUG_TYPE "phi-opt" #include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/IR/Function.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Function.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/Statistic.h" using namespace llvm; STATISTIC(NumPHICycles, "Number of PHI cycles replaced"); diff --git a/contrib/llvm/lib/CodeGen/PHIElimination.cpp b/contrib/llvm/lib/CodeGen/PHIElimination.cpp index e6e23da..5584708 100644 --- a/contrib/llvm/lib/CodeGen/PHIElimination.cpp +++ b/contrib/llvm/lib/CodeGen/PHIElimination.cpp @@ -14,23 +14,24 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "phielim" +#include "llvm/CodeGen/Passes.h" #include "PHIEliminationUtils.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveVariables.h" -#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Function.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/Statistic.h" +#include "llvm/IR/Function.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" #include <algorithm> using namespace llvm; @@ -39,9 +40,16 @@ DisableEdgeSplitting("disable-phi-elim-edge-splitting", cl::init(false), cl::Hidden, cl::desc("Disable critical edge splitting " "during PHI elimination")); +static cl::opt<bool> +SplitAllCriticalEdges("phi-elim-split-all-critical-edges", cl::init(false), + cl::Hidden, cl::desc("Split all critical edges during " + "PHI elimination")); + namespace { class PHIElimination : public MachineFunctionPass { MachineRegisterInfo *MRI; // Machine register information + LiveVariables *LV; + LiveIntervals *LIS; public: static char ID; // Pass identification, replacement for typeid @@ -57,8 +65,8 @@ namespace { /// in predecessor basic blocks. /// bool EliminatePHINodes(MachineFunction &MF, MachineBasicBlock &MBB); - void LowerAtomicPHINode(MachineBasicBlock &MBB, - MachineBasicBlock::iterator AfterPHIsIt); + void LowerPHINode(MachineBasicBlock &MBB, + MachineBasicBlock::iterator AfterPHIsIt); /// analyzePHINodes - Gather information about the PHI nodes in /// here. In particular, we want to map the number of uses of a virtual @@ -70,7 +78,12 @@ namespace { /// Split critical edges where necessary for good coalescer performance. bool SplitPHIEdges(MachineFunction &MF, MachineBasicBlock &MBB, - LiveVariables &LV, MachineLoopInfo *MLI); + MachineLoopInfo *MLI); + + // These functions are temporary abstractions around LiveVariables and + // LiveIntervals, so they can go away when LiveVariables does. + bool isLiveIn(unsigned Reg, MachineBasicBlock *MBB); + bool isLiveOutPastPHIs(unsigned Reg, MachineBasicBlock *MBB); typedef std::pair<unsigned, unsigned> BBVRegPair; typedef DenseMap<BBVRegPair, unsigned> VRegPHIUse; @@ -87,7 +100,7 @@ namespace { }; } -STATISTIC(NumAtomic, "Number of atomic phis lowered"); +STATISTIC(NumLowered, "Number of phis lowered"); STATISTIC(NumCriticalEdgesSplit, "Number of critical edges split"); STATISTIC(NumReused, "Number of reused lowered phis"); @@ -103,6 +116,8 @@ INITIALIZE_PASS_END(PHIElimination, "phi-node-elimination", void PHIElimination::getAnalysisUsage(AnalysisUsage &AU) const { AU.addPreserved<LiveVariables>(); + AU.addPreserved<SlotIndexes>(); + AU.addPreserved<LiveIntervals>(); AU.addPreserved<MachineDominatorTree>(); AU.addPreserved<MachineLoopInfo>(); MachineFunctionPass::getAnalysisUsage(AU); @@ -110,19 +125,20 @@ void PHIElimination::getAnalysisUsage(AnalysisUsage &AU) const { bool PHIElimination::runOnMachineFunction(MachineFunction &MF) { MRI = &MF.getRegInfo(); + LV = getAnalysisIfAvailable<LiveVariables>(); + LIS = getAnalysisIfAvailable<LiveIntervals>(); bool Changed = false; // This pass takes the function out of SSA form. MRI->leaveSSA(); - // Split critical edges to help the coalescer - if (!DisableEdgeSplitting) { - if (LiveVariables *LV = getAnalysisIfAvailable<LiveVariables>()) { - MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>(); - for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) - Changed |= SplitPHIEdges(MF, *I, *LV, MLI); - } + // Split critical edges to help the coalescer. This does not yet support + // updating LiveIntervals, so we disable it. + if (!DisableEdgeSplitting && (LV || LIS)) { + MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>(); + for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) + Changed |= SplitPHIEdges(MF, *I, MLI); } // Populate VRegPHIUseCount @@ -137,14 +153,20 @@ bool PHIElimination::runOnMachineFunction(MachineFunction &MF) { E = ImpDefs.end(); I != E; ++I) { MachineInstr *DefMI = *I; unsigned DefReg = DefMI->getOperand(0).getReg(); - if (MRI->use_nodbg_empty(DefReg)) + if (MRI->use_nodbg_empty(DefReg)) { + if (LIS) + LIS->RemoveMachineInstrFromMaps(DefMI); DefMI->eraseFromParent(); + } } // Clean up the lowered PHI instructions. for (LoweredPHIMap::iterator I = LoweredPHIs.begin(), E = LoweredPHIs.end(); - I != E; ++I) + I != E; ++I) { + if (LIS) + LIS->RemoveMachineInstrFromMaps(I->first); MF.DeleteMachineInstr(I->first); + } LoweredPHIs.clear(); ImpDefs.clear(); @@ -166,7 +188,7 @@ bool PHIElimination::EliminatePHINodes(MachineFunction &MF, MachineBasicBlock::iterator AfterPHIsIt = MBB.SkipPHIsAndLabels(MBB.begin()); while (MBB.front().isPHI()) - LowerAtomicPHINode(MBB, AfterPHIsIt); + LowerPHINode(MBB, AfterPHIsIt); return true; } @@ -193,15 +215,11 @@ static bool isSourceDefinedByImplicitDef(const MachineInstr *MPhi, } -/// LowerAtomicPHINode - Lower the PHI node at the top of the specified block, -/// under the assumption that it needs to be lowered in a way that supports -/// atomic execution of PHIs. This lowering method is always correct all of the -/// time. +/// LowerPHINode - Lower the PHI node at the top of the specified block, /// -void PHIElimination::LowerAtomicPHINode( - MachineBasicBlock &MBB, - MachineBasicBlock::iterator AfterPHIsIt) { - ++NumAtomic; +void PHIElimination::LowerPHINode(MachineBasicBlock &MBB, + MachineBasicBlock::iterator AfterPHIsIt) { + ++NumLowered; // Unlink the PHI node from the basic block, but don't delete the PHI yet. MachineInstr *MPhi = MBB.remove(MBB.begin()); @@ -244,7 +262,6 @@ void PHIElimination::LowerAtomicPHINode( } // Update live variable information if there is any. - LiveVariables *LV = getAnalysisIfAvailable<LiveVariables>(); if (LV) { MachineInstr *PHICopy = prior(AfterPHIsIt); @@ -283,6 +300,48 @@ void PHIElimination::LowerAtomicPHINode( } } + // Update LiveIntervals for the new copy or implicit def. + if (LIS) { + MachineInstr *NewInstr = prior(AfterPHIsIt); + SlotIndex DestCopyIndex = LIS->InsertMachineInstrInMaps(NewInstr); + + SlotIndex MBBStartIndex = LIS->getMBBStartIdx(&MBB); + if (IncomingReg) { + // Add the region from the beginning of MBB to the copy instruction to + // IncomingReg's live interval. + LiveInterval &IncomingLI = LIS->getOrCreateInterval(IncomingReg); + VNInfo *IncomingVNI = IncomingLI.getVNInfoAt(MBBStartIndex); + if (!IncomingVNI) + IncomingVNI = IncomingLI.getNextValue(MBBStartIndex, + LIS->getVNInfoAllocator()); + IncomingLI.addRange(LiveRange(MBBStartIndex, + DestCopyIndex.getRegSlot(), + IncomingVNI)); + } + + LiveInterval &DestLI = LIS->getInterval(DestReg); + assert(DestLI.begin() != DestLI.end() && + "PHIs should have nonempty LiveIntervals."); + if (DestLI.endIndex().isDead()) { + // A dead PHI's live range begins and ends at the start of the MBB, but + // the lowered copy, which will still be dead, needs to begin and end at + // the copy instruction. + VNInfo *OrigDestVNI = DestLI.getVNInfoAt(MBBStartIndex); + assert(OrigDestVNI && "PHI destination should be live at block entry."); + DestLI.removeRange(MBBStartIndex, MBBStartIndex.getDeadSlot()); + DestLI.createDeadDef(DestCopyIndex.getRegSlot(), + LIS->getVNInfoAllocator()); + DestLI.removeValNo(OrigDestVNI); + } else { + // Otherwise, remove the region from the beginning of MBB to the copy + // instruction from DestReg's live interval. + DestLI.removeRange(MBBStartIndex, DestCopyIndex.getRegSlot()); + VNInfo *DestVNI = DestLI.getVNInfoAt(DestCopyIndex.getRegSlot()); + assert(DestVNI && "PHI destination should be live at its definition."); + DestVNI->def = DestCopyIndex.getRegSlot(); + } + } + // Adjust the VRegPHIUseCount map to account for the removal of this PHI node. for (unsigned i = 1; i != MPhi->getNumOperands(); i += 2) --VRegPHIUseCount[BBVRegPair(MPhi->getOperand(i+1).getMBB()->getNumber(), @@ -315,45 +374,44 @@ void PHIElimination::LowerAtomicPHINode( findPHICopyInsertPoint(&opBlock, &MBB, SrcReg); // Insert the copy. + MachineInstr *NewSrcInstr = 0; if (!reusedIncoming && IncomingReg) { if (SrcUndef) { // The source register is undefined, so there is no need for a real // COPY, but we still need to ensure joint dominance by defs. // Insert an IMPLICIT_DEF instruction. - BuildMI(opBlock, InsertPos, MPhi->getDebugLoc(), - TII->get(TargetOpcode::IMPLICIT_DEF), IncomingReg); + NewSrcInstr = BuildMI(opBlock, InsertPos, MPhi->getDebugLoc(), + TII->get(TargetOpcode::IMPLICIT_DEF), + IncomingReg); // Clean up the old implicit-def, if there even was one. if (MachineInstr *DefMI = MRI->getVRegDef(SrcReg)) if (DefMI->isImplicitDef()) ImpDefs.insert(DefMI); } else { - BuildMI(opBlock, InsertPos, MPhi->getDebugLoc(), - TII->get(TargetOpcode::COPY), IncomingReg) - .addReg(SrcReg, 0, SrcSubReg); + NewSrcInstr = BuildMI(opBlock, InsertPos, MPhi->getDebugLoc(), + TII->get(TargetOpcode::COPY), IncomingReg) + .addReg(SrcReg, 0, SrcSubReg); } } - // Now update live variable information if we have it. Otherwise we're done - if (SrcUndef || !LV) continue; - - // We want to be able to insert a kill of the register if this PHI (aka, the - // copy we just inserted) is the last use of the source value. Live - // variable analysis conservatively handles this by saying that the value is - // live until the end of the block the PHI entry lives in. If the value - // really is dead at the PHI copy, there will be no successor blocks which - // have the value live-in. - - // Also check to see if this register is in use by another PHI node which - // has not yet been eliminated. If so, it will be killed at an appropriate - // point later. - - // Is it used by any PHI instructions in this block? - bool ValueIsUsed = VRegPHIUseCount[BBVRegPair(opBlock.getNumber(), SrcReg)]; + // We only need to update the LiveVariables kill of SrcReg if this was the + // last PHI use of SrcReg to be lowered on this CFG edge and it is not live + // out of the predecessor. We can also ignore undef sources. + if (LV && !SrcUndef && + !VRegPHIUseCount[BBVRegPair(opBlock.getNumber(), SrcReg)] && + !LV->isLiveOut(SrcReg, opBlock)) { + // We want to be able to insert a kill of the register if this PHI (aka, + // the copy we just inserted) is the last use of the source value. Live + // variable analysis conservatively handles this by saying that the value + // is live until the end of the block the PHI entry lives in. If the value + // really is dead at the PHI copy, there will be no successor blocks which + // have the value live-in. + + // Okay, if we now know that the value is not live out of the block, we + // can add a kill marker in this block saying that it kills the incoming + // value! - // Okay, if we now know that the value is not live out of the block, we can - // add a kill marker in this block saying that it kills the incoming value! - if (!ValueIsUsed && !LV->isLiveOut(SrcReg, opBlock)) { // In our final twist, we have to decide which instruction kills the // register. In most cases this is the copy, however, terminator // instructions at the end of the block may also use the value. In this @@ -394,11 +452,74 @@ void PHIElimination::LowerAtomicPHINode( unsigned opBlockNum = opBlock.getNumber(); LV->getVarInfo(SrcReg).AliveBlocks.reset(opBlockNum); } + + if (LIS) { + if (NewSrcInstr) { + LIS->InsertMachineInstrInMaps(NewSrcInstr); + LIS->addLiveRangeToEndOfBlock(IncomingReg, NewSrcInstr); + } + + if (!SrcUndef && + !VRegPHIUseCount[BBVRegPair(opBlock.getNumber(), SrcReg)]) { + LiveInterval &SrcLI = LIS->getInterval(SrcReg); + + bool isLiveOut = false; + for (MachineBasicBlock::succ_iterator SI = opBlock.succ_begin(), + SE = opBlock.succ_end(); SI != SE; ++SI) { + SlotIndex startIdx = LIS->getMBBStartIdx(*SI); + VNInfo *VNI = SrcLI.getVNInfoAt(startIdx); + + // Definitions by other PHIs are not truly live-in for our purposes. + if (VNI && VNI->def != startIdx) { + isLiveOut = true; + break; + } + } + + if (!isLiveOut) { + MachineBasicBlock::iterator KillInst = opBlock.end(); + MachineBasicBlock::iterator FirstTerm = opBlock.getFirstTerminator(); + for (MachineBasicBlock::iterator Term = FirstTerm; + Term != opBlock.end(); ++Term) { + if (Term->readsRegister(SrcReg)) + KillInst = Term; + } + + if (KillInst == opBlock.end()) { + // No terminator uses the register. + + if (reusedIncoming || !IncomingReg) { + // We may have to rewind a bit if we didn't just insert a copy. + KillInst = FirstTerm; + while (KillInst != opBlock.begin()) { + --KillInst; + if (KillInst->isDebugValue()) + continue; + if (KillInst->readsRegister(SrcReg)) + break; + } + } else { + // We just inserted this copy. + KillInst = prior(InsertPos); + } + } + assert(KillInst->readsRegister(SrcReg) && + "Cannot find kill instruction"); + + SlotIndex LastUseIndex = LIS->getInstructionIndex(KillInst); + SrcLI.removeRange(LastUseIndex.getRegSlot(), + LIS->getMBBEndIdx(&opBlock)); + } + } + } } // Really delete the PHI instruction now, if it is not in the LoweredPHIs map. - if (reusedIncoming || !IncomingReg) + if (reusedIncoming || !IncomingReg) { + if (LIS) + LIS->RemoveMachineInstrFromMaps(MPhi); MF.DeleteMachineInstr(MPhi); + } } /// analyzePHINodes - Gather information about the PHI nodes in here. In @@ -418,7 +539,6 @@ void PHIElimination::analyzePHINodes(const MachineFunction& MF) { bool PHIElimination::SplitPHIEdges(MachineFunction &MF, MachineBasicBlock &MBB, - LiveVariables &LV, MachineLoopInfo *MLI) { if (MBB.empty() || !MBB.front().isPHI() || MBB.isLandingPad()) return false; // Quick exit for basic blocks without PHIs. @@ -438,10 +558,10 @@ bool PHIElimination::SplitPHIEdges(MachineFunction &MF, // Avoid splitting backedges of loops. It would introduce small // out-of-line blocks into the loop which is very bad for code placement. - if (PreMBB == &MBB) + if (PreMBB == &MBB && !SplitAllCriticalEdges) continue; const MachineLoop *PreLoop = MLI ? MLI->getLoopFor(PreMBB) : 0; - if (IsLoopHeader && PreLoop == CurLoop) + if (IsLoopHeader && PreLoop == CurLoop && !SplitAllCriticalEdges) continue; // LV doesn't consider a phi use live-out, so isLiveOut only returns true @@ -450,7 +570,7 @@ bool PHIElimination::SplitPHIEdges(MachineFunction &MF, // there is a risk it may not be coalesced away. // // If the copy would be a kill, there is no need to split the edge. - if (!LV.isLiveOut(Reg, *PreMBB)) + if (!isLiveOutPastPHIs(Reg, PreMBB) && !SplitAllCriticalEdges) continue; DEBUG(dbgs() << PrintReg(Reg) << " live-out before critical edge BB#" @@ -465,7 +585,7 @@ bool PHIElimination::SplitPHIEdges(MachineFunction &MF, // is likely to be left after coalescing. If we are looking at a loop // exiting edge, split it so we won't insert code in the loop, otherwise // don't bother. - bool ShouldSplit = !LV.isLiveIn(Reg, MBB); + bool ShouldSplit = !isLiveIn(Reg, &MBB) || SplitAllCriticalEdges; // Check for a loop exiting edge. if (!ShouldSplit && CurLoop != PreLoop) { @@ -492,3 +612,33 @@ bool PHIElimination::SplitPHIEdges(MachineFunction &MF, } return Changed; } + +bool PHIElimination::isLiveIn(unsigned Reg, MachineBasicBlock *MBB) { + assert((LV || LIS) && + "isLiveIn() requires either LiveVariables or LiveIntervals"); + if (LIS) + return LIS->isLiveInToMBB(LIS->getInterval(Reg), MBB); + else + return LV->isLiveIn(Reg, *MBB); +} + +bool PHIElimination::isLiveOutPastPHIs(unsigned Reg, MachineBasicBlock *MBB) { + assert((LV || LIS) && + "isLiveOutPastPHIs() requires either LiveVariables or LiveIntervals"); + // LiveVariables considers uses in PHIs to be in the predecessor basic block, + // so that a register used only in a PHI is not live out of the block. In + // contrast, LiveIntervals considers uses in PHIs to be on the edge rather than + // in the predecessor basic block, so that a register used only in a PHI is live + // out of the block. + if (LIS) { + const LiveInterval &LI = LIS->getInterval(Reg); + for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(), + SE = MBB->succ_end(); SI != SE; ++SI) { + if (LI.liveAt(LIS->getMBBStartIdx(*SI))) + return true; + } + return false; + } else { + return LV->isLiveOut(Reg, *MBB); + } +} diff --git a/contrib/llvm/lib/CodeGen/PHIEliminationUtils.cpp b/contrib/llvm/lib/CodeGen/PHIEliminationUtils.cpp index 10bfdcc..e1b56e9 100644 --- a/contrib/llvm/lib/CodeGen/PHIEliminationUtils.cpp +++ b/contrib/llvm/lib/CodeGen/PHIEliminationUtils.cpp @@ -8,10 +8,10 @@ //===----------------------------------------------------------------------===// #include "PHIEliminationUtils.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/ADT/SmallPtrSet.h" using namespace llvm; // findCopyInsertPoint - Find a safe place in MBB to insert a copy from SrcReg diff --git a/contrib/llvm/lib/CodeGen/Passes.cpp b/contrib/llvm/lib/CodeGen/Passes.cpp index 4ea21d4..1af65c8 100644 --- a/contrib/llvm/lib/CodeGen/Passes.cpp +++ b/contrib/llvm/lib/CodeGen/Passes.cpp @@ -12,21 +12,21 @@ // //===---------------------------------------------------------------------===// +#include "llvm/CodeGen/Passes.h" #include "llvm/Analysis/Passes.h" #include "llvm/Analysis/Verifier.h" -#include "llvm/Transforms/Scalar.h" -#include "llvm/PassManager.h" +#include "llvm/Assembly/PrintModulePass.h" #include "llvm/CodeGen/GCStrategy.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetOptions.h" #include "llvm/MC/MCAsmInfo.h" -#include "llvm/Assembly/PrintModulePass.h" +#include "llvm/PassManager.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/Transforms/Scalar.h" using namespace llvm; @@ -39,12 +39,9 @@ static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate", cl::Hidden, static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup", cl::Hidden, cl::desc("Disable pre-register allocation tail duplication")); static cl::opt<bool> DisableBlockPlacement("disable-block-placement", - cl::Hidden, cl::desc("Disable the probability-driven block placement, and " - "re-enable the old code placement pass")); + cl::Hidden, cl::desc("Disable probability-driven block placement")); static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats", cl::Hidden, cl::desc("Collect probability-driven block placement stats")); -static cl::opt<bool> DisableCodePlace("disable-code-place", cl::Hidden, - cl::desc("Disable code placement")); static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden, cl::desc("Disable Stack Slot Coloring")); static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden, @@ -88,7 +85,7 @@ PrintMachineInstrs("print-machineinstrs", cl::ValueOptional, cl::desc("Print machine instrs"), cl::value_desc("pass-name"), cl::init("option-unspecified")); -// Experimental option to run live inteerval analysis early. +// Experimental option to run live interval analysis early. static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden, cl::desc("Run live interval analysis earlier in the pipeline")); @@ -149,10 +146,7 @@ static AnalysisID overridePass(AnalysisID StandardID, AnalysisID TargetID) { return applyDisable(TargetID, DisableEarlyTailDup); if (StandardID == &MachineBlockPlacementID) - return applyDisable(TargetID, DisableCodePlace); - - if (StandardID == &CodePlacementOptID) - return applyDisable(TargetID, DisableCodePlace); + return applyDisable(TargetID, DisableBlockPlacement); if (StandardID == &StackSlotColoringID) return applyDisable(TargetID, DisableSSC); @@ -237,11 +231,10 @@ TargetPassConfig::TargetPassConfig(TargetMachine *tm, PassManagerBase &pm) substitutePass(&EarlyTailDuplicateID, &TailDuplicateID); substitutePass(&PostRAMachineLICMID, &MachineLICMID); - // Disable early if-conversion. Targets that are ready can enable it. - disablePass(&EarlyIfConverterID); - // Temporarily disable experimental passes. - substitutePass(&MachineSchedulerID, 0); + const TargetSubtargetInfo &ST = TM->getSubtarget<TargetSubtargetInfo>(); + if (!ST.enableMachineScheduler()) + disablePass(&MachineSchedulerID); } /// Insert InsertedPassID pass after TargetPassID. @@ -359,7 +352,7 @@ void TargetPassConfig::addIRPasses() { // Run loop strength reduction before anything else. if (getOptLevel() != CodeGenOpt::None && !DisableLSR) { - addPass(createLoopStrengthReducePass(getTargetLowering())); + addPass(createLoopStrengthReducePass()); if (PrintLSR) addPass(createPrintFunctionPass("\n\n*** Code after LSR ***\n", &dbgs())); } @@ -397,12 +390,16 @@ void TargetPassConfig::addPassesToHandleExceptions() { } } -/// Add common passes that perform LLVM IR to IR transforms in preparation for -/// instruction selection. -void TargetPassConfig::addISelPrepare() { +/// Add pass to prepare the LLVM IR for code generation. This should be done +/// before exception handling preparation passes. +void TargetPassConfig::addCodeGenPrepare() { if (getOptLevel() != CodeGenOpt::None && !DisableCGP) addPass(createCodeGenPreparePass(getTargetLowering())); +} +/// Add common passes that perform LLVM IR to IR transforms in preparation for +/// instruction selection. +void TargetPassConfig::addISelPrepare() { addPass(createStackProtectorPass(getTargetLowering())); addPreISel(); @@ -462,8 +459,7 @@ void TargetPassConfig::addMachinePasses() { // Add passes that optimize machine instructions in SSA form. if (getOptLevel() != CodeGenOpt::None) { addMachineSSAOptimization(); - } - else { + } else { // If the target requests it, assign local variables to stack slots relative // to one another and simplify frame index references where possible. addPass(&LocalStackSlotAllocationID); @@ -507,9 +503,10 @@ void TargetPassConfig::addMachinePasses() { } // GC - addPass(&GCMachineCodeAnalysisID); - if (PrintGCInfo) - addPass(createGCInfoPrinter(dbgs())); + if (addGCPasses()) { + if (PrintGCInfo) + addPass(createGCInfoPrinter(dbgs())); + } // Basic block placement. if (getOptLevel() != CodeGenOpt::None) @@ -544,7 +541,12 @@ void TargetPassConfig::addMachineSSAOptimization() { addPass(&DeadMachineInstructionElimID); printAndVerify("After codegen DCE pass"); - addPass(&EarlyIfConverterID); + // Allow targets to insert passes that improve instruction level parallelism, + // like if-conversion. Such passes will typically need dominator trees and + // loop info, just like LICM and CSE below. + if (addILPOpts()) + printAndVerify("After ILP optimizations"); + addPass(&MachineLICMID); addPass(&MachineCSEID); addPass(&MachineSinkingID); @@ -726,18 +728,15 @@ void TargetPassConfig::addMachineLateOptimization() { printAndVerify("After copy propagation pass"); } +/// Add standard GC passes. +bool TargetPassConfig::addGCPasses() { + addPass(&GCMachineCodeAnalysisID); + return true; +} + /// Add standard basic block placement passes. void TargetPassConfig::addBlockPlacement() { - AnalysisID PassID = 0; - if (!DisableBlockPlacement) { - // MachineBlockPlacement is a new pass which subsumes the functionality of - // CodPlacementOpt. The old code placement pass can be restored by - // disabling block placement, but eventually it will be removed. - PassID = addPass(&MachineBlockPlacementID); - } else { - PassID = addPass(&CodePlacementOptID); - } - if (PassID) { + if (addPass(&MachineBlockPlacementID)) { // Run a separate pass to collect block placement statistics. if (EnableBlockPlacementStats) addPass(&MachineBlockPlacementStatsID); diff --git a/contrib/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/contrib/llvm/lib/CodeGen/PeepholeOptimizer.cpp index a795ac8..a7439b5 100644 --- a/contrib/llvm/lib/CodeGen/PeepholeOptimizer.cpp +++ b/contrib/llvm/lib/CodeGen/PeepholeOptimizer.cpp @@ -49,20 +49,26 @@ // v1 = bitcast v0 // = v0 // +// - Optimize Loads: +// +// Loads that can be folded into a later instruction. A load is foldable +// if it loads to virtual registers and the virtual register defined has +// a single use. //===----------------------------------------------------------------------===// #define DEBUG_TYPE "peephole-opt" #include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/Statistic.h" using namespace llvm; // Optimize Extensions @@ -473,6 +479,9 @@ bool PeepholeOptimizer::foldImmediate(MachineInstr *MI, MachineBasicBlock *MBB, } bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) { + DEBUG(dbgs() << "********** PEEPHOLE OPTIMIZER **********\n"); + DEBUG(dbgs() << "********** Function: " << MF.getName() << '\n'); + if (DisablePeephole) return false; @@ -547,6 +556,8 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) { FoldAsLoadDefReg, DefMI); if (FoldMI) { // Update LocalMIs since we replaced MI with FoldMI and deleted DefMI. + DEBUG(dbgs() << "Replacing: " << *MI); + DEBUG(dbgs() << " With: " << *FoldMI); LocalMIs.erase(MI); LocalMIs.erase(DefMI); LocalMIs.insert(FoldMI); diff --git a/contrib/llvm/lib/CodeGen/PostRASchedulerList.cpp b/contrib/llvm/lib/CodeGen/PostRASchedulerList.cpp index d57bc73..53fe273 100644 --- a/contrib/llvm/lib/CodeGen/PostRASchedulerList.cpp +++ b/contrib/llvm/lib/CodeGen/PostRASchedulerList.cpp @@ -19,32 +19,33 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "post-RA-sched" -#include "AntiDepBreaker.h" +#include "llvm/CodeGen/Passes.h" #include "AggressiveAntiDepBreaker.h" +#include "AntiDepBreaker.h" #include "CriticalAntiDepBreaker.h" -#include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/LatencyPriorityQueue.h" -#include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/CodeGen/ScheduleDAGInstrs.h" #include "llvm/CodeGen/ScheduleHazardRecognizer.h" -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/BitVector.h" -#include "llvm/ADT/Statistic.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; STATISTIC(NumNoops, "Number of noops inserted"); @@ -111,9 +112,6 @@ namespace { /// added to the AvailableQueue. std::vector<SUnit*> PendingQueue; - /// Topo - A topological ordering for SUnits. - ScheduleDAGTopologicalSort Topo; - /// HazardRec - The hazard recognizer to use. ScheduleHazardRecognizer *HazardRec; @@ -198,7 +196,7 @@ SchedulePostRATDList::SchedulePostRATDList( AliasAnalysis *AA, const RegisterClassInfo &RCI, TargetSubtargetInfo::AntiDepBreakMode AntiDepMode, SmallVectorImpl<const TargetRegisterClass*> &CriticalPathRCs) - : ScheduleDAGInstrs(MF, MLI, MDT, /*IsPostRA=*/true), Topo(SUnits), AA(AA), + : ScheduleDAGInstrs(MF, MLI, MDT, /*IsPostRA=*/true), AA(AA), LiveRegs(TRI->getNumRegs()) { const TargetMachine &TM = MF.getTarget(); @@ -420,11 +418,11 @@ void SchedulePostRATDList::StartBlockForKills(MachineBasicBlock *BB) { // Start with no live registers. LiveRegs.reset(); - // Determine the live-out physregs for this block. - if (!BB->empty() && BB->back().isReturn()) { - // In a return block, examine the function live-out regs. - for (MachineRegisterInfo::liveout_iterator I = MRI.liveout_begin(), - E = MRI.liveout_end(); I != E; ++I) { + // Examine the live-in regs of all successors. + for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), + SE = BB->succ_end(); SI != SE; ++SI) { + for (MachineBasicBlock::livein_iterator I = (*SI)->livein_begin(), + E = (*SI)->livein_end(); I != E; ++I) { unsigned Reg = *I; LiveRegs.set(Reg); // Repeat, for all subregs. @@ -432,20 +430,6 @@ void SchedulePostRATDList::StartBlockForKills(MachineBasicBlock *BB) { LiveRegs.set(*SubRegs); } } - else { - // In a non-return block, examine the live-in regs of all successors. - for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), - SE = BB->succ_end(); SI != SE; ++SI) { - for (MachineBasicBlock::livein_iterator I = (*SI)->livein_begin(), - E = (*SI)->livein_end(); I != E; ++I) { - unsigned Reg = *I; - LiveRegs.set(Reg); - // Repeat, for all subregs. - for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs) - LiveRegs.set(*SubRegs); - } - } - } } bool SchedulePostRATDList::ToggleKillFlag(MachineInstr *MI, @@ -467,13 +451,10 @@ bool SchedulePostRATDList::ToggleKillFlag(MachineInstr *MI, MO.setIsKill(false); bool AllDead = true; const unsigned SuperReg = MO.getReg(); + MachineInstrBuilder MIB(MF, MI); for (MCSubRegIterator SubRegs(SuperReg, TRI); SubRegs.isValid(); ++SubRegs) { if (LiveRegs.test(*SubRegs)) { - MI->addOperand(MachineOperand::CreateReg(*SubRegs, - true /*IsDef*/, - true /*IsImp*/, - false /*IsKill*/, - false /*IsDead*/)); + MIB.addReg(*SubRegs, RegState::ImplicitDefine); AllDead = false; } } @@ -580,10 +561,14 @@ void SchedulePostRATDList::FixupKills(MachineBasicBlock *MBB) { //===----------------------------------------------------------------------===// /// ReleaseSucc - Decrement the NumPredsLeft count of a successor. Add it to -/// the PendingQueue if the count reaches zero. Also update its cycle bound. +/// the PendingQueue if the count reaches zero. void SchedulePostRATDList::ReleaseSucc(SUnit *SU, SDep *SuccEdge) { SUnit *SuccSU = SuccEdge->getSUnit(); + if (SuccEdge->isWeak()) { + --SuccSU->WeakPredsLeft; + return; + } #ifndef NDEBUG if (SuccSU->NumPredsLeft == 0) { dbgs() << "*** Scheduling failed! ***\n"; @@ -653,8 +638,7 @@ void SchedulePostRATDList::ListScheduleTopDown() { // Add all leaves to Available queue. for (unsigned i = 0, e = SUnits.size(); i != e; ++i) { // It is available if it has no predecessors. - bool available = SUnits[i].Preds.empty(); - if (available) { + if (!SUnits[i].NumPredsLeft && !SUnits[i].isAvailable) { AvailableQueue.push(&SUnits[i]); SUnits[i].isAvailable = true; } diff --git a/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp index 77554d6..e5872df 100644 --- a/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp +++ b/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp @@ -21,25 +21,24 @@ #define DEBUG_TYPE "pei" #include "PrologEpilogInserter.h" -#include "llvm/InlineAsm.h" +#include "llvm/ADT/IndexedMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RegisterScavenging.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetFrameLowering.h" -#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/IR/InlineAsm.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" -#include "llvm/ADT/IndexedMap.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/Target/TargetFrameLowering.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <climits> using namespace llvm; @@ -56,7 +55,6 @@ INITIALIZE_PASS_END(PEI, "prologepilog", "Prologue/Epilogue Insertion & Frame Finalization", false, false) -STATISTIC(NumVirtualFrameRegs, "Number of virtual frame regs encountered"); STATISTIC(NumScavengedRegs, "Number of frame index regs scavenged"); STATISTIC(NumBytesStackSpace, "Number of bytes used for stack in all functions"); @@ -96,12 +94,13 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) { placeCSRSpillsAndRestores(Fn); // Add the code to save and restore the callee saved registers - if (!F->getFnAttributes().hasAttribute(Attributes::Naked)) + if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, + Attribute::Naked)) insertCSRSpillsAndRestores(Fn); // Allow the target machine to make final modifications to the function // before the frame layout is finalized. - TFI->processFunctionBeforeFrameFinalized(Fn); + TFI->processFunctionBeforeFrameFinalized(Fn, RS); // Calculate actual frame offsets for all abstract stack objects... calculateFrameObjectOffsets(Fn); @@ -111,7 +110,8 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) { // called functions. Because of this, calculateCalleeSavedRegisters() // must be called before this function in order to set the AdjustsStack // and MaxCallFrameSize variables. - if (!F->getFnAttributes().hasAttribute(Attributes::Naked)) + if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, + Attribute::Naked)) insertPrologEpilogCode(Fn); // Replace all MO_FrameIndex operands with physical register references @@ -133,24 +133,10 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) { return true; } -#if 0 -void PEI::getAnalysisUsage(AnalysisUsage &AU) const { - AU.setPreservesCFG(); - if (ShrinkWrapping || ShrinkWrapFunc != "") { - AU.addRequired<MachineLoopInfo>(); - AU.addRequired<MachineDominatorTree>(); - } - AU.addPreserved<MachineLoopInfo>(); - AU.addPreserved<MachineDominatorTree>(); - MachineFunctionPass::getAnalysisUsage(AU); -} -#endif - /// calculateCallsInformation - Calculate the MaxCallFrameSize and AdjustsStack /// variables for the function's frame information and eliminate call frame /// pseudo instructions. void PEI::calculateCallsInformation(MachineFunction &Fn) { - const TargetRegisterInfo *RegInfo = Fn.getTarget().getRegisterInfo(); const TargetInstrInfo &TII = *Fn.getTarget().getInstrInfo(); const TargetFrameLowering *TFI = Fn.getTarget().getFrameLowering(); MachineFrameInfo *MFI = Fn.getFrameInfo(); @@ -197,20 +183,20 @@ void PEI::calculateCallsInformation(MachineFunction &Fn) { // here. The sub/add sp instruction pairs are still inserted, but we don't // need to track the SP adjustment for frame index elimination. if (TFI->canSimplifyCallFramePseudos(Fn)) - RegInfo->eliminateCallFramePseudoInstr(Fn, *I->getParent(), I); + TFI->eliminateCallFramePseudoInstr(Fn, *I->getParent(), I); } } /// calculateCalleeSavedRegisters - Scan the function for modified callee saved /// registers. -void PEI::calculateCalleeSavedRegisters(MachineFunction &Fn) { - const TargetRegisterInfo *RegInfo = Fn.getTarget().getRegisterInfo(); - const TargetFrameLowering *TFI = Fn.getTarget().getFrameLowering(); - MachineFrameInfo *MFI = Fn.getFrameInfo(); +void PEI::calculateCalleeSavedRegisters(MachineFunction &F) { + const TargetRegisterInfo *RegInfo = F.getTarget().getRegisterInfo(); + const TargetFrameLowering *TFI = F.getTarget().getFrameLowering(); + MachineFrameInfo *MFI = F.getFrameInfo(); // Get the callee saved register list... - const uint16_t *CSRegs = RegInfo->getCalleeSavedRegs(&Fn); + const uint16_t *CSRegs = RegInfo->getCalleeSavedRegs(&F); // These are used to keep track the callee-save area. Initialize them. MinCSFrameIndex = INT_MAX; @@ -221,13 +207,14 @@ void PEI::calculateCalleeSavedRegisters(MachineFunction &Fn) { return; // In Naked functions we aren't going to save any registers. - if (Fn.getFunction()->getFnAttributes().hasAttribute(Attributes::Naked)) + if (F.getFunction()->getAttributes().hasAttribute(AttributeSet::FunctionIndex, + Attribute::Naked)) return; std::vector<CalleeSavedInfo> CSI; for (unsigned i = 0; CSRegs[i]; ++i) { unsigned Reg = CSRegs[i]; - if (Fn.getRegInfo().isPhysRegUsed(Reg)) { + if (F.getRegInfo().isPhysRegUsed(Reg)) { // If the reg is modified, save it! CSI.push_back(CalleeSavedInfo(Reg)); } @@ -248,7 +235,7 @@ void PEI::calculateCalleeSavedRegisters(MachineFunction &Fn) { const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg); int FrameIdx; - if (RegInfo->hasReservedSpillSlot(Fn, Reg, FrameIdx)) { + if (RegInfo->hasReservedSpillSlot(F, Reg, FrameIdx)) { I->setFrameIdx(FrameIdx); continue; } @@ -560,9 +547,11 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { const TargetRegisterInfo *RegInfo = Fn.getTarget().getRegisterInfo(); if (RS && TFI.hasFP(Fn) && RegInfo->useFPForScavengingIndex(Fn) && !RegInfo->needsStackRealignment(Fn)) { - int SFI = RS->getScavengingFrameIndex(); - if (SFI >= 0) - AdjustStackOffset(MFI, SFI, StackGrowsDown, Offset, MaxAlign); + SmallVector<int, 2> SFIs; + RS->getScavengingFrameIndices(SFIs); + for (SmallVector<int, 2>::iterator I = SFIs.begin(), + IE = SFIs.end(); I != IE; ++I) + AdjustStackOffset(MFI, *I, StackGrowsDown, Offset, MaxAlign); } // FIXME: Once this is working, then enable flag will change to a target @@ -605,7 +594,7 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { continue; if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex) continue; - if (RS && (int)i == RS->getScavengingFrameIndex()) + if (RS && RS->isScavengingFrameIndex((int)i)) continue; if (MFI->isDeadObjectIndex(i)) continue; @@ -627,7 +616,7 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { continue; if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex) continue; - if (RS && (int)i == RS->getScavengingFrameIndex()) + if (RS && RS->isScavengingFrameIndex((int)i)) continue; if (MFI->isDeadObjectIndex(i)) continue; @@ -643,9 +632,11 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { // stack pointer. if (RS && (!TFI.hasFP(Fn) || RegInfo->needsStackRealignment(Fn) || !RegInfo->useFPForScavengingIndex(Fn))) { - int SFI = RS->getScavengingFrameIndex(); - if (SFI >= 0) - AdjustStackOffset(MFI, SFI, StackGrowsDown, Offset, MaxAlign); + SmallVector<int, 2> SFIs; + RS->getScavengingFrameIndices(SFIs); + for (SmallVector<int, 2>::iterator I = SFIs.begin(), + IE = SFIs.end(); I != IE; ++I) + AdjustStackOffset(MFI, *I, StackGrowsDown, Offset, MaxAlign); } if (!TFI.targetHandlesStackFrameRounding()) { @@ -703,6 +694,14 @@ void PEI::insertPrologEpilogCode(MachineFunction &Fn) { // space in small chunks instead of one large contiguous block. if (Fn.getTarget().Options.EnableSegmentedStacks) TFI.adjustForSegmentedStacks(Fn); + + // Emit additional code that is required to explicitly handle the stack in + // HiPE native code (if needed) when loaded in the Erlang/OTP runtime. The + // approach is rather similar to that of Segmented Stacks, but it uses a + // different conditional check and another BIF for allocating more stack + // space. + if (Fn.getFunction()->getCallingConv() == CallingConv::HiPE) + TFI.adjustForHiPEPrologue(Fn); } /// replaceFrameIndices - Replace all MO_FrameIndex operands with physical @@ -749,7 +748,7 @@ void PEI::replaceFrameIndices(MachineFunction &Fn) { MachineBasicBlock::iterator PrevI = BB->end(); if (I != BB->begin()) PrevI = prior(I); - TRI.eliminateCallFramePseudoInstr(Fn, *BB, I); + TFI->eliminateCallFramePseudoInstr(Fn, *BB, I); // Visit the instructions created by eliminateCallFramePseudoInstr(). if (PrevI == BB->end()) @@ -761,34 +760,36 @@ void PEI::replaceFrameIndices(MachineFunction &Fn) { MachineInstr *MI = I; bool DoIncr = true; - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) - if (MI->getOperand(i).isFI()) { - // Some instructions (e.g. inline asm instructions) can have - // multiple frame indices and/or cause eliminateFrameIndex - // to insert more than one instruction. We need the register - // scavenger to go through all of these instructions so that - // it can update its register information. We keep the - // iterator at the point before insertion so that we can - // revisit them in full. - bool AtBeginning = (I == BB->begin()); - if (!AtBeginning) --I; - - // If this instruction has a FrameIndex operand, we need to - // use that target machine register info object to eliminate - // it. - TRI.eliminateFrameIndex(MI, SPAdj, - FrameIndexVirtualScavenging ? NULL : RS); - - // Reset the iterator if we were at the beginning of the BB. - if (AtBeginning) { - I = BB->begin(); - DoIncr = false; - } + for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { + if (!MI->getOperand(i).isFI()) + continue; - MI = 0; - break; + // Some instructions (e.g. inline asm instructions) can have + // multiple frame indices and/or cause eliminateFrameIndex + // to insert more than one instruction. We need the register + // scavenger to go through all of these instructions so that + // it can update its register information. We keep the + // iterator at the point before insertion so that we can + // revisit them in full. + bool AtBeginning = (I == BB->begin()); + if (!AtBeginning) --I; + + // If this instruction has a FrameIndex operand, we need to + // use that target machine register info object to eliminate + // it. + TRI.eliminateFrameIndex(MI, SPAdj, i, + FrameIndexVirtualScavenging ? NULL : RS); + + // Reset the iterator if we were at the beginning of the BB. + if (AtBeginning) { + I = BB->begin(); + DoIncr = false; } + MI = 0; + break; + } + if (DoIncr && I != BB->end()) ++I; // Update register states. @@ -818,14 +819,22 @@ void PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) { E = Fn.end(); BB != E; ++BB) { RS->enterBasicBlock(BB); - unsigned VirtReg = 0; - unsigned ScratchReg = 0; int SPAdj = 0; // The instruction stream may change in the loop, so check BB->end() // directly. for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ) { MachineInstr *MI = I; + MachineBasicBlock::iterator J = llvm::next(I); + MachineBasicBlock::iterator P = I == BB->begin() ? + MachineBasicBlock::iterator(NULL) : llvm::prior(I); + + // RS should process this instruction before we might scavenge at this + // location. This is because we might be replacing a virtual register + // defined by this instruction, and if so, registers killed by this + // instruction are available, and defined registers are not. + RS->forward(I); + for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { if (MI->getOperand(i).isReg()) { MachineOperand &MO = MI->getOperand(i); @@ -835,29 +844,49 @@ void PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) { if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; - ++NumVirtualFrameRegs; - - // Have we already allocated a scratch register for this virtual? - if (Reg != VirtReg) { - // When we first encounter a new virtual register, it - // must be a definition. - assert(MI->getOperand(i).isDef() && - "frame index virtual missing def!"); - // Scavenge a new scratch register - VirtReg = Reg; - const TargetRegisterClass *RC = Fn.getRegInfo().getRegClass(Reg); - ScratchReg = RS->scavengeRegister(RC, I, SPAdj); - ++NumScavengedRegs; - } + // When we first encounter a new virtual register, it + // must be a definition. + assert(MI->getOperand(i).isDef() && + "frame index virtual missing def!"); + // Scavenge a new scratch register + const TargetRegisterClass *RC = Fn.getRegInfo().getRegClass(Reg); + unsigned ScratchReg = RS->scavengeRegister(RC, J, SPAdj); + + ++NumScavengedRegs; + // Replace this reference to the virtual register with the // scratch register. assert (ScratchReg && "Missing scratch register!"); - MI->getOperand(i).setReg(ScratchReg); + Fn.getRegInfo().replaceRegWith(Reg, ScratchReg); + // Because this instruction was processed by the RS before this + // register was allocated, make sure that the RS now records the + // register as being used. + RS->setUsed(ScratchReg); } } - RS->forward(I); - ++I; + + // If the scavenger needed to use one of its spill slots, the + // spill code will have been inserted in between I and J. This is a + // problem because we need the spill code before I: Move I to just + // prior to J. + if (I != llvm::prior(J)) { + BB->splice(J, BB, I); + + // Before we move I, we need to prepare the RS to visit I again. + // Specifically, RS will assert if it sees uses of registers that + // it believes are undefined. Because we have already processed + // register kills in I, when it visits I again, it will believe that + // those registers are undefined. To avoid this situation, unprocess + // the instruction I. + assert(RS->getCurrentPosition() == I && + "The register scavenger has an unexpected position"); + I = P; + RS->unprocess(P); + + // RS->skipTo(I == BB->begin() ? NULL : llvm::prior(I)); + } else + ++I; } } } diff --git a/contrib/llvm/lib/CodeGen/PrologEpilogInserter.h b/contrib/llvm/lib/CodeGen/PrologEpilogInserter.h index 0d140a9..87fff9a 100644 --- a/contrib/llvm/lib/CodeGen/PrologEpilogInserter.h +++ b/contrib/llvm/lib/CodeGen/PrologEpilogInserter.h @@ -22,11 +22,11 @@ #ifndef LLVM_CODEGEN_PEI_H #define LLVM_CODEGEN_PEI_H -#include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SparseBitVector.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/ADT/SparseBitVector.h" -#include "llvm/ADT/DenseMap.h" +#include "llvm/CodeGen/Passes.h" #include "llvm/Target/TargetRegisterInfo.h" namespace llvm { diff --git a/contrib/llvm/lib/CodeGen/PseudoSourceValue.cpp b/contrib/llvm/lib/CodeGen/PseudoSourceValue.cpp index 49599b3..8564911 100644 --- a/contrib/llvm/lib/CodeGen/PseudoSourceValue.cpp +++ b/contrib/llvm/lib/CodeGen/PseudoSourceValue.cpp @@ -11,14 +11,14 @@ // //===----------------------------------------------------------------------===// -#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" -#include "llvm/DerivedTypes.h" -#include "llvm/LLVMContext.h" +#include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Support/Mutex.h" +#include "llvm/Support/raw_ostream.h" #include <map> using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/RegAllocBase.cpp b/contrib/llvm/lib/CodeGen/RegAllocBase.cpp index 993dbc7..c035590 100644 --- a/contrib/llvm/lib/CodeGen/RegAllocBase.cpp +++ b/contrib/llvm/lib/CodeGen/RegAllocBase.cpp @@ -14,14 +14,14 @@ #define DEBUG_TYPE "regalloc" #include "RegAllocBase.h" -#include "LiveRegMatrix.h" #include "Spiller.h" -#include "VirtRegMap.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveRangeEdit.h" +#include "llvm/CodeGen/LiveRegMatrix.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" #ifndef NDEBUG diff --git a/contrib/llvm/lib/CodeGen/RegAllocBase.h b/contrib/llvm/lib/CodeGen/RegAllocBase.h index db0c8e1..064e40f 100644 --- a/contrib/llvm/lib/CodeGen/RegAllocBase.h +++ b/contrib/llvm/lib/CodeGen/RegAllocBase.h @@ -37,9 +37,9 @@ #ifndef LLVM_CODEGEN_REGALLOCBASE #define LLVM_CODEGEN_REGALLOCBASE -#include "LiveIntervalUnion.h" -#include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/ADT/OwningPtr.h" +#include "llvm/CodeGen/LiveIntervalUnion.h" +#include "llvm/CodeGen/RegisterClassInfo.h" namespace llvm { diff --git a/contrib/llvm/lib/CodeGen/RegAllocBasic.cpp b/contrib/llvm/lib/CodeGen/RegAllocBasic.cpp index 8a49609..0b6dc68 100644 --- a/contrib/llvm/lib/CodeGen/RegAllocBasic.cpp +++ b/contrib/llvm/lib/CodeGen/RegAllocBasic.cpp @@ -13,30 +13,28 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "regalloc" +#include "llvm/CodeGen/Passes.h" #include "AllocationOrder.h" -#include "RegAllocBase.h" #include "LiveDebugVariables.h" +#include "RegAllocBase.h" #include "Spiller.h" -#include "VirtRegMap.h" -#include "LiveRegMatrix.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/PassAnalysisSupport.h" #include "llvm/CodeGen/CalcSpillWeights.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveRangeEdit.h" +#include "llvm/CodeGen/LiveRegMatrix.h" #include "llvm/CodeGen/LiveStackAnalysis.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" -#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/PassAnalysisSupport.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" - +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <cstdlib> #include <queue> diff --git a/contrib/llvm/lib/CodeGen/RegAllocFast.cpp b/contrib/llvm/lib/CodeGen/RegAllocFast.cpp index 8892216..bb9c05c 100644 --- a/contrib/llvm/lib/CodeGen/RegAllocFast.cpp +++ b/contrib/llvm/lib/CodeGen/RegAllocFast.cpp @@ -13,28 +13,28 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "regalloc" -#include "llvm/BasicBlock.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/IndexedMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/SparseSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/RegisterClassInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/IndexedMap.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/SparseSet.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" #include <algorithm> using namespace llvm; @@ -113,12 +113,27 @@ namespace { // PhysRegState - One of the RegState enums, or a virtreg. std::vector<unsigned> PhysRegState; + // Set of register units. typedef SparseSet<unsigned> UsedInInstrSet; - // UsedInInstr - Set of physregs that are used in the current instruction, - // and so cannot be allocated. + // Set of register units that are used in the current instruction, and so + // cannot be allocated. UsedInInstrSet UsedInInstr; + // Mark a physreg as used in this instruction. + void markRegUsedInInstr(unsigned PhysReg) { + for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) + UsedInInstr.insert(*Units); + } + + // Check if a physreg or any of its aliases are used in this instruction. + bool isRegUsedInInstr(unsigned PhysReg) const { + for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) + if (UsedInInstr.count(*Units)) + return true; + return false; + } + // SkippedInstrs - Descriptors of instructions whose clobber list was // ignored because all registers were spilled. It is still necessary to // mark all the clobbered registers as used by the function. @@ -177,7 +192,6 @@ namespace { unsigned VirtReg, unsigned Hint); void spillAll(MachineBasicBlock::iterator MI); bool setPhysReg(MachineInstr *MI, unsigned OpNum, unsigned PhysReg); - void addRetOperands(MachineBasicBlock *MBB); }; char RAFast::ID = 0; } @@ -334,7 +348,7 @@ void RAFast::usePhysReg(MachineOperand &MO) { unsigned PhysReg = MO.getReg(); assert(TargetRegisterInfo::isPhysicalRegister(PhysReg) && "Bad usePhysReg operand"); - + markRegUsedInInstr(PhysReg); switch (PhysRegState[PhysReg]) { case regDisabled: break; @@ -342,7 +356,6 @@ void RAFast::usePhysReg(MachineOperand &MO) { PhysRegState[PhysReg] = regFree; // Fall through case regFree: - UsedInInstr.insert(PhysReg); MO.setIsKill(); return; default: @@ -362,13 +375,11 @@ void RAFast::usePhysReg(MachineOperand &MO) { "Instruction is not using a subregister of a reserved register"); // Leave the superregister in the working set. PhysRegState[Alias] = regFree; - UsedInInstr.insert(Alias); MO.getParent()->addRegisterKilled(Alias, TRI, true); return; case regFree: if (TRI->isSuperRegister(PhysReg, Alias)) { // Leave the superregister in the working set. - UsedInInstr.insert(Alias); MO.getParent()->addRegisterKilled(Alias, TRI, true); return; } @@ -382,7 +393,6 @@ void RAFast::usePhysReg(MachineOperand &MO) { // All aliases are disabled, bring register into working set. PhysRegState[PhysReg] = regFree; - UsedInInstr.insert(PhysReg); MO.setIsKill(); } @@ -391,7 +401,7 @@ void RAFast::usePhysReg(MachineOperand &MO) { /// reserved instead of allocated. void RAFast::definePhysReg(MachineInstr *MI, unsigned PhysReg, RegState NewState) { - UsedInInstr.insert(PhysReg); + markRegUsedInInstr(PhysReg); switch (unsigned VirtReg = PhysRegState[PhysReg]) { case regDisabled: break; @@ -431,7 +441,7 @@ void RAFast::definePhysReg(MachineInstr *MI, unsigned PhysReg, // can be allocated directly. // Returns spillImpossible when PhysReg or an alias can't be spilled. unsigned RAFast::calcSpillCost(unsigned PhysReg) const { - if (UsedInInstr.count(PhysReg)) { + if (isRegUsedInInstr(PhysReg)) { DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " is already used in instr.\n"); return spillImpossible; } @@ -456,8 +466,6 @@ unsigned RAFast::calcSpillCost(unsigned PhysReg) const { unsigned Cost = 0; for (MCRegAliasIterator AI(PhysReg, TRI, false); AI.isValid(); ++AI) { unsigned Alias = *AI; - if (UsedInInstr.count(Alias)) - return spillImpossible; switch (unsigned VirtReg = PhysRegState[Alias]) { case regDisabled: break; @@ -527,12 +535,12 @@ RAFast::LiveRegMap::iterator RAFast::allocVirtReg(MachineInstr *MI, } } - ArrayRef<unsigned> AO = RegClassInfo.getOrder(RC); + ArrayRef<MCPhysReg> AO = RegClassInfo.getOrder(RC); // First try to find a completely free register. - for (ArrayRef<unsigned>::iterator I = AO.begin(), E = AO.end(); I != E; ++I) { + for (ArrayRef<MCPhysReg>::iterator I = AO.begin(), E = AO.end(); I != E; ++I){ unsigned PhysReg = *I; - if (PhysRegState[PhysReg] == regFree && !UsedInInstr.count(PhysReg)) { + if (PhysRegState[PhysReg] == regFree && !isRegUsedInInstr(PhysReg)) { assignVirtToPhysReg(*LRI, PhysReg); return LRI; } @@ -542,7 +550,7 @@ RAFast::LiveRegMap::iterator RAFast::allocVirtReg(MachineInstr *MI, << RC->getName() << "\n"); unsigned BestReg = 0, BestCost = spillImpossible; - for (ArrayRef<unsigned>::iterator I = AO.begin(), E = AO.end(); I != E; ++I) { + for (ArrayRef<MCPhysReg>::iterator I = AO.begin(), E = AO.end(); I != E; ++I){ unsigned Cost = calcSpillCost(*I); DEBUG(dbgs() << "\tRegister: " << PrintReg(*I, TRI) << "\n"); DEBUG(dbgs() << "\tCost: " << Cost << "\n"); @@ -598,7 +606,7 @@ RAFast::defineVirtReg(MachineInstr *MI, unsigned OpNum, LRI->LastUse = MI; LRI->LastOpNum = OpNum; LRI->Dirty = true; - UsedInInstr.insert(LRI->PhysReg); + markRegUsedInInstr(LRI->PhysReg); return LRI; } @@ -648,7 +656,7 @@ RAFast::reloadVirtReg(MachineInstr *MI, unsigned OpNum, assert(LRI->PhysReg && "Register not assigned"); LRI->LastUse = MI; LRI->LastOpNum = OpNum; - UsedInInstr.insert(LRI->PhysReg); + markRegUsedInInstr(LRI->PhysReg); return LRI; } @@ -709,8 +717,8 @@ void RAFast::handleThroughOperands(MachineInstr *MI, if (!MO.isReg() || !MO.isDef()) continue; unsigned Reg = MO.getReg(); if (!Reg || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue; + markRegUsedInInstr(Reg); for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) { - UsedInInstr.insert(*AI); if (ThroughRegs.count(PhysRegState[*AI])) definePhysReg(MI, *AI, regFree); } @@ -766,67 +774,12 @@ void RAFast::handleThroughOperands(MachineInstr *MI, if (!Reg || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue; DEBUG(dbgs() << "\tSetting " << PrintReg(Reg, TRI) << " as used in instr\n"); - UsedInInstr.insert(Reg); + markRegUsedInInstr(Reg); } // Also mark PartialDefs as used to avoid reallocation. for (unsigned i = 0, e = PartialDefs.size(); i != e; ++i) - UsedInInstr.insert(PartialDefs[i]); -} - -/// addRetOperand - ensure that a return instruction has an operand for each -/// value live out of the function. -/// -/// Things marked both call and return are tail calls; do not do this for them. -/// The tail callee need not take the same registers as input that it produces -/// as output, and there are dependencies for its input registers elsewhere. -/// -/// FIXME: This should be done as part of instruction selection, and this helper -/// should be deleted. Until then, we use custom logic here to create the proper -/// operand under all circumstances. We can't use addRegisterKilled because that -/// doesn't make sense for undefined values. We can't simply avoid calling it -/// for undefined values, because we must ensure that the operand always exists. -void RAFast::addRetOperands(MachineBasicBlock *MBB) { - if (MBB->empty() || !MBB->back().isReturn() || MBB->back().isCall()) - return; - - MachineInstr *MI = &MBB->back(); - - for (MachineRegisterInfo::liveout_iterator - I = MBB->getParent()->getRegInfo().liveout_begin(), - E = MBB->getParent()->getRegInfo().liveout_end(); I != E; ++I) { - unsigned Reg = *I; - assert(TargetRegisterInfo::isPhysicalRegister(Reg) && - "Cannot have a live-out virtual register."); - - bool hasDef = PhysRegState[Reg] == regReserved; - - // Check if this register already has an operand. - bool Found = false; - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || !MO.isUse()) - continue; - - unsigned OperReg = MO.getReg(); - if (!TargetRegisterInfo::isPhysicalRegister(OperReg)) - continue; - - if (OperReg == Reg || TRI->isSuperRegister(OperReg, Reg)) { - // If the ret already has an operand for this physreg or a superset, - // don't duplicate it. Set the kill flag if the value is defined. - if (hasDef && !MO.isKill()) - MO.setIsKill(); - Found = true; - break; - } - } - if (!Found) - MI->addOperand(MachineOperand::CreateReg(Reg, - false /*IsDef*/, - true /*IsImp*/, - hasDef/*IsKill*/)); - } + markRegUsedInInstr(PartialDefs[i]); } void RAFast::AllocateBasicBlock() { @@ -1025,7 +978,7 @@ void RAFast::AllocateBasicBlock() { for (UsedInInstrSet::iterator I = UsedInInstr.begin(), E = UsedInInstr.end(); I != E; ++I) - MRI->setPhysRegUsed(*I); + MRI->setRegUnitUsed(*I); // Track registers defined by instruction - early clobbers and tied uses at // this point. @@ -1038,8 +991,7 @@ void RAFast::AllocateBasicBlock() { if (!Reg || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue; // Look for physreg defs and tied uses. if (!MO.isDef() && !MI->isRegTiedToDefOperand(i)) continue; - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - UsedInInstr.insert(*AI); + markRegUsedInInstr(Reg); } } @@ -1091,7 +1043,7 @@ void RAFast::AllocateBasicBlock() { for (UsedInInstrSet::iterator I = UsedInInstr.begin(), E = UsedInInstr.end(); I != E; ++I) - MRI->setPhysRegUsed(*I); + MRI->setRegUnitUsed(*I); if (CopyDst && CopyDst == CopySrc && CopyDstSub == CopySrcSub) { DEBUG(dbgs() << "-- coalescing: " << *MI); @@ -1111,9 +1063,6 @@ void RAFast::AllocateBasicBlock() { MBB->erase(Coalesced[i]); NumCopies += Coalesced.size(); - // addRetOperands must run after we've seen all defs in this block. - addRetOperands(MBB); - DEBUG(MBB->dump()); } @@ -1130,7 +1079,7 @@ bool RAFast::runOnMachineFunction(MachineFunction &Fn) { MRI->freezeReservedRegs(Fn); RegClassInfo.runOnMachineFunction(Fn); UsedInInstr.clear(); - UsedInInstr.setUniverse(TRI->getNumRegs()); + UsedInInstr.setUniverse(TRI->getNumRegUnits()); assert(!MRI->isSSA() && "regalloc requires leaving SSA"); diff --git a/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp b/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp index 06f69c1e..6d84176 100644 --- a/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp +++ b/contrib/llvm/lib/CodeGen/RegAllocGreedy.cpp @@ -13,36 +13,34 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "regalloc" +#include "llvm/CodeGen/Passes.h" #include "AllocationOrder.h" #include "InterferenceCache.h" #include "LiveDebugVariables.h" -#include "LiveRegMatrix.h" #include "RegAllocBase.h" -#include "Spiller.h" #include "SpillPlacement.h" +#include "Spiller.h" #include "SplitKit.h" -#include "VirtRegMap.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/PassAnalysisSupport.h" #include "llvm/CodeGen/CalcSpillWeights.h" #include "llvm/CodeGen/EdgeBundles.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveRangeEdit.h" +#include "llvm/CodeGen/LiveRegMatrix.h" #include "llvm/CodeGen/LiveStackAnalysis.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" -#include "llvm/Target/TargetOptions.h" +#include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/PassAnalysisSupport.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Support/Timer.h" - +#include "llvm/Support/raw_ostream.h" #include <queue> using namespace llvm; @@ -414,7 +412,7 @@ void RAGreedy::enqueue(LiveInterval *LI) { Prio = (1u << 31) + Size; // Boost ranges that have a physical register hint. - if (TargetRegisterInfo::isPhysicalRegister(VRM->getRegAllocPref(Reg))) + if (VRM->hasKnownPreference(Reg)) Prio |= (1u << 30); } @@ -443,7 +441,7 @@ unsigned RAGreedy::tryAssign(LiveInterval &VirtReg, while ((PhysReg = Order.next())) if (!Matrix->checkInterference(VirtReg, PhysReg)) break; - if (!PhysReg || Order.isHint(PhysReg)) + if (!PhysReg || Order.isHint()) return PhysReg; // PhysReg is available, but there may be a better choice. @@ -633,16 +631,33 @@ unsigned RAGreedy::tryEvict(LiveInterval &VirtReg, // Keep track of the cheapest interference seen so far. EvictionCost BestCost(~0u); unsigned BestPhys = 0; + unsigned OrderLimit = Order.getOrder().size(); // When we are just looking for a reduced cost per use, don't break any // hints, and only evict smaller spill weights. if (CostPerUseLimit < ~0u) { BestCost.BrokenHints = 0; BestCost.MaxWeight = VirtReg.weight; + + // Check of any registers in RC are below CostPerUseLimit. + const TargetRegisterClass *RC = MRI->getRegClass(VirtReg.reg); + unsigned MinCost = RegClassInfo.getMinCost(RC); + if (MinCost >= CostPerUseLimit) { + DEBUG(dbgs() << RC->getName() << " minimum cost = " << MinCost + << ", no cheaper registers to be found.\n"); + return 0; + } + + // It is normal for register classes to have a long tail of registers with + // the same cost. We don't need to look at them if they're too expensive. + if (TRI->getCostPerUse(Order.getOrder().back()) >= CostPerUseLimit) { + OrderLimit = RegClassInfo.getLastCostChange(RC); + DEBUG(dbgs() << "Only trying the first " << OrderLimit << " regs.\n"); + } } Order.rewind(); - while (unsigned PhysReg = Order.next()) { + while (unsigned PhysReg = Order.nextWithDups(OrderLimit)) { if (TRI->getCostPerUse(PhysReg) >= CostPerUseLimit) continue; // The first use of a callee-saved register in a function has cost 1. @@ -662,7 +677,7 @@ unsigned RAGreedy::tryEvict(LiveInterval &VirtReg, BestPhys = PhysReg; // Stop if the hint can be used. - if (Order.isHint(PhysReg)) + if (Order.isHint()) break; } diff --git a/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp b/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp index 02ebce7..607edac 100644 --- a/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp +++ b/contrib/llvm/lib/CodeGen/RegAllocPBQP.cpp @@ -31,24 +31,24 @@ #define DEBUG_TYPE "regalloc" -#include "Spiller.h" -#include "VirtRegMap.h" +#include "llvm/CodeGen/RegAllocPBQP.h" #include "RegisterCoalescer.h" -#include "llvm/Module.h" +#include "Spiller.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/CalcSpillWeights.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveRangeEdit.h" #include "llvm/CodeGen/LiveStackAnalysis.h" -#include "llvm/CodeGen/RegAllocPBQP.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/PBQP/HeuristicSolver.h" #include "llvm/CodeGen/PBQP/Graph.h" +#include "llvm/CodeGen/PBQP/HeuristicSolver.h" #include "llvm/CodeGen/PBQP/Heuristics/Briggs.h" #include "llvm/CodeGen/RegAllocRegistry.h" +#include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" @@ -526,7 +526,7 @@ void RegAllocPBQP::finalizeAlloc() const { itr != end; ++itr) { LiveInterval *li = &lis->getInterval(*itr); - unsigned physReg = vrm->getRegAllocPref(li->reg); + unsigned physReg = mri->getSimpleHint(li->reg); if (physReg == 0) { const TargetRegisterClass *liRC = mri->getRegClass(li->reg); diff --git a/contrib/llvm/lib/CodeGen/RegisterClassInfo.cpp b/contrib/llvm/lib/CodeGen/RegisterClassInfo.cpp index 805d235..87382d8 100644 --- a/contrib/llvm/lib/CodeGen/RegisterClassInfo.cpp +++ b/contrib/llvm/lib/CodeGen/RegisterClassInfo.cpp @@ -18,10 +18,10 @@ #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; @@ -44,7 +44,7 @@ void RegisterClassInfo::runOnMachineFunction(const MachineFunction &mf) { } // Does this MF have different CSRs? - const uint16_t *CSR = TRI->getCalleeSavedRegs(MF); + const MCPhysReg *CSR = TRI->getCalleeSavedRegs(MF); if (Update || CSR != CalleeSaved) { // Build a CSRNum map. Every CSR alias gets an entry pointing to the last // overlapping CSR. @@ -79,30 +79,47 @@ void RegisterClassInfo::compute(const TargetRegisterClass *RC) const { unsigned NumRegs = RC->getNumRegs(); if (!RCI.Order) - RCI.Order.reset(new unsigned[NumRegs]); + RCI.Order.reset(new MCPhysReg[NumRegs]); unsigned N = 0; - SmallVector<unsigned, 16> CSRAlias; + SmallVector<MCPhysReg, 16> CSRAlias; + unsigned MinCost = 0xff; + unsigned LastCost = ~0u; + unsigned LastCostChange = 0; // FIXME: Once targets reserve registers instead of removing them from the // allocation order, we can simply use begin/end here. - ArrayRef<uint16_t> RawOrder = RC->getRawAllocationOrder(*MF); + ArrayRef<MCPhysReg> RawOrder = RC->getRawAllocationOrder(*MF); for (unsigned i = 0; i != RawOrder.size(); ++i) { unsigned PhysReg = RawOrder[i]; // Remove reserved registers from the allocation order. if (Reserved.test(PhysReg)) continue; + unsigned Cost = TRI->getCostPerUse(PhysReg); + MinCost = std::min(MinCost, Cost); + if (CSRNum[PhysReg]) // PhysReg aliases a CSR, save it for later. CSRAlias.push_back(PhysReg); - else + else { + if (Cost != LastCost) + LastCostChange = N; RCI.Order[N++] = PhysReg; + LastCost = Cost; + } } RCI.NumRegs = N + CSRAlias.size(); assert (RCI.NumRegs <= NumRegs && "Allocation order larger than regclass"); // CSR aliases go after the volatile registers, preserve the target's order. - std::copy(CSRAlias.begin(), CSRAlias.end(), &RCI.Order[N]); + for (unsigned i = 0, e = CSRAlias.size(); i != e; ++i) { + unsigned PhysReg = CSRAlias[i]; + unsigned Cost = TRI->getCostPerUse(PhysReg); + if (Cost != LastCost) + LastCostChange = N; + RCI.Order[N++] = PhysReg; + LastCost = Cost; + } // Register allocator stress test. Clip register class to N registers. if (StressRA && RCI.NumRegs > StressRA) @@ -113,6 +130,9 @@ void RegisterClassInfo::compute(const TargetRegisterClass *RC) const { if (Super != RC && getNumAllocatableRegs(Super) > RCI.NumRegs) RCI.ProperSubClass = true; + RCI.MinCost = uint8_t(MinCost); + RCI.LastCostChange = LastCostChange; + DEBUG({ dbgs() << "AllocationOrder(" << RC->getName() << ") = ["; for (unsigned I = 0; I != RCI.NumRegs; ++I) diff --git a/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp b/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp index 2538f10..d85646d 100644 --- a/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp +++ b/contrib/llvm/lib/CodeGen/RegisterCoalescer.cpp @@ -15,36 +15,30 @@ #define DEBUG_TYPE "regalloc" #include "RegisterCoalescer.h" -#include "LiveDebugVariables.h" -#include "VirtRegMap.h" - -#include "llvm/Pass.h" -#include "llvm/Value.h" #include "llvm/ADT/OwningPtr.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" -#include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveRangeEdit.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegisterClassInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/IR/Value.h" +#include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" #include <algorithm> #include <cmath> using namespace llvm; @@ -63,6 +57,17 @@ EnableJoining("join-liveintervals", cl::desc("Coalesce copies (default=true)"), cl::init(true)); +// Temporary flag to test critical edge unsplitting. +static cl::opt<bool> +EnableJoinSplits("join-splitedges", + cl::desc("Coalesce copies on split edges (default=subtarget)"), cl::Hidden); + +// Temporary flag to test global copy optimization. +static cl::opt<cl::boolOrDefault> +EnableGlobalCopies("join-globalcopies", + cl::desc("Coalesce copies that span blocks (default=subtarget)"), + cl::init(cl::BOU_UNSET), cl::Hidden); + static cl::opt<bool> VerifyCoalescing("verify-coalescing", cl::desc("Verify machine instrs before and after register coalescing"), @@ -77,13 +82,21 @@ namespace { const TargetRegisterInfo* TRI; const TargetInstrInfo* TII; LiveIntervals *LIS; - LiveDebugVariables *LDV; const MachineLoopInfo* Loops; AliasAnalysis *AA; RegisterClassInfo RegClassInfo; + /// \brief True if the coalescer should aggressively coalesce global copies + /// in favor of keeping local copies. + bool JoinGlobalCopies; + + /// \brief True if the coalescer should aggressively coalesce fall-thru + /// blocks exclusively containing copies. + bool JoinSplitEdges; + /// WorkList - Copy instructions yet to be coalesced. SmallVector<MachineInstr*, 8> WorkList; + SmallVector<MachineInstr*, 8> LocalWorkList; /// ErasedInstrs - Set of instruction pointers that have been erased, and /// that may be present in WorkList. @@ -101,6 +114,9 @@ namespace { /// LiveRangeEdit callback. void LRE_WillEraseInstruction(MachineInstr *MI); + /// coalesceLocals - coalesce the LocalWorkList. + void coalesceLocals(); + /// joinAllIntervals - join compatible live intervals void joinAllIntervals(); @@ -108,9 +124,9 @@ namespace { /// copies that cannot yet be coalesced into WorkList. void copyCoalesceInMBB(MachineBasicBlock *MBB); - /// copyCoalesceWorkList - Try to coalesce all copies in WorkList after - /// position From. Return true if any progress was made. - bool copyCoalesceWorkList(unsigned From = 0); + /// copyCoalesceWorkList - Try to coalesce all copies in CurrList. Return + /// true if any progress was made. + bool copyCoalesceWorkList(MutableArrayRef<MachineInstr*> CurrList); /// joinCopy - Attempt to join intervals corresponding to SrcReg/DstReg, /// which are the src/dst of the copy instruction CopyMI. This returns @@ -150,11 +166,10 @@ namespace { /// reMaterializeTrivialDef - If the source of a copy is defined by a /// trivial computation, replace the copy by rematerialize the definition. - bool reMaterializeTrivialDef(LiveInterval &SrcInt, unsigned DstReg, - MachineInstr *CopyMI); + bool reMaterializeTrivialDef(CoalescerPair &CP, MachineInstr *CopyMI); /// canJoinPhys - Return true if a physreg copy should be joined. - bool canJoinPhys(CoalescerPair &CP); + bool canJoinPhys(const CoalescerPair &CP); /// updateRegDefsUses - Replace all defs and uses of SrcReg to DstReg and /// update the subregister number if it is not zero. If DstReg is a @@ -189,7 +204,6 @@ char &llvm::RegisterCoalescerID = RegisterCoalescer::ID; INITIALIZE_PASS_BEGIN(RegisterCoalescer, "simple-register-coalescing", "Simple Register Coalescing", false, false) INITIALIZE_PASS_DEPENDENCY(LiveIntervals) -INITIALIZE_PASS_DEPENDENCY(LiveDebugVariables) INITIALIZE_PASS_DEPENDENCY(SlotIndexes) INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) INITIALIZE_AG_DEPENDENCY(AliasAnalysis) @@ -217,6 +231,23 @@ static bool isMoveInstr(const TargetRegisterInfo &tri, const MachineInstr *MI, return true; } +// Return true if this block should be vacated by the coalescer to eliminate +// branches. The important cases to handle in the coalescer are critical edges +// split during phi elimination which contain only copies. Simple blocks that +// contain non-branches should also be vacated, but this can be handled by an +// earlier pass similar to early if-conversion. +static bool isSplitEdge(const MachineBasicBlock *MBB) { + if (MBB->pred_size() != 1 || MBB->succ_size() != 1) + return false; + + for (MachineBasicBlock::const_iterator MII = MBB->begin(), E = MBB->end(); + MII != E; ++MII) { + if (!MII->isCopyLike() && !MII->isUnconditionalBranch()) + return false; + } + return true; +} + bool CoalescerPair::setRegisters(const MachineInstr *MI) { SrcReg = DstReg = 0; SrcIdx = DstIdx = 0; @@ -358,8 +389,6 @@ void RegisterCoalescer::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired<AliasAnalysis>(); AU.addRequired<LiveIntervals>(); AU.addPreserved<LiveIntervals>(); - AU.addRequired<LiveDebugVariables>(); - AU.addPreserved<LiveDebugVariables>(); AU.addPreserved<SlotIndexes>(); AU.addRequired<MachineLoopInfo>(); AU.addPreserved<MachineLoopInfo>(); @@ -701,9 +730,14 @@ bool RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP, /// reMaterializeTrivialDef - If the source of a copy is defined by a trivial /// computation, replace the copy by rematerialize the definition. -bool RegisterCoalescer::reMaterializeTrivialDef(LiveInterval &SrcInt, - unsigned DstReg, +bool RegisterCoalescer::reMaterializeTrivialDef(CoalescerPair &CP, MachineInstr *CopyMI) { + unsigned SrcReg = CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg(); + unsigned DstReg = CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg(); + if (TargetRegisterInfo::isPhysicalRegister(SrcReg)) + return false; + + LiveInterval &SrcInt = LIS->getInterval(SrcReg); SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI).getRegSlot(true); LiveInterval::iterator SrcLR = SrcInt.FindLiveRangeContaining(CopyIdx); assert(SrcLR != SrcInt.end() && "Live range not found!"); @@ -724,13 +758,17 @@ bool RegisterCoalescer::reMaterializeTrivialDef(LiveInterval &SrcInt, const MCInstrDesc &MCID = DefMI->getDesc(); if (MCID.getNumDefs() != 1) return false; + // Only support subregister destinations when the def is read-undef. + MachineOperand &DstOperand = CopyMI->getOperand(0); + if (DstOperand.getSubReg() && !DstOperand.isUndef()) + return false; if (!DefMI->isImplicitDef()) { // Make sure the copy destination register class fits the instruction // definition register class. The mismatch can happen as a result of earlier // extract_subreg, insert_subreg, subreg_to_reg coalescing. const TargetRegisterClass *RC = TII->getRegClass(MCID, 0, TRI, *MF); if (TargetRegisterInfo::isVirtualRegister(DstReg)) { - if (MRI->getRegClass(DstReg) != RC) + if (!MRI->constrainRegClass(DstReg, RC)) return false; } else if (!RC->contains(DstReg)) return false; @@ -742,6 +780,12 @@ bool RegisterCoalescer::reMaterializeTrivialDef(LiveInterval &SrcInt, TII->reMaterialize(*MBB, MII, DstReg, 0, DefMI, *TRI); MachineInstr *NewMI = prior(MII); + // The original DefMI may have been a subregister def, but the full register + // class of its destination matches the destination of CopyMI, and CopyMI is + // either a full register def or is read-undef. Therefore we can clear the + // subregister index on the rematerialized instruction. + NewMI->getOperand(0).setSubReg(0); + // NewMI may have dead implicit defs (E.g. EFLAGS for MOV<bits>r0 on X86). // We need to remember these so we can add intervals once we insert // NewMI into SlotIndexes. @@ -847,9 +891,6 @@ void RegisterCoalescer::updateRegDefsUses(unsigned SrcReg, bool DstIsPhys = TargetRegisterInfo::isPhysicalRegister(DstReg); LiveInterval *DstInt = DstIsPhys ? 0 : &LIS->getInterval(DstReg); - // Update LiveDebugVariables. - LDV->renameRegister(SrcReg, DstReg, SubIdx); - SmallPtrSet<MachineInstr*, 8> Visited; for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(SrcReg); MachineInstr *UseMI = I.skipInstruction();) { @@ -896,7 +937,7 @@ void RegisterCoalescer::updateRegDefsUses(unsigned SrcReg, } /// canJoinPhys - Return true if a copy involving a physreg should be joined. -bool RegisterCoalescer::canJoinPhys(CoalescerPair &CP) { +bool RegisterCoalescer::canJoinPhys(const CoalescerPair &CP) { /// Always join simple intervals that are defined by a single copy from a /// reserved register. This doesn't increase register pressure, so it is /// always beneficial. @@ -974,9 +1015,7 @@ bool RegisterCoalescer::joinCopy(MachineInstr *CopyMI, bool &Again) { if (!canJoinPhys(CP)) { // Before giving up coalescing, if definition of source is defined by // trivial computation, try rematerializing it. - if (!CP.isFlipped() && - reMaterializeTrivialDef(LIS->getInterval(CP.getSrcReg()), - CP.getDstReg(), CopyMI)) + if (reMaterializeTrivialDef(CP, CopyMI)) return true; return false; } @@ -1009,9 +1048,7 @@ bool RegisterCoalescer::joinCopy(MachineInstr *CopyMI, bool &Again) { // If definition of source is defined by trivial computation, try // rematerializing it. - if (!CP.isFlipped() && - reMaterializeTrivialDef(LIS->getInterval(CP.getSrcReg()), - CP.getDstReg(), CopyMI)) + if (reMaterializeTrivialDef(CP, CopyMI)) return true; // If we can eliminate the copy without merging the live ranges, do so now. @@ -1246,8 +1283,18 @@ class JoinVals { // Value in the other live range that overlaps this def, if any. VNInfo *OtherVNI; - // Is this value an IMPLICIT_DEF? - bool IsImplicitDef; + // Is this value an IMPLICIT_DEF that can be erased? + // + // IMPLICIT_DEF values should only exist at the end of a basic block that + // is a predecessor to a phi-value. These IMPLICIT_DEF instructions can be + // safely erased if they are overlapping a live value in the other live + // interval. + // + // Weird control flow graphs and incomplete PHI handling in + // ProcessImplicitDefs can very rarely create IMPLICIT_DEF values with + // longer live ranges. Such IMPLICIT_DEF values should be treated like + // normal values. + bool ErasableImplicitDef; // True when the live range of this value will be pruned because of an // overlapping CR_Replace value in the other live range. @@ -1257,8 +1304,8 @@ class JoinVals { bool PrunedComputed; Val() : Resolution(CR_Keep), WriteLanes(0), ValidLanes(0), - RedefVNI(0), OtherVNI(0), IsImplicitDef(false), Pruned(false), - PrunedComputed(false) {} + RedefVNI(0), OtherVNI(0), ErasableImplicitDef(false), + Pruned(false), PrunedComputed(false) {} bool isAnalyzed() const { return WriteLanes != 0; } }; @@ -1396,7 +1443,10 @@ JoinVals::analyzeValue(unsigned ValNo, JoinVals &Other) { // An IMPLICIT_DEF writes undef values. if (DefMI->isImplicitDef()) { - V.IsImplicitDef = true; + // We normally expect IMPLICIT_DEF values to be live only until the end + // of their block. If the value is really live longer and gets pruned in + // another block, this flag is cleared again. + V.ErasableImplicitDef = true; V.ValidLanes &= ~V.WriteLanes; } } @@ -1449,7 +1499,22 @@ JoinVals::analyzeValue(unsigned ValNo, JoinVals &Other) { // We have overlapping values, or possibly a kill of Other. // Recursively compute assignments up the dominator tree. Other.computeAssignment(V.OtherVNI->id, *this); - const Val &OtherV = Other.Vals[V.OtherVNI->id]; + Val &OtherV = Other.Vals[V.OtherVNI->id]; + + // Check if OtherV is an IMPLICIT_DEF that extends beyond its basic block. + // This shouldn't normally happen, but ProcessImplicitDefs can leave such + // IMPLICIT_DEF instructions behind, and there is nothing wrong with it + // technically. + // + // WHen it happens, treat that IMPLICIT_DEF as a normal value, and don't try + // to erase the IMPLICIT_DEF instruction. + if (OtherV.ErasableImplicitDef && DefMI && + DefMI->getParent() != Indexes->getMBBFromIndex(V.OtherVNI->def)) { + DEBUG(dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->def + << " extends into BB#" << DefMI->getParent()->getNumber() + << ", keeping it.\n"); + OtherV.ErasableImplicitDef = false; + } // Allow overlapping PHI values. Any real interference would show up in a // predecessor, the PHI itself can't introduce any conflicts. @@ -1758,7 +1823,8 @@ void JoinVals::pruneValues(JoinVals &Other, // predecessors, so the instruction should simply go away once its value // has been replaced. Val &OtherV = Other.Vals[Vals[i].OtherVNI->id]; - bool EraseImpDef = OtherV.IsImplicitDef && OtherV.Resolution == CR_Keep; + bool EraseImpDef = OtherV.ErasableImplicitDef && + OtherV.Resolution == CR_Keep; if (!Def.isBlock()) { // Remove <def,read-undef> flags. This def is now a partial redef. // Also remove <def,dead> flags since the joined live range will @@ -1807,7 +1873,7 @@ void JoinVals::eraseInstrs(SmallPtrSet<MachineInstr*, 8> &ErasedInstrs, // If an IMPLICIT_DEF value is pruned, it doesn't serve a purpose any // longer. The IMPLICIT_DEF instructions are only inserted by // PHIElimination to guarantee that all PHI predecessors have a value. - if (!Vals[i].IsImplicitDef || !Vals[i].Pruned) + if (!Vals[i].ErasableImplicitDef || !Vals[i].Pruned) break; // Remove value number i from LI. Note that this VNInfo is still present // in NewVNInfo, so it will appear as an unused value number in the final @@ -1904,47 +1970,77 @@ bool RegisterCoalescer::joinIntervals(CoalescerPair &CP) { } namespace { - // DepthMBBCompare - Comparison predicate that sort first based on the loop - // depth of the basic block (the unsigned), and then on the MBB number. - struct DepthMBBCompare { - typedef std::pair<unsigned, MachineBasicBlock*> DepthMBBPair; - bool operator()(const DepthMBBPair &LHS, const DepthMBBPair &RHS) const { - // Deeper loops first - if (LHS.first != RHS.first) - return LHS.first > RHS.first; - - // Prefer blocks that are more connected in the CFG. This takes care of - // the most difficult copies first while intervals are short. - unsigned cl = LHS.second->pred_size() + LHS.second->succ_size(); - unsigned cr = RHS.second->pred_size() + RHS.second->succ_size(); - if (cl != cr) - return cl > cr; - - // As a last resort, sort by block number. - return LHS.second->getNumber() < RHS.second->getNumber(); - } - }; +// Information concerning MBB coalescing priority. +struct MBBPriorityInfo { + MachineBasicBlock *MBB; + unsigned Depth; + bool IsSplit; + + MBBPriorityInfo(MachineBasicBlock *mbb, unsigned depth, bool issplit) + : MBB(mbb), Depth(depth), IsSplit(issplit) {} +}; +} + +// C-style comparator that sorts first based on the loop depth of the basic +// block (the unsigned), and then on the MBB number. +// +// EnableGlobalCopies assumes that the primary sort key is loop depth. +static int compareMBBPriority(const void *L, const void *R) { + const MBBPriorityInfo *LHS = static_cast<const MBBPriorityInfo*>(L); + const MBBPriorityInfo *RHS = static_cast<const MBBPriorityInfo*>(R); + // Deeper loops first + if (LHS->Depth != RHS->Depth) + return LHS->Depth > RHS->Depth ? -1 : 1; + + // Try to unsplit critical edges next. + if (LHS->IsSplit != RHS->IsSplit) + return LHS->IsSplit ? -1 : 1; + + // Prefer blocks that are more connected in the CFG. This takes care of + // the most difficult copies first while intervals are short. + unsigned cl = LHS->MBB->pred_size() + LHS->MBB->succ_size(); + unsigned cr = RHS->MBB->pred_size() + RHS->MBB->succ_size(); + if (cl != cr) + return cl > cr ? -1 : 1; + + // As a last resort, sort by block number. + return LHS->MBB->getNumber() < RHS->MBB->getNumber() ? -1 : 1; +} + +/// \returns true if the given copy uses or defines a local live range. +static bool isLocalCopy(MachineInstr *Copy, const LiveIntervals *LIS) { + if (!Copy->isCopy()) + return false; + + unsigned SrcReg = Copy->getOperand(1).getReg(); + unsigned DstReg = Copy->getOperand(0).getReg(); + if (TargetRegisterInfo::isPhysicalRegister(SrcReg) + || TargetRegisterInfo::isPhysicalRegister(DstReg)) + return false; + + return LIS->intervalIsInOneMBB(LIS->getInterval(SrcReg)) + || LIS->intervalIsInOneMBB(LIS->getInterval(DstReg)); } // Try joining WorkList copies starting from index From. // Null out any successful joins. -bool RegisterCoalescer::copyCoalesceWorkList(unsigned From) { - assert(From <= WorkList.size() && "Out of range"); +bool RegisterCoalescer:: +copyCoalesceWorkList(MutableArrayRef<MachineInstr*> CurrList) { bool Progress = false; - for (unsigned i = From, e = WorkList.size(); i != e; ++i) { - if (!WorkList[i]) + for (unsigned i = 0, e = CurrList.size(); i != e; ++i) { + if (!CurrList[i]) continue; // Skip instruction pointers that have already been erased, for example by // dead code elimination. - if (ErasedInstrs.erase(WorkList[i])) { - WorkList[i] = 0; + if (ErasedInstrs.erase(CurrList[i])) { + CurrList[i] = 0; continue; } bool Again = false; - bool Success = joinCopy(WorkList[i], Again); + bool Success = joinCopy(CurrList[i], Again); Progress |= Success; if (Success || !Again) - WorkList[i] = 0; + CurrList[i] = 0; } return Progress; } @@ -1956,52 +2052,74 @@ RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) { // Collect all copy-like instructions in MBB. Don't start coalescing anything // yet, it might invalidate the iterator. const unsigned PrevSize = WorkList.size(); - for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end(); - MII != E; ++MII) - if (MII->isCopyLike()) - WorkList.push_back(MII); - + if (JoinGlobalCopies) { + // Coalesce copies bottom-up to coalesce local defs before local uses. They + // are not inherently easier to resolve, but slightly preferable until we + // have local live range splitting. In particular this is required by + // cmp+jmp macro fusion. + for (MachineBasicBlock::reverse_iterator + MII = MBB->rbegin(), E = MBB->rend(); MII != E; ++MII) { + if (!MII->isCopyLike()) + continue; + if (isLocalCopy(&(*MII), LIS)) + LocalWorkList.push_back(&(*MII)); + else + WorkList.push_back(&(*MII)); + } + } + else { + for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end(); + MII != E; ++MII) + if (MII->isCopyLike()) + WorkList.push_back(MII); + } // Try coalescing the collected copies immediately, and remove the nulls. // This prevents the WorkList from getting too large since most copies are // joinable on the first attempt. - if (copyCoalesceWorkList(PrevSize)) + MutableArrayRef<MachineInstr*> + CurrList(WorkList.begin() + PrevSize, WorkList.end()); + if (copyCoalesceWorkList(CurrList)) WorkList.erase(std::remove(WorkList.begin() + PrevSize, WorkList.end(), (MachineInstr*)0), WorkList.end()); } +void RegisterCoalescer::coalesceLocals() { + copyCoalesceWorkList(LocalWorkList); + for (unsigned j = 0, je = LocalWorkList.size(); j != je; ++j) { + if (LocalWorkList[j]) + WorkList.push_back(LocalWorkList[j]); + } + LocalWorkList.clear(); +} + void RegisterCoalescer::joinAllIntervals() { DEBUG(dbgs() << "********** JOINING INTERVALS ***********\n"); - assert(WorkList.empty() && "Old data still around."); - - if (Loops->empty()) { - // If there are no loops in the function, join intervals in function order. - for (MachineFunction::iterator I = MF->begin(), E = MF->end(); - I != E; ++I) - copyCoalesceInMBB(I); - } else { - // Otherwise, join intervals in inner loops before other intervals. - // Unfortunately we can't just iterate over loop hierarchy here because - // there may be more MBB's than BB's. Collect MBB's for sorting. - - // Join intervals in the function prolog first. We want to join physical - // registers with virtual registers before the intervals got too long. - std::vector<std::pair<unsigned, MachineBasicBlock*> > MBBs; - for (MachineFunction::iterator I = MF->begin(), E = MF->end();I != E;++I){ - MachineBasicBlock *MBB = I; - MBBs.push_back(std::make_pair(Loops->getLoopDepth(MBB), I)); + assert(WorkList.empty() && LocalWorkList.empty() && "Old data still around."); + + std::vector<MBBPriorityInfo> MBBs; + MBBs.reserve(MF->size()); + for (MachineFunction::iterator I = MF->begin(), E = MF->end();I != E;++I){ + MachineBasicBlock *MBB = I; + MBBs.push_back(MBBPriorityInfo(MBB, Loops->getLoopDepth(MBB), + JoinSplitEdges && isSplitEdge(MBB))); + } + array_pod_sort(MBBs.begin(), MBBs.end(), compareMBBPriority); + + // Coalesce intervals in MBB priority order. + unsigned CurrDepth = UINT_MAX; + for (unsigned i = 0, e = MBBs.size(); i != e; ++i) { + // Try coalescing the collected local copies for deeper loops. + if (JoinGlobalCopies && MBBs[i].Depth < CurrDepth) { + coalesceLocals(); + CurrDepth = MBBs[i].Depth; } - - // Sort by loop depth. - std::sort(MBBs.begin(), MBBs.end(), DepthMBBCompare()); - - // Finally, join intervals in loop nest order. - for (unsigned i = 0, e = MBBs.size(); i != e; ++i) - copyCoalesceInMBB(MBBs[i].second); + copyCoalesceInMBB(MBBs[i].MBB); } + coalesceLocals(); // Joining intervals can allow other intervals to be joined. Iteratively join // until we make no progress. - while (copyCoalesceWorkList()) + while (copyCoalesceWorkList(WorkList)) /* empty */ ; } @@ -2019,10 +2137,20 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { TRI = TM->getRegisterInfo(); TII = TM->getInstrInfo(); LIS = &getAnalysis<LiveIntervals>(); - LDV = &getAnalysis<LiveDebugVariables>(); AA = &getAnalysis<AliasAnalysis>(); Loops = &getAnalysis<MachineLoopInfo>(); + const TargetSubtargetInfo &ST = TM->getSubtarget<TargetSubtargetInfo>(); + if (EnableGlobalCopies == cl::BOU_UNSET) + JoinGlobalCopies = ST.enableMachineScheduler(); + else + JoinGlobalCopies = (EnableGlobalCopies == cl::BOU_TRUE); + + // The MachineScheduler does not currently require JoinSplitEdges. This will + // either be enabled unconditionally or replaced by a more general live range + // splitting optimization. + JoinSplitEdges = EnableJoinSplits; + DEBUG(dbgs() << "********** SIMPLE REGISTER COALESCING **********\n" << "********** Function: " << MF->getName() << '\n'); @@ -2054,7 +2182,6 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { } DEBUG(dump()); - DEBUG(LDV->dump()); if (VerifyCoalescing) MF->verify(this, "After register coalescing"); return true; diff --git a/contrib/llvm/lib/CodeGen/RegisterPressure.cpp b/contrib/llvm/lib/CodeGen/RegisterPressure.cpp index 543c426..97f22e1 100644 --- a/contrib/llvm/lib/CodeGen/RegisterPressure.cpp +++ b/contrib/llvm/lib/CodeGen/RegisterPressure.cpp @@ -12,25 +12,22 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CodeGen/RegisterPressure.h" #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RegisterClassInfo.h" -#include "llvm/CodeGen/RegisterPressure.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; -/// Increase register pressure for each set impacted by this register class. +/// Increase pressure for each pressure set provided by TargetRegisterInfo. static void increaseSetPressure(std::vector<unsigned> &CurrSetPressure, std::vector<unsigned> &MaxSetPressure, - const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) { - unsigned Weight = TRI->getRegClassWeight(RC).RegWeight; - for (const int *PSet = TRI->getRegClassPressureSets(RC); - *PSet != -1; ++PSet) { + const int *PSet, unsigned Weight) { + for (; *PSet != -1; ++PSet) { CurrSetPressure[*PSet] += Weight; if (&CurrSetPressure != &MaxSetPressure && CurrSetPressure[*PSet] > MaxSetPressure[*PSet]) { @@ -39,32 +36,57 @@ static void increaseSetPressure(std::vector<unsigned> &CurrSetPressure, } } -/// Decrease register pressure for each set impacted by this register class. +/// Decrease pressure for each pressure set provided by TargetRegisterInfo. static void decreaseSetPressure(std::vector<unsigned> &CurrSetPressure, - const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) { - unsigned Weight = TRI->getRegClassWeight(RC).RegWeight; - for (const int *PSet = TRI->getRegClassPressureSets(RC); - *PSet != -1; ++PSet) { + const int *PSet, unsigned Weight) { + for (; *PSet != -1; ++PSet) { assert(CurrSetPressure[*PSet] >= Weight && "register pressure underflow"); CurrSetPressure[*PSet] -= Weight; } } /// Directly increase pressure only within this RegisterPressure result. -void RegisterPressure::increase(const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) { - increaseSetPressure(MaxSetPressure, MaxSetPressure, RC, TRI); +void RegisterPressure::increase(unsigned Reg, const TargetRegisterInfo *TRI, + const MachineRegisterInfo *MRI) { + if (TargetRegisterInfo::isVirtualRegister(Reg)) { + const TargetRegisterClass *RC = MRI->getRegClass(Reg); + increaseSetPressure(MaxSetPressure, MaxSetPressure, + TRI->getRegClassPressureSets(RC), + TRI->getRegClassWeight(RC).RegWeight); + } + else { + increaseSetPressure(MaxSetPressure, MaxSetPressure, + TRI->getRegUnitPressureSets(Reg), + TRI->getRegUnitWeight(Reg)); + } } /// Directly decrease pressure only within this RegisterPressure result. -void RegisterPressure::decrease(const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) { - decreaseSetPressure(MaxSetPressure, RC, TRI); +void RegisterPressure::decrease(unsigned Reg, const TargetRegisterInfo *TRI, + const MachineRegisterInfo *MRI) { + if (TargetRegisterInfo::isVirtualRegister(Reg)) { + const TargetRegisterClass *RC = MRI->getRegClass(Reg); + decreaseSetPressure(MaxSetPressure, TRI->getRegClassPressureSets(RC), + TRI->getRegClassWeight(RC).RegWeight); + } + else { + decreaseSetPressure(MaxSetPressure, TRI->getRegUnitPressureSets(Reg), + TRI->getRegUnitWeight(Reg)); + } } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +static void dumpSetPressure(const std::vector<unsigned> &SetPressure, + const TargetRegisterInfo *TRI) { + for (unsigned i = 0, e = SetPressure.size(); i < e; ++i) { + if (SetPressure[i] != 0) + dbgs() << TRI->getRegPressureSetName(i) << "=" << SetPressure[i] << '\n'; + } +} + void RegisterPressure::dump(const TargetRegisterInfo *TRI) const { + dbgs() << "Max Pressure: "; + dumpSetPressure(MaxSetPressure, TRI); dbgs() << "Live In: "; for (unsigned i = 0, e = LiveInRegs.size(); i < e; ++i) dbgs() << PrintReg(LiveInRegs[i], TRI) << " "; @@ -73,42 +95,47 @@ void RegisterPressure::dump(const TargetRegisterInfo *TRI) const { for (unsigned i = 0, e = LiveOutRegs.size(); i < e; ++i) dbgs() << PrintReg(LiveOutRegs[i], TRI) << " "; dbgs() << '\n'; - for (unsigned i = 0, e = MaxSetPressure.size(); i < e; ++i) { - if (MaxSetPressure[i] != 0) - dbgs() << TRI->getRegPressureSetName(i) << "=" << MaxSetPressure[i] - << '\n'; - } -} -#endif - -/// Increase the current pressure as impacted by these physical registers and -/// bump the high water mark if needed. -void RegPressureTracker::increasePhysRegPressure(ArrayRef<unsigned> Regs) { - for (unsigned I = 0, E = Regs.size(); I != E; ++I) - increaseSetPressure(CurrSetPressure, P.MaxSetPressure, - TRI->getMinimalPhysRegClass(Regs[I]), TRI); } -/// Simply decrease the current pressure as impacted by these physcial -/// registers. -void RegPressureTracker::decreasePhysRegPressure(ArrayRef<unsigned> Regs) { - for (unsigned I = 0, E = Regs.size(); I != E; ++I) - decreaseSetPressure(CurrSetPressure, TRI->getMinimalPhysRegClass(Regs[I]), - TRI); +void RegPressureTracker::dump() const { + dbgs() << "Curr Pressure: "; + dumpSetPressure(CurrSetPressure, TRI); + P.dump(TRI); } +#endif -/// Increase the current pressure as impacted by these virtual registers and -/// bump the high water mark if needed. -void RegPressureTracker::increaseVirtRegPressure(ArrayRef<unsigned> Regs) { - for (unsigned I = 0, E = Regs.size(); I != E; ++I) - increaseSetPressure(CurrSetPressure, P.MaxSetPressure, - MRI->getRegClass(Regs[I]), TRI); +/// Increase the current pressure as impacted by these registers and bump +/// the high water mark if needed. +void RegPressureTracker::increaseRegPressure(ArrayRef<unsigned> Regs) { + for (unsigned I = 0, E = Regs.size(); I != E; ++I) { + if (TargetRegisterInfo::isVirtualRegister(Regs[I])) { + const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]); + increaseSetPressure(CurrSetPressure, P.MaxSetPressure, + TRI->getRegClassPressureSets(RC), + TRI->getRegClassWeight(RC).RegWeight); + } + else { + increaseSetPressure(CurrSetPressure, P.MaxSetPressure, + TRI->getRegUnitPressureSets(Regs[I]), + TRI->getRegUnitWeight(Regs[I])); + } + } } -/// Simply decrease the current pressure as impacted by these virtual registers. -void RegPressureTracker::decreaseVirtRegPressure(ArrayRef<unsigned> Regs) { - for (unsigned I = 0, E = Regs.size(); I != E; ++I) - decreaseSetPressure(CurrSetPressure, MRI->getRegClass(Regs[I]), TRI); +/// Simply decrease the current pressure as impacted by these registers. +void RegPressureTracker::decreaseRegPressure(ArrayRef<unsigned> Regs) { + for (unsigned I = 0, E = Regs.size(); I != E; ++I) { + if (TargetRegisterInfo::isVirtualRegister(Regs[I])) { + const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]); + decreaseSetPressure(CurrSetPressure, + TRI->getRegClassPressureSets(RC), + TRI->getRegClassWeight(RC).RegWeight); + } + else { + decreaseSetPressure(CurrSetPressure, TRI->getRegUnitPressureSets(Regs[I]), + TRI->getRegUnitWeight(Regs[I])); + } + } } /// Clear the result so it can be used for another round of pressure tracking. @@ -160,6 +187,12 @@ void RegionPressure::openBottom(MachineBasicBlock::const_iterator PrevBottom) { LiveInRegs.clear(); } +const LiveInterval *RegPressureTracker::getInterval(unsigned Reg) const { + if (TargetRegisterInfo::isVirtualRegister(Reg)) + return &LIS->getInterval(Reg); + return LIS->getCachedRegUnit(Reg); +} + /// Setup the RegPressureTracker. /// /// TODO: Add support for pressure without LiveIntervals. @@ -181,9 +214,6 @@ void RegPressureTracker::init(const MachineFunction *mf, } CurrPos = pos; - while (CurrPos != MBB->end() && CurrPos->isDebugValue()) - ++CurrPos; - CurrSetPressure.assign(TRI->getNumRegPressureSets(), 0); if (RequireIntervals) @@ -192,10 +222,10 @@ void RegPressureTracker::init(const MachineFunction *mf, static_cast<RegionPressure&>(P).reset(); P.MaxSetPressure = CurrSetPressure; - LivePhysRegs.clear(); - LivePhysRegs.setUniverse(TRI->getNumRegs()); - LiveVirtRegs.clear(); - LiveVirtRegs.setUniverse(MRI->getNumVirtRegs()); + LiveRegs.PhysRegs.clear(); + LiveRegs.PhysRegs.setUniverse(TRI->getNumRegs()); + LiveRegs.VirtRegs.clear(); + LiveRegs.VirtRegs.setUniverse(MRI->getNumVirtRegs()); } /// Does this pressure result have a valid top position and live ins. @@ -214,19 +244,28 @@ bool RegPressureTracker::isBottomClosed() const { MachineBasicBlock::const_iterator()); } + +SlotIndex RegPressureTracker::getCurrSlot() const { + MachineBasicBlock::const_iterator IdxPos = CurrPos; + while (IdxPos != MBB->end() && IdxPos->isDebugValue()) + ++IdxPos; + if (IdxPos == MBB->end()) + return LIS->getMBBEndIdx(MBB); + return LIS->getInstructionIndex(IdxPos).getRegSlot(); +} + /// Set the boundary for the top of the region and summarize live ins. void RegPressureTracker::closeTop() { if (RequireIntervals) - static_cast<IntervalPressure&>(P).TopIdx = - LIS->getInstructionIndex(CurrPos).getRegSlot(); + static_cast<IntervalPressure&>(P).TopIdx = getCurrSlot(); else static_cast<RegionPressure&>(P).TopPos = CurrPos; assert(P.LiveInRegs.empty() && "inconsistent max pressure result"); - P.LiveInRegs.reserve(LivePhysRegs.size() + LiveVirtRegs.size()); - P.LiveInRegs.append(LivePhysRegs.begin(), LivePhysRegs.end()); + P.LiveInRegs.reserve(LiveRegs.PhysRegs.size() + LiveRegs.VirtRegs.size()); + P.LiveInRegs.append(LiveRegs.PhysRegs.begin(), LiveRegs.PhysRegs.end()); for (SparseSet<unsigned>::const_iterator I = - LiveVirtRegs.begin(), E = LiveVirtRegs.end(); I != E; ++I) + LiveRegs.VirtRegs.begin(), E = LiveRegs.VirtRegs.end(); I != E; ++I) P.LiveInRegs.push_back(*I); std::sort(P.LiveInRegs.begin(), P.LiveInRegs.end()); P.LiveInRegs.erase(std::unique(P.LiveInRegs.begin(), P.LiveInRegs.end()), @@ -236,19 +275,15 @@ void RegPressureTracker::closeTop() { /// Set the boundary for the bottom of the region and summarize live outs. void RegPressureTracker::closeBottom() { if (RequireIntervals) - if (CurrPos == MBB->end()) - static_cast<IntervalPressure&>(P).BottomIdx = LIS->getMBBEndIdx(MBB); - else - static_cast<IntervalPressure&>(P).BottomIdx = - LIS->getInstructionIndex(CurrPos).getRegSlot(); + static_cast<IntervalPressure&>(P).BottomIdx = getCurrSlot(); else static_cast<RegionPressure&>(P).BottomPos = CurrPos; assert(P.LiveOutRegs.empty() && "inconsistent max pressure result"); - P.LiveOutRegs.reserve(LivePhysRegs.size() + LiveVirtRegs.size()); - P.LiveOutRegs.append(LivePhysRegs.begin(), LivePhysRegs.end()); + P.LiveOutRegs.reserve(LiveRegs.PhysRegs.size() + LiveRegs.VirtRegs.size()); + P.LiveOutRegs.append(LiveRegs.PhysRegs.begin(), LiveRegs.PhysRegs.end()); for (SparseSet<unsigned>::const_iterator I = - LiveVirtRegs.begin(), E = LiveVirtRegs.end(); I != E; ++I) + LiveRegs.VirtRegs.begin(), E = LiveRegs.VirtRegs.end(); I != E; ++I) P.LiveOutRegs.push_back(*I); std::sort(P.LiveOutRegs.begin(), P.LiveOutRegs.end()); P.LiveOutRegs.erase(std::unique(P.LiveOutRegs.begin(), P.LiveOutRegs.end()), @@ -258,7 +293,7 @@ void RegPressureTracker::closeBottom() { /// Finalize the region boundaries and record live ins and live outs. void RegPressureTracker::closeRegion() { if (!isTopClosed() && !isBottomClosed()) { - assert(LivePhysRegs.empty() && LiveVirtRegs.empty() && + assert(LiveRegs.PhysRegs.empty() && LiveRegs.VirtRegs.empty() && "no region boundary"); return; } @@ -269,151 +304,97 @@ void RegPressureTracker::closeRegion() { // If both top and bottom are closed, do nothing. } -/// Return true if Reg aliases a register in Regs SparseSet. -static bool hasRegAlias(unsigned Reg, SparseSet<unsigned> &Regs, - const TargetRegisterInfo *TRI) { - assert(!TargetRegisterInfo::isVirtualRegister(Reg) && "only for physregs"); - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - if (Regs.count(*AI)) - return true; - return false; -} - -/// Return true if Reg aliases a register in unsorted Regs SmallVector. -/// This is only valid for physical registers. -static SmallVectorImpl<unsigned>::iterator -findRegAlias(unsigned Reg, SmallVectorImpl<unsigned> &Regs, - const TargetRegisterInfo *TRI) { - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) { - SmallVectorImpl<unsigned>::iterator I = - std::find(Regs.begin(), Regs.end(), *AI); - if (I != Regs.end()) - return I; - } - return Regs.end(); -} - -/// Return true if Reg can be inserted into Regs SmallVector. For virtual -/// register, do a linear search. For physical registers check for aliases. -static SmallVectorImpl<unsigned>::iterator -findReg(unsigned Reg, bool isVReg, SmallVectorImpl<unsigned> &Regs, - const TargetRegisterInfo *TRI) { - if(isVReg) - return std::find(Regs.begin(), Regs.end(), Reg); - return findRegAlias(Reg, Regs, TRI); +/// \brief Convenient wrapper for checking membership in RegisterOperands. +static bool containsReg(ArrayRef<unsigned> Regs, unsigned Reg) { + return std::find(Regs.begin(), Regs.end(), Reg) != Regs.end(); } /// Collect this instruction's unique uses and defs into SmallVectors for /// processing defs and uses in order. -template<bool isVReg> -struct RegisterOperands { +class RegisterOperands { + const TargetRegisterInfo *TRI; + const MachineRegisterInfo *MRI; + +public: SmallVector<unsigned, 8> Uses; SmallVector<unsigned, 8> Defs; SmallVector<unsigned, 8> DeadDefs; + RegisterOperands(const TargetRegisterInfo *tri, + const MachineRegisterInfo *mri): TRI(tri), MRI(mri) {} + /// Push this operand's register onto the correct vector. - void collect(const MachineOperand &MO, const TargetRegisterInfo *TRI) { - if (MO.readsReg()) { - if (findReg(MO.getReg(), isVReg, Uses, TRI) == Uses.end()) - Uses.push_back(MO.getReg()); - } + void collect(const MachineOperand &MO) { + if (!MO.isReg() || !MO.getReg()) + return; + if (MO.readsReg()) + pushRegUnits(MO.getReg(), Uses); if (MO.isDef()) { - if (MO.isDead()) { - if (findReg(MO.getReg(), isVReg, DeadDefs, TRI) == DeadDefs.end()) - DeadDefs.push_back(MO.getReg()); + if (MO.isDead()) + pushRegUnits(MO.getReg(), DeadDefs); + else + pushRegUnits(MO.getReg(), Defs); + } + } + +protected: + void pushRegUnits(unsigned Reg, SmallVectorImpl<unsigned> &Regs) { + if (TargetRegisterInfo::isVirtualRegister(Reg)) { + if (containsReg(Regs, Reg)) + return; + Regs.push_back(Reg); + } + else if (MRI->isAllocatable(Reg)) { + for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units) { + if (containsReg(Regs, *Units)) + continue; + Regs.push_back(*Units); } - else if (findReg(MO.getReg(), isVReg, Defs, TRI) == Defs.end()) - Defs.push_back(MO.getReg()); } } }; -typedef RegisterOperands<false> PhysRegOperands; -typedef RegisterOperands<true> VirtRegOperands; /// Collect physical and virtual register operands. static void collectOperands(const MachineInstr *MI, - PhysRegOperands &PhysRegOpers, - VirtRegOperands &VirtRegOpers, - const TargetRegisterInfo *TRI, - const MachineRegisterInfo *MRI) { - for(ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI) { - const MachineOperand &MO = *OperI; - if (!MO.isReg() || !MO.getReg()) - continue; + RegisterOperands &RegOpers) { + for (ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI) + RegOpers.collect(*OperI); - if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) - VirtRegOpers.collect(MO, TRI); - else if (MRI->isAllocatable(MO.getReg())) - PhysRegOpers.collect(MO, TRI); - } // Remove redundant physreg dead defs. - for (unsigned i = PhysRegOpers.DeadDefs.size(); i > 0; --i) { - unsigned Reg = PhysRegOpers.DeadDefs[i-1]; - if (findRegAlias(Reg, PhysRegOpers.Defs, TRI) != PhysRegOpers.Defs.end()) - PhysRegOpers.DeadDefs.erase(&PhysRegOpers.DeadDefs[i-1]); - } + SmallVectorImpl<unsigned>::iterator I = + std::remove_if(RegOpers.DeadDefs.begin(), RegOpers.DeadDefs.end(), + std::bind1st(std::ptr_fun(containsReg), RegOpers.Defs)); + RegOpers.DeadDefs.erase(I, RegOpers.DeadDefs.end()); } /// Force liveness of registers. void RegPressureTracker::addLiveRegs(ArrayRef<unsigned> Regs) { for (unsigned i = 0, e = Regs.size(); i != e; ++i) { - if (TargetRegisterInfo::isVirtualRegister(Regs[i])) { - if (LiveVirtRegs.insert(Regs[i]).second) - increaseVirtRegPressure(Regs[i]); - } - else { - if (!hasRegAlias(Regs[i], LivePhysRegs, TRI)) { - LivePhysRegs.insert(Regs[i]); - increasePhysRegPressure(Regs[i]); - } - } + if (LiveRegs.insert(Regs[i])) + increaseRegPressure(Regs[i]); } } -/// Add PhysReg to the live in set and increase max pressure. -void RegPressureTracker::discoverPhysLiveIn(unsigned Reg) { - assert(!LivePhysRegs.count(Reg) && "avoid bumping max pressure twice"); - if (findRegAlias(Reg, P.LiveInRegs, TRI) != P.LiveInRegs.end()) +/// Add Reg to the live in set and increase max pressure. +void RegPressureTracker::discoverLiveIn(unsigned Reg) { + assert(!LiveRegs.contains(Reg) && "avoid bumping max pressure twice"); + if (containsReg(P.LiveInRegs, Reg)) return; // At live in discovery, unconditionally increase the high water mark. P.LiveInRegs.push_back(Reg); - P.increase(TRI->getMinimalPhysRegClass(Reg), TRI); + P.increase(Reg, TRI, MRI); } -/// Add PhysReg to the live out set and increase max pressure. -void RegPressureTracker::discoverPhysLiveOut(unsigned Reg) { - assert(!LivePhysRegs.count(Reg) && "avoid bumping max pressure twice"); - if (findRegAlias(Reg, P.LiveOutRegs, TRI) != P.LiveOutRegs.end()) +/// Add Reg to the live out set and increase max pressure. +void RegPressureTracker::discoverLiveOut(unsigned Reg) { + assert(!LiveRegs.contains(Reg) && "avoid bumping max pressure twice"); + if (containsReg(P.LiveOutRegs, Reg)) return; // At live out discovery, unconditionally increase the high water mark. P.LiveOutRegs.push_back(Reg); - P.increase(TRI->getMinimalPhysRegClass(Reg), TRI); -} - -/// Add VirtReg to the live in set and increase max pressure. -void RegPressureTracker::discoverVirtLiveIn(unsigned Reg) { - assert(!LiveVirtRegs.count(Reg) && "avoid bumping max pressure twice"); - if (std::find(P.LiveInRegs.begin(), P.LiveInRegs.end(), Reg) != - P.LiveInRegs.end()) - return; - - // At live in discovery, unconditionally increase the high water mark. - P.LiveInRegs.push_back(Reg); - P.increase(MRI->getRegClass(Reg), TRI); -} - -/// Add VirtReg to the live out set and increase max pressure. -void RegPressureTracker::discoverVirtLiveOut(unsigned Reg) { - assert(!LiveVirtRegs.count(Reg) && "avoid bumping max pressure twice"); - if (std::find(P.LiveOutRegs.begin(), P.LiveOutRegs.end(), Reg) != - P.LiveOutRegs.end()) - return; - - // At live out discovery, unconditionally increase the high water mark. - P.LiveOutRegs.push_back(Reg); - P.increase(MRI->getRegClass(Reg), TRI); + P.increase(Reg, TRI, MRI); } /// Recede across the previous instruction. @@ -447,52 +428,35 @@ bool RegPressureTracker::recede() { if (RequireIntervals && isTopClosed()) static_cast<IntervalPressure&>(P).openTop(SlotIdx); - PhysRegOperands PhysRegOpers; - VirtRegOperands VirtRegOpers; - collectOperands(CurrPos, PhysRegOpers, VirtRegOpers, TRI, MRI); + RegisterOperands RegOpers(TRI, MRI); + collectOperands(CurrPos, RegOpers); // Boost pressure for all dead defs together. - increasePhysRegPressure(PhysRegOpers.DeadDefs); - increaseVirtRegPressure(VirtRegOpers.DeadDefs); - decreasePhysRegPressure(PhysRegOpers.DeadDefs); - decreaseVirtRegPressure(VirtRegOpers.DeadDefs); + increaseRegPressure(RegOpers.DeadDefs); + decreaseRegPressure(RegOpers.DeadDefs); // Kill liveness at live defs. // TODO: consider earlyclobbers? - for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Defs[i]; - if (LivePhysRegs.erase(Reg)) - decreasePhysRegPressure(Reg); - else - discoverPhysLiveOut(Reg); - } - for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Defs[i]; - if (LiveVirtRegs.erase(Reg)) - decreaseVirtRegPressure(Reg); + for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) { + unsigned Reg = RegOpers.Defs[i]; + if (LiveRegs.erase(Reg)) + decreaseRegPressure(Reg); else - discoverVirtLiveOut(Reg); + discoverLiveOut(Reg); } // Generate liveness for uses. - for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Uses[i]; - if (!hasRegAlias(Reg, LivePhysRegs, TRI)) { - increasePhysRegPressure(Reg); - LivePhysRegs.insert(Reg); - } - } - for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Uses[i]; - if (!LiveVirtRegs.count(Reg)) { + for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) { + unsigned Reg = RegOpers.Uses[i]; + if (!LiveRegs.contains(Reg)) { // Adjust liveouts if LiveIntervals are available. if (RequireIntervals) { - const LiveInterval *LI = &LIS->getInterval(Reg); - if (!LI->killedAt(SlotIdx)) - discoverVirtLiveOut(Reg); + const LiveInterval *LI = getInterval(Reg); + if (LI && !LI->killedAt(SlotIdx)) + discoverLiveOut(Reg); } - increaseVirtRegPressure(Reg); - LiveVirtRegs.insert(Reg); + increaseRegPressure(Reg); + LiveRegs.insert(Reg); } } return true; @@ -510,7 +474,7 @@ bool RegPressureTracker::advance() { SlotIndex SlotIdx; if (RequireIntervals) - SlotIdx = LIS->getInstructionIndex(CurrPos).getRegSlot(); + SlotIdx = getCurrSlot(); // Open the bottom of the region using slot indexes. if (isBottomClosed()) { @@ -520,57 +484,43 @@ bool RegPressureTracker::advance() { static_cast<RegionPressure&>(P).openBottom(CurrPos); } - PhysRegOperands PhysRegOpers; - VirtRegOperands VirtRegOpers; - collectOperands(CurrPos, PhysRegOpers, VirtRegOpers, TRI, MRI); - - // Kill liveness at last uses. - for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Uses[i]; - if (!hasRegAlias(Reg, LivePhysRegs, TRI)) - discoverPhysLiveIn(Reg); - else { - // Allocatable physregs are always single-use before regalloc. - decreasePhysRegPressure(Reg); - LivePhysRegs.erase(Reg); - } - } - for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Uses[i]; + RegisterOperands RegOpers(TRI, MRI); + collectOperands(CurrPos, RegOpers); + + for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) { + unsigned Reg = RegOpers.Uses[i]; + // Discover live-ins. + bool isLive = LiveRegs.contains(Reg); + if (!isLive) + discoverLiveIn(Reg); + // Kill liveness at last uses. + bool lastUse = false; if (RequireIntervals) { - const LiveInterval *LI = &LIS->getInterval(Reg); - if (LI->killedAt(SlotIdx)) { - if (LiveVirtRegs.erase(Reg)) - decreaseVirtRegPressure(Reg); - else - discoverVirtLiveIn(Reg); - } + const LiveInterval *LI = getInterval(Reg); + lastUse = LI && LI->killedAt(SlotIdx); } - else if (!LiveVirtRegs.count(Reg)) { - discoverVirtLiveIn(Reg); - increaseVirtRegPressure(Reg); + else { + // Allocatable physregs are always single-use before register rewriting. + lastUse = !TargetRegisterInfo::isVirtualRegister(Reg); } + if (lastUse && isLive) { + LiveRegs.erase(Reg); + decreaseRegPressure(Reg); + } + else if (!lastUse && !isLive) + increaseRegPressure(Reg); } // Generate liveness for defs. - for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Defs[i]; - if (!hasRegAlias(Reg, LivePhysRegs, TRI)) { - increasePhysRegPressure(Reg); - LivePhysRegs.insert(Reg); - } - } - for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Defs[i]; - if (LiveVirtRegs.insert(Reg).second) - increaseVirtRegPressure(Reg); + for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) { + unsigned Reg = RegOpers.Defs[i]; + if (LiveRegs.insert(Reg)) + increaseRegPressure(Reg); } // Boost pressure for all dead defs together. - increasePhysRegPressure(PhysRegOpers.DeadDefs); - increaseVirtRegPressure(VirtRegOpers.DeadDefs); - decreasePhysRegPressure(PhysRegOpers.DeadDefs); - decreaseVirtRegPressure(VirtRegOpers.DeadDefs); + increaseRegPressure(RegOpers.DeadDefs); + decreaseRegPressure(RegOpers.DeadDefs); // Find the next instruction. do @@ -661,39 +611,28 @@ static void computeMaxPressureDelta(ArrayRef<unsigned> OldMaxPressureVec, /// This is intended for speculative queries. It leaves pressure inconsistent /// with the current position, so must be restored by the caller. void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) { + assert(!MI->isDebugValue() && "Expect a nondebug instruction."); + // Account for register pressure similar to RegPressureTracker::recede(). - PhysRegOperands PhysRegOpers; - VirtRegOperands VirtRegOpers; - collectOperands(MI, PhysRegOpers, VirtRegOpers, TRI, MRI); + RegisterOperands RegOpers(TRI, MRI); + collectOperands(MI, RegOpers); // Boost max pressure for all dead defs together. // Since CurrSetPressure and MaxSetPressure - increasePhysRegPressure(PhysRegOpers.DeadDefs); - increaseVirtRegPressure(VirtRegOpers.DeadDefs); - decreasePhysRegPressure(PhysRegOpers.DeadDefs); - decreaseVirtRegPressure(VirtRegOpers.DeadDefs); + increaseRegPressure(RegOpers.DeadDefs); + decreaseRegPressure(RegOpers.DeadDefs); // Kill liveness at live defs. - for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Defs[i]; - if (!findReg(Reg, false, PhysRegOpers.Uses, TRI)) - decreasePhysRegPressure(PhysRegOpers.Defs); - } - for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Defs[i]; - if (!findReg(Reg, true, VirtRegOpers.Uses, TRI)) - decreaseVirtRegPressure(VirtRegOpers.Defs); + for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) { + unsigned Reg = RegOpers.Defs[i]; + if (!containsReg(RegOpers.Uses, Reg)) + decreaseRegPressure(Reg); } // Generate liveness for uses. - for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Uses[i]; - if (!hasRegAlias(Reg, LivePhysRegs, TRI)) - increasePhysRegPressure(Reg); - } - for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Uses[i]; - if (!LiveVirtRegs.count(Reg)) - increaseVirtRegPressure(Reg); + for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) { + unsigned Reg = RegOpers.Uses[i]; + if (!LiveRegs.contains(Reg)) + increaseRegPressure(Reg); } } @@ -740,6 +679,8 @@ static bool findUseBetween(unsigned Reg, UI = MRI->use_nodbg_begin(Reg), UE = MRI->use_nodbg_end(); UI != UE; UI.skipInstruction()) { const MachineInstr* MI = &*UI; + if (MI->isDebugValue()) + continue; SlotIndex InstSlot = LIS->getInstructionIndex(MI).getRegSlot(); if (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx) return true; @@ -754,38 +695,42 @@ static bool findUseBetween(unsigned Reg, /// This is intended for speculative queries. It leaves pressure inconsistent /// with the current position, so must be restored by the caller. void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) { + assert(!MI->isDebugValue() && "Expect a nondebug instruction."); + // Account for register pressure similar to RegPressureTracker::recede(). - PhysRegOperands PhysRegOpers; - VirtRegOperands VirtRegOpers; - collectOperands(MI, PhysRegOpers, VirtRegOpers, TRI, MRI); + RegisterOperands RegOpers(TRI, MRI); + collectOperands(MI, RegOpers); // Kill liveness at last uses. Assume allocatable physregs are single-use // rather than checking LiveIntervals. - decreasePhysRegPressure(PhysRegOpers.Uses); - if (RequireIntervals) { - SlotIndex SlotIdx = LIS->getInstructionIndex(MI).getRegSlot(); - for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Uses[i]; - const LiveInterval *LI = &LIS->getInterval(Reg); - // FIXME: allow the caller to pass in the list of vreg uses that remain to - // be bottom-scheduled to avoid searching uses at each query. - SlotIndex CurrIdx = LIS->getInstructionIndex(CurrPos).getRegSlot(); - if (LI->killedAt(SlotIdx) + SlotIndex SlotIdx; + if (RequireIntervals) + SlotIdx = LIS->getInstructionIndex(MI).getRegSlot(); + + for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) { + unsigned Reg = RegOpers.Uses[i]; + if (RequireIntervals) { + // FIXME: allow the caller to pass in the list of vreg uses that remain + // to be bottom-scheduled to avoid searching uses at each query. + SlotIndex CurrIdx = getCurrSlot(); + const LiveInterval *LI = getInterval(Reg); + if (LI && LI->killedAt(SlotIdx) && !findUseBetween(Reg, CurrIdx, SlotIdx, MRI, LIS)) { - decreaseVirtRegPressure(Reg); + decreaseRegPressure(Reg); } } + else if (!TargetRegisterInfo::isVirtualRegister(Reg)) { + // Allocatable physregs are always single-use before register rewriting. + decreaseRegPressure(Reg); + } } // Generate liveness for defs. - increasePhysRegPressure(PhysRegOpers.Defs); - increaseVirtRegPressure(VirtRegOpers.Defs); + increaseRegPressure(RegOpers.Defs); // Boost pressure for all dead defs together. - increasePhysRegPressure(PhysRegOpers.DeadDefs); - increaseVirtRegPressure(VirtRegOpers.DeadDefs); - decreasePhysRegPressure(PhysRegOpers.DeadDefs); - decreaseVirtRegPressure(VirtRegOpers.DeadDefs); + increaseRegPressure(RegOpers.DeadDefs); + decreaseRegPressure(RegOpers.DeadDefs); } /// Consider the pressure increase caused by traversing this instruction diff --git a/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp b/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp index 5ec6564..07ace7a 100644 --- a/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp +++ b/contrib/llvm/lib/CodeGen/RegisterScavenging.cpp @@ -16,21 +16,17 @@ #define DEBUG_TYPE "reg-scavenging" #include "llvm/CodeGen/RegisterScavenging.h" +#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; /// setUsed - Set the register and its sub-registers as being used. @@ -43,15 +39,17 @@ void RegScavenger::setUsed(unsigned Reg) { bool RegScavenger::isAliasUsed(unsigned Reg) const { for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - if (isUsed(*AI)) + if (isUsed(*AI, *AI == Reg)) return true; return false; } void RegScavenger::initRegState() { - ScavengedReg = 0; - ScavengedRC = NULL; - ScavengeRestore = NULL; + for (SmallVector<ScavengedInfo, 2>::iterator I = Scavenged.begin(), + IE = Scavenged.end(); I != IE; ++I) { + I->Reg = 0; + I->Restore = NULL; + } // All registers started out unused. RegsAvailable.set(); @@ -112,27 +110,11 @@ void RegScavenger::addRegWithSubRegs(BitVector &BV, unsigned Reg) { BV.set(*SubRegs); } -void RegScavenger::forward() { - // Move ptr forward. - if (!Tracking) { - MBBI = MBB->begin(); - Tracking = true; - } else { - assert(MBBI != MBB->end() && "Already past the end of the basic block!"); - MBBI = llvm::next(MBBI); - } - assert(MBBI != MBB->end() && "Already at the end of the basic block!"); +void RegScavenger::determineKillsAndDefs() { + assert(Tracking && "Must be tracking to determine kills and defs"); MachineInstr *MI = MBBI; - - if (MI == ScavengeRestore) { - ScavengedReg = 0; - ScavengedRC = NULL; - ScavengeRestore = NULL; - } - - if (MI->isDebugValue()) - return; + assert(!MI->isDebugValue() && "Debug values have no kills or defs"); // Find out which registers are early clobbered, killed, defined, and marked // def-dead in this instruction. @@ -149,7 +131,7 @@ void RegScavenger::forward() { if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); - if (!Reg || isReserved(Reg)) + if (!Reg || TargetRegisterInfo::isVirtualRegister(Reg) || isReserved(Reg)) continue; if (MO.isUse()) { @@ -166,6 +148,54 @@ void RegScavenger::forward() { addRegWithSubRegs(DefRegs, Reg); } } +} + +void RegScavenger::unprocess() { + assert(Tracking && "Cannot unprocess because we're not tracking"); + + MachineInstr *MI = MBBI; + if (MI->isDebugValue()) + return; + + determineKillsAndDefs(); + + // Commit the changes. + setUsed(KillRegs); + setUnused(DefRegs); + + if (MBBI == MBB->begin()) { + MBBI = MachineBasicBlock::iterator(NULL); + Tracking = false; + } else + --MBBI; +} + +void RegScavenger::forward() { + // Move ptr forward. + if (!Tracking) { + MBBI = MBB->begin(); + Tracking = true; + } else { + assert(MBBI != MBB->end() && "Already past the end of the basic block!"); + MBBI = llvm::next(MBBI); + } + assert(MBBI != MBB->end() && "Already at the end of the basic block!"); + + MachineInstr *MI = MBBI; + + for (SmallVector<ScavengedInfo, 2>::iterator I = Scavenged.begin(), + IE = Scavenged.end(); I != IE; ++I) { + if (I->Restore != MI) + continue; + + I->Reg = 0; + I->Restore = NULL; + } + + if (MI->isDebugValue()) + return; + + determineKillsAndDefs(); // Verify uses and defs. #ifndef NDEBUG @@ -174,7 +204,7 @@ void RegScavenger::forward() { if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); - if (!Reg || isReserved(Reg)) + if (!Reg || TargetRegisterInfo::isVirtualRegister(Reg) || isReserved(Reg)) continue; if (MO.isUse()) { if (MO.isUndef()) @@ -320,6 +350,16 @@ unsigned RegScavenger::findSurvivorReg(MachineBasicBlock::iterator StartMI, return Survivor; } +static unsigned getFrameIndexOperandNum(MachineInstr *MI) { + unsigned i = 0; + while (!MI->getOperand(i).isFI()) { + ++i; + assert(i < MI->getNumOperands() && + "Instr doesn't have FrameIndex operand!"); + } + return i; +} + unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC, MachineBasicBlock::iterator I, int SPAdj) { @@ -354,33 +394,47 @@ unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC, return SReg; } - assert(ScavengedReg == 0 && - "Scavenger slot is live, unable to scavenge another register!"); + // Find an available scavenging slot. + unsigned SI; + for (SI = 0; SI < Scavenged.size(); ++SI) + if (Scavenged[SI].Reg == 0) + break; + + if (SI == Scavenged.size()) { + // We need to scavenge a register but have no spill slot, the target + // must know how to do it (if not, we'll assert below). + Scavenged.push_back(ScavengedInfo()); + } // Avoid infinite regress - ScavengedReg = SReg; + Scavenged[SI].Reg = SReg; // If the target knows how to save/restore the register, let it do so; // otherwise, use the emergency stack spill slot. if (!TRI->saveScavengerRegister(*MBB, I, UseMI, RC, SReg)) { // Spill the scavenged register before I. - assert(ScavengingFrameIndex >= 0 && + assert(Scavenged[SI].FrameIndex >= 0 && "Cannot scavenge register without an emergency spill slot!"); - TII->storeRegToStackSlot(*MBB, I, SReg, true, ScavengingFrameIndex, RC,TRI); + TII->storeRegToStackSlot(*MBB, I, SReg, true, Scavenged[SI].FrameIndex, + RC, TRI); MachineBasicBlock::iterator II = prior(I); - TRI->eliminateFrameIndex(II, SPAdj, this); + + unsigned FIOperandNum = getFrameIndexOperandNum(II); + TRI->eliminateFrameIndex(II, SPAdj, FIOperandNum, this); // Restore the scavenged register before its use (or first terminator). - TII->loadRegFromStackSlot(*MBB, UseMI, SReg, ScavengingFrameIndex, RC, TRI); + TII->loadRegFromStackSlot(*MBB, UseMI, SReg, Scavenged[SI].FrameIndex, + RC, TRI); II = prior(UseMI); - TRI->eliminateFrameIndex(II, SPAdj, this); + + FIOperandNum = getFrameIndexOperandNum(II); + TRI->eliminateFrameIndex(II, SPAdj, FIOperandNum, this); } - ScavengeRestore = prior(UseMI); + Scavenged[SI].Restore = prior(UseMI); // Doing this here leads to infinite regress. - // ScavengedReg = SReg; - ScavengedRC = RC; + // Scavenged[SI].Reg = SReg; DEBUG(dbgs() << "Scavenged register (with spill): " << TRI->getName(SReg) << "\n"); diff --git a/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp b/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp index 9a65071..07e5b47 100644 --- a/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp +++ b/contrib/llvm/lib/CodeGen/ScheduleDAG.cpp @@ -16,12 +16,12 @@ #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/CodeGen/ScheduleHazardRecognizer.h" #include "llvm/CodeGen/SelectionDAGNodes.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <climits> using namespace llvm; @@ -62,10 +62,14 @@ const MCInstrDesc *ScheduleDAG::getNodeDesc(const SDNode *Node) const { /// addPred - This adds the specified edge as a pred of the current node if /// not already. It also adds the current node as a successor of the /// specified node. -bool SUnit::addPred(const SDep &D) { +bool SUnit::addPred(const SDep &D, bool Required) { // If this node already has this depenence, don't add a redundant one. for (SmallVector<SDep, 4>::iterator I = Preds.begin(), E = Preds.end(); I != E; ++I) { + // Zero-latency weak edges may be added purely for heuristic ordering. Don't + // add them if another kind of edge already exists. + if (!Required && I->getSUnit() == D.getSUnit()) + return false; if (I->overlaps(D)) { // Extend the latency if needed. Equivalent to removePred(I) + addPred(D). if (I->getLatency() < D.getLatency()) { @@ -97,12 +101,22 @@ bool SUnit::addPred(const SDep &D) { ++N->NumSuccs; } if (!N->isScheduled) { - assert(NumPredsLeft < UINT_MAX && "NumPredsLeft will overflow!"); - ++NumPredsLeft; + if (D.isWeak()) { + ++WeakPredsLeft; + } + else { + assert(NumPredsLeft < UINT_MAX && "NumPredsLeft will overflow!"); + ++NumPredsLeft; + } } if (!isScheduled) { - assert(N->NumSuccsLeft < UINT_MAX && "NumSuccsLeft will overflow!"); - ++N->NumSuccsLeft; + if (D.isWeak()) { + ++N->WeakSuccsLeft; + } + else { + assert(N->NumSuccsLeft < UINT_MAX && "NumSuccsLeft will overflow!"); + ++N->NumSuccsLeft; + } } Preds.push_back(D); N->Succs.push_back(P); @@ -121,20 +135,14 @@ void SUnit::removePred(const SDep &D) { for (SmallVector<SDep, 4>::iterator I = Preds.begin(), E = Preds.end(); I != E; ++I) if (*I == D) { - bool FoundSucc = false; // Find the corresponding successor in N. SDep P = D; P.setSUnit(this); SUnit *N = D.getSUnit(); - for (SmallVector<SDep, 4>::iterator II = N->Succs.begin(), - EE = N->Succs.end(); II != EE; ++II) - if (*II == P) { - FoundSucc = true; - N->Succs.erase(II); - break; - } - assert(FoundSucc && "Mismatching preds / succs lists!"); - (void)FoundSucc; + SmallVectorImpl<SDep>::iterator Succ = std::find(N->Succs.begin(), + N->Succs.end(), P); + assert(Succ != N->Succs.end() && "Mismatching preds / succs lists!"); + N->Succs.erase(Succ); Preds.erase(I); // Update the bookkeeping. if (P.getKind() == SDep::Data) { @@ -144,12 +152,20 @@ void SUnit::removePred(const SDep &D) { --N->NumSuccs; } if (!N->isScheduled) { - assert(NumPredsLeft > 0 && "NumPredsLeft will underflow!"); - --NumPredsLeft; + if (D.isWeak()) + --WeakPredsLeft; + else { + assert(NumPredsLeft > 0 && "NumPredsLeft will underflow!"); + --NumPredsLeft; + } } if (!isScheduled) { - assert(N->NumSuccsLeft > 0 && "NumSuccsLeft will underflow!"); - --N->NumSuccsLeft; + if (D.isWeak()) + --N->WeakSuccsLeft; + else { + assert(N->NumSuccsLeft > 0 && "NumSuccsLeft will underflow!"); + --N->NumSuccsLeft; + } } if (P.getLatency() != 0) { this->setDepthDirty(); @@ -279,6 +295,21 @@ void SUnit::ComputeHeight() { } while (!WorkList.empty()); } +void SUnit::biasCriticalPath() { + if (NumPreds < 2) + return; + + SUnit::pred_iterator BestI = Preds.begin(); + unsigned MaxDepth = BestI->getSUnit()->getDepth(); + for (SUnit::pred_iterator + I = llvm::next(BestI), E = Preds.end(); I != E; ++I) { + if (I->getKind() == SDep::Data && I->getSUnit()->getDepth() > MaxDepth) + BestI = I; + } + if (BestI != Preds.begin()) + std::swap(*Preds.begin(), *BestI); +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// SUnit - Scheduling unit. It's an wrapper around either a single SDNode or /// a group of nodes flagged together. @@ -292,10 +323,14 @@ void SUnit::dumpAll(const ScheduleDAG *G) const { dbgs() << " # preds left : " << NumPredsLeft << "\n"; dbgs() << " # succs left : " << NumSuccsLeft << "\n"; + if (WeakPredsLeft) + dbgs() << " # weak preds left : " << WeakPredsLeft << "\n"; + if (WeakSuccsLeft) + dbgs() << " # weak succs left : " << WeakSuccsLeft << "\n"; dbgs() << " # rdefs left : " << NumRegDefsLeft << "\n"; dbgs() << " Latency : " << Latency << "\n"; - dbgs() << " Depth : " << Depth << "\n"; - dbgs() << " Height : " << Height << "\n"; + dbgs() << " Depth : " << getDepth() << "\n"; + dbgs() << " Height : " << getHeight() << "\n"; if (Preds.size() != 0) { dbgs() << " Predecessors:\n"; @@ -332,6 +367,8 @@ void SUnit::dumpAll(const ScheduleDAG *G) const { if (I->isArtificial()) dbgs() << " *"; dbgs() << ": Latency=" << I->getLatency(); + if (I->isAssignedRegDep()) + dbgs() << " Reg=" << PrintReg(I->getReg(), G->TRI); dbgs() << "\n"; } } @@ -429,6 +466,8 @@ void ScheduleDAGTopologicalSort::InitDAGTopologicalSorting() { Node2Index.resize(DAGSize); // Initialize the data structures. + if (ExitSU) + WorkList.push_back(ExitSU); for (unsigned i = 0, e = DAGSize; i != e; ++i) { SUnit *SU = &SUnits[i]; int NodeNum = SU->NodeNum; @@ -448,11 +487,12 @@ void ScheduleDAGTopologicalSort::InitDAGTopologicalSorting() { while (!WorkList.empty()) { SUnit *SU = WorkList.back(); WorkList.pop_back(); - Allocate(SU->NodeNum, --Id); + if (SU->NodeNum < DAGSize) + Allocate(SU->NodeNum, --Id); for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); I != E; ++I) { SUnit *SU = I->getSUnit(); - if (!--Node2Index[SU->NodeNum]) + if (SU->NodeNum < DAGSize && !--Node2Index[SU->NodeNum]) // If all dependencies of the node are processed already, // then the node can be computed now. WorkList.push_back(SU); @@ -513,7 +553,10 @@ void ScheduleDAGTopologicalSort::DFS(const SUnit *SU, int UpperBound, WorkList.pop_back(); Visited.set(SU->NodeNum); for (int I = SU->Succs.size()-1; I >= 0; --I) { - int s = SU->Succs[I].getSUnit()->NodeNum; + unsigned s = SU->Succs[I].getSUnit()->NodeNum; + // Edges to non-SUnits are allowed but ignored (e.g. ExitSU). + if (s >= Node2Index.size()) + continue; if (Node2Index[s] == UpperBound) { HasLoop = true; return; @@ -554,15 +597,16 @@ void ScheduleDAGTopologicalSort::Shift(BitVector& Visited, int LowerBound, } -/// WillCreateCycle - Returns true if adding an edge from SU to TargetSU will -/// create a cycle. -bool ScheduleDAGTopologicalSort::WillCreateCycle(SUnit *SU, SUnit *TargetSU) { - if (IsReachable(TargetSU, SU)) +/// WillCreateCycle - Returns true if adding an edge to TargetSU from SU will +/// create a cycle. If so, it is not safe to call AddPred(TargetSU, SU). +bool ScheduleDAGTopologicalSort::WillCreateCycle(SUnit *TargetSU, SUnit *SU) { + // Is SU reachable from TargetSU via successor edges? + if (IsReachable(SU, TargetSU)) return true; - for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); - I != E; ++I) + for (SUnit::pred_iterator + I = TargetSU->Preds.begin(), E = TargetSU->Preds.end(); I != E; ++I) if (I->isAssignedRegDep() && - IsReachable(TargetSU, I->getSUnit())) + IsReachable(SU, I->getSUnit())) return true; return false; } @@ -592,6 +636,7 @@ void ScheduleDAGTopologicalSort::Allocate(int n, int index) { } ScheduleDAGTopologicalSort:: -ScheduleDAGTopologicalSort(std::vector<SUnit> &sunits) : SUnits(sunits) {} +ScheduleDAGTopologicalSort(std::vector<SUnit> &sunits, SUnit *exitsu) + : SUnits(sunits), ExitSU(exitsu) {} ScheduleHazardRecognizer::~ScheduleHazardRecognizer() {} diff --git a/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp index a4d4a93..71e7a21 100644 --- a/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/contrib/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -12,8 +12,11 @@ // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "sched-instrs" -#include "llvm/Operator.h" +#define DEBUG_TYPE "misched" +#include "llvm/CodeGen/ScheduleDAGInstrs.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" @@ -22,19 +25,17 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/CodeGen/RegisterPressure.h" -#include "llvm/CodeGen/ScheduleDAGILP.h" -#include "llvm/CodeGen/ScheduleDAGInstrs.h" +#include "llvm/CodeGen/ScheduleDFS.h" +#include "llvm/IR/Operator.h" #include "llvm/MC/MCInstrItineraries.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetSubtargetInfo.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; static cl::opt<bool> EnableAASchedMI("enable-aa-sched-mi", cl::Hidden, @@ -66,7 +67,7 @@ static const Value *getUnderlyingObjectFromInt(const Value *V) { // regular getUnderlyingObjectFromInt. if (U->getOpcode() == Instruction::PtrToInt) return U->getOperand(0); - // If we find an add of a constant or a multiplied value, it's + // If we find an add of a constant, a multiplied value, or a phi, it's // likely that the other operand will lead us to the base // object. We don't have to worry about the case where the // object address is somehow being computed by the multiply, @@ -74,7 +75,8 @@ static const Value *getUnderlyingObjectFromInt(const Value *V) { // identifiable object. if (U->getOpcode() != Instruction::Add || (!isa<ConstantInt>(U->getOperand(1)) && - Operator::getOpcode(U->getOperand(1)) != Instruction::Mul)) + Operator::getOpcode(U->getOperand(1)) != Instruction::Mul && + !isa<PHINode>(U->getOperand(1)))) return V; V = U->getOperand(0); } else { @@ -84,56 +86,77 @@ static const Value *getUnderlyingObjectFromInt(const Value *V) { } while (1); } -/// getUnderlyingObject - This is a wrapper around GetUnderlyingObject +/// getUnderlyingObjects - This is a wrapper around GetUnderlyingObjects /// and adds support for basic ptrtoint+arithmetic+inttoptr sequences. -static const Value *getUnderlyingObject(const Value *V) { - // First just call Value::getUnderlyingObject to let it do what it does. +static void getUnderlyingObjects(const Value *V, + SmallVectorImpl<Value *> &Objects) { + SmallPtrSet<const Value*, 16> Visited; + SmallVector<const Value *, 4> Working(1, V); do { - V = GetUnderlyingObject(V); - // If it found an inttoptr, use special code to continue climing. - if (Operator::getOpcode(V) != Instruction::IntToPtr) - break; - const Value *O = getUnderlyingObjectFromInt(cast<User>(V)->getOperand(0)); - // If that succeeded in finding a pointer, continue the search. - if (!O->getType()->isPointerTy()) - break; - V = O; - } while (1); - return V; + V = Working.pop_back_val(); + + SmallVector<Value *, 4> Objs; + GetUnderlyingObjects(const_cast<Value *>(V), Objs); + + for (SmallVector<Value *, 4>::iterator I = Objs.begin(), IE = Objs.end(); + I != IE; ++I) { + V = *I; + if (!Visited.insert(V)) + continue; + if (Operator::getOpcode(V) == Instruction::IntToPtr) { + const Value *O = + getUnderlyingObjectFromInt(cast<User>(V)->getOperand(0)); + if (O->getType()->isPointerTy()) { + Working.push_back(O); + continue; + } + } + Objects.push_back(const_cast<Value *>(V)); + } + } while (!Working.empty()); } -/// getUnderlyingObjectForInstr - If this machine instr has memory reference +/// getUnderlyingObjectsForInstr - If this machine instr has memory reference /// information and it can be tracked to a normal reference to a known -/// object, return the Value for that object. Otherwise return null. -static const Value *getUnderlyingObjectForInstr(const MachineInstr *MI, - const MachineFrameInfo *MFI, - bool &MayAlias) { - MayAlias = true; +/// object, return the Value for that object. +static void getUnderlyingObjectsForInstr(const MachineInstr *MI, + const MachineFrameInfo *MFI, + SmallVectorImpl<std::pair<const Value *, bool> > &Objects) { if (!MI->hasOneMemOperand() || !(*MI->memoperands_begin())->getValue() || (*MI->memoperands_begin())->isVolatile()) - return 0; + return; const Value *V = (*MI->memoperands_begin())->getValue(); if (!V) - return 0; - - V = getUnderlyingObject(V); - if (const PseudoSourceValue *PSV = dyn_cast<PseudoSourceValue>(V)) { - // For now, ignore PseudoSourceValues which may alias LLVM IR values - // because the code that uses this function has no way to cope with - // such aliases. - if (PSV->isAliased(MFI)) - return 0; - - MayAlias = PSV->mayAlias(MFI); - return V; - } + return; + + SmallVector<Value *, 4> Objs; + getUnderlyingObjects(V, Objs); - if (isIdentifiedObject(V)) - return V; + for (SmallVector<Value *, 4>::iterator I = Objs.begin(), IE = Objs.end(); + I != IE; ++I) { + bool MayAlias = true; + V = *I; + + if (const PseudoSourceValue *PSV = dyn_cast<PseudoSourceValue>(V)) { + // For now, ignore PseudoSourceValues which may alias LLVM IR values + // because the code that uses this function has no way to cope with + // such aliases. - return 0; + if (PSV->isAliased(MFI)) { + Objects.clear(); + return; + } + + MayAlias = PSV->mayAlias(MFI); + } else if (!isIdentifiedObject(V)) { + Objects.clear(); + return; + } + + Objects.push_back(std::make_pair(V, MayAlias)); + } } void ScheduleDAGInstrs::startBlock(MachineBasicBlock *bb) { @@ -145,20 +168,6 @@ void ScheduleDAGInstrs::finishBlock() { BB = 0; } -/// Initialize the map with the number of registers. -void Reg2SUnitsMap::setRegLimit(unsigned Limit) { - PhysRegSet.setUniverse(Limit); - SUnits.resize(Limit); -} - -/// Clear the map without deallocating storage. -void Reg2SUnitsMap::clear() { - for (const_iterator I = reg_begin(), E = reg_end(); I != E; ++I) { - SUnits[*I].clear(); - } - PhysRegSet.clear(); -} - /// Initialize the DAG and common scheduler state for the current scheduling /// region. This does not actually create the DAG, only clears it. The /// scheduling driver may call BuildSchedGraph multiple times per scheduling @@ -205,10 +214,11 @@ void ScheduleDAGInstrs::addSchedBarrierDeps() { if (Reg == 0) continue; if (TRI->isPhysicalRegister(Reg)) - Uses[Reg].push_back(PhysRegSUOper(&ExitSU, -1)); + Uses.insert(PhysRegSUOper(&ExitSU, -1, Reg)); else { assert(!IsPostRA && "Virtual register encountered after regalloc."); - addVRegUseDeps(&ExitSU, i); + if (MO.readsReg()) // ignore undef operands + addVRegUseDeps(&ExitSU, i); } } } else { @@ -221,7 +231,7 @@ void ScheduleDAGInstrs::addSchedBarrierDeps() { E = (*SI)->livein_end(); I != E; ++I) { unsigned Reg = *I; if (!Uses.contains(Reg)) - Uses[Reg].push_back(PhysRegSUOper(&ExitSU, -1)); + Uses.insert(PhysRegSUOper(&ExitSU, -1, Reg)); } } } @@ -239,27 +249,31 @@ void ScheduleDAGInstrs::addPhysRegDataDeps(SUnit *SU, unsigned OperIdx) { Alias.isValid(); ++Alias) { if (!Uses.contains(*Alias)) continue; - std::vector<PhysRegSUOper> &UseList = Uses[*Alias]; - for (unsigned i = 0, e = UseList.size(); i != e; ++i) { - SUnit *UseSU = UseList[i].SU; + for (Reg2SUnitsMap::iterator I = Uses.find(*Alias); I != Uses.end(); ++I) { + SUnit *UseSU = I->SU; if (UseSU == SU) continue; - SDep dep(SU, SDep::Data, *Alias); - // Adjust the dependence latency using operand def/use information, // then allow the target to perform its own adjustments. - int UseOp = UseList[i].OpIdx; - MachineInstr *RegUse = UseOp < 0 ? 0 : UseSU->getInstr(); - dep.setLatency( + int UseOp = I->OpIdx; + MachineInstr *RegUse = 0; + SDep Dep; + if (UseOp < 0) + Dep = SDep(SU, SDep::Artificial); + else { + Dep = SDep(SU, SDep::Data, *Alias); + RegUse = UseSU->getInstr(); + Dep.setMinLatency( + SchedModel.computeOperandLatency(SU->getInstr(), OperIdx, + RegUse, UseOp, /*FindMin=*/true)); + } + Dep.setLatency( SchedModel.computeOperandLatency(SU->getInstr(), OperIdx, RegUse, UseOp, /*FindMin=*/false)); - dep.setMinLatency( - SchedModel.computeOperandLatency(SU->getInstr(), OperIdx, - RegUse, UseOp, /*FindMin=*/true)); - ST.adjustSchedDependency(SU, UseSU, dep); - UseSU->addPred(dep); + ST.adjustSchedDependency(SU, UseSU, Dep); + UseSU->addPred(Dep); } } } @@ -282,9 +296,8 @@ void ScheduleDAGInstrs::addPhysRegDeps(SUnit *SU, unsigned OperIdx) { Alias.isValid(); ++Alias) { if (!Defs.contains(*Alias)) continue; - std::vector<PhysRegSUOper> &DefList = Defs[*Alias]; - for (unsigned i = 0, e = DefList.size(); i != e; ++i) { - SUnit *DefSU = DefList[i].SU; + for (Reg2SUnitsMap::iterator I = Defs.find(*Alias); I != Defs.end(); ++I) { + SUnit *DefSU = I->SU; if (DefSU == &ExitSU) continue; if (DefSU != SU && @@ -308,33 +321,37 @@ void ScheduleDAGInstrs::addPhysRegDeps(SUnit *SU, unsigned OperIdx) { // Either insert a new Reg2SUnits entry with an empty SUnits list, or // retrieve the existing SUnits list for this register's uses. // Push this SUnit on the use list. - Uses[MO.getReg()].push_back(PhysRegSUOper(SU, OperIdx)); + Uses.insert(PhysRegSUOper(SU, OperIdx, MO.getReg())); } else { addPhysRegDataDeps(SU, OperIdx); - - // Either insert a new Reg2SUnits entry with an empty SUnits list, or - // retrieve the existing SUnits list for this register's defs. - std::vector<PhysRegSUOper> &DefList = Defs[MO.getReg()]; + unsigned Reg = MO.getReg(); // clear this register's use list - if (Uses.contains(MO.getReg())) - Uses[MO.getReg()].clear(); - - if (!MO.isDead()) - DefList.clear(); - - // Calls will not be reordered because of chain dependencies (see - // below). Since call operands are dead, calls may continue to be added - // to the DefList making dependence checking quadratic in the size of - // the block. Instead, we leave only one call at the back of the - // DefList. - if (SU->isCall) { - while (!DefList.empty() && DefList.back().SU->isCall) - DefList.pop_back(); + if (Uses.contains(Reg)) + Uses.eraseAll(Reg); + + if (!MO.isDead()) { + Defs.eraseAll(Reg); + } else if (SU->isCall) { + // Calls will not be reordered because of chain dependencies (see + // below). Since call operands are dead, calls may continue to be added + // to the DefList making dependence checking quadratic in the size of + // the block. Instead, we leave only one call at the back of the + // DefList. + Reg2SUnitsMap::RangePair P = Defs.equal_range(Reg); + Reg2SUnitsMap::iterator B = P.first; + Reg2SUnitsMap::iterator I = P.second; + for (bool isBegin = I == B; !isBegin; /* empty */) { + isBegin = (--I) == B; + if (!I->SU->isCall) + break; + I = Defs.erase(I); + } } + // Defs are pushed in the order they are visited and never reordered. - DefList.push_back(PhysRegSUOper(SU, OperIdx)); + Defs.insert(PhysRegSUOper(SU, OperIdx, Reg)); } } @@ -445,23 +462,29 @@ static inline bool isUnsafeMemoryObject(MachineInstr *MI, if ((*MI->memoperands_begin())->isVolatile() || MI->hasUnmodeledSideEffects()) return true; - const Value *V = (*MI->memoperands_begin())->getValue(); if (!V) return true; - V = getUnderlyingObject(V); - if (const PseudoSourceValue *PSV = dyn_cast<PseudoSourceValue>(V)) { - // Similarly to getUnderlyingObjectForInstr: - // For now, ignore PseudoSourceValues which may alias LLVM IR values - // because the code that uses this function has no way to cope with - // such aliases. - if (PSV->isAliased(MFI)) + SmallVector<Value *, 4> Objs; + getUnderlyingObjects(V, Objs); + for (SmallVector<Value *, 4>::iterator I = Objs.begin(), + IE = Objs.end(); I != IE; ++I) { + V = *I; + + if (const PseudoSourceValue *PSV = dyn_cast<PseudoSourceValue>(V)) { + // Similarly to getUnderlyingObjectForInstr: + // For now, ignore PseudoSourceValues which may alias LLVM IR values + // because the code that uses this function has no way to cope with + // such aliases. + if (PSV->isAliased(MFI)) + return true; + } + + // Does this pointer refer to a distinct and identifiable object? + if (!isIdentifiedObject(V)) return true; } - // Does this pointer refer to a distinct and identifiable object? - if (!isIdentifiedObject(V)) - return true; return false; } @@ -680,8 +703,8 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, // so that they can be given more precise dependencies. We track // separately the known memory locations that may alias and those // that are known not to alias - std::map<const Value *, SUnit *> AliasMemDefs, NonAliasMemDefs; - std::map<const Value *, std::vector<SUnit *> > AliasMemUses, NonAliasMemUses; + MapVector<const Value *, SUnit *> AliasMemDefs, NonAliasMemDefs; + MapVector<const Value *, std::vector<SUnit *> > AliasMemUses, NonAliasMemUses; std::set<SUnit*> RejectMemNodes; // Remove any stale debug info; sometimes BuildSchedGraph is called again @@ -691,8 +714,8 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, assert(Defs.empty() && Uses.empty() && "Only BuildGraph should update Defs/Uses"); - Defs.setRegLimit(TRI->getNumRegs()); - Uses.setRegLimit(TRI->getNumRegs()); + Defs.setUniverse(TRI->getNumRegs()); + Uses.setUniverse(TRI->getNumRegs()); assert(VRegDefs.empty() && "Only BuildSchedGraph may access VRegDefs"); // FIXME: Allow SparseSet to reserve space for the creation of virtual @@ -705,17 +728,17 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, addSchedBarrierDeps(); // Walk the list of instructions, from bottom moving up. - MachineInstr *PrevMI = NULL; + MachineInstr *DbgMI = NULL; for (MachineBasicBlock::iterator MII = RegionEnd, MIE = RegionBegin; MII != MIE; --MII) { MachineInstr *MI = prior(MII); - if (MI && PrevMI) { - DbgValues.push_back(std::make_pair(PrevMI, MI)); - PrevMI = NULL; + if (MI && DbgMI) { + DbgValues.push_back(std::make_pair(DbgMI, MI)); + DbgMI = NULL; } if (MI->isDebugValue()) { - PrevMI = MI; + DbgMI = MI; continue; } if (RPTracker) { @@ -723,13 +746,14 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, assert(RPTracker->getPos() == prior(MII) && "RPTracker can't find MI"); } - assert((!MI->isTerminator() || CanHandleTerminators) && !MI->isLabel() && + assert((CanHandleTerminators || (!MI->isTerminator() && !MI->isLabel())) && "Cannot schedule terminators or labels!"); SUnit *SU = MISUnitMap[MI]; assert(SU && "No SUnit mapped to this MI"); // Add register-based dependencies (data, anti, and output). + bool HasVRegDef = false; for (unsigned j = 0, n = MI->getNumOperands(); j != n; ++j) { const MachineOperand &MO = MI->getOperand(j); if (!MO.isReg()) continue; @@ -740,12 +764,26 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, addPhysRegDeps(SU, j); else { assert(!IsPostRA && "Virtual register encountered!"); - if (MO.isDef()) + if (MO.isDef()) { + HasVRegDef = true; addVRegDefDeps(SU, j); + } else if (MO.readsReg()) // ignore undef operands addVRegUseDeps(SU, j); } } + // If we haven't seen any uses in this scheduling region, create a + // dependence edge to ExitSU to model the live-out latency. This is required + // for vreg defs with no in-region use, and prefetches with no vreg def. + // + // FIXME: NumDataSuccs would be more precise than NumSuccs here. This + // check currently relies on being called before adding chain deps. + if (SU->NumSuccs == 0 && SU->Latency > 1 + && (HasVRegDef || MI->mayLoad())) { + SDep Dep(SU, SDep::Artificial); + Dep.setLatency(SU->Latency - 1); + ExitSU.addPred(Dep); + } // Add chain dependencies. // Chain dependencies used to enforce memory order should have @@ -760,11 +798,11 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, if (isGlobalMemoryObject(AA, MI)) { // Be conservative with these and add dependencies on all memory // references, even those that are known to not alias. - for (std::map<const Value *, SUnit *>::iterator I = + for (MapVector<const Value *, SUnit *>::iterator I = NonAliasMemDefs.begin(), E = NonAliasMemDefs.end(); I != E; ++I) { I->second->addPred(SDep(SU, SDep::Barrier)); } - for (std::map<const Value *, std::vector<SUnit *> >::iterator I = + for (MapVector<const Value *, std::vector<SUnit *> >::iterator I = NonAliasMemUses.begin(), E = NonAliasMemUses.end(); I != E; ++I) { for (unsigned i = 0, e = I->second.size(); i != e; ++i) { SDep Dep(SU, SDep::Barrier); @@ -798,10 +836,10 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k) addChainDependency(AA, MFI, SU, PendingLoads[k], RejectMemNodes, TrueMemOrderLatency); - for (std::map<const Value *, SUnit *>::iterator I = AliasMemDefs.begin(), + for (MapVector<const Value *, SUnit *>::iterator I = AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I) addChainDependency(AA, MFI, SU, I->second, RejectMemNodes); - for (std::map<const Value *, std::vector<SUnit *> >::iterator I = + for (MapVector<const Value *, std::vector<SUnit *> >::iterator I = AliasMemUses.begin(), E = AliasMemUses.end(); I != E; ++I) { for (unsigned i = 0, e = I->second.size(); i != e; ++i) addChainDependency(AA, MFI, SU, I->second[i], RejectMemNodes, @@ -813,60 +851,70 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, AliasMemDefs.clear(); AliasMemUses.clear(); } else if (MI->mayStore()) { - bool MayAlias = true; - if (const Value *V = getUnderlyingObjectForInstr(MI, MFI, MayAlias)) { + SmallVector<std::pair<const Value *, bool>, 4> Objs; + getUnderlyingObjectsForInstr(MI, MFI, Objs); + + if (Objs.empty()) { + // Treat all other stores conservatively. + goto new_alias_chain; + } + + bool MayAlias = false; + for (SmallVector<std::pair<const Value *, bool>, 4>::iterator + K = Objs.begin(), KE = Objs.end(); K != KE; ++K) { + const Value *V = K->first; + bool ThisMayAlias = K->second; + if (ThisMayAlias) + MayAlias = true; + // A store to a specific PseudoSourceValue. Add precise dependencies. // Record the def in MemDefs, first adding a dep if there is // an existing def. - std::map<const Value *, SUnit *>::iterator I = - ((MayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V)); - std::map<const Value *, SUnit *>::iterator IE = - ((MayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end()); + MapVector<const Value *, SUnit *>::iterator I = + ((ThisMayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V)); + MapVector<const Value *, SUnit *>::iterator IE = + ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end()); if (I != IE) { - addChainDependency(AA, MFI, SU, I->second, RejectMemNodes, - 0, true); + addChainDependency(AA, MFI, SU, I->second, RejectMemNodes, 0, true); I->second = SU; } else { - if (MayAlias) + if (ThisMayAlias) AliasMemDefs[V] = SU; else NonAliasMemDefs[V] = SU; } // Handle the uses in MemUses, if there are any. - std::map<const Value *, std::vector<SUnit *> >::iterator J = - ((MayAlias) ? AliasMemUses.find(V) : NonAliasMemUses.find(V)); - std::map<const Value *, std::vector<SUnit *> >::iterator JE = - ((MayAlias) ? AliasMemUses.end() : NonAliasMemUses.end()); + MapVector<const Value *, std::vector<SUnit *> >::iterator J = + ((ThisMayAlias) ? AliasMemUses.find(V) : NonAliasMemUses.find(V)); + MapVector<const Value *, std::vector<SUnit *> >::iterator JE = + ((ThisMayAlias) ? AliasMemUses.end() : NonAliasMemUses.end()); if (J != JE) { for (unsigned i = 0, e = J->second.size(); i != e; ++i) addChainDependency(AA, MFI, SU, J->second[i], RejectMemNodes, TrueMemOrderLatency, true); J->second.clear(); } - if (MayAlias) { - // Add dependencies from all the PendingLoads, i.e. loads - // with no underlying object. - for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k) - addChainDependency(AA, MFI, SU, PendingLoads[k], RejectMemNodes, - TrueMemOrderLatency); - // Add dependence on alias chain, if needed. - if (AliasChain) - addChainDependency(AA, MFI, SU, AliasChain, RejectMemNodes); - // But we also should check dependent instructions for the - // SU in question. - adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes, - TrueMemOrderLatency); - } - // Add dependence on barrier chain, if needed. - // There is no point to check aliasing on barrier event. Even if - // SU and barrier _could_ be reordered, they should not. In addition, - // we have lost all RejectMemNodes below barrier. - if (BarrierChain) - BarrierChain->addPred(SDep(SU, SDep::Barrier)); - } else { - // Treat all other stores conservatively. - goto new_alias_chain; } + if (MayAlias) { + // Add dependencies from all the PendingLoads, i.e. loads + // with no underlying object. + for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k) + addChainDependency(AA, MFI, SU, PendingLoads[k], RejectMemNodes, + TrueMemOrderLatency); + // Add dependence on alias chain, if needed. + if (AliasChain) + addChainDependency(AA, MFI, SU, AliasChain, RejectMemNodes); + // But we also should check dependent instructions for the + // SU in question. + adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes, + TrueMemOrderLatency); + } + // Add dependence on barrier chain, if needed. + // There is no point to check aliasing on barrier event. Even if + // SU and barrier _could_ be reordered, they should not. In addition, + // we have lost all RejectMemNodes below barrier. + if (BarrierChain) + BarrierChain->addPred(SDep(SU, SDep::Barrier)); if (!ExitSU.isPred(SU)) // Push store's up a bit to avoid them getting in between cmp @@ -877,28 +925,41 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, if (MI->isInvariantLoad(AA)) { // Invariant load, no chain dependencies needed! } else { - if (const Value *V = - getUnderlyingObjectForInstr(MI, MFI, MayAlias)) { - // A load from a specific PseudoSourceValue. Add precise dependencies. - std::map<const Value *, SUnit *>::iterator I = - ((MayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V)); - std::map<const Value *, SUnit *>::iterator IE = - ((MayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end()); - if (I != IE) - addChainDependency(AA, MFI, SU, I->second, RejectMemNodes, 0, true); - if (MayAlias) - AliasMemUses[V].push_back(SU); - else - NonAliasMemUses[V].push_back(SU); - } else { + SmallVector<std::pair<const Value *, bool>, 4> Objs; + getUnderlyingObjectsForInstr(MI, MFI, Objs); + + if (Objs.empty()) { // A load with no underlying object. Depend on all // potentially aliasing stores. - for (std::map<const Value *, SUnit *>::iterator I = + for (MapVector<const Value *, SUnit *>::iterator I = AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I) addChainDependency(AA, MFI, SU, I->second, RejectMemNodes); PendingLoads.push_back(SU); MayAlias = true; + } else { + MayAlias = false; + } + + for (SmallVector<std::pair<const Value *, bool>, 4>::iterator + J = Objs.begin(), JE = Objs.end(); J != JE; ++J) { + const Value *V = J->first; + bool ThisMayAlias = J->second; + + if (ThisMayAlias) + MayAlias = true; + + // A load from a specific PseudoSourceValue. Add precise dependencies. + MapVector<const Value *, SUnit *>::iterator I = + ((ThisMayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V)); + MapVector<const Value *, SUnit *>::iterator IE = + ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end()); + if (I != IE) + addChainDependency(AA, MFI, SU, I->second, RejectMemNodes, 0, true); + if (ThisMayAlias) + AliasMemUses[V].push_back(SU); + else + NonAliasMemUses[V].push_back(SU); } if (MayAlias) adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes, /*Latency=*/0); @@ -910,8 +971,8 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, } } } - if (PrevMI) - FirstDbgValue = PrevMI; + if (DbgMI) + FirstDbgValue = DbgMI; Defs.clear(); Uses.clear(); @@ -933,7 +994,7 @@ std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const { else if (SU == &ExitSU) oss << "<exit>"; else - SU->getInstr()->print(oss); + SU->getInstr()->print(oss, &TM, /*SkipOpers=*/true); return oss.str(); } @@ -943,6 +1004,203 @@ std::string ScheduleDAGInstrs::getDAGName() const { return "dag." + BB->getFullName(); } +//===----------------------------------------------------------------------===// +// SchedDFSResult Implementation +//===----------------------------------------------------------------------===// + +namespace llvm { +/// \brief Internal state used to compute SchedDFSResult. +class SchedDFSImpl { + SchedDFSResult &R; + + /// Join DAG nodes into equivalence classes by their subtree. + IntEqClasses SubtreeClasses; + /// List PredSU, SuccSU pairs that represent data edges between subtrees. + std::vector<std::pair<const SUnit*, const SUnit*> > ConnectionPairs; + + struct RootData { + unsigned NodeID; + unsigned ParentNodeID; // Parent node (member of the parent subtree). + unsigned SubInstrCount; // Instr count in this tree only, not children. + + RootData(unsigned id): NodeID(id), + ParentNodeID(SchedDFSResult::InvalidSubtreeID), + SubInstrCount(0) {} + + unsigned getSparseSetIndex() const { return NodeID; } + }; + + SparseSet<RootData> RootSet; + +public: + SchedDFSImpl(SchedDFSResult &r): R(r), SubtreeClasses(R.DFSNodeData.size()) { + RootSet.setUniverse(R.DFSNodeData.size()); + } + + /// Return true if this node been visited by the DFS traversal. + /// + /// During visitPostorderNode the Node's SubtreeID is assigned to the Node + /// ID. Later, SubtreeID is updated but remains valid. + bool isVisited(const SUnit *SU) const { + return R.DFSNodeData[SU->NodeNum].SubtreeID + != SchedDFSResult::InvalidSubtreeID; + } + + /// Initialize this node's instruction count. We don't need to flag the node + /// visited until visitPostorder because the DAG cannot have cycles. + void visitPreorder(const SUnit *SU) { + R.DFSNodeData[SU->NodeNum].InstrCount = + SU->getInstr()->isTransient() ? 0 : 1; + } + + /// Called once for each node after all predecessors are visited. Revisit this + /// node's predecessors and potentially join them now that we know the ILP of + /// the other predecessors. + void visitPostorderNode(const SUnit *SU) { + // Mark this node as the root of a subtree. It may be joined with its + // successors later. + R.DFSNodeData[SU->NodeNum].SubtreeID = SU->NodeNum; + RootData RData(SU->NodeNum); + RData.SubInstrCount = SU->getInstr()->isTransient() ? 0 : 1; + + // If any predecessors are still in their own subtree, they either cannot be + // joined or are large enough to remain separate. If this parent node's + // total instruction count is not greater than a child subtree by at least + // the subtree limit, then try to join it now since splitting subtrees is + // only useful if multiple high-pressure paths are possible. + unsigned InstrCount = R.DFSNodeData[SU->NodeNum].InstrCount; + for (SUnit::const_pred_iterator + PI = SU->Preds.begin(), PE = SU->Preds.end(); PI != PE; ++PI) { + if (PI->getKind() != SDep::Data) + continue; + unsigned PredNum = PI->getSUnit()->NodeNum; + if ((InstrCount - R.DFSNodeData[PredNum].InstrCount) < R.SubtreeLimit) + joinPredSubtree(*PI, SU, /*CheckLimit=*/false); + + // Either link or merge the TreeData entry from the child to the parent. + if (R.DFSNodeData[PredNum].SubtreeID == PredNum) { + // If the predecessor's parent is invalid, this is a tree edge and the + // current node is the parent. + if (RootSet[PredNum].ParentNodeID == SchedDFSResult::InvalidSubtreeID) + RootSet[PredNum].ParentNodeID = SU->NodeNum; + } + else if (RootSet.count(PredNum)) { + // The predecessor is not a root, but is still in the root set. This + // must be the new parent that it was just joined to. Note that + // RootSet[PredNum].ParentNodeID may either be invalid or may still be + // set to the original parent. + RData.SubInstrCount += RootSet[PredNum].SubInstrCount; + RootSet.erase(PredNum); + } + } + RootSet[SU->NodeNum] = RData; + } + + /// Called once for each tree edge after calling visitPostOrderNode on the + /// predecessor. Increment the parent node's instruction count and + /// preemptively join this subtree to its parent's if it is small enough. + void visitPostorderEdge(const SDep &PredDep, const SUnit *Succ) { + R.DFSNodeData[Succ->NodeNum].InstrCount + += R.DFSNodeData[PredDep.getSUnit()->NodeNum].InstrCount; + joinPredSubtree(PredDep, Succ); + } + + /// Add a connection for cross edges. + void visitCrossEdge(const SDep &PredDep, const SUnit *Succ) { + ConnectionPairs.push_back(std::make_pair(PredDep.getSUnit(), Succ)); + } + + /// Set each node's subtree ID to the representative ID and record connections + /// between trees. + void finalize() { + SubtreeClasses.compress(); + R.DFSTreeData.resize(SubtreeClasses.getNumClasses()); + assert(SubtreeClasses.getNumClasses() == RootSet.size() + && "number of roots should match trees"); + for (SparseSet<RootData>::const_iterator + RI = RootSet.begin(), RE = RootSet.end(); RI != RE; ++RI) { + unsigned TreeID = SubtreeClasses[RI->NodeID]; + if (RI->ParentNodeID != SchedDFSResult::InvalidSubtreeID) + R.DFSTreeData[TreeID].ParentTreeID = SubtreeClasses[RI->ParentNodeID]; + R.DFSTreeData[TreeID].SubInstrCount = RI->SubInstrCount; + // Note that SubInstrCount may be greater than InstrCount if we joined + // subtrees across a cross edge. InstrCount will be attributed to the + // original parent, while SubInstrCount will be attributed to the joined + // parent. + } + R.SubtreeConnections.resize(SubtreeClasses.getNumClasses()); + R.SubtreeConnectLevels.resize(SubtreeClasses.getNumClasses()); + DEBUG(dbgs() << R.getNumSubtrees() << " subtrees:\n"); + for (unsigned Idx = 0, End = R.DFSNodeData.size(); Idx != End; ++Idx) { + R.DFSNodeData[Idx].SubtreeID = SubtreeClasses[Idx]; + DEBUG(dbgs() << " SU(" << Idx << ") in tree " + << R.DFSNodeData[Idx].SubtreeID << '\n'); + } + for (std::vector<std::pair<const SUnit*, const SUnit*> >::const_iterator + I = ConnectionPairs.begin(), E = ConnectionPairs.end(); + I != E; ++I) { + unsigned PredTree = SubtreeClasses[I->first->NodeNum]; + unsigned SuccTree = SubtreeClasses[I->second->NodeNum]; + if (PredTree == SuccTree) + continue; + unsigned Depth = I->first->getDepth(); + addConnection(PredTree, SuccTree, Depth); + addConnection(SuccTree, PredTree, Depth); + } + } + +protected: + /// Join the predecessor subtree with the successor that is its DFS + /// parent. Apply some heuristics before joining. + bool joinPredSubtree(const SDep &PredDep, const SUnit *Succ, + bool CheckLimit = true) { + assert(PredDep.getKind() == SDep::Data && "Subtrees are for data edges"); + + // Check if the predecessor is already joined. + const SUnit *PredSU = PredDep.getSUnit(); + unsigned PredNum = PredSU->NodeNum; + if (R.DFSNodeData[PredNum].SubtreeID != PredNum) + return false; + + // Four is the magic number of successors before a node is considered a + // pinch point. + unsigned NumDataSucs = 0; + for (SUnit::const_succ_iterator SI = PredSU->Succs.begin(), + SE = PredSU->Succs.end(); SI != SE; ++SI) { + if (SI->getKind() == SDep::Data) { + if (++NumDataSucs >= 4) + return false; + } + } + if (CheckLimit && R.DFSNodeData[PredNum].InstrCount > R.SubtreeLimit) + return false; + R.DFSNodeData[PredNum].SubtreeID = Succ->NodeNum; + SubtreeClasses.join(Succ->NodeNum, PredNum); + return true; + } + + /// Called by finalize() to record a connection between trees. + void addConnection(unsigned FromTree, unsigned ToTree, unsigned Depth) { + if (!Depth) + return; + + do { + SmallVectorImpl<SchedDFSResult::Connection> &Connections = + R.SubtreeConnections[FromTree]; + for (SmallVectorImpl<SchedDFSResult::Connection>::iterator + I = Connections.begin(), E = Connections.end(); I != E; ++I) { + if (I->TreeID == ToTree) { + I->Level = std::max(I->Level, Depth); + return; + } + } + Connections.push_back(SchedDFSResult::Connection(ToTree, Depth)); + FromTree = R.DFSTreeData[FromTree].ParentTreeID; + } while (FromTree != SchedDFSResult::InvalidSubtreeID); + } +}; +} // namespace llvm + namespace { /// \brief Manage the stack used by a reverse depth-first search over the DAG. class SchedDAGReverseDFS { @@ -955,7 +1213,10 @@ public: } void advance() { ++DFSStack.back().second; } - void backtrack() { DFSStack.pop_back(); } + const SDep *backtrack() { + DFSStack.pop_back(); + return DFSStack.empty() ? 0 : llvm::prior(DFSStack.back().second); + } const SUnit *getCurr() const { return DFSStack.back().first; } @@ -967,57 +1228,83 @@ public: }; } // anonymous -void ScheduleDAGILP::resize(unsigned NumSUnits) { - ILPValues.resize(NumSUnits); -} - -ILPValue ScheduleDAGILP::getILP(const SUnit *SU) { - return ILPValues[SU->NodeNum]; -} - -// A leaf node has an ILP of 1/1. -static ILPValue initILP(const SUnit *SU) { - unsigned Cnt = SU->getInstr()->isTransient() ? 0 : 1; - return ILPValue(Cnt, 1 + SU->getDepth()); +static bool hasDataSucc(const SUnit *SU) { + for (SUnit::const_succ_iterator + SI = SU->Succs.begin(), SE = SU->Succs.end(); SI != SE; ++SI) { + if (SI->getKind() == SDep::Data && !SI->getSUnit()->isBoundaryNode()) + return true; + } + return false; } /// Compute an ILP metric for all nodes in the subDAG reachable via depth-first /// search from this root. -void ScheduleDAGILP::computeILP(const SUnit *Root) { +void SchedDFSResult::compute(ArrayRef<SUnit> SUnits) { if (!IsBottomUp) llvm_unreachable("Top-down ILP metric is unimplemnted"); - SchedDAGReverseDFS DFS; - // Mark a node visited by validating it. - ILPValues[Root->NodeNum] = initILP(Root); - DFS.follow(Root); - for (;;) { - // Traverse the leftmost path as far as possible. - while (DFS.getPred() != DFS.getPredEnd()) { - const SUnit *PredSU = DFS.getPred()->getSUnit(); - DFS.advance(); - // If the pred is already valid, skip it. - if (ILPValues[PredSU->NodeNum].isValid()) - continue; - ILPValues[PredSU->NodeNum] = initILP(PredSU); - DFS.follow(PredSU); + SchedDFSImpl Impl(*this); + for (ArrayRef<SUnit>::const_iterator + SI = SUnits.begin(), SE = SUnits.end(); SI != SE; ++SI) { + const SUnit *SU = &*SI; + if (Impl.isVisited(SU) || hasDataSucc(SU)) + continue; + + SchedDAGReverseDFS DFS; + Impl.visitPreorder(SU); + DFS.follow(SU); + for (;;) { + // Traverse the leftmost path as far as possible. + while (DFS.getPred() != DFS.getPredEnd()) { + const SDep &PredDep = *DFS.getPred(); + DFS.advance(); + // Ignore non-data edges. + if (PredDep.getKind() != SDep::Data + || PredDep.getSUnit()->isBoundaryNode()) { + continue; + } + // An already visited edge is a cross edge, assuming an acyclic DAG. + if (Impl.isVisited(PredDep.getSUnit())) { + Impl.visitCrossEdge(PredDep, DFS.getCurr()); + continue; + } + Impl.visitPreorder(PredDep.getSUnit()); + DFS.follow(PredDep.getSUnit()); + } + // Visit the top of the stack in postorder and backtrack. + const SUnit *Child = DFS.getCurr(); + const SDep *PredDep = DFS.backtrack(); + Impl.visitPostorderNode(Child); + if (PredDep) + Impl.visitPostorderEdge(*PredDep, DFS.getCurr()); + if (DFS.isComplete()) + break; } - // Visit the top of the stack in postorder and backtrack. - unsigned PredCount = ILPValues[DFS.getCurr()->NodeNum].InstrCount; - DFS.backtrack(); - if (DFS.isComplete()) - break; - // Add the recently finished predecessor's bottom-up descendent count. - ILPValues[DFS.getCurr()->NodeNum].InstrCount += PredCount; + } + Impl.finalize(); +} + +/// The root of the given SubtreeID was just scheduled. For all subtrees +/// connected to this tree, record the depth of the connection so that the +/// nearest connected subtrees can be prioritized. +void SchedDFSResult::scheduleTree(unsigned SubtreeID) { + for (SmallVectorImpl<Connection>::const_iterator + I = SubtreeConnections[SubtreeID].begin(), + E = SubtreeConnections[SubtreeID].end(); I != E; ++I) { + SubtreeConnectLevels[I->TreeID] = + std::max(SubtreeConnectLevels[I->TreeID], I->Level); + DEBUG(dbgs() << " Tree: " << I->TreeID + << " @" << SubtreeConnectLevels[I->TreeID] << '\n'); } } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void ILPValue::print(raw_ostream &OS) const { - if (!isValid()) + OS << InstrCount << " / " << Length << " = "; + if (!Length) OS << "BADILP"; - OS << InstrCount << " / " << Cycles << " = " - << format("%g", ((double)InstrCount / Cycles)); + else + OS << format("%g", ((double)InstrCount / Length)); } void ILPValue::dump() const { diff --git a/contrib/llvm/lib/CodeGen/ScheduleDAGPrinter.cpp b/contrib/llvm/lib/CodeGen/ScheduleDAGPrinter.cpp index 6e781b1..8ddb3e8 100644 --- a/contrib/llvm/lib/CodeGen/ScheduleDAGPrinter.cpp +++ b/contrib/llvm/lib/CodeGen/ScheduleDAGPrinter.cpp @@ -11,19 +11,19 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Constants.h" -#include "llvm/Assembly/Writer.h" #include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Assembly/Writer.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetMachine.h" +#include "llvm/IR/Constants.h" #include "llvm/Support/Debug.h" #include "llvm/Support/GraphWriter.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/StringExtras.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <fstream> using namespace llvm; @@ -41,6 +41,10 @@ namespace llvm { return true; } + static bool isNodeHidden(const SUnit *Node) { + return (Node->NumPreds > 10 || Node->NumSuccs > 10); + } + static bool hasNodeAddressLabel(const SUnit *Node, const ScheduleDAG *Graph) { return true; diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 37d7731..eb16095 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -18,22 +18,23 @@ #define DEBUG_TYPE "dagcombine" #include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/DerivedTypes.h" -#include "llvm/LLVMContext.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" #include <algorithm> using namespace llvm; @@ -291,6 +292,10 @@ namespace { unsigned SrcValueAlign2, const MDNode *TBAAInfo2) const; + /// isAlias - Return true if there is any possibility that the two addresses + /// overlap. + bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1); + /// FindAliasInfo - Extracts the relevant alias information from the memory /// node. Returns true if the operand was a load. bool FindAliasInfo(SDNode *N, @@ -1178,7 +1183,7 @@ SDValue DAGCombiner::combine(SDNode *N) { // Expose the DAG combiner to the target combiner impls. TargetLowering::DAGCombinerInfo - DagCombineInfo(DAG, !LegalTypes, !LegalOperations, false, this); + DagCombineInfo(DAG, Level, false, this); RV = TLI.PerformDAGCombine(N, DagCombineInfo); } @@ -1377,6 +1382,12 @@ SDValue DAGCombiner::visitADD(SDNode *N) { if (VT.isVector()) { SDValue FoldedVOp = SimplifyVBinOp(N); if (FoldedVOp.getNode()) return FoldedVOp; + + // fold (add x, 0) -> x, vector edition + if (ISD::isBuildVectorAllZeros(N1.getNode())) + return N0; + if (ISD::isBuildVectorAllZeros(N0.getNode())) + return N1; } // fold (add x, undef) -> undef @@ -1620,6 +1631,10 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (VT.isVector()) { SDValue FoldedVOp = SimplifyVBinOp(N); if (FoldedVOp.getNode()) return FoldedVOp; + + // fold (sub x, 0) -> x, vector edition + if (ISD::isBuildVectorAllZeros(N1.getNode())) + return N0; } // fold (sub x, x) -> 0 @@ -2423,6 +2438,18 @@ SDValue DAGCombiner::visitAND(SDNode *N) { if (VT.isVector()) { SDValue FoldedVOp = SimplifyVBinOp(N); if (FoldedVOp.getNode()) return FoldedVOp; + + // fold (and x, 0) -> 0, vector edition + if (ISD::isBuildVectorAllZeros(N0.getNode())) + return N0; + if (ISD::isBuildVectorAllZeros(N1.getNode())) + return N1; + + // fold (and x, -1) -> x, vector edition + if (ISD::isBuildVectorAllOnes(N0.getNode())) + return N1; + if (ISD::isBuildVectorAllOnes(N1.getNode())) + return N0; } // fold (and x, undef) -> 0 @@ -2606,7 +2633,10 @@ SDValue DAGCombiner::visitAND(SDNode *N) { bool isInteger = LL.getValueType().isInteger(); ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger); if (Result != ISD::SETCC_INVALID && - (!LegalOperations || TLI.isCondCodeLegal(Result, LL.getValueType()))) + (!LegalOperations || + (TLI.isCondCodeLegal(Result, LL.getSimpleValueType()) && + TLI.isOperationLegal(ISD::SETCC, + TLI.getSetCCResultType(N0.getSimpleValueType()))))) return DAG.getSetCC(N->getDebugLoc(), N0.getValueType(), LL, LR, Result); } @@ -2766,7 +2796,6 @@ SDValue DAGCombiner::visitAND(SDNode *N) { } } } - return SDValue(); } @@ -2959,7 +2988,8 @@ SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) { SDValue N00 = N0.getOperand(0); SDValue N01 = N0.getOperand(1); - if (N1.getOpcode() == ISD::OR) { + if (N1.getOpcode() == ISD::OR && + N00.getNumOperands() == 2 && N01.getNumOperands() == 2) { // (or (or (and), (and)), (or (and), (and))) SDValue N000 = N00.getOperand(0); if (!isBSwapHWordElement(N000, Parts)) @@ -3021,6 +3051,18 @@ SDValue DAGCombiner::visitOR(SDNode *N) { if (VT.isVector()) { SDValue FoldedVOp = SimplifyVBinOp(N); if (FoldedVOp.getNode()) return FoldedVOp; + + // fold (or x, 0) -> x, vector edition + if (ISD::isBuildVectorAllZeros(N0.getNode())) + return N1; + if (ISD::isBuildVectorAllZeros(N1.getNode())) + return N0; + + // fold (or x, -1) -> -1, vector edition + if (ISD::isBuildVectorAllOnes(N0.getNode())) + return N0; + if (ISD::isBuildVectorAllOnes(N1.getNode())) + return N1; } // fold (or x, undef) -> -1 @@ -3103,7 +3145,10 @@ SDValue DAGCombiner::visitOR(SDNode *N) { bool isInteger = LL.getValueType().isInteger(); ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger); if (Result != ISD::SETCC_INVALID && - (!LegalOperations || TLI.isCondCodeLegal(Result, LL.getValueType()))) + (!LegalOperations || + (TLI.isCondCodeLegal(Result, LL.getSimpleValueType()) && + TLI.isOperationLegal(ISD::SETCC, + TLI.getSetCCResultType(N0.getValueType()))))) return DAG.getSetCC(N->getDebugLoc(), N0.getValueType(), LL, LR, Result); } @@ -3330,6 +3375,12 @@ SDValue DAGCombiner::visitXOR(SDNode *N) { if (VT.isVector()) { SDValue FoldedVOp = SimplifyVBinOp(N); if (FoldedVOp.getNode()) return FoldedVOp; + + // fold (xor x, 0) -> x, vector edition + if (ISD::isBuildVectorAllZeros(N0.getNode())) + return N1; + if (ISD::isBuildVectorAllZeros(N1.getNode())) + return N0; } // fold (xor undef, undef) -> 0. This is a common idiom (misuse). @@ -3360,7 +3411,8 @@ SDValue DAGCombiner::visitXOR(SDNode *N) { ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), isInt); - if (!LegalOperations || TLI.isCondCodeLegal(NotCC, LHS.getValueType())) { + if (!LegalOperations || + TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) { switch (N0.getOpcode()) { default: llvm_unreachable("Unhandled SetCC Equivalent!"); @@ -4444,8 +4496,8 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) { NegOne, DAG.getConstant(0, VT), cast<CondCodeSDNode>(N0.getOperand(2))->get(), true); if (SCC.getNode()) return SCC; - if (!LegalOperations || - TLI.isOperationLegal(ISD::SETCC, TLI.getSetCCResultType(VT))) + if (!VT.isVector() && (!LegalOperations || + TLI.isOperationLegal(ISD::SETCC, TLI.getSetCCResultType(VT)))) return DAG.getNode(ISD::SELECT, N->getDebugLoc(), VT, DAG.getSetCC(N->getDebugLoc(), TLI.getSetCCResultType(VT), @@ -5025,11 +5077,15 @@ SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) { // At this point, we must have a load or else we can't do the transform. if (!isa<LoadSDNode>(N0)) return SDValue(); + // Because a SRL must be assumed to *need* to zero-extend the high bits + // (as opposed to anyext the high bits), we can't combine the zextload + // lowering of SRL and an sextload. + if (cast<LoadSDNode>(N0)->getExtensionType() == ISD::SEXTLOAD) + return SDValue(); + // If the shift amount is larger than the input type then we're not // accessing any of the loaded bytes. If the load was a zextload/extload // then the result of the shift+trunc is zero/undef (handled elsewhere). - // If the load was a sextload then the result is a splat of the sign bit - // of the extended byte. This is not worth optimizing for. if (ShAmt >= cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits()) return SDValue(); } @@ -5048,16 +5104,26 @@ SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) { // If we haven't found a load, we can't narrow it. Don't transform one with // multiple uses, this would require adding a new load. - if (!isa<LoadSDNode>(N0) || !N0.hasOneUse() || - // Don't change the width of a volatile load. - cast<LoadSDNode>(N0)->isVolatile()) + if (!isa<LoadSDNode>(N0) || !N0.hasOneUse()) + return SDValue(); + + // Don't change the width of a volatile load. + LoadSDNode *LN0 = cast<LoadSDNode>(N0); + if (LN0->isVolatile()) return SDValue(); // Verify that we are actually reducing a load width here. - if (cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits() < EVTBits) + if (LN0->getMemoryVT().getSizeInBits() < EVTBits) + return SDValue(); + + // For the transform to be legal, the load must produce only two values + // (the value loaded and the chain). Don't transform a pre-increment + // load, for example, which produces an extra value. Otherwise the + // transformation is not equivalent, and the downstream logic to replace + // uses gets things wrong. + if (LN0->getNumValues() > 2) return SDValue(); - LoadSDNode *LN0 = cast<LoadSDNode>(N0); EVT PtrType = N0.getOperand(1).getValueType(); if (PtrType == MVT::Untyped || PtrType.isExtended()) @@ -5101,8 +5167,15 @@ SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) { EVT ShImmTy = getShiftAmountTy(Result.getValueType()); if (!isUIntN(ShImmTy.getSizeInBits(), ShLeftAmt)) ShImmTy = VT; - Result = DAG.getNode(ISD::SHL, N0.getDebugLoc(), VT, - Result, DAG.getConstant(ShLeftAmt, ShImmTy)); + // If the shift amount is as large as the result size (but, presumably, + // no larger than the source) then the useful bits of the result are + // zero; we can't simply return the shortened shift, because the result + // of that operation is undefined. + if (ShLeftAmt >= VT.getSizeInBits()) + Result = DAG.getConstant(0, VT); + else + Result = DAG.getNode(ISD::SHL, N0.getDebugLoc(), VT, + Result, DAG.getConstant(ShLeftAmt, ShImmTy)); } // Return the new loaded value. @@ -5187,6 +5260,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) { LN0->getAlignment()); CombineTo(N, ExtLoad); CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); + AddToWorkList(ExtLoad.getNode()); return SDValue(N, 0); // Return N so it doesn't get rechecked! } // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use @@ -5287,6 +5361,38 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { } } + // Fold a series of buildvector, bitcast, and truncate if possible. + // For example fold + // (2xi32 trunc (bitcast ((4xi32)buildvector x, x, y, y) 2xi64)) to + // (2xi32 (buildvector x, y)). + if (Level == AfterLegalizeVectorOps && VT.isVector() && + N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && + N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR && + N0.getOperand(0).hasOneUse()) { + + SDValue BuildVect = N0.getOperand(0); + EVT BuildVectEltTy = BuildVect.getValueType().getVectorElementType(); + EVT TruncVecEltTy = VT.getVectorElementType(); + + // Check that the element types match. + if (BuildVectEltTy == TruncVecEltTy) { + // Now we only need to compute the offset of the truncated elements. + unsigned BuildVecNumElts = BuildVect.getNumOperands(); + unsigned TruncVecNumElts = VT.getVectorNumElements(); + unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts; + + assert((BuildVecNumElts % TruncVecNumElts) == 0 && + "Invalid number of elements"); + + SmallVector<SDValue, 8> Opnds; + for (unsigned i = 0, e = BuildVecNumElts; i != e; i += TruncEltOffset) + Opnds.push_back(BuildVect.getOperand(i)); + + return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT, &Opnds[0], + Opnds.size()); + } + } + // See if we can simplify the input to this truncate through knowledge that // only the low bits are being used. // For example "trunc (or (shl x, 8), y)" // -> trunc y @@ -5729,14 +5835,25 @@ SDValue DAGCombiner::visitFADD(SDNode *N) { DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0.getOperand(1), N1)); + // No FP constant should be created after legalization as Instruction + // Selection pass has hard time in dealing with FP constant. + // + // We don't need test this condition for transformation like following, as + // the DAG being transformed implies it is legal to take FP constant as + // operand. + // + // (fadd (fmul c, x), x) -> (fmul c+1, x) + // + bool AllowNewFpConst = (Level < AfterLegalizeDAG); + // If allow, fold (fadd (fneg x), x) -> 0.0 - if (DAG.getTarget().Options.UnsafeFPMath && + if (AllowNewFpConst && DAG.getTarget().Options.UnsafeFPMath && N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1) { return DAG.getConstantFP(0.0, VT); } // If allow, fold (fadd x, (fneg x)) -> 0.0 - if (DAG.getTarget().Options.UnsafeFPMath && + if (AllowNewFpConst && DAG.getTarget().Options.UnsafeFPMath && N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0) { return DAG.getConstantFP(0.0, VT); } @@ -5769,13 +5886,6 @@ SDValue DAGCombiner::visitFADD(SDNode *N) { N1, NewCFP); } - // (fadd (fadd x, x), x) -> (fmul 3.0, x) - if (!CFP00 && !CFP01 && N0.getOperand(0) == N0.getOperand(1) && - N0.getOperand(0) == N1) { - return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, - N1, DAG.getConstantFP(3.0, VT)); - } - // (fadd (fmul c, x), (fadd x, x)) -> (fmul c+2, x) if (CFP00 && !CFP01 && N1.getOpcode() == ISD::FADD && N1.getOperand(0) == N1.getOperand(1) && @@ -5821,12 +5931,6 @@ SDValue DAGCombiner::visitFADD(SDNode *N) { N0, NewCFP); } - // (fadd x, (fadd x, x)) -> (fmul 3.0, x) - if (!CFP10 && !CFP11 && N1.getOperand(0) == N1.getOperand(1) && - N1.getOperand(0) == N0) { - return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, - N0, DAG.getConstantFP(3.0, VT)); - } // (fadd (fadd x, x), (fmul c, x)) -> (fmul c+2, x) if (CFP10 && !CFP11 && N1.getOpcode() == ISD::FADD && @@ -5851,8 +5955,29 @@ SDValue DAGCombiner::visitFADD(SDNode *N) { } } + if (N0.getOpcode() == ISD::FADD && AllowNewFpConst) { + ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N0.getOperand(0)); + // (fadd (fadd x, x), x) -> (fmul 3.0, x) + if (!CFP && N0.getOperand(0) == N0.getOperand(1) && + (N0.getOperand(0) == N1)) { + return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, + N1, DAG.getConstantFP(3.0, VT)); + } + } + + if (N1.getOpcode() == ISD::FADD && AllowNewFpConst) { + ConstantFPSDNode *CFP10 = dyn_cast<ConstantFPSDNode>(N1.getOperand(0)); + // (fadd x, (fadd x, x)) -> (fmul 3.0, x) + if (!CFP10 && N1.getOperand(0) == N1.getOperand(1) && + N1.getOperand(0) == N0) { + return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, + N0, DAG.getConstantFP(3.0, VT)); + } + } + // (fadd (fadd x, x), (fadd x, x)) -> (fmul 4.0, x) - if (N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD && + if (AllowNewFpConst && + N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD && N0.getOperand(0) == N0.getOperand(1) && N1.getOperand(0) == N1.getOperand(1) && N0.getOperand(0) == N1.getOperand(0)) { @@ -6596,7 +6721,8 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) { // fold a brcond with a setcc condition into a BR_CC node if BR_CC is legal // on the target. if (N1.getOpcode() == ISD::SETCC && - TLI.isOperationLegalOrCustom(ISD::BR_CC, MVT::Other)) { + TLI.isOperationLegalOrCustom(ISD::BR_CC, + N1.getOperand(0).getValueType())) { return DAG.getNode(ISD::BR_CC, N->getDebugLoc(), MVT::Other, Chain, N1.getOperand(2), N1.getOperand(0), N1.getOperand(1), N2); @@ -6682,18 +6808,24 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) { if (Op0.getOpcode() == Op1.getOpcode()) { // Avoid missing important xor optimizations. SDValue Tmp = visitXOR(TheXor); - if (Tmp.getNode() && Tmp.getNode() != TheXor) { - DEBUG(dbgs() << "\nReplacing.8 "; - TheXor->dump(&DAG); - dbgs() << "\nWith: "; - Tmp.getNode()->dump(&DAG); - dbgs() << '\n'); - WorkListRemover DeadNodes(*this); - DAG.ReplaceAllUsesOfValueWith(N1, Tmp); - removeFromWorkList(TheXor); - DAG.DeleteNode(TheXor); - return DAG.getNode(ISD::BRCOND, N->getDebugLoc(), - MVT::Other, Chain, Tmp, N2); + if (Tmp.getNode()) { + if (Tmp.getNode() != TheXor) { + DEBUG(dbgs() << "\nReplacing.8 "; + TheXor->dump(&DAG); + dbgs() << "\nWith: "; + Tmp.getNode()->dump(&DAG); + dbgs() << '\n'); + WorkListRemover DeadNodes(*this); + DAG.ReplaceAllUsesOfValueWith(N1, Tmp); + removeFromWorkList(TheXor); + DAG.DeleteNode(TheXor); + return DAG.getNode(ISD::BRCOND, N->getDebugLoc(), + MVT::Other, Chain, Tmp, N2); + } + + // visitXOR has changed XOR's operands or replaced the XOR completely, + // bail out. + return SDValue(N, 0); } } @@ -6772,7 +6904,7 @@ static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, } else return false; - AddrMode AM; + TargetLowering::AddrMode AM; if (N->getOpcode() == ISD::ADD) { ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); if (Offset) @@ -6841,6 +6973,16 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) { ISD::MemIndexedMode AM = ISD::UNINDEXED; if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG)) return false; + + // Backends without true r+i pre-indexed forms may need to pass a + // constant base with a variable offset so that constant coercion + // will work with the patterns in canonical form. + bool Swapped = false; + if (isa<ConstantSDNode>(BasePtr)) { + std::swap(BasePtr, Offset); + Swapped = true; + } + // Don't create a indexed load / store with zero offset. if (isa<ConstantSDNode>(Offset) && cast<ConstantSDNode>(Offset)->isNullValue()) @@ -6866,6 +7008,48 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) { return false; } + // If the offset is a constant, there may be other adds of constants that + // can be folded with this one. We should do this to avoid having to keep + // a copy of the original base pointer. + SmallVector<SDNode *, 16> OtherUses; + if (isa<ConstantSDNode>(Offset)) + for (SDNode::use_iterator I = BasePtr.getNode()->use_begin(), + E = BasePtr.getNode()->use_end(); I != E; ++I) { + SDNode *Use = *I; + if (Use == Ptr.getNode()) + continue; + + if (Use->isPredecessorOf(N)) + continue; + + if (Use->getOpcode() != ISD::ADD && Use->getOpcode() != ISD::SUB) { + OtherUses.clear(); + break; + } + + SDValue Op0 = Use->getOperand(0), Op1 = Use->getOperand(1); + if (Op1.getNode() == BasePtr.getNode()) + std::swap(Op0, Op1); + assert(Op0.getNode() == BasePtr.getNode() && + "Use of ADD/SUB but not an operand"); + + if (!isa<ConstantSDNode>(Op1)) { + OtherUses.clear(); + break; + } + + // FIXME: In some cases, we can be smarter about this. + if (Op1.getValueType() != Offset.getValueType()) { + OtherUses.clear(); + break; + } + + OtherUses.push_back(Use); + } + + if (Swapped) + std::swap(BasePtr, Offset); + // Now check for #3 and #4. bool RealUse = false; @@ -6915,6 +7099,43 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) { // Finally, since the node is now dead, remove it from the graph. DAG.DeleteNode(N); + if (Swapped) + std::swap(BasePtr, Offset); + + // Replace other uses of BasePtr that can be updated to use Ptr + for (unsigned i = 0, e = OtherUses.size(); i != e; ++i) { + unsigned OffsetIdx = 1; + if (OtherUses[i]->getOperand(OffsetIdx).getNode() == BasePtr.getNode()) + OffsetIdx = 0; + assert(OtherUses[i]->getOperand(!OffsetIdx).getNode() == + BasePtr.getNode() && "Expected BasePtr operand"); + + APInt OV = + cast<ConstantSDNode>(Offset)->getAPIntValue(); + if (AM == ISD::PRE_DEC) + OV = -OV; + + ConstantSDNode *CN = + cast<ConstantSDNode>(OtherUses[i]->getOperand(OffsetIdx)); + APInt CNV = CN->getAPIntValue(); + if (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 1) + CNV += OV; + else + CNV -= OV; + + SDValue NewOp1 = Result.getValue(isLoad ? 1 : 0); + SDValue NewOp2 = DAG.getConstant(CNV, CN->getValueType(0)); + if (OffsetIdx == 0) + std::swap(NewOp1, NewOp2); + + SDValue NewUse = DAG.getNode(OtherUses[i]->getOpcode(), + OtherUses[i]->getDebugLoc(), + OtherUses[i]->getValueType(0), NewOp1, NewOp2); + DAG.ReplaceAllUsesOfValueWith(SDValue(OtherUses[i], 0), NewUse); + removeFromWorkList(OtherUses[i]); + DAG.DeleteNode(OtherUses[i]); + } + // Replace the uses of Ptr with uses of the updated base value. DAG.ReplaceAllUsesOfValueWith(Ptr, Result.getValue(isLoad ? 1 : 0)); removeFromWorkList(Ptr.getNode()); @@ -7123,12 +7344,15 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) { // Try to infer better alignment information than the load already has. if (OptLevel != CodeGenOpt::None && LD->isUnindexed()) { if (unsigned Align = DAG.InferPtrAlignment(Ptr)) { - if (Align > LD->getAlignment()) - return DAG.getExtLoad(LD->getExtensionType(), N->getDebugLoc(), + if (Align > LD->getMemOperand()->getBaseAlignment()) { + SDValue NewLoad = + DAG.getExtLoad(LD->getExtensionType(), N->getDebugLoc(), LD->getValueType(0), Chain, Ptr, LD->getPointerInfo(), LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), Align); + return CombineTo(N, NewLoad, SDValue(NewLoad.getNode(), 1), true); + } } } @@ -7386,7 +7610,8 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) { // start at the previous one. if (ShAmt % NewBW) ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW; - APInt Mask = APInt::getBitsSet(BitWidth, ShAmt, ShAmt + NewBW); + APInt Mask = APInt::getBitsSet(BitWidth, ShAmt, + std::min(BitWidth, ShAmt + NewBW)); if ((Imm & Mask) == Imm) { APInt NewImm = (Imm & Mask).lshr(ShAmt).trunc(NewBW); if (Opc == ISD::AND) @@ -7486,16 +7711,82 @@ SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) { return SDValue(); } -/// Returns the base pointer and an integer offset from that object. -static std::pair<SDValue, int64_t> GetPointerBaseAndOffset(SDValue Ptr) { - if (Ptr->getOpcode() == ISD::ADD && isa<ConstantSDNode>(Ptr->getOperand(1))) { - int64_t Offset = cast<ConstantSDNode>(Ptr->getOperand(1))->getSExtValue(); - SDValue Base = Ptr->getOperand(0); - return std::make_pair(Base, Offset); +/// Helper struct to parse and store a memory address as base + index + offset. +/// We ignore sign extensions when it is safe to do so. +/// The following two expressions are not equivalent. To differentiate we need +/// to store whether there was a sign extension involved in the index +/// computation. +/// (load (i64 add (i64 copyfromreg %c) +/// (i64 signextend (add (i8 load %index) +/// (i8 1)))) +/// vs +/// +/// (load (i64 add (i64 copyfromreg %c) +/// (i64 signextend (i32 add (i32 signextend (i8 load %index)) +/// (i32 1))))) +struct BaseIndexOffset { + SDValue Base; + SDValue Index; + int64_t Offset; + bool IsIndexSignExt; + + BaseIndexOffset() : Offset(0), IsIndexSignExt(false) {} + + BaseIndexOffset(SDValue Base, SDValue Index, int64_t Offset, + bool IsIndexSignExt) : + Base(Base), Index(Index), Offset(Offset), IsIndexSignExt(IsIndexSignExt) {} + + bool equalBaseIndex(const BaseIndexOffset &Other) { + return Other.Base == Base && Other.Index == Index && + Other.IsIndexSignExt == IsIndexSignExt; } - return std::make_pair(Ptr, 0); -} + /// Parses tree in Ptr for base, index, offset addresses. + static BaseIndexOffset match(SDValue Ptr) { + bool IsIndexSignExt = false; + + // Just Base or possibly anything else. + if (Ptr->getOpcode() != ISD::ADD) + return BaseIndexOffset(Ptr, SDValue(), 0, IsIndexSignExt); + + // Base + offset. + if (isa<ConstantSDNode>(Ptr->getOperand(1))) { + int64_t Offset = cast<ConstantSDNode>(Ptr->getOperand(1))->getSExtValue(); + return BaseIndexOffset(Ptr->getOperand(0), SDValue(), Offset, + IsIndexSignExt); + } + + // Look at Base + Index + Offset cases. + SDValue Base = Ptr->getOperand(0); + SDValue IndexOffset = Ptr->getOperand(1); + + // Skip signextends. + if (IndexOffset->getOpcode() == ISD::SIGN_EXTEND) { + IndexOffset = IndexOffset->getOperand(0); + IsIndexSignExt = true; + } + + // Either the case of Base + Index (no offset) or something else. + if (IndexOffset->getOpcode() != ISD::ADD) + return BaseIndexOffset(Base, IndexOffset, 0, IsIndexSignExt); + + // Now we have the case of Base + Index + offset. + SDValue Index = IndexOffset->getOperand(0); + SDValue Offset = IndexOffset->getOperand(1); + + if (!isa<ConstantSDNode>(Offset)) + return BaseIndexOffset(Ptr, SDValue(), 0, IsIndexSignExt); + + // Ignore signextends. + if (Index->getOpcode() == ISD::SIGN_EXTEND) { + Index = Index->getOperand(0); + IsIndexSignExt = true; + } else IsIndexSignExt = false; + + int64_t Off = cast<ConstantSDNode>(Offset)->getSExtValue(); + return BaseIndexOffset(Base, Index, Off, IsIndexSignExt); + } +}; /// Holds a pointer to an LSBaseSDNode as well as information on where it /// is located in a sequence of memory operations connected by a chain. @@ -7522,6 +7813,8 @@ struct ConsecutiveMemoryChainSorter { bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { EVT MemVT = St->getMemoryVT(); int64_t ElementSizeBytes = MemVT.getSizeInBits()/8; + bool NoVectors = DAG.getMachineFunction().getFunction()->getAttributes(). + hasAttribute(AttributeSet::FunctionIndex, Attribute::NoImplicitFloat); // Don't merge vectors into wider inputs. if (MemVT.isVector() || !MemVT.isSimple()) @@ -7540,19 +7833,26 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { if (Chain->hasOneUse() && Chain->use_begin()->getOpcode() == ISD::STORE) return false; - // This holds the base pointer and the offset in bytes from the base pointer. - std::pair<SDValue, int64_t> BasePtr = - GetPointerBaseAndOffset(St->getBasePtr()); + // This holds the base pointer, index, and the offset in bytes from the base + // pointer. + BaseIndexOffset BasePtr = BaseIndexOffset::match(St->getBasePtr()); // We must have a base and an offset. - if (!BasePtr.first.getNode()) + if (!BasePtr.Base.getNode()) return false; // Do not handle stores to undef base pointers. - if (BasePtr.first.getOpcode() == ISD::UNDEF) + if (BasePtr.Base.getOpcode() == ISD::UNDEF) return false; + // Save the LoadSDNodes that we find in the chain. + // We need to make sure that these nodes do not interfere with + // any of the store nodes. + SmallVector<LSBaseSDNode*, 8> AliasLoadNodes; + + // Save the StoreSDNodes that we find in the chain. SmallVector<MemOpLink, 8> StoreNodes; + // Walk up the chain and look for nodes with offsets from the same // base pointer. Stop when reaching an instruction with a different kind // or instruction which has a different base pointer. @@ -7564,11 +7864,10 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { break; // Find the base pointer and offset for this memory node. - std::pair<SDValue, int64_t> Ptr = - GetPointerBaseAndOffset(Index->getBasePtr()); + BaseIndexOffset Ptr = BaseIndexOffset::match(Index->getBasePtr()); // Check that the base pointer is the same as the original one. - if (Ptr.first.getNode() != BasePtr.first.getNode()) + if (!Ptr.equalBaseIndex(BasePtr)) break; // Check that the alignment is the same. @@ -7594,10 +7893,28 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { break; // We found a potential memory operand to merge. - StoreNodes.push_back(MemOpLink(Index, Ptr.second, Seq++)); - - // Move up the chain to the next memory operation. - Index = dyn_cast<StoreSDNode>(Index->getChain().getNode()); + StoreNodes.push_back(MemOpLink(Index, Ptr.Offset, Seq++)); + + // Find the next memory operand in the chain. If the next operand in the + // chain is a store then move up and continue the scan with the next + // memory operand. If the next operand is a load save it and use alias + // information to check if it interferes with anything. + SDNode *NextInChain = Index->getChain().getNode(); + while (1) { + if (StoreSDNode *STn = dyn_cast<StoreSDNode>(NextInChain)) { + // We found a store node. Use it for the next iteration. + Index = STn; + break; + } else if (LoadSDNode *Ldn = dyn_cast<LoadSDNode>(NextInChain)) { + // Save the load node for later. Continue the scan. + AliasLoadNodes.push_back(Ldn); + NextInChain = Ldn->getChain().getNode(); + continue; + } else { + Index = NULL; + break; + } + } } // Check if there is anything to merge. @@ -7612,9 +7929,25 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { // store memory address. unsigned LastConsecutiveStore = 0; int64_t StartAddress = StoreNodes[0].OffsetFromBase; - for (unsigned i=1; i<StoreNodes.size(); ++i) { - int64_t CurrAddress = StoreNodes[i].OffsetFromBase; - if (CurrAddress - StartAddress != (ElementSizeBytes * i)) + for (unsigned i = 0, e = StoreNodes.size(); i < e; ++i) { + + // Check that the addresses are consecutive starting from the second + // element in the list of stores. + if (i > 0) { + int64_t CurrAddress = StoreNodes[i].OffsetFromBase; + if (CurrAddress - StartAddress != (ElementSizeBytes * i)) + break; + } + + bool Alias = false; + // Check if this store interferes with any of the loads that we found. + for (unsigned ld = 0, lde = AliasLoadNodes.size(); ld < lde; ++ld) + if (isAlias(AliasLoadNodes[ld], StoreNodes[i].MemNode)) { + Alias = true; + break; + } + // We found a load that alias with this store. Stop the sequence. + if (Alias) break; // Mark this node as useful. @@ -7647,6 +7980,14 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { EVT StoreTy = EVT::getIntegerVT(*DAG.getContext(), StoreBW); if (TLI.isTypeLegal(StoreTy)) LastLegalType = i+1; + // Or check whether a truncstore is legal. + else if (TLI.getTypeAction(*DAG.getContext(), StoreTy) == + TargetLowering::TypePromoteInteger) { + EVT LegalizedStoredValueTy = + TLI.getTypeToTransformTo(*DAG.getContext(), StoredVal.getValueType()); + if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy)) + LastLegalType = i+1; + } // Find a legal type for the vector store. EVT Ty = EVT::getVectorVT(*DAG.getContext(), MemVT, i+1); @@ -7654,15 +7995,16 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { LastLegalVectorType = i + 1; } - // We only use vectors if the constant is known to be zero. - if (NonZero) + // We only use vectors if the constant is known to be zero and the + // function is not marked with the noimplicitfloat attribute. + if (NonZero || NoVectors) LastLegalVectorType = 0; // Check if we found a legal integer type to store. if (LastLegalType == 0 && LastLegalVectorType == 0) return false; - bool UseVector = LastLegalVectorType > LastLegalType; + bool UseVector = (LastLegalVectorType > LastLegalType) && !NoVectors; unsigned NumElem = UseVector ? LastLegalVectorType : LastLegalType; // Make sure we have something to merge. @@ -7756,7 +8098,7 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { // Find acceptable loads. Loads need to have the same chain (token factor), // must not be zext, volatile, indexed, and they must be consecutive. - SDValue LdBasePtr; + BaseIndexOffset LdBasePtr; for (unsigned i=0; i<LastConsecutiveStore+1; ++i) { StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode); LoadSDNode *Ld = dyn_cast<LoadSDNode>(St->getValue()); @@ -7782,21 +8124,19 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { if (Ld->getMemoryVT() != MemVT) break; - std::pair<SDValue, int64_t> LdPtr = - GetPointerBaseAndOffset(Ld->getBasePtr()); - + BaseIndexOffset LdPtr = BaseIndexOffset::match(Ld->getBasePtr()); // If this is not the first ptr that we check. - if (LdBasePtr.getNode()) { + if (LdBasePtr.Base.getNode()) { // The base ptr must be the same. - if (LdPtr.first != LdBasePtr) + if (!LdPtr.equalBaseIndex(LdBasePtr)) break; } else { // Check that all other base pointers are the same as this one. - LdBasePtr = LdPtr.first; + LdBasePtr = LdPtr; } // We found a potential memory operand to merge. - LoadNodes.push_back(MemOpLink(Ld, LdPtr.second, 0)); + LoadNodes.push_back(MemOpLink(Ld, LdPtr.Offset, 0)); } if (LoadNodes.size() < 2) @@ -7815,7 +8155,7 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { // All loads much share the same chain. if (LoadNodes[i].MemNode->getChain() != FirstChain) break; - + int64_t CurrAddress = LoadNodes[i].OffsetFromBase; if (CurrAddress - StartAddress != (ElementSizeBytes * i)) break; @@ -7831,11 +8171,22 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { StoreTy = EVT::getIntegerVT(*DAG.getContext(), StoreBW); if (TLI.isTypeLegal(StoreTy)) LastLegalIntegerType = i + 1; + // Or check whether a truncstore and extload is legal. + else if (TLI.getTypeAction(*DAG.getContext(), StoreTy) == + TargetLowering::TypePromoteInteger) { + EVT LegalizedStoredValueTy = + TLI.getTypeToTransformTo(*DAG.getContext(), StoreTy); + if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy) && + TLI.isLoadExtLegal(ISD::ZEXTLOAD, StoreTy) && + TLI.isLoadExtLegal(ISD::SEXTLOAD, StoreTy) && + TLI.isLoadExtLegal(ISD::EXTLOAD, StoreTy)) + LastLegalIntegerType = i+1; + } } // Only use vector types if the vector type is larger than the integer type. // If they are the same, use integers. - bool UseVectorTy = LastLegalVectorType > LastLegalIntegerType; + bool UseVectorTy = LastLegalVectorType > LastLegalIntegerType && !NoVectors; unsigned LastLegalType = std::max(LastLegalVectorType, LastLegalIntegerType); // We add +1 here because the LastXXX variables refer to location while @@ -8116,8 +8467,21 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { // Only perform this optimization before the types are legal, because we // don't want to perform this optimization on every DAGCombine invocation. - if (!LegalTypes && MergeConsecutiveStores(ST)) - return SDValue(N, 0); + if (!LegalTypes) { + bool EverChanged = false; + + do { + // There can be multiple store sequences on the same chain. + // Keep trying to merge store sequences until we are unable to do so + // or until we merge the last store on the chain. + bool Changed = MergeConsecutiveStores(ST); + EverChanged |= Changed; + if (!Changed) break; + } while (ST->getOpcode() != ISD::DELETED_NODE); + + if (EverChanged) + return SDValue(N, 0); + } return ReduceLoadOpStoreWidth(N); } @@ -8514,11 +8878,8 @@ SDValue DAGCombiner::reduceBuildVecConvertToConvertBuildVec(SDNode *N) { if (Opcode == ISD::DELETED_NODE && (Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP)) { Opcode = Opc; - // If not supported by target, bail out. - if (TLI.getOperationAction(Opcode, VT) != TargetLowering::Legal && - TLI.getOperationAction(Opcode, VT) != TargetLowering::Custom) - return SDValue(); } + if (Opc != Opcode) return SDValue(); @@ -8543,6 +8904,10 @@ SDValue DAGCombiner::reduceBuildVecConvertToConvertBuildVec(SDNode *N) { assert(SrcVT != MVT::Other && "Cannot determine source type!"); EVT NVT = EVT::getVectorVT(*DAG.getContext(), SrcVT, NumInScalars); + + if (!TLI.isOperationLegalOrCustom(Opcode, NVT)) + return SDValue(); + SmallVector<SDValue, 8> Opnds; for (unsigned i = 0; i != NumInScalars; ++i) { SDValue In = N->getOperand(i); @@ -8707,12 +9072,32 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { EVT NVT = N->getValueType(0); SDValue V = N->getOperand(0); + if (V->getOpcode() == ISD::CONCAT_VECTORS) { + // Combine: + // (extract_subvec (concat V1, V2, ...), i) + // Into: + // Vi if possible + // Only operand 0 is checked as 'concat' assumes all inputs of the same type. + if (V->getOperand(0).getValueType() != NVT) + return SDValue(); + unsigned Idx = dyn_cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); + unsigned NumElems = NVT.getVectorNumElements(); + assert((Idx % NumElems) == 0 && + "IDX in concat is not a multiple of the result vector length."); + return V->getOperand(Idx / NumElems); + } + + // Skip bitcasting + if (V->getOpcode() == ISD::BITCAST) + V = V.getOperand(0); + if (V->getOpcode() == ISD::INSERT_SUBVECTOR) { + DebugLoc dl = N->getDebugLoc(); // Handle only simple case where vector being inserted and vector // being extracted are of same type, and are half size of larger vectors. EVT BigVT = V->getOperand(0).getValueType(); EVT SmallVT = V->getOperand(1).getValueType(); - if (NVT != SmallVT || NVT.getSizeInBits()*2 != BigVT.getSizeInBits()) + if (!NVT.bitsEq(SmallVT) || NVT.getSizeInBits()*2 != BigVT.getSizeInBits()) return SDValue(); // Only handle cases where both indexes are constants with the same type. @@ -8725,30 +9110,18 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { // Combine: // (extract_subvec (insert_subvec V1, V2, InsIdx), ExtIdx) // Into: - // indices are equal => V1 + // indices are equal or bit offsets are equal => V1 // otherwise => (extract_subvec V1, ExtIdx) - if (InsIdx->getZExtValue() == ExtIdx->getZExtValue()) - return V->getOperand(1); - return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), NVT, - V->getOperand(0), N->getOperand(1)); + if (InsIdx->getZExtValue() * SmallVT.getScalarType().getSizeInBits() == + ExtIdx->getZExtValue() * NVT.getScalarType().getSizeInBits()) + return DAG.getNode(ISD::BITCAST, dl, NVT, V->getOperand(1)); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, + DAG.getNode(ISD::BITCAST, dl, + N->getOperand(0).getValueType(), + V->getOperand(0)), N->getOperand(1)); } } - if (V->getOpcode() == ISD::CONCAT_VECTORS) { - // Combine: - // (extract_subvec (concat V1, V2, ...), i) - // Into: - // Vi if possible - // Only operand 0 is checked as 'concat' assumes all inputs of the same type. - if (V->getOperand(0).getValueType() != NVT) - return SDValue(); - unsigned Idx = dyn_cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); - unsigned NumElems = NVT.getVectorNumElements(); - assert((Idx % NumElems) == 0 && - "IDX in concat is not a multiple of the result vector length."); - return V->getOperand(Idx / NumElems); - } - return SDValue(); } @@ -8992,11 +9365,6 @@ SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) { /// SimplifyVBinOp - Visit a binary vector operation, like ADD. SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) { - // After legalize, the target may be depending on adds and other - // binary ops to provide legal ways to construct constants or other - // things. Simplifying them may result in a loss of legality. - if (LegalOperations) return SDValue(); - assert(N->getValueType(0).isVector() && "SimplifyVBinOp only works on vectors!"); @@ -9066,11 +9434,6 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) { /// SimplifyVUnaryOp - Visit a binary vector operation, like FABS/FNEG. SDValue DAGCombiner::SimplifyVUnaryOp(SDNode *N) { - // After legalize, the target may be depending on adds and other - // binary ops to provide legal ways to construct constants or other - // things. Simplifying them may result in a loss of legality. - if (LegalOperations) return SDValue(); - assert(N->getValueType(0).isVector() && "SimplifyVUnaryOp only works on vectors!"); @@ -9173,7 +9536,9 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS, // src value info, don't do the transformation if the memory // locations are not in the default address space. LLD->getPointerInfo().getAddrSpace() != 0 || - RLD->getPointerInfo().getAddrSpace() != 0) + RLD->getPointerInfo().getAddrSpace() != 0 || + !TLI.isOperationLegalOrCustom(TheSelect->getOpcode(), + LLD->getBasePtr().getValueType())) return false; // Check that the select condition doesn't reach either load. If so, @@ -9537,7 +9902,7 @@ SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, DebugLoc DL, bool foldBooleans) { TargetLowering::DAGCombinerInfo - DagCombineInfo(DAG, !LegalTypes, !LegalOperations, false, this); + DagCombineInfo(DAG, Level, false, this); return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL); } @@ -9680,6 +10045,23 @@ bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1, return true; } +bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) { + SDValue Ptr0, Ptr1; + int64_t Size0, Size1; + const Value *SrcValue0, *SrcValue1; + int SrcValueOffset0, SrcValueOffset1; + unsigned SrcValueAlign0, SrcValueAlign1; + const MDNode *SrcTBAAInfo0, *SrcTBAAInfo1; + FindAliasInfo(Op0, Ptr0, Size0, SrcValue0, SrcValueOffset0, + SrcValueAlign0, SrcTBAAInfo0); + FindAliasInfo(Op1, Ptr1, Size1, SrcValue1, SrcValueOffset1, + SrcValueAlign1, SrcTBAAInfo1); + return isAlias(Ptr0, Size0, SrcValue0, SrcValueOffset0, + SrcValueAlign0, SrcTBAAInfo0, + Ptr1, Size1, SrcValue1, SrcValueOffset1, + SrcValueAlign1, SrcTBAAInfo1); +} + /// FindAliasInfo - Extracts the relevant alias information from the memory /// node. Returns true if the operand was a load. bool DAGCombiner::FindAliasInfo(SDNode *N, diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp index 4854cf7..9ac738e 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -40,27 +40,27 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "isel" -#include "llvm/DebugInfo.h" -#include "llvm/Function.h" -#include "llvm/GlobalVariable.h" -#include "llvm/Instructions.h" -#include "llvm/IntrinsicInst.h" -#include "llvm/Operator.h" -#include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/FastISel.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/Loads.h" +#include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Analysis/Loads.h" -#include "llvm/DataLayout.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Operator.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLibraryInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Debug.h" -#include "llvm/ADT/Statistic.h" using namespace llvm; STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by " @@ -87,6 +87,27 @@ void FastISel::startNewBlock() { LastLocalValue = EmitStartPt; } +bool FastISel::LowerArguments() { + if (!FuncInfo.CanLowerReturn) + // Fallback to SDISel argument lowering code to deal with sret pointer + // parameter. + return false; + + if (!FastLowerArguments()) + return false; + + // Enter non-dead arguments into ValueMap for uses in non-entry BBs. + for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(), + E = FuncInfo.Fn->arg_end(); I != E; ++I) { + if (!I->use_empty()) { + DenseMap<const Value *, unsigned>::iterator VI = LocalValueMap.find(I); + assert(VI != LocalValueMap.end() && "Missed an argument?"); + FuncInfo.ValueMap[I] = VI->second; + } + } + return true; +} + void FastISel::flushLocalValueMap() { LocalValueMap.clear(); LastLocalValue = EmitStartPt; @@ -675,6 +696,13 @@ bool FastISel::SelectCall(const User *I) { UpdateValueMap(Call, ResultReg); return true; } + case Intrinsic::expect: { + unsigned ResultReg = getRegForValue(Call->getArgOperand(0)); + if (ResultReg == 0) + return false; + UpdateValueMap(Call, ResultReg); + return true; + } } // Usually, it does not make sense to initialize a value, @@ -684,7 +712,7 @@ bool FastISel::SelectCall(const User *I) { // all the values which have already been materialized, // appear after the call. It also makes sense to skip intrinsics // since they tend to be inlined. - if (!isa<IntrinsicInst>(F)) + if (!isa<IntrinsicInst>(Call)) flushLocalValueMap(); // An arbitrary call. Bail. @@ -737,15 +765,15 @@ bool FastISel::SelectBitCast(const User *I) { } // Bitcasts of other values become reg-reg copies or BITCAST operators. - EVT SrcVT = TLI.getValueType(I->getOperand(0)->getType()); - EVT DstVT = TLI.getValueType(I->getType()); - - if (SrcVT == MVT::Other || !SrcVT.isSimple() || - DstVT == MVT::Other || !DstVT.isSimple() || - !TLI.isTypeLegal(SrcVT) || !TLI.isTypeLegal(DstVT)) + EVT SrcEVT = TLI.getValueType(I->getOperand(0)->getType()); + EVT DstEVT = TLI.getValueType(I->getType()); + if (SrcEVT == MVT::Other || DstEVT == MVT::Other || + !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT)) // Unhandled type. Halt "fast" selection and bail. return false; + MVT SrcVT = SrcEVT.getSimpleVT(); + MVT DstVT = DstEVT.getSimpleVT(); unsigned Op0 = getRegForValue(I->getOperand(0)); if (Op0 == 0) // Unhandled operand. Halt "fast" selection and bail. @@ -755,7 +783,7 @@ bool FastISel::SelectBitCast(const User *I) { // First, try to perform the bitcast by inserting a reg-reg copy. unsigned ResultReg = 0; - if (SrcVT.getSimpleVT() == DstVT.getSimpleVT()) { + if (SrcVT == DstVT) { const TargetRegisterClass* SrcClass = TLI.getRegClassFor(SrcVT); const TargetRegisterClass* DstClass = TLI.getRegClassFor(DstVT); // Don't attempt a cross-class copy. It will likely fail. @@ -768,8 +796,7 @@ bool FastISel::SelectBitCast(const User *I) { // If the reg-reg copy failed, select a BITCAST opcode. if (!ResultReg) - ResultReg = FastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), - ISD::BITCAST, Op0, Op0IsKill); + ResultReg = FastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill); if (!ResultReg) return false; @@ -837,7 +864,8 @@ FastISel::SelectInstruction(const Instruction *I) { void FastISel::FastEmitBranch(MachineBasicBlock *MSucc, DebugLoc DL) { - if (FuncInfo.MBB->getBasicBlock()->size() > 1 && FuncInfo.MBB->isLayoutSuccessor(MSucc)) { + if (FuncInfo.MBB->getBasicBlock()->size() > 1 && + FuncInfo.MBB->isLayoutSuccessor(MSucc)) { // For more accurate line information if this is the only instruction // in the block then emit it, otherwise we have the unconditional // fall-through case, which needs no instructions. @@ -1068,6 +1096,10 @@ FastISel::FastISel(FunctionLoweringInfo &funcInfo, FastISel::~FastISel() {} +bool FastISel::FastLowerArguments() { + return false; +} + unsigned FastISel::FastEmit_(MVT, MVT, unsigned) { return 0; @@ -1151,6 +1183,8 @@ unsigned FastISel::FastEmit_ri_(MVT VT, unsigned Opcode, IntegerType *ITy = IntegerType::get(FuncInfo.Fn->getContext(), VT.getSizeInBits()); MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm)); + assert (MaterialReg != 0 && "Unable to materialize imm."); + if (MaterialReg == 0) return 0; } return FastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp index a418290..b46edad 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -13,29 +13,29 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "function-lowering-info" -#include "llvm/ADT/PostOrderIterator.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" -#include "llvm/DebugInfo.h" -#include "llvm/DerivedTypes.h" -#include "llvm/Function.h" -#include "llvm/Instructions.h" -#include "llvm/IntrinsicInst.h" -#include "llvm/LLVMContext.h" -#include "llvm/Module.h" +#include "llvm/ADT/PostOrderIterator.h" #include "llvm/CodeGen/Analysis.h" -#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetOptions.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <algorithm> using namespace llvm; @@ -66,8 +66,7 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf) { // Check whether the function can return without sret-demotion. SmallVector<ISD::OutputArg, 4> Outs; - GetReturnInfo(Fn->getReturnType(), - Fn->getAttributes().getRetAttributes(), Outs, TLI); + GetReturnInfo(Fn->getReturnType(), Fn->getAttributes(), Outs, TLI); CanLowerReturn = TLI.CanLowerReturn(Fn->getCallingConv(), *MF, Fn->isVarArg(), Outs, Fn->getContext()); @@ -208,7 +207,7 @@ void FunctionLoweringInfo::clear() { } /// CreateReg - Allocate a single virtual register for the given type. -unsigned FunctionLoweringInfo::CreateReg(EVT VT) { +unsigned FunctionLoweringInfo::CreateReg(MVT VT) { return RegInfo->createVirtualRegister(TLI.getRegClassFor(VT)); } @@ -226,7 +225,7 @@ unsigned FunctionLoweringInfo::CreateRegs(Type *Ty) { unsigned FirstReg = 0; for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { EVT ValueVT = ValueVTs[Value]; - EVT RegisterVT = TLI.getRegisterType(Ty->getContext(), ValueVT); + MVT RegisterVT = TLI.getRegisterType(Ty->getContext(), ValueVT); unsigned NumRegs = TLI.getNumRegisters(Ty->getContext(), ValueVT); for (unsigned i = 0; i != NumRegs; ++i) { diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index a8381b2..3b1abd7 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -16,18 +16,18 @@ #define DEBUG_TYPE "instr-emitter" #include "InstrEmitter.h" #include "SDNodeDbgValue.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/ADT/Statistic.h" +#include "llvm/IR/DataLayout.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; /// MinRCSize - Smallest register class we allow when constraining virtual @@ -99,7 +99,7 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned, // the CopyToReg'd destination register instead of creating a new vreg. bool MatchReg = true; const TargetRegisterClass *UseRC = NULL; - EVT VT = Node->getValueType(ResNo); + MVT VT = Node->getSimpleValueType(ResNo); // Stick to the preferred register classes for legal types. if (TLI->isTypeLegal(VT)) @@ -124,7 +124,7 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned, SDValue Op = User->getOperand(i); if (Op.getNode() != Node || Op.getResNo() != ResNo) continue; - EVT VT = Node->getValueType(Op.getResNo()); + MVT VT = Node->getSimpleValueType(Op.getResNo()); if (VT == MVT::Other || VT == MVT::Glue) continue; Match = false; @@ -203,7 +203,8 @@ unsigned InstrEmitter::getDstOfOnlyCopyToRegUse(SDNode *Node, return 0; } -void InstrEmitter::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, +void InstrEmitter::CreateVirtualRegisters(SDNode *Node, + MachineInstrBuilder &MIB, const MCInstrDesc &II, bool IsClone, bool IsCloned, DenseMap<SDValue, unsigned> &VRBaseMap) { @@ -222,7 +223,7 @@ void InstrEmitter::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, unsigned NumResults = CountResults(Node); VRBase = cast<RegisterSDNode>(Node->getOperand(i-NumResults))->getReg(); assert(TargetRegisterInfo::isPhysicalRegister(VRBase)); - MI->addOperand(MachineOperand::CreateReg(VRBase, true)); + MIB.addReg(VRBase, RegState::Define); } if (!VRBase && !IsClone && !IsCloned) @@ -237,7 +238,7 @@ void InstrEmitter::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, const TargetRegisterClass *RegRC = MRI->getRegClass(Reg); if (RegRC == RC) { VRBase = Reg; - MI->addOperand(MachineOperand::CreateReg(Reg, true)); + MIB.addReg(VRBase, RegState::Define); break; } } @@ -249,7 +250,7 @@ void InstrEmitter::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, if (VRBase == 0) { assert(RC && "Isn't a register operand!"); VRBase = MRI->createVirtualRegister(RC); - MI->addOperand(MachineOperand::CreateReg(VRBase, true)); + MIB.addReg(VRBase, RegState::Define); } SDValue Op(Node, i); @@ -272,7 +273,8 @@ unsigned InstrEmitter::getVR(SDValue Op, // IMPLICIT_DEF can produce any type of result so its MCInstrDesc // does not include operand register class info. if (!VReg) { - const TargetRegisterClass *RC = TLI->getRegClassFor(Op.getValueType()); + const TargetRegisterClass *RC = + TLI->getRegClassFor(Op.getSimpleValueType()); VReg = MRI->createVirtualRegister(RC); } BuildMI(*MBB, InsertPos, Op.getDebugLoc(), @@ -290,7 +292,8 @@ unsigned InstrEmitter::getVR(SDValue Op, /// specified machine instr. Insert register copies if the register is /// not in the required register class. void -InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, +InstrEmitter::AddRegisterOperand(MachineInstrBuilder &MIB, + SDValue Op, unsigned IIOpNum, const MCInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, @@ -302,7 +305,7 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, unsigned VReg = getVR(Op, VRBaseMap); assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?"); - const MCInstrDesc &MCID = MI->getDesc(); + const MCInstrDesc &MCID = MIB->getDesc(); bool isOptDef = IIOpNum < MCID.getNumOperands() && MCID.OpInfo[IIOpNum].isOptionalDef(); @@ -334,56 +337,53 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, !IsDebug && !(IsClone || IsCloned); if (isKill) { - unsigned Idx = MI->getNumOperands(); + unsigned Idx = MIB->getNumOperands(); while (Idx > 0 && - MI->getOperand(Idx-1).isReg() && MI->getOperand(Idx-1).isImplicit()) + MIB->getOperand(Idx-1).isReg() && + MIB->getOperand(Idx-1).isImplicit()) --Idx; - bool isTied = MI->getDesc().getOperandConstraint(Idx, MCOI::TIED_TO) != -1; + bool isTied = MCID.getOperandConstraint(Idx, MCOI::TIED_TO) != -1; if (isTied) isKill = false; } - MI->addOperand(MachineOperand::CreateReg(VReg, isOptDef, - false/*isImp*/, isKill, - false/*isDead*/, false/*isUndef*/, - false/*isEarlyClobber*/, - 0/*SubReg*/, IsDebug)); + MIB.addReg(VReg, getDefRegState(isOptDef) | getKillRegState(isKill) | + getDebugRegState(IsDebug)); } /// AddOperand - Add the specified operand to the specified machine instr. II /// specifies the instruction information for the node, and IIOpNum is the /// operand number (in the II) that we are adding. -void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op, +void InstrEmitter::AddOperand(MachineInstrBuilder &MIB, + SDValue Op, unsigned IIOpNum, const MCInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, bool IsDebug, bool IsClone, bool IsCloned) { if (Op.isMachineOpcode()) { - AddRegisterOperand(MI, Op, IIOpNum, II, VRBaseMap, + AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap, IsDebug, IsClone, IsCloned); } else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { - MI->addOperand(MachineOperand::CreateImm(C->getSExtValue())); + MIB.addImm(C->getSExtValue()); } else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Op)) { - const ConstantFP *CFP = F->getConstantFPValue(); - MI->addOperand(MachineOperand::CreateFPImm(CFP)); + MIB.addFPImm(F->getConstantFPValue()); } else if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Op)) { // Turn additional physreg operands into implicit uses on non-variadic // instructions. This is used by call and return instructions passing // arguments in registers. bool Imp = II && (IIOpNum >= II->getNumOperands() && !II->isVariadic()); - MI->addOperand(MachineOperand::CreateReg(R->getReg(), false, Imp)); + MIB.addReg(R->getReg(), getImplRegState(Imp)); } else if (RegisterMaskSDNode *RM = dyn_cast<RegisterMaskSDNode>(Op)) { - MI->addOperand(MachineOperand::CreateRegMask(RM->getRegMask())); + MIB.addRegMask(RM->getRegMask()); } else if (GlobalAddressSDNode *TGA = dyn_cast<GlobalAddressSDNode>(Op)) { - MI->addOperand(MachineOperand::CreateGA(TGA->getGlobal(), TGA->getOffset(), - TGA->getTargetFlags())); + MIB.addGlobalAddress(TGA->getGlobal(), TGA->getOffset(), + TGA->getTargetFlags()); } else if (BasicBlockSDNode *BBNode = dyn_cast<BasicBlockSDNode>(Op)) { - MI->addOperand(MachineOperand::CreateMBB(BBNode->getBasicBlock())); + MIB.addMBB(BBNode->getBasicBlock()); } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) { - MI->addOperand(MachineOperand::CreateFI(FI->getIndex())); + MIB.addFrameIndex(FI->getIndex()); } else if (JumpTableSDNode *JT = dyn_cast<JumpTableSDNode>(Op)) { - MI->addOperand(MachineOperand::CreateJTI(JT->getIndex(), - JT->getTargetFlags())); + MIB.addJumpTableIndex(JT->getIndex(), JT->getTargetFlags()); } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op)) { int Offset = CP->getOffset(); unsigned Align = CP->getAlignment(); @@ -403,30 +403,26 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op, Idx = MCP->getConstantPoolIndex(CP->getMachineCPVal(), Align); else Idx = MCP->getConstantPoolIndex(CP->getConstVal(), Align); - MI->addOperand(MachineOperand::CreateCPI(Idx, Offset, - CP->getTargetFlags())); + MIB.addConstantPoolIndex(Idx, Offset, CP->getTargetFlags()); } else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op)) { - MI->addOperand(MachineOperand::CreateES(ES->getSymbol(), - ES->getTargetFlags())); + MIB.addExternalSymbol(ES->getSymbol(), ES->getTargetFlags()); } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Op)) { - MI->addOperand(MachineOperand::CreateBA(BA->getBlockAddress(), - BA->getOffset(), - BA->getTargetFlags())); + MIB.addBlockAddress(BA->getBlockAddress(), + BA->getOffset(), + BA->getTargetFlags()); } else if (TargetIndexSDNode *TI = dyn_cast<TargetIndexSDNode>(Op)) { - MI->addOperand(MachineOperand::CreateTargetIndex(TI->getIndex(), - TI->getOffset(), - TI->getTargetFlags())); + MIB.addTargetIndex(TI->getIndex(), TI->getOffset(), TI->getTargetFlags()); } else { assert(Op.getValueType() != MVT::Other && Op.getValueType() != MVT::Glue && "Chain and glue operands should occur at end of operand list!"); - AddRegisterOperand(MI, Op, IIOpNum, II, VRBaseMap, + AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap, IsDebug, IsClone, IsCloned); } } unsigned InstrEmitter::ConstrainForSubReg(unsigned VReg, unsigned SubIdx, - EVT VT, DebugLoc DL) { + MVT VT, DebugLoc DL) { const TargetRegisterClass *VRC = MRI->getRegClass(VReg); const TargetRegisterClass *RC = TRI->getSubClassWithSubReg(VRC, SubIdx); @@ -477,7 +473,8 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, // constraints on the %dst register, COPY can target all legal register // classes. unsigned SubIdx = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); - const TargetRegisterClass *TRC = TLI->getRegClassFor(Node->getValueType(0)); + const TargetRegisterClass *TRC = + TLI->getRegClassFor(Node->getSimpleValueType(0)); unsigned VReg = getVR(Node->getOperand(0), VRBaseMap); MachineInstr *DefMI = MRI->getVRegDef(VReg); @@ -500,7 +497,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, // constrain its register class or issue a COPY to a compatible register // class. VReg = ConstrainForSubReg(VReg, SubIdx, - Node->getOperand(0).getValueType(), + Node->getOperand(0).getSimpleValueType(), Node->getDebugLoc()); // Create the destreg if it is missing. @@ -532,7 +529,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, // // There is no constraint on the %src register class. // - const TargetRegisterClass *SRC = TLI->getRegClassFor(Node->getValueType(0)); + const TargetRegisterClass *SRC = TLI->getRegClassFor(Node->getSimpleValueType(0)); SRC = TRI->getSubClassWithSubReg(SRC, SubIdx); assert(SRC && "No register class supports VT and SubIdx for INSERT_SUBREG"); @@ -540,22 +537,22 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, VRBase = MRI->createVirtualRegister(SRC); // Create the insert_subreg or subreg_to_reg machine instruction. - MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), TII->get(Opc)); - MI->addOperand(MachineOperand::CreateReg(VRBase, true)); + MachineInstrBuilder MIB = + BuildMI(*MF, Node->getDebugLoc(), TII->get(Opc), VRBase); // If creating a subreg_to_reg, then the first input operand // is an implicit value immediate, otherwise it's a register if (Opc == TargetOpcode::SUBREG_TO_REG) { const ConstantSDNode *SD = cast<ConstantSDNode>(N0); - MI->addOperand(MachineOperand::CreateImm(SD->getZExtValue())); + MIB.addImm(SD->getZExtValue()); } else - AddOperand(MI, N0, 0, 0, VRBaseMap, /*IsDebug=*/false, + AddOperand(MIB, N0, 0, 0, VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned); // Add the subregster being inserted - AddOperand(MI, N1, 0, 0, VRBaseMap, /*IsDebug=*/false, + AddOperand(MIB, N1, 0, 0, VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned); - MI->addOperand(MachineOperand::CreateImm(SubIdx)); - MBB->insert(InsertPos, MI); + MIB.addImm(SubIdx); + MBB->insert(InsertPos, MIB); } else llvm_unreachable("Node is not insert_subreg, extract_subreg, or subreg_to_reg"); @@ -596,12 +593,11 @@ void InstrEmitter::EmitRegSequence(SDNode *Node, unsigned DstRCIdx = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue(); const TargetRegisterClass *RC = TRI->getRegClass(DstRCIdx); unsigned NewVReg = MRI->createVirtualRegister(TRI->getAllocatableClass(RC)); - MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), - TII->get(TargetOpcode::REG_SEQUENCE), NewVReg); + const MCInstrDesc &II = TII->get(TargetOpcode::REG_SEQUENCE); + MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II, NewVReg); unsigned NumOps = Node->getNumOperands(); assert((NumOps & 1) == 1 && "REG_SEQUENCE must have an odd number of operands!"); - const MCInstrDesc &II = TII->get(TargetOpcode::REG_SEQUENCE); for (unsigned i = 1; i != NumOps; ++i) { SDValue Op = Node->getOperand(i); if ((i & 1) == 0) { @@ -620,11 +616,11 @@ void InstrEmitter::EmitRegSequence(SDNode *Node, } } } - AddOperand(MI, Op, i+1, &II, VRBaseMap, /*IsDebug=*/false, + AddOperand(MIB, Op, i+1, &II, VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned); } - MBB->insert(InsertPos, MI); + MBB->insert(InsertPos, MIB); SDValue Op(Node, 0); bool isNew = VRBaseMap.insert(std::make_pair(Op, NewVReg)).second; (void)isNew; // Silence compiler warning. @@ -661,7 +657,7 @@ InstrEmitter::EmitDbgValue(SDDbgValue *SD, if (I==VRBaseMap.end()) MIB.addReg(0U); // undef else - AddOperand(&*MIB, Op, (*MIB).getNumOperands(), &II, VRBaseMap, + AddOperand(MIB, Op, (*MIB).getNumOperands(), &II, VRBaseMap, /*IsDebug=*/true, /*IsClone=*/false, /*IsCloned=*/false); } else if (SD->getKind() == SDDbgValue::CONST) { const Value *V = SD->getConst(); @@ -737,12 +733,12 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, #endif // Create the new machine instruction. - MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), II); + MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II); // Add result register values for things that are defined by this // instruction. if (NumResults) - CreateVirtualRegisters(Node, MI, II, IsClone, IsCloned, VRBaseMap); + CreateVirtualRegisters(Node, MIB, II, IsClone, IsCloned, VRBaseMap); // Emit all of the actual operands of this instruction, adding them to the // instruction as appropriate. @@ -751,17 +747,17 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, "Unable to cope with optional defs and phys regs defs!"); unsigned NumSkip = HasOptPRefs ? II.getNumDefs() - NumResults : 0; for (unsigned i = NumSkip; i != NodeOperands; ++i) - AddOperand(MI, Node->getOperand(i), i-NumSkip+II.getNumDefs(), &II, + AddOperand(MIB, Node->getOperand(i), i-NumSkip+II.getNumDefs(), &II, VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned); // Transfer all of the memory reference descriptions of this instruction. - MI->setMemRefs(cast<MachineSDNode>(Node)->memoperands_begin(), + MIB.setMemRefs(cast<MachineSDNode>(Node)->memoperands_begin(), cast<MachineSDNode>(Node)->memoperands_end()); // Insert the instruction into position in the block. This needs to // happen before any custom inserter hook is called so that the // hook knows where in the block to insert the replacement code. - MBB->insert(InsertPos, MI); + MBB->insert(InsertPos, MIB); // The MachineInstr may also define physregs instead of virtregs. These // physreg values can reach other instructions in different ways: @@ -819,13 +815,13 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, // Finally mark unused registers as dead. if (!UsedRegs.empty() || II.getImplicitDefs()) - MI->setPhysRegsDeadExcept(UsedRegs, *TRI); + MIB->setPhysRegsDeadExcept(UsedRegs, *TRI); // Run post-isel target hook to adjust this instruction if needed. #ifdef NDEBUG if (II.hasPostISelHook()) #endif - TLI->AdjustInstrPostInstrSelection(MI, Node); + TLI->AdjustInstrPostInstrSelection(MIB, Node); } /// EmitSpecialNode - Generate machine code for a target-independent node and @@ -889,20 +885,20 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, --NumOps; // Ignore the glue operand. // Create the inline asm machine instruction. - MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), - TII->get(TargetOpcode::INLINEASM)); + MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), + TII->get(TargetOpcode::INLINEASM)); // Add the asm string as an external symbol operand. SDValue AsmStrV = Node->getOperand(InlineAsm::Op_AsmString); const char *AsmStr = cast<ExternalSymbolSDNode>(AsmStrV)->getSymbol(); - MI->addOperand(MachineOperand::CreateES(AsmStr)); + MIB.addExternalSymbol(AsmStr); // Add the HasSideEffect, isAlignStack, AsmDialect, MayLoad and MayStore // bits. int64_t ExtraInfo = cast<ConstantSDNode>(Node->getOperand(InlineAsm::Op_ExtraInfo))-> getZExtValue(); - MI->addOperand(MachineOperand::CreateImm(ExtraInfo)); + MIB.addImm(ExtraInfo); // Remember to operand index of the group flags. SmallVector<unsigned, 8> GroupIdx; @@ -913,8 +909,8 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue(); const unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); - GroupIdx.push_back(MI->getNumOperands()); - MI->addOperand(MachineOperand::CreateImm(Flags)); + GroupIdx.push_back(MIB->getNumOperands()); + MIB.addImm(Flags); ++i; // Skip the ID value. switch (InlineAsm::getKind(Flags)) { @@ -925,20 +921,16 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, // FIXME: Add dead flags for physical and virtual registers defined. // For now, mark physical register defs as implicit to help fast // regalloc. This makes inline asm look a lot like calls. - MI->addOperand(MachineOperand::CreateReg(Reg, true, - /*isImp=*/ TargetRegisterInfo::isPhysicalRegister(Reg))); + MIB.addReg(Reg, RegState::Define | + getImplRegState(TargetRegisterInfo::isPhysicalRegister(Reg))); } break; case InlineAsm::Kind_RegDefEarlyClobber: case InlineAsm::Kind_Clobber: for (unsigned j = 0; j != NumVals; ++j, ++i) { unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); - MI->addOperand(MachineOperand::CreateReg(Reg, /*isDef=*/ true, - /*isImp=*/ TargetRegisterInfo::isPhysicalRegister(Reg), - /*isKill=*/ false, - /*isDead=*/ false, - /*isUndef=*/false, - /*isEarlyClobber=*/ true)); + MIB.addReg(Reg, RegState::Define | RegState::EarlyClobber | + getImplRegState(TargetRegisterInfo::isPhysicalRegister(Reg))); } break; case InlineAsm::Kind_RegUse: // Use of register. @@ -947,7 +939,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, // The addressing mode has been selected, just add all of the // operands to the machine instruction. for (unsigned j = 0; j != NumVals; ++j, ++i) - AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap, + AddOperand(MIB, Node->getOperand(i), 0, 0, VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned); // Manually set isTied bits. @@ -957,7 +949,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, unsigned DefIdx = GroupIdx[DefGroup] + 1; unsigned UseIdx = GroupIdx.back() + 1; for (unsigned j = 0; j != NumVals; ++j) - MI->tieOperands(DefIdx + j, UseIdx + j); + MIB->tieOperands(DefIdx + j, UseIdx + j); } } break; @@ -968,9 +960,9 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, SDValue MDV = Node->getOperand(InlineAsm::Op_MDNode); const MDNode *MD = cast<MDNodeSDNode>(MDV)->getMD(); if (MD) - MI->addOperand(MachineOperand::CreateMetadata(MD)); + MIB.addMetadata(MD); - MBB->insert(InsertPos, MI); + MBB->insert(InsertPos, MIB); break; } } diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h b/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h index 9eddee9..a9c2203 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h @@ -16,12 +16,13 @@ #ifndef INSTREMITTER_H #define INSTREMITTER_H -#include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/SelectionDAG.h" namespace llvm { +class MachineInstrBuilder; class MCInstrDesc; class SDDbgValue; @@ -48,7 +49,8 @@ class InstrEmitter { unsigned getDstOfOnlyCopyToRegUse(SDNode *Node, unsigned ResNo) const; - void CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, + void CreateVirtualRegisters(SDNode *Node, + MachineInstrBuilder &MIB, const MCInstrDesc &II, bool IsClone, bool IsCloned, DenseMap<SDValue, unsigned> &VRBaseMap); @@ -61,7 +63,8 @@ class InstrEmitter { /// AddRegisterOperand - Add the specified register as an operand to the /// specified machine instr. Insert register copies if the register is /// not in the required register class. - void AddRegisterOperand(MachineInstr *MI, SDValue Op, + void AddRegisterOperand(MachineInstrBuilder &MIB, + SDValue Op, unsigned IIOpNum, const MCInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, @@ -71,7 +74,8 @@ class InstrEmitter { /// specifies the instruction information for the node, and IIOpNum is the /// operand number (in the II) that we are adding. IIOpNum and II are used for /// assertions only. - void AddOperand(MachineInstr *MI, SDValue Op, + void AddOperand(MachineInstrBuilder &MIB, + SDValue Op, unsigned IIOpNum, const MCInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, @@ -81,7 +85,7 @@ class InstrEmitter { /// supports SubIdx sub-registers. Emit a copy if that isn't possible. /// Return the virtual register to use. unsigned ConstrainForSubReg(unsigned VReg, unsigned SubIdx, - EVT VT, DebugLoc DL); + MVT VT, DebugLoc DL); /// EmitSubregNode - Generate machine code for subreg nodes. /// diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index abf40b7..51cc254 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -11,26 +11,27 @@ // //===----------------------------------------------------------------------===// -#include "llvm/CallingConv.h" -#include "llvm/Constants.h" -#include "llvm/DebugInfo.h" -#include "llvm/DerivedTypes.h" -#include "llvm/LLVMContext.h" +#include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Triple.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" -#include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/Target/TargetFrameLowering.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetMachine.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/CallingConv.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/Target/TargetFrameLowering.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; //===----------------------------------------------------------------------===// @@ -101,6 +102,7 @@ private: SDNode *Node, bool isSigned); SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, + RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128); SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8, @@ -109,6 +111,7 @@ private: RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128); void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); + void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, DebugLoc dl); SDValue ExpandBUILD_VECTOR(SDNode *Node); @@ -321,7 +324,7 @@ static void ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, // Do a (aligned) store to a stack slot, then copy from the stack slot // to the final destination using (unaligned) integer loads and stores. EVT StoredVT = ST->getMemoryVT(); - EVT RegVT = + MVT RegVT = TLI.getRegisterType(*DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), StoredVT.getSizeInBits())); @@ -447,7 +450,7 @@ ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, // Copy the value to a (aligned) stack slot using (unaligned) integer // loads and stores, then do a (aligned) load from the stack slot. - EVT RegVT = TLI.getRegisterType(*DAG.getContext(), intVT); + MVT RegVT = TLI.getRegisterType(*DAG.getContext(), intVT); unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8; unsigned RegBytes = RegVT.getSizeInBits() / 8; unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes; @@ -710,7 +713,7 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { { SDValue Value = ST->getValue(); - EVT VT = Value.getValueType(); + MVT VT = Value.getSimpleValueType(); switch (TLI.getOperationAction(ISD::STORE, VT)) { default: llvm_unreachable("This action is not supported yet!"); case TargetLowering::Legal: @@ -731,9 +734,10 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { return; } case TargetLowering::Promote: { - assert(VT.isVector() && "Unknown legal promote case!"); - Value = DAG.getNode(ISD::BITCAST, dl, - TLI.getTypeToPromoteTo(ISD::STORE, VT), Value); + MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); + assert(NVT.getSizeInBits() == VT.getSizeInBits() && + "Can only promote stores to same size type"); + Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), isVolatile, @@ -817,7 +821,8 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); ReplaceNode(SDValue(Node, 0), Result); } else { - switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { + switch (TLI.getTruncStoreAction(ST->getValue().getSimpleValueType(), + StVT.getSimpleVT())) { default: llvm_unreachable("This action is not supported yet!"); case TargetLowering::Legal: // If this is an unaligned store and the target doesn't support it, @@ -862,7 +867,7 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { ISD::LoadExtType ExtType = LD->getExtensionType(); if (ExtType == ISD::NON_EXTLOAD) { - EVT VT = Node->getValueType(0); + MVT VT = Node->getSimpleValueType(0); SDValue RVal = SDValue(Node, 0); SDValue RChain = SDValue(Node, 1); @@ -889,10 +894,9 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { break; } case TargetLowering::Promote: { - // Only promote a load of vector type to another. - assert(VT.isVector() && "Cannot promote this load!"); - // Change base type to a different vector type. - EVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); + MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); + assert(NVT.getSizeInBits() == VT.getSizeInBits() && + "Can only promote loads to same size type"); SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(), LD->isVolatile(), LD->isNonTemporal(), @@ -1037,7 +1041,7 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { Chain = Ch; } else { bool isCustom = false; - switch (TLI.getLoadExtAction(ExtType, SrcVT)) { + switch (TLI.getLoadExtAction(ExtType, SrcVT.getSimpleVT())) { default: llvm_unreachable("This action is not supported yet!"); case TargetLowering::Custom: isCustom = true; @@ -1184,7 +1188,7 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : Node->getOpcode() == ISD::SETCC ? 2 : 1; unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0; - EVT OpVT = Node->getOperand(CompareOperand).getValueType(); + MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); Action = TLI.getCondCodeAction(CCCode, OpVT); @@ -1591,7 +1595,7 @@ void SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, DebugLoc dl) { - EVT OpVT = LHS.getValueType(); + MVT OpVT = LHS.getSimpleValueType(); ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); switch (TLI.getCondCodeAction(CCCode, OpVT)) { default: llvm_unreachable("Unknown condition code action!"); @@ -1869,7 +1873,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, // isTailCall may be true since the callee does not reference caller stack // frame. Check if it's in the right position. SDValue TCChain = InChain; - bool isTailCall = isInTailCallPosition(DAG, Node, TCChain, TLI); + bool isTailCall = TLI.isInTailCallPosition(DAG, Node, TCChain); if (isTailCall) InChain = TCChain; @@ -1956,6 +1960,7 @@ SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, + RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128) { RTLIB::Libcall LC; switch (Node->getValueType(0).getSimpleVT().SimpleTy) { @@ -1963,6 +1968,7 @@ SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, case MVT::f32: LC = Call_F32; break; case MVT::f64: LC = Call_F64; break; case MVT::f80: LC = Call_F80; break; + case MVT::f128: LC = Call_F128; break; case MVT::ppcf128: LC = Call_PPCF128; break; } return ExpandLibCall(LC, Node, false); @@ -2091,6 +2097,120 @@ SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, Results.push_back(Rem); } +/// isSinCosLibcallAvailable - Return true if sincos libcall is available. +static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { + RTLIB::Libcall LC; + switch (Node->getValueType(0).getSimpleVT().SimpleTy) { + default: llvm_unreachable("Unexpected request for libcall!"); + case MVT::f32: LC = RTLIB::SINCOS_F32; break; + case MVT::f64: LC = RTLIB::SINCOS_F64; break; + case MVT::f80: LC = RTLIB::SINCOS_F80; break; + case MVT::f128: LC = RTLIB::SINCOS_F128; break; + case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; + } + return TLI.getLibcallName(LC) != 0; +} + +/// canCombineSinCosLibcall - Return true if sincos libcall is available and +/// can be used to combine sin and cos. +static bool canCombineSinCosLibcall(SDNode *Node, const TargetLowering &TLI, + const TargetMachine &TM) { + if (!isSinCosLibcallAvailable(Node, TLI)) + return false; + // GNU sin/cos functions set errno while sincos does not. Therefore + // combining sin and cos is only safe if unsafe-fpmath is enabled. + bool isGNU = Triple(TM.getTargetTriple()).getEnvironment() == Triple::GNU; + if (isGNU && !TM.Options.UnsafeFPMath) + return false; + return true; +} + +/// useSinCos - Only issue sincos libcall if both sin and cos are +/// needed. +static bool useSinCos(SDNode *Node) { + unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN + ? ISD::FCOS : ISD::FSIN; + + SDValue Op0 = Node->getOperand(0); + for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), + UE = Op0.getNode()->use_end(); UI != UE; ++UI) { + SDNode *User = *UI; + if (User == Node) + continue; + // The other user might have been turned into sincos already. + if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) + return true; + } + return false; +} + +/// ExpandSinCosLibCall - Issue libcalls to sincos to compute sin / cos +/// pairs. +void +SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, + SmallVectorImpl<SDValue> &Results) { + RTLIB::Libcall LC; + switch (Node->getValueType(0).getSimpleVT().SimpleTy) { + default: llvm_unreachable("Unexpected request for libcall!"); + case MVT::f32: LC = RTLIB::SINCOS_F32; break; + case MVT::f64: LC = RTLIB::SINCOS_F64; break; + case MVT::f80: LC = RTLIB::SINCOS_F80; break; + case MVT::f128: LC = RTLIB::SINCOS_F128; break; + case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; + } + + // The input chain to this libcall is the entry node of the function. + // Legalizing the call will automatically add the previous call to the + // dependence. + SDValue InChain = DAG.getEntryNode(); + + EVT RetVT = Node->getValueType(0); + Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); + + TargetLowering::ArgListTy Args; + TargetLowering::ArgListEntry Entry; + + // Pass the argument. + Entry.Node = Node->getOperand(0); + Entry.Ty = RetTy; + Entry.isSExt = false; + Entry.isZExt = false; + Args.push_back(Entry); + + // Pass the return address of sin. + SDValue SinPtr = DAG.CreateStackTemporary(RetVT); + Entry.Node = SinPtr; + Entry.Ty = RetTy->getPointerTo(); + Entry.isSExt = false; + Entry.isZExt = false; + Args.push_back(Entry); + + // Also pass the return address of the cos. + SDValue CosPtr = DAG.CreateStackTemporary(RetVT); + Entry.Node = CosPtr; + Entry.Ty = RetTy->getPointerTo(); + Entry.isSExt = false; + Entry.isZExt = false; + Args.push_back(Entry); + + SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), + TLI.getPointerTy()); + + DebugLoc dl = Node->getDebugLoc(); + TargetLowering:: + CallLoweringInfo CLI(InChain, Type::getVoidTy(*DAG.getContext()), + false, false, false, false, + 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, + /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, + Callee, Args, DAG, dl); + std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); + + Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, + MachinePointerInfo(), false, false, false, 0)); + Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, + MachinePointerInfo(), false, false, false, 0)); +} + /// ExpandLegalINT_TO_FP - This function is responsible for legalizing a /// INT_TO_FP operation of the specified operand when the target requests that /// we expand it. At this point, we know that the result and operand types are @@ -2419,18 +2539,6 @@ SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, DebugLoc dl) { } } -/// SplatByte - Distribute ByteVal over NumBits bits. -// FIXME: Move this helper to a common place. -static APInt SplatByte(unsigned NumBits, uint8_t ByteVal) { - APInt Val = APInt(NumBits, ByteVal); - unsigned Shift = 8; - for (unsigned i = NumBits; i > 8; i >>= 1) { - Val = (Val << Shift) | Val; - Shift <<= 1; - } - return Val; -} - /// ExpandBitCount - Expand the specified bitcount instruction into operations. /// SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, @@ -2448,10 +2556,10 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, // This is the "best" algorithm from // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel - SDValue Mask55 = DAG.getConstant(SplatByte(Len, 0x55), VT); - SDValue Mask33 = DAG.getConstant(SplatByte(Len, 0x33), VT); - SDValue Mask0F = DAG.getConstant(SplatByte(Len, 0x0F), VT); - SDValue Mask01 = DAG.getConstant(SplatByte(Len, 0x01), VT); + SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), VT); + SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), VT); + SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), VT); + SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), VT); // v = v - ((v >> 1) & 0x55555555...) Op = DAG.getNode(ISD::SUB, dl, VT, Op, @@ -2801,7 +2909,8 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { SDValue True, False; EVT VT = Node->getOperand(0).getValueType(); EVT NVT = Node->getValueType(0); - APFloat apf(APInt::getNullValue(VT.getSizeInBits())); + APFloat apf(DAG.EVTToAPFloatSemantics(VT), + APInt::getNullValue(VT.getSizeInBits())); APInt x = APInt::getSignBit(NVT.getSizeInBits()); (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven); Tmp1 = DAG.getConstantFP(apf, VT); @@ -3032,77 +3141,114 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { } case ISD::FSQRT: Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, - RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128)); + RTLIB::SQRT_F80, RTLIB::SQRT_F128, + RTLIB::SQRT_PPCF128)); break; case ISD::FSIN: - Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, - RTLIB::SIN_F80, RTLIB::SIN_PPCF128)); + case ISD::FCOS: { + EVT VT = Node->getValueType(0); + bool isSIN = Node->getOpcode() == ISD::FSIN; + // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / + // fcos which share the same operand and both are used. + if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || + canCombineSinCosLibcall(Node, TLI, TM)) + && useSinCos(Node)) { + SDVTList VTs = DAG.getVTList(VT, VT); + Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); + if (!isSIN) + Tmp1 = Tmp1.getValue(1); + Results.push_back(Tmp1); + } else if (isSIN) { + Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, + RTLIB::SIN_F80, RTLIB::SIN_F128, + RTLIB::SIN_PPCF128)); + } else { + Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, + RTLIB::COS_F80, RTLIB::COS_F128, + RTLIB::COS_PPCF128)); + } break; - case ISD::FCOS: - Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, - RTLIB::COS_F80, RTLIB::COS_PPCF128)); + } + case ISD::FSINCOS: + // Expand into sincos libcall. + ExpandSinCosLibCall(Node, Results); break; case ISD::FLOG: Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, - RTLIB::LOG_F80, RTLIB::LOG_PPCF128)); + RTLIB::LOG_F80, RTLIB::LOG_F128, + RTLIB::LOG_PPCF128)); break; case ISD::FLOG2: Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, - RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128)); + RTLIB::LOG2_F80, RTLIB::LOG2_F128, + RTLIB::LOG2_PPCF128)); break; case ISD::FLOG10: Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, - RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128)); + RTLIB::LOG10_F80, RTLIB::LOG10_F128, + RTLIB::LOG10_PPCF128)); break; case ISD::FEXP: Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, - RTLIB::EXP_F80, RTLIB::EXP_PPCF128)); + RTLIB::EXP_F80, RTLIB::EXP_F128, + RTLIB::EXP_PPCF128)); break; case ISD::FEXP2: Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, - RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128)); + RTLIB::EXP2_F80, RTLIB::EXP2_F128, + RTLIB::EXP2_PPCF128)); break; case ISD::FTRUNC: Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, - RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128)); + RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, + RTLIB::TRUNC_PPCF128)); break; case ISD::FFLOOR: Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, - RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128)); + RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, + RTLIB::FLOOR_PPCF128)); break; case ISD::FCEIL: Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, - RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128)); + RTLIB::CEIL_F80, RTLIB::CEIL_F128, + RTLIB::CEIL_PPCF128)); break; case ISD::FRINT: Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, - RTLIB::RINT_F80, RTLIB::RINT_PPCF128)); + RTLIB::RINT_F80, RTLIB::RINT_F128, + RTLIB::RINT_PPCF128)); break; case ISD::FNEARBYINT: Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, RTLIB::NEARBYINT_F64, RTLIB::NEARBYINT_F80, + RTLIB::NEARBYINT_F128, RTLIB::NEARBYINT_PPCF128)); break; case ISD::FPOWI: Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, - RTLIB::POWI_F80, RTLIB::POWI_PPCF128)); + RTLIB::POWI_F80, RTLIB::POWI_F128, + RTLIB::POWI_PPCF128)); break; case ISD::FPOW: Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, - RTLIB::POW_F80, RTLIB::POW_PPCF128)); + RTLIB::POW_F80, RTLIB::POW_F128, + RTLIB::POW_PPCF128)); break; case ISD::FDIV: Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, - RTLIB::DIV_F80, RTLIB::DIV_PPCF128)); + RTLIB::DIV_F80, RTLIB::DIV_F128, + RTLIB::DIV_PPCF128)); break; case ISD::FREM: Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, - RTLIB::REM_F80, RTLIB::REM_PPCF128)); + RTLIB::REM_F80, RTLIB::REM_F128, + RTLIB::REM_PPCF128)); break; case ISD::FMA: Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, - RTLIB::FMA_F80, RTLIB::FMA_PPCF128)); + RTLIB::FMA_F80, RTLIB::FMA_F128, + RTLIB::FMA_PPCF128)); break; case ISD::FP16_TO_FP32: Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); @@ -3158,7 +3304,6 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { case ISD::UREM: case ISD::SREM: { EVT VT = Node->getValueType(0); - SDVTList VTs = DAG.getVTList(VT, VT); bool isSigned = Node->getOpcode() == ISD::SREM; unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; @@ -3169,6 +3314,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { // If div is legal, it's better to do the normal expansion !TLI.isOperationLegalOrCustom(DivOpc, Node->getValueType(0)) && useDivRem(Node, isSigned, false))) { + SDVTList VTs = DAG.getVTList(VT, VT); Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1); } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) { // X % Y -> X-X/Y*Y @@ -3486,8 +3632,19 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { // Otherwise, SETCC for the given comparison type must be completely // illegal; expand it into a SELECT_CC. EVT VT = Node->getValueType(0); + int TrueValue; + switch (TLI.getBooleanContents(VT.isVector())) { + case TargetLowering::ZeroOrOneBooleanContent: + case TargetLowering::UndefinedBooleanContent: + TrueValue = 1; + break; + case TargetLowering::ZeroOrNegativeOneBooleanContent: + TrueValue = -1; + break; + } Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, - DAG.getConstant(1, VT), DAG.getConstant(0, VT), Tmp3); + DAG.getConstant(TrueValue, VT), DAG.getConstant(0, VT), + Tmp3); Results.push_back(Tmp1); break; } @@ -3575,13 +3732,13 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { void SelectionDAGLegalize::PromoteNode(SDNode *Node) { SmallVector<SDValue, 8> Results; - EVT OVT = Node->getValueType(0); + MVT OVT = Node->getSimpleValueType(0); if (Node->getOpcode() == ISD::UINT_TO_FP || Node->getOpcode() == ISD::SINT_TO_FP || Node->getOpcode() == ISD::SETCC) { - OVT = Node->getOperand(0).getValueType(); + OVT = Node->getOperand(0).getSimpleValueType(); } - EVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); + MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); DebugLoc dl = Node->getDebugLoc(); SDValue Tmp1, Tmp2, Tmp3; switch (Node->getOpcode()) { diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index 92dc5a9..de217d8 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -29,11 +29,13 @@ static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, + RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128) { return VT == MVT::f32 ? Call_F32 : VT == MVT::f64 ? Call_F64 : VT == MVT::f80 ? Call_F80 : + VT == MVT::f128 ? Call_F128 : VT == MVT::ppcf128 ? Call_PPCF128 : RTLIB::UNKNOWN_LIBCALL; } @@ -152,23 +154,25 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)), GetSoftenedFloat(N->getOperand(1)) }; - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::ADD_F32, - RTLIB::ADD_F64, - RTLIB::ADD_F80, - RTLIB::ADD_PPCF128), - NVT, Ops, 2, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::ADD_F32, + RTLIB::ADD_F64, + RTLIB::ADD_F80, + RTLIB::ADD_F128, + RTLIB::ADD_PPCF128), + NVT, Ops, 2, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::CEIL_F32, - RTLIB::CEIL_F64, - RTLIB::CEIL_F80, - RTLIB::CEIL_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::CEIL_F32, + RTLIB::CEIL_F64, + RTLIB::CEIL_F80, + RTLIB::CEIL_F128, + RTLIB::CEIL_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) { @@ -216,90 +220,98 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) { SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::COS_F32, - RTLIB::COS_F64, - RTLIB::COS_F80, - RTLIB::COS_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::COS_F32, + RTLIB::COS_F64, + RTLIB::COS_F80, + RTLIB::COS_F128, + RTLIB::COS_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)), GetSoftenedFloat(N->getOperand(1)) }; - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::DIV_F32, - RTLIB::DIV_F64, - RTLIB::DIV_F80, - RTLIB::DIV_PPCF128), - NVT, Ops, 2, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::DIV_F32, + RTLIB::DIV_F64, + RTLIB::DIV_F80, + RTLIB::DIV_F128, + RTLIB::DIV_PPCF128), + NVT, Ops, 2, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::EXP_F32, - RTLIB::EXP_F64, - RTLIB::EXP_F80, - RTLIB::EXP_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::EXP_F32, + RTLIB::EXP_F64, + RTLIB::EXP_F80, + RTLIB::EXP_F128, + RTLIB::EXP_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::EXP2_F32, - RTLIB::EXP2_F64, - RTLIB::EXP2_F80, - RTLIB::EXP2_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::EXP2_F32, + RTLIB::EXP2_F64, + RTLIB::EXP2_F80, + RTLIB::EXP2_F128, + RTLIB::EXP2_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::FLOOR_F32, - RTLIB::FLOOR_F64, - RTLIB::FLOOR_F80, - RTLIB::FLOOR_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::FLOOR_F32, + RTLIB::FLOOR_F64, + RTLIB::FLOOR_F80, + RTLIB::FLOOR_F128, + RTLIB::FLOOR_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::LOG_F32, - RTLIB::LOG_F64, - RTLIB::LOG_F80, - RTLIB::LOG_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::LOG_F32, + RTLIB::LOG_F64, + RTLIB::LOG_F80, + RTLIB::LOG_F128, + RTLIB::LOG_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::LOG2_F32, - RTLIB::LOG2_F64, - RTLIB::LOG2_F80, - RTLIB::LOG2_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::LOG2_F32, + RTLIB::LOG2_F64, + RTLIB::LOG2_F80, + RTLIB::LOG2_F128, + RTLIB::LOG2_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::LOG10_F32, - RTLIB::LOG10_F64, - RTLIB::LOG10_F80, - RTLIB::LOG10_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::LOG10_F32, + RTLIB::LOG10_F64, + RTLIB::LOG10_F80, + RTLIB::LOG10_F128, + RTLIB::LOG10_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) { @@ -307,35 +319,38 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) { SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0)), GetSoftenedFloat(N->getOperand(1)), GetSoftenedFloat(N->getOperand(2)) }; - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::FMA_F32, - RTLIB::FMA_F64, - RTLIB::FMA_F80, - RTLIB::FMA_PPCF128), - NVT, Ops, 3, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::FMA_F32, + RTLIB::FMA_F64, + RTLIB::FMA_F80, + RTLIB::FMA_F128, + RTLIB::FMA_PPCF128), + NVT, Ops, 3, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)), GetSoftenedFloat(N->getOperand(1)) }; - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::MUL_F32, - RTLIB::MUL_F64, - RTLIB::MUL_F80, - RTLIB::MUL_PPCF128), - NVT, Ops, 2, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::MUL_F32, + RTLIB::MUL_F64, + RTLIB::MUL_F80, + RTLIB::MUL_F128, + RTLIB::MUL_PPCF128), + NVT, Ops, 2, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::NEARBYINT_F32, - RTLIB::NEARBYINT_F64, - RTLIB::NEARBYINT_F80, - RTLIB::NEARBYINT_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::NEARBYINT_F32, + RTLIB::NEARBYINT_F64, + RTLIB::NEARBYINT_F80, + RTLIB::NEARBYINT_F128, + RTLIB::NEARBYINT_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) { @@ -343,12 +358,13 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) { // Expand Y = FNEG(X) -> Y = SUB -0.0, X SDValue Ops[2] = { DAG.getConstantFP(-0.0, N->getValueType(0)), GetSoftenedFloat(N->getOperand(0)) }; - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::SUB_F32, - RTLIB::SUB_F64, - RTLIB::SUB_F80, - RTLIB::SUB_PPCF128), - NVT, Ops, 2, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::SUB_F32, + RTLIB::SUB_F64, + RTLIB::SUB_F80, + RTLIB::SUB_F128, + RTLIB::SUB_PPCF128), + NVT, Ops, 2, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { @@ -356,7 +372,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { SDValue Op = N->getOperand(0); RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0)); assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); - return MakeLibCall(LC, NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, LC, NVT, &Op, 1, false, N->getDebugLoc()); } // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special @@ -364,8 +380,8 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP32(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = N->getOperand(0); - return MakeLibCall(RTLIB::FPEXT_F16_F32, NVT, &Op, 1, false, - N->getDebugLoc()); + return TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, NVT, &Op, 1, false, + N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { @@ -373,19 +389,20 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { SDValue Op = N->getOperand(0); RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0)); assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); - return MakeLibCall(LC, NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, LC, NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)), GetSoftenedFloat(N->getOperand(1)) }; - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::POW_F32, - RTLIB::POW_F64, - RTLIB::POW_F80, - RTLIB::POW_PPCF128), - NVT, Ops, 2, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::POW_F32, + RTLIB::POW_F64, + RTLIB::POW_F80, + RTLIB::POW_F128, + RTLIB::POW_PPCF128), + NVT, Ops, 2, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) { @@ -393,80 +410,87 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) { "Unsupported power type!"); EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)), N->getOperand(1) }; - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::POWI_F32, - RTLIB::POWI_F64, - RTLIB::POWI_F80, - RTLIB::POWI_PPCF128), - NVT, Ops, 2, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::POWI_F32, + RTLIB::POWI_F64, + RTLIB::POWI_F80, + RTLIB::POWI_F128, + RTLIB::POWI_PPCF128), + NVT, Ops, 2, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)), GetSoftenedFloat(N->getOperand(1)) }; - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::REM_F32, - RTLIB::REM_F64, - RTLIB::REM_F80, - RTLIB::REM_PPCF128), - NVT, Ops, 2, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::REM_F32, + RTLIB::REM_F64, + RTLIB::REM_F80, + RTLIB::REM_F128, + RTLIB::REM_PPCF128), + NVT, Ops, 2, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::RINT_F32, - RTLIB::RINT_F64, - RTLIB::RINT_F80, - RTLIB::RINT_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::RINT_F32, + RTLIB::RINT_F64, + RTLIB::RINT_F80, + RTLIB::RINT_F128, + RTLIB::RINT_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::SIN_F32, - RTLIB::SIN_F64, - RTLIB::SIN_F80, - RTLIB::SIN_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::SIN_F32, + RTLIB::SIN_F64, + RTLIB::SIN_F80, + RTLIB::SIN_F128, + RTLIB::SIN_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::SQRT_F32, - RTLIB::SQRT_F64, - RTLIB::SQRT_F80, - RTLIB::SQRT_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::SQRT_F32, + RTLIB::SQRT_F64, + RTLIB::SQRT_F80, + RTLIB::SQRT_F128, + RTLIB::SQRT_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0)), GetSoftenedFloat(N->getOperand(1)) }; - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::SUB_F32, - RTLIB::SUB_F64, - RTLIB::SUB_F80, - RTLIB::SUB_PPCF128), - NVT, Ops, 2, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::SUB_F32, + RTLIB::SUB_F64, + RTLIB::SUB_F80, + RTLIB::SUB_F128, + RTLIB::SUB_PPCF128), + NVT, Ops, 2, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::TRUNC_F32, - RTLIB::TRUNC_F64, - RTLIB::TRUNC_F80, - RTLIB::TRUNC_PPCF128), - NVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::TRUNC_F32, + RTLIB::TRUNC_F64, + RTLIB::TRUNC_F80, + RTLIB::TRUNC_F128, + RTLIB::TRUNC_PPCF128), + NVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) { @@ -559,8 +583,9 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) { // Sign/zero extend the argument if the libcall takes a larger type. SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); - return MakeLibCall(LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT), - &Op, 1, false, dl); + return TLI.makeLibCall(DAG, LC, + TLI.getTypeToTransformTo(*DAG.getContext(), RVT), + &Op, 1, false, dl); } @@ -607,92 +632,6 @@ bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { return false; } -/// SoftenSetCCOperands - Soften the operands of a comparison. This code is -/// shared among BR_CC, SELECT_CC, and SETCC handlers. -void DAGTypeLegalizer::SoftenSetCCOperands(SDValue &NewLHS, SDValue &NewRHS, - ISD::CondCode &CCCode, DebugLoc dl) { - SDValue LHSInt = GetSoftenedFloat(NewLHS); - SDValue RHSInt = GetSoftenedFloat(NewRHS); - EVT VT = NewLHS.getValueType(); - - assert((VT == MVT::f32 || VT == MVT::f64) && "Unsupported setcc type!"); - - // Expand into one or more soft-fp libcall(s). - RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL; - switch (CCCode) { - case ISD::SETEQ: - case ISD::SETOEQ: - LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64; - break; - case ISD::SETNE: - case ISD::SETUNE: - LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : RTLIB::UNE_F64; - break; - case ISD::SETGE: - case ISD::SETOGE: - LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64; - break; - case ISD::SETLT: - case ISD::SETOLT: - LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; - break; - case ISD::SETLE: - case ISD::SETOLE: - LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64; - break; - case ISD::SETGT: - case ISD::SETOGT: - LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64; - break; - case ISD::SETUO: - LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64; - break; - case ISD::SETO: - LC1 = (VT == MVT::f32) ? RTLIB::O_F32 : RTLIB::O_F64; - break; - default: - LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64; - switch (CCCode) { - case ISD::SETONE: - // SETONE = SETOLT | SETOGT - LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; - // Fallthrough - case ISD::SETUGT: - LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64; - break; - case ISD::SETUGE: - LC2 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64; - break; - case ISD::SETULT: - LC2 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; - break; - case ISD::SETULE: - LC2 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64; - break; - case ISD::SETUEQ: - LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64; - break; - default: llvm_unreachable("Do not know how to soften this setcc!"); - } - } - - // Use the target specific return value for comparions lib calls. - EVT RetVT = TLI.getCmpLibcallReturnType(); - SDValue Ops[2] = { LHSInt, RHSInt }; - NewLHS = MakeLibCall(LC1, RetVT, Ops, 2, false/*sign irrelevant*/, dl); - NewRHS = DAG.getConstant(0, RetVT); - CCCode = TLI.getCmpLibcallCC(LC1); - if (LC2 != RTLIB::UNKNOWN_LIBCALL) { - SDValue Tmp = DAG.getNode(ISD::SETCC, dl, TLI.getSetCCResultType(RetVT), - NewLHS, NewRHS, DAG.getCondCode(CCCode)); - NewLHS = MakeLibCall(LC2, RetVT, Ops, 2, false/*sign irrelevant*/, dl); - NewLHS = DAG.getNode(ISD::SETCC, dl, TLI.getSetCCResultType(RetVT), NewLHS, - NewRHS, DAG.getCondCode(TLI.getCmpLibcallCC(LC2))); - NewLHS = DAG.getNode(ISD::OR, dl, Tmp.getValueType(), Tmp, NewLHS); - NewRHS = SDValue(); - } -} - SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) { return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), N->getValueType(0), GetSoftenedFloat(N->getOperand(0))); @@ -706,15 +645,19 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) { assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall"); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(LC, RVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, LC, RVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) { SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); - SoftenSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); - // If SoftenSetCCOperands returned a scalar, we need to compare the result + EVT VT = NewLHS.getValueType(); + NewLHS = GetSoftenedFloat(NewLHS); + NewRHS = GetSoftenedFloat(NewRHS); + TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, N->getDebugLoc()); + + // If softenSetCCOperands returned a scalar, we need to compare the result // against zero to select between true and false values. if (NewRHS.getNode() == 0) { NewRHS = DAG.getConstant(0, NewLHS.getValueType()); @@ -733,7 +676,7 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_SINT(SDNode *N) { RTLIB::Libcall LC = RTLIB::getFPTOSINT(N->getOperand(0).getValueType(), RVT); assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_TO_SINT!"); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(LC, RVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, LC, RVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_UINT(SDNode *N) { @@ -741,22 +684,26 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_UINT(SDNode *N) { RTLIB::Libcall LC = RTLIB::getFPTOUINT(N->getOperand(0).getValueType(), RVT); assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_TO_UINT!"); SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(LC, RVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, LC, RVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatOp_FP32_TO_FP16(SDNode *N) { EVT RVT = N->getValueType(0); RTLIB::Libcall LC = RTLIB::FPROUND_F32_F16; SDValue Op = GetSoftenedFloat(N->getOperand(0)); - return MakeLibCall(LC, RVT, &Op, 1, false, N->getDebugLoc()); + return TLI.makeLibCall(DAG, LC, RVT, &Op, 1, false, N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); - SoftenSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); - // If SoftenSetCCOperands returned a scalar, we need to compare the result + EVT VT = NewLHS.getValueType(); + NewLHS = GetSoftenedFloat(NewLHS); + NewRHS = GetSoftenedFloat(NewRHS); + TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, N->getDebugLoc()); + + // If softenSetCCOperands returned a scalar, we need to compare the result // against zero to select between true and false values. if (NewRHS.getNode() == 0) { NewRHS = DAG.getConstant(0, NewLHS.getValueType()); @@ -773,9 +720,13 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) { SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); - SoftenSetCCOperands(NewLHS, NewRHS, CCCode, N->getDebugLoc()); - // If SoftenSetCCOperands returned a scalar, use it. + EVT VT = NewLHS.getValueType(); + NewLHS = GetSoftenedFloat(NewLHS); + NewRHS = GetSoftenedFloat(NewRHS); + TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, N->getDebugLoc()); + + // If softenSetCCOperands returned a scalar, use it. if (NewRHS.getNode() == 0) { assert(NewLHS.getValueType() == N->getValueType(0) && "Unexpected setcc expansion!"); @@ -873,6 +824,7 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break; case ISD::SINT_TO_FP: case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break; + case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break; } // If Lo/Hi is null, the sub-method took care of registering results etc. @@ -886,9 +838,11 @@ void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, assert(NVT.getSizeInBits() == integerPartWidth && "Do not know how to expand this float constant!"); APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt(); - Lo = DAG.getConstantFP(APFloat(APInt(integerPartWidth, C.getRawData()[1])), + Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), + APInt(integerPartWidth, C.getRawData()[1])), NVT); - Hi = DAG.getConstantFP(APFloat(APInt(integerPartWidth, C.getRawData()[0])), + Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), + APInt(integerPartWidth, C.getRawData()[0])), NVT); } @@ -910,7 +864,8 @@ void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::ADD_F32, RTLIB::ADD_F64, - RTLIB::ADD_F80, RTLIB::ADD_PPCF128), + RTLIB::ADD_F80, RTLIB::ADD_F128, + RTLIB::ADD_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -919,7 +874,8 @@ void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::CEIL_F32, RTLIB::CEIL_F64, - RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128), + RTLIB::CEIL_F80, RTLIB::CEIL_F128, + RTLIB::CEIL_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -930,6 +886,7 @@ void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N, RTLIB::COPYSIGN_F32, RTLIB::COPYSIGN_F64, RTLIB::COPYSIGN_F80, + RTLIB::COPYSIGN_F128, RTLIB::COPYSIGN_PPCF128), N, false); GetPairElements(Call, Lo, Hi); @@ -939,7 +896,8 @@ void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::COS_F32, RTLIB::COS_F64, - RTLIB::COS_F80, RTLIB::COS_PPCF128), + RTLIB::COS_F80, RTLIB::COS_F128, + RTLIB::COS_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -947,13 +905,14 @@ void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N, void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; - SDValue Call = MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::DIV_F32, - RTLIB::DIV_F64, - RTLIB::DIV_F80, - RTLIB::DIV_PPCF128), - N->getValueType(0), Ops, 2, false, - N->getDebugLoc()); + SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::DIV_F32, + RTLIB::DIV_F64, + RTLIB::DIV_F80, + RTLIB::DIV_F128, + RTLIB::DIV_PPCF128), + N->getValueType(0), Ops, 2, false, + N->getDebugLoc()); GetPairElements(Call, Lo, Hi); } @@ -961,7 +920,8 @@ void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::EXP_F32, RTLIB::EXP_F64, - RTLIB::EXP_F80, RTLIB::EXP_PPCF128), + RTLIB::EXP_F80, RTLIB::EXP_F128, + RTLIB::EXP_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -970,7 +930,8 @@ void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::EXP2_F32, RTLIB::EXP2_F64, - RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128), + RTLIB::EXP2_F80, RTLIB::EXP2_F128, + RTLIB::EXP2_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -978,8 +939,9 @@ void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N, void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), - RTLIB::FLOOR_F32,RTLIB::FLOOR_F64, - RTLIB::FLOOR_F80,RTLIB::FLOOR_PPCF128), + RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, + RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, + RTLIB::FLOOR_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -988,7 +950,8 @@ void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::LOG_F32, RTLIB::LOG_F64, - RTLIB::LOG_F80, RTLIB::LOG_PPCF128), + RTLIB::LOG_F80, RTLIB::LOG_F128, + RTLIB::LOG_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -997,7 +960,8 @@ void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::LOG2_F32, RTLIB::LOG2_F64, - RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128), + RTLIB::LOG2_F80, RTLIB::LOG2_F128, + RTLIB::LOG2_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -1005,8 +969,9 @@ void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N, void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), - RTLIB::LOG10_F32,RTLIB::LOG10_F64, - RTLIB::LOG10_F80,RTLIB::LOG10_PPCF128), + RTLIB::LOG10_F32, RTLIB::LOG10_F64, + RTLIB::LOG10_F80, RTLIB::LOG10_F128, + RTLIB::LOG10_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -1014,26 +979,28 @@ void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N, void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Ops[3] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) }; - SDValue Call = MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::FMA_F32, - RTLIB::FMA_F64, - RTLIB::FMA_F80, - RTLIB::FMA_PPCF128), - N->getValueType(0), Ops, 3, false, - N->getDebugLoc()); + SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::FMA_F32, + RTLIB::FMA_F64, + RTLIB::FMA_F80, + RTLIB::FMA_F128, + RTLIB::FMA_PPCF128), + N->getValueType(0), Ops, 3, false, + N->getDebugLoc()); GetPairElements(Call, Lo, Hi); } void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; - SDValue Call = MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::MUL_F32, - RTLIB::MUL_F64, - RTLIB::MUL_F80, - RTLIB::MUL_PPCF128), - N->getValueType(0), Ops, 2, false, - N->getDebugLoc()); + SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::MUL_F32, + RTLIB::MUL_F64, + RTLIB::MUL_F80, + RTLIB::MUL_F128, + RTLIB::MUL_PPCF128), + N->getValueType(0), Ops, 2, false, + N->getDebugLoc()); GetPairElements(Call, Lo, Hi); } @@ -1043,6 +1010,7 @@ void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N, RTLIB::NEARBYINT_F32, RTLIB::NEARBYINT_F64, RTLIB::NEARBYINT_F80, + RTLIB::NEARBYINT_F128, RTLIB::NEARBYINT_PPCF128), N, false); GetPairElements(Call, Lo, Hi); @@ -1060,14 +1028,16 @@ void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo, SDValue &Hi) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); Hi = DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), NVT, N->getOperand(0)); - Lo = DAG.getConstantFP(APFloat(APInt(NVT.getSizeInBits(), 0)), NVT); + Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), + APInt(NVT.getSizeInBits(), 0)), NVT); } void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::POW_F32, RTLIB::POW_F64, - RTLIB::POW_F80, RTLIB::POW_PPCF128), + RTLIB::POW_F80, RTLIB::POW_F128, + RTLIB::POW_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -1076,7 +1046,18 @@ void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::POWI_F32, RTLIB::POWI_F64, - RTLIB::POWI_F80, RTLIB::POWI_PPCF128), + RTLIB::POWI_F80, RTLIB::POWI_F128, + RTLIB::POWI_PPCF128), + N, false); + GetPairElements(Call, Lo, Hi); +} + +void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N, + SDValue &Lo, SDValue &Hi) { + SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), + RTLIB::REM_F32, RTLIB::REM_F64, + RTLIB::REM_F80, RTLIB::REM_F128, + RTLIB::REM_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -1085,7 +1066,8 @@ void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::RINT_F32, RTLIB::RINT_F64, - RTLIB::RINT_F80, RTLIB::RINT_PPCF128), + RTLIB::RINT_F80, RTLIB::RINT_F128, + RTLIB::RINT_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -1094,7 +1076,8 @@ void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::SIN_F32, RTLIB::SIN_F64, - RTLIB::SIN_F80, RTLIB::SIN_PPCF128), + RTLIB::SIN_F80, RTLIB::SIN_F128, + RTLIB::SIN_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -1103,7 +1086,8 @@ void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::SQRT_F32, RTLIB::SQRT_F64, - RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128), + RTLIB::SQRT_F80, RTLIB::SQRT_F128, + RTLIB::SQRT_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -1111,13 +1095,14 @@ void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N, void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; - SDValue Call = MakeLibCall(GetFPLibCall(N->getValueType(0), - RTLIB::SUB_F32, - RTLIB::SUB_F64, - RTLIB::SUB_F80, - RTLIB::SUB_PPCF128), - N->getValueType(0), Ops, 2, false, - N->getDebugLoc()); + SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), + RTLIB::SUB_F32, + RTLIB::SUB_F64, + RTLIB::SUB_F80, + RTLIB::SUB_F128, + RTLIB::SUB_PPCF128), + N->getValueType(0), Ops, 2, false, + N->getDebugLoc()); GetPairElements(Call, Lo, Hi); } @@ -1125,7 +1110,8 @@ void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Call = LibCallify(GetFPLibCall(N->getValueType(0), RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, - RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128), + RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, + RTLIB::TRUNC_PPCF128), N, false); GetPairElements(Call, Lo, Hi); } @@ -1155,7 +1141,8 @@ void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo, Chain = Hi.getValue(1); // The low part is zero. - Lo = DAG.getConstantFP(APFloat(APInt(NVT.getSizeInBits(), 0)), NVT); + Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), + APInt(NVT.getSizeInBits(), 0)), NVT); // Modified the chain - switch anything that used the old chain to use the // new one. @@ -1179,7 +1166,8 @@ void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, // The integer can be represented exactly in an f64. Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, MVT::i32, Src); - Lo = DAG.getConstantFP(APFloat(APInt(NVT.getSizeInBits(), 0)), NVT); + Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), + APInt(NVT.getSizeInBits(), 0)), NVT); Hi = DAG.getNode(ISD::SINT_TO_FP, dl, NVT, Src); } else { RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; @@ -1193,7 +1181,7 @@ void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, } assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); - Hi = MakeLibCall(LC, VT, &Src, 1, true, dl); + Hi = TLI.makeLibCall(DAG, LC, VT, &Src, 1, true, dl); GetPairElements(Hi, Lo, Hi); } @@ -1225,7 +1213,8 @@ void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, } Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, - DAG.getConstantFP(APFloat(APInt(128, Parts)), + DAG.getConstantFP(APFloat(APFloat::PPCDoubleDouble, + APInt(128, Parts)), MVT::ppcf128)); Lo = DAG.getNode(ISD::SELECT_CC, dl, VT, Src, DAG.getConstant(0, SrcVT), Lo, Hi, DAG.getCondCode(ISD::SETLT)); @@ -1364,7 +1353,7 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_SINT(SDNode *N) { RTLIB::Libcall LC = RTLIB::getFPTOSINT(N->getOperand(0).getValueType(), RVT); assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_TO_SINT!"); - return MakeLibCall(LC, RVT, &N->getOperand(0), 1, false, dl); + return TLI.makeLibCall(DAG, LC, RVT, &N->getOperand(0), 1, false, dl); } SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_UINT(SDNode *N) { @@ -1377,7 +1366,7 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_UINT(SDNode *N) { assert(N->getOperand(0).getValueType() == MVT::ppcf128 && "Logic only correct for ppcf128!"); const uint64_t TwoE31[] = {0x41e0000000000000LL, 0}; - APFloat APF = APFloat(APInt(128, TwoE31)); + APFloat APF = APFloat(APFloat::PPCDoubleDouble, APInt(128, TwoE31)); SDValue Tmp = DAG.getConstantFP(APF, MVT::ppcf128); // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X // FIXME: generated code sucks. @@ -1396,7 +1385,8 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_UINT(SDNode *N) { RTLIB::Libcall LC = RTLIB::getFPTOUINT(N->getOperand(0).getValueType(), RVT); assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_TO_UINT!"); - return MakeLibCall(LC, N->getValueType(0), &N->getOperand(0), 1, false, dl); + return TLI.makeLibCall(DAG, LC, N->getValueType(0), &N->getOperand(0), 1, + false, dl); } SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) { diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index a370fae..d19c13b 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -19,7 +19,7 @@ //===----------------------------------------------------------------------===// #include "LegalizeTypes.h" -#include "llvm/DerivedTypes.h" +#include "llvm/IR/DerivedTypes.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -515,7 +515,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { // Only use the result of getSetCCResultType if it is legal, // otherwise just use the promoted result type (NVT). if (!TLI.isTypeLegal(SVT)) - SVT = NVT; + SVT = NVT; DebugLoc dl = N->getDebugLoc(); assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() && @@ -531,9 +531,10 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { } SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { - return DAG.getNode(ISD::SHL, N->getDebugLoc(), - TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), - GetPromotedInteger(N->getOperand(0)), N->getOperand(1)); + SDValue Res = GetPromotedInteger(N->getOperand(0)); + SDValue Amt = N->getOperand(1); + Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt; + return DAG.getNode(ISD::SHL, N->getDebugLoc(), Res.getValueType(), Res, Amt); } SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { @@ -549,22 +550,23 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { SDValue LHS = GetPromotedInteger(N->getOperand(0)); SDValue RHS = GetPromotedInteger(N->getOperand(1)); return DAG.getNode(N->getOpcode(), N->getDebugLoc(), - LHS.getValueType(), LHS, RHS); + LHS.getValueType(), LHS, RHS); } SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { // The input value must be properly sign extended. SDValue Res = SExtPromotedInteger(N->getOperand(0)); - return DAG.getNode(ISD::SRA, N->getDebugLoc(), - Res.getValueType(), Res, N->getOperand(1)); + SDValue Amt = N->getOperand(1); + Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt; + return DAG.getNode(ISD::SRA, N->getDebugLoc(), Res.getValueType(), Res, Amt); } SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { // The input value must be properly zero extended. - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); SDValue Res = ZExtPromotedInteger(N->getOperand(0)); - return DAG.getNode(ISD::SRL, N->getDebugLoc(), NVT, Res, N->getOperand(1)); + SDValue Amt = N->getOperand(1); + Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt; + return DAG.getNode(ISD::SRL, N->getDebugLoc(), Res.getValueType(), Res, Amt); } SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { @@ -703,7 +705,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { EVT VT = N->getValueType(0); DebugLoc dl = N->getDebugLoc(); - EVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); + MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); // The argument is passed as NumRegs registers of type RegVT. @@ -1767,7 +1769,8 @@ void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, SDValue Op = N->getOperand(0); RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); - SplitInteger(MakeLibCall(LC, VT, &Op, 1, true/*irrelevant*/, dl), Lo, Hi); + SplitInteger(TLI.makeLibCall(DAG, LC, VT, &Op, 1, true/*irrelevant*/, dl), + Lo, Hi); } void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, @@ -1777,7 +1780,8 @@ void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, SDValue Op = N->getOperand(0); RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); - SplitInteger(MakeLibCall(LC, VT, &Op, 1, false/*irrelevant*/, dl), Lo, Hi); + SplitInteger(TLI.makeLibCall(DAG, LC, VT, &Op, 1, false/*irrelevant*/, dl), + Lo, Hi); } void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, @@ -1992,7 +1996,8 @@ void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported MUL!"); SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; - SplitInteger(MakeLibCall(LC, VT, Ops, 2, true/*irrelevant*/, dl), Lo, Hi); + SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true/*irrelevant*/, dl), + Lo, Hi); } void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, @@ -2054,7 +2059,7 @@ void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; - SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi); + SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true, dl), Lo, Hi); } void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, @@ -2092,9 +2097,20 @@ void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, // Expand the subcomponents. SDValue LHSL, LHSH; GetExpandedInteger(N->getOperand(0), LHSL, LHSH); - - SDValue Ops[] = { LHSL, LHSH, N->getOperand(1) }; EVT VT = LHSL.getValueType(); + + // If the shift amount operand is coming from a vector legalization it may + // have an illegal type. Fix that first by casting the operand, otherwise + // the new SHL_PARTS operation would need further legalization. + SDValue ShiftOp = N->getOperand(1); + EVT ShiftTy = TLI.getShiftAmountTy(VT); + assert(ShiftTy.getScalarType().getSizeInBits() >= + Log2_32_Ceil(VT.getScalarType().getSizeInBits()) && + "ShiftAmountTy is too small to cover the range of this type!"); + if (ShiftOp.getValueType() != ShiftTy) + ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); + + SDValue Ops[] = { LHSL, LHSH, ShiftOp }; Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops, 3); Hi = Lo.getValue(1); return; @@ -2138,7 +2154,7 @@ void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; - SplitInteger(MakeLibCall(LC, VT, Ops, 2, isSigned, dl), Lo, Hi); + SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, isSigned, dl), Lo, Hi); return; } @@ -2221,7 +2237,7 @@ void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; - SplitInteger(MakeLibCall(LC, VT, Ops, 2, true, dl), Lo, Hi); + SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true, dl), Lo, Hi); } void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, @@ -2361,7 +2377,7 @@ void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; - SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi); + SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, false, dl), Lo, Hi); } void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, @@ -2381,7 +2397,7 @@ void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; - SplitInteger(MakeLibCall(LC, VT, Ops, 2, false, dl), Lo, Hi); + SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, false, dl), Lo, Hi); } void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, @@ -2549,7 +2565,7 @@ void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, // NOTE: on targets without efficient SELECT of bools, we can always use // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) - TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, true, NULL); + TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, NULL); SDValue Tmp1, Tmp2; Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo.getValueType()), LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl); @@ -2668,7 +2684,7 @@ SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); assert(LC != RTLIB::UNKNOWN_LIBCALL && "Don't know how to expand this SINT_TO_FP!"); - return MakeLibCall(LC, DstVT, &Op, 1, true, N->getDebugLoc()); + return TLI.makeLibCall(DAG, LC, DstVT, &Op, 1, true, N->getDebugLoc()); } SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { @@ -2764,17 +2780,6 @@ SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), N->getValueType(0), InL); } -static const fltSemantics *EVTToAPFloatSemantics(EVT VT) { - switch (VT.getSimpleVT().SimpleTy) { - default: llvm_unreachable("Unknown FP format"); - case MVT::f32: return &APFloat::IEEEsingle; - case MVT::f64: return &APFloat::IEEEdouble; - case MVT::f80: return &APFloat::x87DoubleExtended; - case MVT::f128: return &APFloat::IEEEquad; - case MVT::ppcf128: return &APFloat::PPCDoubleDouble; - } -} - SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { SDValue Op = N->getOperand(0); EVT SrcVT = Op.getValueType(); @@ -2784,8 +2789,8 @@ SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { // The following optimization is valid only if every value in SrcVT (when // treated as signed) is representable in DstVT. Check that the mantissa // size of DstVT is >= than the number of bits in SrcVT -1. - const fltSemantics *sem = EVTToAPFloatSemantics(DstVT); - if (APFloat::semanticsPrecision(*sem) >= SrcVT.getSizeInBits()-1 && + const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT); + if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 && TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){ // Do a signed conversion then adjust the result. SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op); @@ -2846,7 +2851,7 @@ SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); assert(LC != RTLIB::UNKNOWN_LIBCALL && "Don't know how to expand this UINT_TO_FP!"); - return MakeLibCall(LC, DstVT, &Op, 1, true, dl); + return TLI.makeLibCall(DAG, LC, DstVT, &Op, 1, true, dl); } SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp index 644e36e..b6436bf 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp @@ -14,9 +14,9 @@ //===----------------------------------------------------------------------===// #include "LegalizeTypes.h" -#include "llvm/CallingConv.h" -#include "llvm/DataLayout.h" #include "llvm/ADT/SetVector.h" +#include "llvm/IR/CallingConv.h" +#include "llvm/IR/DataLayout.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" @@ -735,6 +735,9 @@ void DAGTypeLegalizer::SetPromotedInteger(SDValue Op, SDValue Result) { SDValue &OpEntry = PromotedIntegers[Op]; assert(OpEntry.getNode() == 0 && "Node is already promoted!"); OpEntry = Result; + + // Propagate node ordering + DAG.AssignOrdering(Result.getNode(), DAG.GetOrdering(Op.getNode())); } void DAGTypeLegalizer::SetSoftenedFloat(SDValue Op, SDValue Result) { @@ -746,6 +749,9 @@ void DAGTypeLegalizer::SetSoftenedFloat(SDValue Op, SDValue Result) { SDValue &OpEntry = SoftenedFloats[Op]; assert(OpEntry.getNode() == 0 && "Node is already converted to integer!"); OpEntry = Result; + + // Propagate node ordering + DAG.AssignOrdering(Result.getNode(), DAG.GetOrdering(Op.getNode())); } void DAGTypeLegalizer::SetScalarizedVector(SDValue Op, SDValue Result) { @@ -760,6 +766,9 @@ void DAGTypeLegalizer::SetScalarizedVector(SDValue Op, SDValue Result) { SDValue &OpEntry = ScalarizedVectors[Op]; assert(OpEntry.getNode() == 0 && "Node is already scalarized!"); OpEntry = Result; + + // Propagate node ordering + DAG.AssignOrdering(Result.getNode(), DAG.GetOrdering(Op.getNode())); } void DAGTypeLegalizer::GetExpandedInteger(SDValue Op, SDValue &Lo, @@ -787,6 +796,10 @@ void DAGTypeLegalizer::SetExpandedInteger(SDValue Op, SDValue Lo, assert(Entry.first.getNode() == 0 && "Node already expanded"); Entry.first = Lo; Entry.second = Hi; + + // Propagate ordering + DAG.AssignOrdering(Lo.getNode(), DAG.GetOrdering(Op.getNode())); + DAG.AssignOrdering(Hi.getNode(), DAG.GetOrdering(Op.getNode())); } void DAGTypeLegalizer::GetExpandedFloat(SDValue Op, SDValue &Lo, @@ -814,6 +827,10 @@ void DAGTypeLegalizer::SetExpandedFloat(SDValue Op, SDValue Lo, assert(Entry.first.getNode() == 0 && "Node already expanded"); Entry.first = Lo; Entry.second = Hi; + + // Propagate ordering + DAG.AssignOrdering(Lo.getNode(), DAG.GetOrdering(Op.getNode())); + DAG.AssignOrdering(Hi.getNode(), DAG.GetOrdering(Op.getNode())); } void DAGTypeLegalizer::GetSplitVector(SDValue Op, SDValue &Lo, @@ -843,6 +860,10 @@ void DAGTypeLegalizer::SetSplitVector(SDValue Op, SDValue Lo, assert(Entry.first.getNode() == 0 && "Node already split"); Entry.first = Lo; Entry.second = Hi; + + // Propagate ordering + DAG.AssignOrdering(Lo.getNode(), DAG.GetOrdering(Op.getNode())); + DAG.AssignOrdering(Hi.getNode(), DAG.GetOrdering(Op.getNode())); } void DAGTypeLegalizer::SetWidenedVector(SDValue Op, SDValue Result) { @@ -854,6 +875,9 @@ void DAGTypeLegalizer::SetWidenedVector(SDValue Op, SDValue Result) { SDValue &OpEntry = WidenedVectors[Op]; assert(OpEntry.getNode() == 0 && "Node already widened!"); OpEntry = Result; + + // Propagate node ordering + DAG.AssignOrdering(Result.getNode(), DAG.GetOrdering(Op.getNode())); } @@ -919,8 +943,11 @@ bool DAGTypeLegalizer::CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult) { // Make everything that once used N's values now use those in Results instead. assert(Results.size() == N->getNumValues() && "Custom lowering returned the wrong number of results!"); - for (unsigned i = 0, e = Results.size(); i != e; ++i) + for (unsigned i = 0, e = Results.size(); i != e; ++i) { ReplaceValueWith(SDValue(N, i), Results[i]); + // Propagate node ordering + DAG.AssignOrdering(Results[i].getNode(), DAG.GetOrdering(N)); + } return true; } @@ -1020,50 +1047,20 @@ SDValue DAGTypeLegalizer::LibCallify(RTLIB::Libcall LC, SDNode *N, unsigned NumOps = N->getNumOperands(); DebugLoc dl = N->getDebugLoc(); if (NumOps == 0) { - return MakeLibCall(LC, N->getValueType(0), 0, 0, isSigned, dl); + return TLI.makeLibCall(DAG, LC, N->getValueType(0), 0, 0, isSigned, dl); } else if (NumOps == 1) { SDValue Op = N->getOperand(0); - return MakeLibCall(LC, N->getValueType(0), &Op, 1, isSigned, dl); + return TLI.makeLibCall(DAG, LC, N->getValueType(0), &Op, 1, isSigned, dl); } else if (NumOps == 2) { SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; - return MakeLibCall(LC, N->getValueType(0), Ops, 2, isSigned, dl); + return TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops, 2, isSigned, dl); } SmallVector<SDValue, 8> Ops(NumOps); for (unsigned i = 0; i < NumOps; ++i) Ops[i] = N->getOperand(i); - return MakeLibCall(LC, N->getValueType(0), &Ops[0], NumOps, isSigned, dl); -} - -/// MakeLibCall - Generate a libcall taking the given operands as arguments and -/// returning a result of type RetVT. -SDValue DAGTypeLegalizer::MakeLibCall(RTLIB::Libcall LC, EVT RetVT, - const SDValue *Ops, unsigned NumOps, - bool isSigned, DebugLoc dl) { - TargetLowering::ArgListTy Args; - Args.reserve(NumOps); - - TargetLowering::ArgListEntry Entry; - for (unsigned i = 0; i != NumOps; ++i) { - Entry.Node = Ops[i]; - Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext()); - Entry.isSExt = isSigned; - Entry.isZExt = !isSigned; - Args.push_back(Entry); - } - SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), - TLI.getPointerTy()); - - Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); - TargetLowering:: - CallLoweringInfo CLI(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false, - false, 0, TLI.getLibcallCallingConv(LC), - /*isTailCall=*/false, - /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, - Callee, Args, DAG, dl); - std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI); - - return CallInfo.first; + return TLI.makeLibCall(DAG, LC, N->getValueType(0), + &Ops[0], NumOps, isSigned, dl); } // ExpandChainLibCall - Expand a node into a call to a libcall. Similar to diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 20b7ce6..54ea926 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -17,12 +17,12 @@ #define SELECTIONDAG_LEGALIZETYPES_H #define DEBUG_TYPE "legalize-types" -#include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/Target/TargetLowering.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/CodeGen/SelectionDAG.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" +#include "llvm/Target/TargetLowering.h" namespace llvm { @@ -80,35 +80,35 @@ private: /// PromotedIntegers - For integer nodes that are below legal width, this map /// indicates what promoted value to use. - DenseMap<SDValue, SDValue> PromotedIntegers; + SmallDenseMap<SDValue, SDValue, 8> PromotedIntegers; /// ExpandedIntegers - For integer nodes that need to be expanded this map /// indicates which operands are the expanded version of the input. - DenseMap<SDValue, std::pair<SDValue, SDValue> > ExpandedIntegers; + SmallDenseMap<SDValue, std::pair<SDValue, SDValue>, 8> ExpandedIntegers; /// SoftenedFloats - For floating point nodes converted to integers of /// the same size, this map indicates the converted value to use. - DenseMap<SDValue, SDValue> SoftenedFloats; + SmallDenseMap<SDValue, SDValue, 8> SoftenedFloats; /// ExpandedFloats - For float nodes that need to be expanded this map /// indicates which operands are the expanded version of the input. - DenseMap<SDValue, std::pair<SDValue, SDValue> > ExpandedFloats; + SmallDenseMap<SDValue, std::pair<SDValue, SDValue>, 8> ExpandedFloats; /// ScalarizedVectors - For nodes that are <1 x ty>, this map indicates the /// scalar value of type 'ty' to use. - DenseMap<SDValue, SDValue> ScalarizedVectors; + SmallDenseMap<SDValue, SDValue, 8> ScalarizedVectors; /// SplitVectors - For nodes that need to be split this map indicates /// which operands are the expanded version of the input. - DenseMap<SDValue, std::pair<SDValue, SDValue> > SplitVectors; + SmallDenseMap<SDValue, std::pair<SDValue, SDValue>, 8> SplitVectors; /// WidenedVectors - For vector nodes that need to be widened, indicates /// the widened value to use. - DenseMap<SDValue, SDValue> WidenedVectors; + SmallDenseMap<SDValue, SDValue, 8> WidenedVectors; /// ReplacedValues - For values that have been replaced with another, /// indicates the replacement value to use. - DenseMap<SDValue, SDValue> ReplacedValues; + SmallDenseMap<SDValue, SDValue, 8> ReplacedValues; /// Worklist - This defines a worklist of nodes to process. In order to be /// pushed onto this worklist, all operands of a node must have already been @@ -159,9 +159,6 @@ private: SDValue GetVectorElementPointer(SDValue VecPtr, EVT EltVT, SDValue Index); SDValue JoinIntegers(SDValue Lo, SDValue Hi); SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned); - SDValue MakeLibCall(RTLIB::Libcall LC, EVT RetVT, - const SDValue *Ops, unsigned NumOps, bool isSigned, - DebugLoc dl); std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); @@ -433,9 +430,6 @@ private: SDValue SoftenFloatOp_SETCC(SDNode *N); SDValue SoftenFloatOp_STORE(SDNode *N, unsigned OpNo); - void SoftenSetCCOperands(SDValue &NewLHS, SDValue &NewRHS, - ISD::CondCode &CCCode, DebugLoc dl); - //===--------------------------------------------------------------------===// // Float Expansion Support: LegalizeFloatTypes.cpp //===--------------------------------------------------------------------===// @@ -471,6 +465,7 @@ private: void ExpandFloatRes_FP_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FPOW (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FPOWI (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandFloatRes_FREM (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FRINT (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FSIN (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandFloatRes_FSQRT (SDNode *N, SDValue &Lo, SDValue &Hi); @@ -536,6 +531,7 @@ private: // Vector Operand Scalarization: <1 x ty> -> ty. bool ScalarizeVectorOperand(SDNode *N, unsigned OpNo); SDValue ScalarizeVecOp_BITCAST(SDNode *N); + SDValue ScalarizeVecOp_EXTEND(SDNode *N); SDValue ScalarizeVecOp_CONCAT_VECTORS(SDNode *N); SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N); SDValue ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo); @@ -578,6 +574,7 @@ private: // Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>. bool SplitVectorOperand(SDNode *N, unsigned OpNo); + SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo); SDValue SplitVecOp_UnaryOp(SDNode *N); SDValue SplitVecOp_BITCAST(SDNode *N); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp index 6bcb3b2..222d1c0 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp @@ -20,7 +20,7 @@ //===----------------------------------------------------------------------===// #include "LegalizeTypes.h" -#include "llvm/DataLayout.h" +#include "llvm/IR/DataLayout.h" using namespace llvm; //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 22f8d51..c6e066e 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -40,7 +40,7 @@ class VectorLegalizer { /// LegalizedNodes - For nodes that are of legal width, and that have more /// than one use, this map indicates what regularized operand to use. This /// allows us to avoid legalizing the same thing more than once. - DenseMap<SDValue, SDValue> LegalizedNodes; + SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes; // Adds a node to the translation cache void AddLegalizedOperand(SDValue From, SDValue To) { @@ -61,6 +61,8 @@ class VectorLegalizer { // Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if // SINT_TO_FLOAT and SHR on vectors isn't legal. SDValue ExpandUINT_TO_FLOAT(SDValue Op); + // Implement expansion for SIGN_EXTEND_INREG using SRL and SRA. + SDValue ExpandSEXTINREG(SDValue Op); // Implement vselect in terms of XOR, AND, OR when blend is not supported // by the target. SDValue ExpandVSELECT(SDValue Op); @@ -83,6 +85,25 @@ class VectorLegalizer { }; bool VectorLegalizer::Run() { + // Before we start legalizing vector nodes, check if there are any vectors. + bool HasVectors = false; + for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), + E = prior(DAG.allnodes_end()); I != llvm::next(E); ++I) { + // Check if the values of the nodes contain vectors. We don't need to check + // the operands because we are going to check their values at some point. + for (SDNode::value_iterator J = I->value_begin(), E = I->value_end(); + J != E; ++J) + HasVectors |= J->isVector(); + + // If we found a vector node we can start the legalization. + if (HasVectors) + break; + } + + // If this basic block has no vectors then no need to legalize vectors. + if (!HasVectors) + return false; + // The legalize process is inherently a bottom-up recursive process (users // legalize their uses before themselves). Given infinite stack space, we // could just start legalizing on the root and traverse the whole graph. In @@ -142,9 +163,9 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { } else if (Op.getOpcode() == ISD::STORE) { StoreSDNode *ST = cast<StoreSDNode>(Op.getNode()); EVT StVT = ST->getMemoryVT(); - EVT ValVT = ST->getValue().getValueType(); + MVT ValVT = ST->getValue().getSimpleValueType(); if (StVT.isVector() && ST->isTruncatingStore()) - switch (TLI.getTruncStoreAction(ValVT, StVT)) { + switch (TLI.getTruncStoreAction(ValVT, StVT.getSimpleVT())) { default: llvm_unreachable("This action is not supported yet!"); case TargetLowering::Legal: return TranslateLegalizeResults(Op, Result); @@ -221,6 +242,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { case ISD::FRINT: case ISD::FNEARBYINT: case ISD::FFLOOR: + case ISD::FP_ROUND: + case ISD::FP_EXTEND: case ISD::FMA: case ISD::SIGN_EXTEND_INREG: QueryType = Node->getValueType(0); @@ -260,7 +283,9 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { // FALL THROUGH } case TargetLowering::Expand: - if (Node->getOpcode() == ISD::VSELECT) + if (Node->getOpcode() == ISD::SIGN_EXTEND_INREG) + Result = ExpandSEXTINREG(Op); + else if (Node->getOpcode() == ISD::VSELECT) Result = ExpandVSELECT(Op); else if (Node->getOpcode() == ISD::SELECT) Result = ExpandSELECT(Op); @@ -291,10 +316,10 @@ SDValue VectorLegalizer::PromoteVectorOp(SDValue Op) { // Vector "promotion" is basically just bitcasting and doing the operation // in a different type. For example, x86 promotes ISD::AND on v2i32 to // v1i64. - EVT VT = Op.getValueType(); + MVT VT = Op.getSimpleValueType(); assert(Op.getNode()->getNumValues() == 1 && "Can't promote a vector with multiple results!"); - EVT NVT = TLI.getTypeToPromoteTo(Op.getOpcode(), VT); + MVT NVT = TLI.getTypeToPromoteTo(Op.getOpcode(), VT); DebugLoc dl = Op.getDebugLoc(); SmallVector<SDValue, 4> Operands(Op.getNumOperands()); @@ -357,30 +382,135 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) { EVT SrcVT = LD->getMemoryVT(); ISD::LoadExtType ExtType = LD->getExtensionType(); - SmallVector<SDValue, 8> LoadVals; + SmallVector<SDValue, 8> Vals; SmallVector<SDValue, 8> LoadChains; unsigned NumElem = SrcVT.getVectorNumElements(); - unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8; - for (unsigned Idx=0; Idx<NumElem; Idx++) { - SDValue ScalarLoad = DAG.getExtLoad(ExtType, dl, - Op.getNode()->getValueType(0).getScalarType(), - Chain, BasePTR, LD->getPointerInfo().getWithOffset(Idx * Stride), - SrcVT.getScalarType(), - LD->isVolatile(), LD->isNonTemporal(), - LD->getAlignment()); + EVT SrcEltVT = SrcVT.getScalarType(); + EVT DstEltVT = Op.getNode()->getValueType(0).getScalarType(); + + if (SrcVT.getVectorNumElements() > 1 && !SrcEltVT.isByteSized()) { + // When elements in a vector is not byte-addressable, we cannot directly + // load each element by advancing pointer, which could only address bytes. + // Instead, we load all significant words, mask bits off, and concatenate + // them to form each element. Finally, they are extended to destination + // scalar type to build the destination vector. + EVT WideVT = TLI.getPointerTy(); + + assert(WideVT.isRound() && + "Could not handle the sophisticated case when the widest integer is" + " not power of 2."); + assert(WideVT.bitsGE(SrcEltVT) && + "Type is not legalized?"); + + unsigned WideBytes = WideVT.getStoreSize(); + unsigned Offset = 0; + unsigned RemainingBytes = SrcVT.getStoreSize(); + SmallVector<SDValue, 8> LoadVals; + + while (RemainingBytes > 0) { + SDValue ScalarLoad; + unsigned LoadBytes = WideBytes; + + if (RemainingBytes >= LoadBytes) { + ScalarLoad = DAG.getLoad(WideVT, dl, Chain, BasePTR, + LD->getPointerInfo().getWithOffset(Offset), + LD->isVolatile(), LD->isNonTemporal(), + LD->isInvariant(), LD->getAlignment()); + } else { + EVT LoadVT = WideVT; + while (RemainingBytes < LoadBytes) { + LoadBytes >>= 1; // Reduce the load size by half. + LoadVT = EVT::getIntegerVT(*DAG.getContext(), LoadBytes << 3); + } + ScalarLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, WideVT, Chain, BasePTR, + LD->getPointerInfo().getWithOffset(Offset), + LoadVT, LD->isVolatile(), + LD->isNonTemporal(), LD->getAlignment()); + } - BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, - DAG.getIntPtrConstant(Stride)); + RemainingBytes -= LoadBytes; + Offset += LoadBytes; + BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, + DAG.getIntPtrConstant(LoadBytes)); + + LoadVals.push_back(ScalarLoad.getValue(0)); + LoadChains.push_back(ScalarLoad.getValue(1)); + } + + // Extract bits, pack and extend/trunc them into destination type. + unsigned SrcEltBits = SrcEltVT.getSizeInBits(); + SDValue SrcEltBitMask = DAG.getConstant((1U << SrcEltBits) - 1, WideVT); + + unsigned BitOffset = 0; + unsigned WideIdx = 0; + unsigned WideBits = WideVT.getSizeInBits(); + + for (unsigned Idx = 0; Idx != NumElem; ++Idx) { + SDValue Lo, Hi, ShAmt; + + if (BitOffset < WideBits) { + ShAmt = DAG.getConstant(BitOffset, TLI.getShiftAmountTy(WideVT)); + Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt); + Lo = DAG.getNode(ISD::AND, dl, WideVT, Lo, SrcEltBitMask); + } - LoadVals.push_back(ScalarLoad.getValue(0)); - LoadChains.push_back(ScalarLoad.getValue(1)); + BitOffset += SrcEltBits; + if (BitOffset >= WideBits) { + WideIdx++; + Offset -= WideBits; + if (Offset > 0) { + ShAmt = DAG.getConstant(SrcEltBits - Offset, + TLI.getShiftAmountTy(WideVT)); + Hi = DAG.getNode(ISD::SHL, dl, WideVT, LoadVals[WideIdx], ShAmt); + Hi = DAG.getNode(ISD::AND, dl, WideVT, Hi, SrcEltBitMask); + } + } + + if (Hi.getNode()) + Lo = DAG.getNode(ISD::OR, dl, WideVT, Lo, Hi); + + switch (ExtType) { + default: llvm_unreachable("Unknown extended-load op!"); + case ISD::EXTLOAD: + Lo = DAG.getAnyExtOrTrunc(Lo, dl, DstEltVT); + break; + case ISD::ZEXTLOAD: + Lo = DAG.getZExtOrTrunc(Lo, dl, DstEltVT); + break; + case ISD::SEXTLOAD: + ShAmt = DAG.getConstant(WideBits - SrcEltBits, + TLI.getShiftAmountTy(WideVT)); + Lo = DAG.getNode(ISD::SHL, dl, WideVT, Lo, ShAmt); + Lo = DAG.getNode(ISD::SRA, dl, WideVT, Lo, ShAmt); + Lo = DAG.getSExtOrTrunc(Lo, dl, DstEltVT); + break; + } + Vals.push_back(Lo); + } + } else { + unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8; + + for (unsigned Idx=0; Idx<NumElem; Idx++) { + SDValue ScalarLoad = DAG.getExtLoad(ExtType, dl, + Op.getNode()->getValueType(0).getScalarType(), + Chain, BasePTR, LD->getPointerInfo().getWithOffset(Idx * Stride), + SrcVT.getScalarType(), + LD->isVolatile(), LD->isNonTemporal(), + LD->getAlignment()); + + BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, + DAG.getIntPtrConstant(Stride)); + + Vals.push_back(ScalarLoad.getValue(0)); + LoadChains.push_back(ScalarLoad.getValue(1)); + } } SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LoadChains[0], LoadChains.size()); SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl, - Op.getNode()->getValueType(0), &LoadVals[0], LoadVals.size()); + Op.getNode()->getValueType(0), &Vals[0], Vals.size()); AddLegalizedOperand(Op.getValue(0), Value); AddLegalizedOperand(Op.getValue(1), NewChain); @@ -499,6 +629,26 @@ SDValue VectorLegalizer::ExpandSELECT(SDValue Op) { return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val); } +SDValue VectorLegalizer::ExpandSEXTINREG(SDValue Op) { + EVT VT = Op.getValueType(); + + // Make sure that the SRA and SHL instructions are available. + if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand || + TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand) + return DAG.UnrollVectorOp(Op.getNode()); + + DebugLoc DL = Op.getDebugLoc(); + EVT OrigTy = cast<VTSDNode>(Op->getOperand(1))->getVT(); + + unsigned BW = VT.getScalarType().getSizeInBits(); + unsigned OrigBW = OrigTy.getScalarType().getSizeInBits(); + SDValue ShiftSz = DAG.getConstant(BW - OrigBW, VT); + + Op = Op.getOperand(0); + Op = DAG.getNode(ISD::SHL, DL, VT, Op, ShiftSz); + return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz); +} + SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) { // Implement VSELECT in terms of XOR, AND, OR // on platforms which do not support blend natively. diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index d51a6eb..5ec8535 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -21,7 +21,7 @@ //===----------------------------------------------------------------------===// #include "LegalizeTypes.h" -#include "llvm/DataLayout.h" +#include "llvm/IR/DataLayout.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -365,6 +365,11 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { case ISD::BITCAST: Res = ScalarizeVecOp_BITCAST(N); break; + case ISD::ANY_EXTEND: + case ISD::ZERO_EXTEND: + case ISD::SIGN_EXTEND: + Res = ScalarizeVecOp_EXTEND(N); + break; case ISD::CONCAT_VECTORS: Res = ScalarizeVecOp_CONCAT_VECTORS(N); break; @@ -400,6 +405,21 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { N->getValueType(0), Elt); } +/// ScalarizeVecOp_EXTEND - If the value to extend is a vector that needs +/// to be scalarized, it must be <1 x ty>. Extend the element instead. +SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTEND(SDNode *N) { + assert(N->getValueType(0).getVectorNumElements() == 1 && + "Unexected vector type!"); + SDValue Elt = GetScalarizedVector(N->getOperand(0)); + SmallVector<SDValue, 1> Ops(1); + Ops[0] = DAG.getNode(N->getOpcode(), N->getDebugLoc(), + N->getValueType(0).getScalarType(), Elt); + // Revectorize the result so the types line up with what the uses of this + // expression expect. + return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), + &Ops[0], 1); +} + /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - /// use a BUILD_VECTOR instead. SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { @@ -1030,7 +1050,9 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { case ISD::STORE: Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); break; - + case ISD::VSELECT: + Res = SplitVecOp_VSELECT(N, OpNo); + break; case ISD::CTTZ: case ISD::CTLZ: case ISD::CTPOP: @@ -1064,6 +1086,58 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { return false; } +SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { + // The only possibility for an illegal operand is the mask, since result type + // legalization would have handled this node already otherwise. + assert(OpNo == 0 && "Illegal operand must be mask"); + + SDValue Mask = N->getOperand(0); + SDValue Src0 = N->getOperand(1); + SDValue Src1 = N->getOperand(2); + DebugLoc DL = N->getDebugLoc(); + EVT MaskVT = Mask.getValueType(); + assert(MaskVT.isVector() && "VSELECT without a vector mask?"); + + SDValue Lo, Hi; + GetSplitVector(N->getOperand(0), Lo, Hi); + assert(Lo.getValueType() == Hi.getValueType() && + "Lo and Hi have differing types");; + + unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); + unsigned HiNumElts = Hi.getValueType().getVectorNumElements(); + assert(LoNumElts == HiNumElts && "Asymmetric vector split?"); + + LLVMContext &Ctx = *DAG.getContext(); + SDValue Zero = DAG.getIntPtrConstant(0); + SDValue LoElts = DAG.getIntPtrConstant(LoNumElts); + EVT Src0VT = Src0.getValueType(); + EVT Src0EltTy = Src0VT.getVectorElementType(); + EVT MaskEltTy = MaskVT.getVectorElementType(); + + EVT LoOpVT = EVT::getVectorVT(Ctx, Src0EltTy, LoNumElts); + EVT LoMaskVT = EVT::getVectorVT(Ctx, MaskEltTy, LoNumElts); + EVT HiOpVT = EVT::getVectorVT(Ctx, Src0EltTy, HiNumElts); + EVT HiMaskVT = EVT::getVectorVT(Ctx, MaskEltTy, HiNumElts); + + SDValue LoOp0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoOpVT, Src0, Zero); + SDValue LoOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoOpVT, Src1, Zero); + + SDValue HiOp0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiOpVT, Src0, LoElts); + SDValue HiOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiOpVT, Src1, LoElts); + + SDValue LoMask = + DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoMaskVT, Mask, Zero); + SDValue HiMask = + DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiMaskVT, Mask, LoElts); + + SDValue LoSelect = + DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); + SDValue HiSelect = + DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); + + return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); +} + SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { // The result has a legal vector type, but the input needs splitting. EVT ResVT = N->getValueType(0); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp index c3794d5..473e138 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp @@ -21,13 +21,13 @@ #define DEBUG_TYPE "scheduler" #include "llvm/CodeGen/ResourcePriorityQueue.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/SelectionDAGNodes.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; @@ -94,9 +94,9 @@ ResourcePriorityQueue::numberRCValPredInSU(SUnit *SU, unsigned RCId) { continue; for (unsigned i = 0, e = ScegN->getNumValues(); i != e; ++i) { - EVT VT = ScegN->getValueType(i); + MVT VT = ScegN->getSimpleValueType(i); if (TLI->isTypeLegal(VT) - && (TLI->getRegClassFor(VT)->getID() == RCId)) { + && (TLI->getRegClassFor(VT)->getID() == RCId)) { NumberDeps++; break; } @@ -132,9 +132,9 @@ unsigned ResourcePriorityQueue::numberRCValSuccInSU(SUnit *SU, for (unsigned i = 0, e = ScegN->getNumOperands(); i != e; ++i) { const SDValue &Op = ScegN->getOperand(i); - EVT VT = Op.getNode()->getValueType(Op.getResNo()); + MVT VT = Op.getNode()->getSimpleValueType(Op.getResNo()); if (TLI->isTypeLegal(VT) - && (TLI->getRegClassFor(VT)->getID() == RCId)) { + && (TLI->getRegClassFor(VT)->getID() == RCId)) { NumberDeps++; break; } @@ -332,7 +332,7 @@ signed ResourcePriorityQueue::rawRegPressureDelta(SUnit *SU, unsigned RCId) { // Gen estimate. for (unsigned i = 0, e = SU->getNode()->getNumValues(); i != e; ++i) { - EVT VT = SU->getNode()->getValueType(i); + MVT VT = SU->getNode()->getSimpleValueType(i); if (TLI->isTypeLegal(VT) && TLI->getRegClassFor(VT) && TLI->getRegClassFor(VT)->getID() == RCId) @@ -341,7 +341,7 @@ signed ResourcePriorityQueue::rawRegPressureDelta(SUnit *SU, unsigned RCId) { // Kill estimate. for (unsigned i = 0, e = SU->getNode()->getNumOperands(); i != e; ++i) { const SDValue &Op = SU->getNode()->getOperand(i); - EVT VT = Op.getNode()->getValueType(Op.getResNo()); + MVT VT = Op.getNode()->getSimpleValueType(Op.getResNo()); if (isa<ConstantSDNode>(Op.getNode())) continue; @@ -485,7 +485,7 @@ void ResourcePriorityQueue::scheduledNode(SUnit *SU) { if (ScegN->isMachineOpcode()) { // Estimate generated regs. for (unsigned i = 0, e = ScegN->getNumValues(); i != e; ++i) { - EVT VT = ScegN->getValueType(i); + MVT VT = ScegN->getSimpleValueType(i); if (TLI->isTypeLegal(VT)) { const TargetRegisterClass *RC = TLI->getRegClassFor(VT); @@ -496,7 +496,7 @@ void ResourcePriorityQueue::scheduledNode(SUnit *SU) { // Estimate killed regs. for (unsigned i = 0, e = ScegN->getNumOperands(); i != e; ++i) { const SDValue &Op = ScegN->getOperand(i); - EVT VT = Op.getNode()->getValueType(Op.getResNo()); + MVT VT = Op.getNode()->getSimpleValueType(Op.getResNo()); if (TLI->isTypeLegal(VT)) { const TargetRegisterClass *RC = TLI->getRegClassFor(VT); @@ -604,10 +604,8 @@ SUnit *ResourcePriorityQueue::pop() { std::vector<SUnit *>::iterator Best = Queue.begin(); if (!DisableDFASched) { signed BestCost = SUSchedulingCost(*Best); - for (std::vector<SUnit *>::iterator I = Queue.begin(), + for (std::vector<SUnit *>::iterator I = llvm::next(Queue.begin()), E = Queue.end(); I != E; ++I) { - if (*I == *Best) - continue; if (SUSchedulingCost(*I) > BestCost) { BestCost = SUSchedulingCost(*I); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h b/contrib/llvm/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h index 2dcb229..4af7172 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h @@ -15,8 +15,8 @@ #define LLVM_CODEGEN_SDNODEDBGVALUE_H #include "llvm/ADT/SmallVector.h" -#include "llvm/Support/DebugLoc.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/DebugLoc.h" namespace llvm { diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SDNodeOrdering.h b/contrib/llvm/lib/CodeGen/SelectionDAG/SDNodeOrdering.h index d2269f8..7e7b897 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SDNodeOrdering.h +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SDNodeOrdering.h @@ -33,8 +33,10 @@ class SDNodeOrdering { public: SDNodeOrdering() {} - void add(const SDNode *Node, unsigned O) { - OrderMap[Node] = O; + void add(const SDNode *Node, unsigned NewOrder) { + unsigned &OldOrder = OrderMap[Node]; + if (OldOrder == 0 || (OldOrder > 0 && NewOrder < OldOrder)) + OldOrder = NewOrder; } void remove(const SDNode *Node) { DenseMap<const SDNode*, unsigned>::iterator Itr = OrderMap.find(Node); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp index 2ecdd89..d1f36cb 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp @@ -12,20 +12,20 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "pre-RA-sched" -#include "ScheduleDAGSDNodes.h" -#include "InstrEmitter.h" -#include "llvm/InlineAsm.h" #include "llvm/CodeGen/SchedulerRegistry.h" -#include "llvm/CodeGen/SelectionDAGISel.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Support/Debug.h" +#include "InstrEmitter.h" +#include "ScheduleDAGSDNodes.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/CodeGen/SelectionDAGISel.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; STATISTIC(NumUnfolds, "Number of nodes unfolded"); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index c554569..c009cfc 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -16,22 +16,23 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "pre-RA-sched" -#include "ScheduleDAGSDNodes.h" -#include "llvm/InlineAsm.h" #include "llvm/CodeGen/SchedulerRegistry.h" -#include "llvm/CodeGen/SelectionDAGISel.h" -#include "llvm/CodeGen/ScheduleHazardRecognizer.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetLowering.h" +#include "ScheduleDAGSDNodes.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/ScheduleHazardRecognizer.h" +#include "llvm/CodeGen/SelectionDAGISel.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/InlineAsm.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <climits> using namespace llvm; @@ -142,6 +143,12 @@ private: std::vector<SUnit*> LiveRegDefs; std::vector<SUnit*> LiveRegGens; + // Collect interferences between physical register use/defs. + // Each interference is an SUnit and set of physical registers. + SmallVector<SUnit*, 4> Interferences; + typedef DenseMap<SUnit*, SmallVector<unsigned, 4> > LRegsMapT; + LRegsMapT LRegsMap; + /// Topo - A topological ordering for SUnits which permits fast IsReachable /// and similar queries. ScheduleDAGTopologicalSort Topo; @@ -156,7 +163,7 @@ public: CodeGenOpt::Level OptLevel) : ScheduleDAGSDNodes(mf), NeedLatency(needlatency), AvailableQueue(availqueue), CurCycle(0), - Topo(SUnits) { + Topo(SUnits, NULL) { const TargetMachine &tm = mf.getTarget(); if (DisableSchedCycles || !NeedLatency) @@ -225,6 +232,8 @@ private: SmallVector<SUnit*, 2>&); bool DelayForLiveRegsBottomUp(SUnit*, SmallVector<unsigned, 4>&); + void releaseInterferences(unsigned Reg = 0); + SUnit *PickNodeToScheduleBottomUp(); void ListScheduleBottomUp(); @@ -268,14 +277,23 @@ static void GetCostForDef(const ScheduleDAGSDNodes::RegDefIter &RegDefPos, const TargetRegisterInfo *TRI, unsigned &RegClass, unsigned &Cost, const MachineFunction &MF) { - EVT VT = RegDefPos.GetValue(); + MVT VT = RegDefPos.GetValue(); // Special handling for untyped values. These values can only come from // the expansion of custom DAG-to-DAG patterns. if (VT == MVT::Untyped) { const SDNode *Node = RegDefPos.GetNode(); - unsigned Opcode = Node->getMachineOpcode(); + // Special handling for CopyFromReg of untyped values. + if (!Node->isMachineOpcode() && Node->getOpcode() == ISD::CopyFromReg) { + unsigned Reg = cast<RegisterSDNode>(Node->getOperand(1))->getReg(); + const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(Reg); + RegClass = RC->getID(); + Cost = 1; + return; + } + + unsigned Opcode = Node->getMachineOpcode(); if (Opcode == TargetOpcode::REG_SEQUENCE) { unsigned DstRCIdx = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue(); const TargetRegisterClass *RC = TRI->getRegClass(DstRCIdx); @@ -312,6 +330,7 @@ void ScheduleDAGRRList::Schedule() { LiveRegDefs.resize(TRI->getNumRegs() + 1, NULL); LiveRegGens.resize(TRI->getNumRegs() + 1, NULL); CallSeqEndForStart.clear(); + assert(Interferences.empty() && LRegsMap.empty() && "stale Interferences"); // Build the scheduling graph. BuildSchedGraph(NULL); @@ -725,6 +744,7 @@ void ScheduleDAGRRList::ScheduleNodeBottomUp(SUnit *SU) { --NumLiveRegs; LiveRegDefs[I->getReg()] = NULL; LiveRegGens[I->getReg()] = NULL; + releaseInterferences(I->getReg()); } } // Release the special call resource dependence, if this is the beginning @@ -739,6 +759,7 @@ void ScheduleDAGRRList::ScheduleNodeBottomUp(SUnit *SU) { --NumLiveRegs; LiveRegDefs[CallResource] = NULL; LiveRegGens[CallResource] = NULL; + releaseInterferences(CallResource); } } @@ -794,6 +815,7 @@ void ScheduleDAGRRList::UnscheduleNodeBottomUp(SUnit *SU) { --NumLiveRegs; LiveRegDefs[I->getReg()] = NULL; LiveRegGens[I->getReg()] = NULL; + releaseInterferences(I->getReg()); } } @@ -821,6 +843,7 @@ void ScheduleDAGRRList::UnscheduleNodeBottomUp(SUnit *SU) { --NumLiveRegs; LiveRegDefs[CallResource] = NULL; LiveRegGens[CallResource] = NULL; + releaseInterferences(CallResource); } } @@ -881,9 +904,6 @@ void ScheduleDAGRRList::BacktrackBottomUp(SUnit *SU, SUnit *BtSU) { SUnit *OldSU = Sequence.back(); while (true) { Sequence.pop_back(); - if (SU->isSucc(OldSU)) - // Don't try to remove SU from AvailableQueue. - SU->isAvailable = false; // FIXME: use ready cycle instead of height CurCycle = OldSU->getHeight(); UnscheduleNodeBottomUp(OldSU); @@ -1305,34 +1325,60 @@ DelayForLiveRegsBottomUp(SUnit *SU, SmallVector<unsigned, 4> &LRegs) { return !LRegs.empty(); } +void ScheduleDAGRRList::releaseInterferences(unsigned Reg) { + // Add the nodes that aren't ready back onto the available list. + for (unsigned i = Interferences.size(); i > 0; --i) { + SUnit *SU = Interferences[i-1]; + LRegsMapT::iterator LRegsPos = LRegsMap.find(SU); + if (Reg) { + SmallVector<unsigned, 4> &LRegs = LRegsPos->second; + if (std::find(LRegs.begin(), LRegs.end(), Reg) == LRegs.end()) + continue; + } + SU->isPending = false; + // The interfering node may no longer be available due to backtracking. + // Furthermore, it may have been made available again, in which case it is + // now already in the AvailableQueue. + if (SU->isAvailable && !SU->NodeQueueId) { + DEBUG(dbgs() << " Repushing SU #" << SU->NodeNum << '\n'); + AvailableQueue->push(SU); + } + if (i < Interferences.size()) + Interferences[i-1] = Interferences.back(); + Interferences.pop_back(); + LRegsMap.erase(LRegsPos); + } +} + /// Return a node that can be scheduled in this cycle. Requirements: /// (1) Ready: latency has been satisfied /// (2) No Hazards: resources are available /// (3) No Interferences: may unschedule to break register interferences. SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() { - SmallVector<SUnit*, 4> Interferences; - DenseMap<SUnit*, SmallVector<unsigned, 4> > LRegsMap; - - SUnit *CurSU = AvailableQueue->pop(); + SUnit *CurSU = AvailableQueue->empty() ? 0 : AvailableQueue->pop(); while (CurSU) { SmallVector<unsigned, 4> LRegs; if (!DelayForLiveRegsBottomUp(CurSU, LRegs)) break; - LRegsMap.insert(std::make_pair(CurSU, LRegs)); - - CurSU->isPending = true; // This SU is not in AvailableQueue right now. - Interferences.push_back(CurSU); + DEBUG(dbgs() << " Interfering reg " << + (LRegs[0] == TRI->getNumRegs() ? "CallResource" + : TRI->getName(LRegs[0])) + << " SU #" << CurSU->NodeNum << '\n'); + std::pair<LRegsMapT::iterator, bool> LRegsPair = + LRegsMap.insert(std::make_pair(CurSU, LRegs)); + if (LRegsPair.second) { + CurSU->isPending = true; // This SU is not in AvailableQueue right now. + Interferences.push_back(CurSU); + } + else { + assert(CurSU->isPending && "Intereferences are pending"); + // Update the interference with current live regs. + LRegsPair.first->second = LRegs; + } CurSU = AvailableQueue->pop(); } - if (CurSU) { - // Add the nodes that aren't ready back onto the available list. - for (unsigned i = 0, e = Interferences.size(); i != e; ++i) { - Interferences[i]->isPending = false; - assert(Interferences[i]->isAvailable && "must still be available"); - AvailableQueue->push(Interferences[i]); - } + if (CurSU) return CurSU; - } // All candidates are delayed due to live physical reg dependencies. // Try backtracking, code duplication, or inserting cross class copies @@ -1353,6 +1399,7 @@ SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() { } } if (!WillCreateCycle(TrySU, BtSU)) { + // BacktrackBottomUp mutates Interferences! BacktrackBottomUp(TrySU, BtSU); // Force the current node to be scheduled before the node that @@ -1362,19 +1409,19 @@ SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() { if (!BtSU->isPending) AvailableQueue->remove(BtSU); } + DEBUG(dbgs() << "ARTIFICIAL edge from SU(" << BtSU->NodeNum << ") to SU(" + << TrySU->NodeNum << ")\n"); AddPred(TrySU, SDep(BtSU, SDep::Artificial)); // If one or more successors has been unscheduled, then the current - // node is no longer avaialable. Schedule a successor that's now - // available instead. - if (!TrySU->isAvailable) { + // node is no longer available. + if (!TrySU->isAvailable) CurSU = AvailableQueue->pop(); - } else { + AvailableQueue->remove(TrySU); CurSU = TrySU; - TrySU->isPending = false; - Interferences.erase(Interferences.begin()+i); } + // Interferences has been mutated. We must break. break; } } @@ -1425,17 +1472,7 @@ SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() { TrySU->isAvailable = false; CurSU = NewDef; } - assert(CurSU && "Unable to resolve live physical register dependencies!"); - - // Add the nodes that aren't ready back onto the available list. - for (unsigned i = 0, e = Interferences.size(); i != e; ++i) { - Interferences[i]->isPending = false; - // May no longer be available due to backtracking. - if (Interferences[i]->isAvailable) { - AvailableQueue->push(Interferences[i]); - } - } return CurSU; } @@ -1456,7 +1493,7 @@ void ScheduleDAGRRList::ListScheduleBottomUp() { // While Available queue is not empty, grab the node with the highest // priority. If it is not ready put it back. Schedule the node. Sequence.reserve(SUnits.size()); - while (!AvailableQueue->empty()) { + while (!AvailableQueue->empty() || !Interferences.empty()) { DEBUG(dbgs() << "\nExamining Available:\n"; AvailableQueue->dump(this)); @@ -1939,7 +1976,7 @@ bool RegReductionPQBase::MayReduceRegPressure(SUnit *SU) const { unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); for (unsigned i = 0; i != NumDefs; ++i) { - EVT VT = N->getValueType(i); + MVT VT = N->getSimpleValueType(i); if (!N->hasAnyUseOfValue(i)) continue; unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); @@ -1973,7 +2010,7 @@ int RegReductionPQBase::RegPressureDiff(SUnit *SU, unsigned &LiveUses) const { } for (ScheduleDAGSDNodes::RegDefIter RegDefPos(PredSU, scheduleDAG); RegDefPos.IsValid(); RegDefPos.Advance()) { - EVT VT = RegDefPos.GetValue(); + MVT VT = RegDefPos.GetValue(); unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); if (RegPressure[RCId] >= RegLimit[RCId]) ++PDiff; @@ -1986,7 +2023,7 @@ int RegReductionPQBase::RegPressureDiff(SUnit *SU, unsigned &LiveUses) const { unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); for (unsigned i = 0; i != NumDefs; ++i) { - EVT VT = N->getValueType(i); + MVT VT = N->getSimpleValueType(i); if (!N->hasAnyUseOfValue(i)) continue; unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); @@ -2097,7 +2134,7 @@ void RegReductionPQBase::unscheduledNode(SUnit *SU) { const SDNode *PN = PredSU->getNode(); if (!PN->isMachineOpcode()) { if (PN->getOpcode() == ISD::CopyFromReg) { - EVT VT = PN->getValueType(0); + MVT VT = PN->getSimpleValueType(0); unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); } @@ -2109,14 +2146,14 @@ void RegReductionPQBase::unscheduledNode(SUnit *SU) { if (POpc == TargetOpcode::EXTRACT_SUBREG || POpc == TargetOpcode::INSERT_SUBREG || POpc == TargetOpcode::SUBREG_TO_REG) { - EVT VT = PN->getValueType(0); + MVT VT = PN->getSimpleValueType(0); unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); continue; } unsigned NumDefs = TII->get(PN->getMachineOpcode()).getNumDefs(); for (unsigned i = 0; i != NumDefs; ++i) { - EVT VT = PN->getValueType(i); + MVT VT = PN->getSimpleValueType(i); if (!PN->hasAnyUseOfValue(i)) continue; unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); @@ -2133,7 +2170,7 @@ void RegReductionPQBase::unscheduledNode(SUnit *SU) { if (SU->NumSuccs && N->isMachineOpcode()) { unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs(); for (unsigned i = NumDefs, e = N->getNumValues(); i != e; ++i) { - EVT VT = N->getValueType(i); + MVT VT = N->getSimpleValueType(i); if (VT == MVT::Glue || VT == MVT::Other) continue; if (!N->hasAnyUseOfValue(i)) diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index a197fcb..b22440d 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -13,26 +13,26 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "pre-RA-sched" -#include "SDNodeDbgValue.h" #include "ScheduleDAGSDNodes.h" #include "InstrEmitter.h" -#include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/MC/MCInstrItineraries.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetSubtargetInfo.h" +#include "SDNodeDbgValue.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/MC/MCInstrItineraries.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; STATISTIC(LoadsClustered, "Number of loads clustered together"); @@ -562,7 +562,7 @@ void ScheduleDAGSDNodes::RegDefIter::Advance() { for (;DefIdx < NodeNumDefs; ++DefIdx) { if (!Node->hasAnyUseOfValue(DefIdx)) continue; - ValueType = Node->getValueType(DefIdx); + ValueType = Node->getSimpleValueType(DefIdx); ++DefIdx; return; // Found a normal regdef. } diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h index 907356f..2ff37e0 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h @@ -15,8 +15,8 @@ #ifndef SCHEDULEDAGSDNODES_H #define SCHEDULEDAGSDNODES_H +#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/ScheduleDAG.h" -#include "llvm/CodeGen/SelectionDAG.h" namespace llvm { /// ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs. @@ -135,13 +135,13 @@ namespace llvm { const SDNode *Node; unsigned DefIdx; unsigned NodeNumDefs; - EVT ValueType; + MVT ValueType; public: RegDefIter(const SUnit *SU, const ScheduleDAGSDNodes *SD); bool IsValid() const { return Node != NULL; } - EVT GetValue() const { + MVT GetValue() const { assert(IsValid() && "bad iterator"); return ValueType; } diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp index 30f03ac..58aa1fe 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp @@ -19,19 +19,19 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "pre-RA-sched" +#include "llvm/CodeGen/SchedulerRegistry.h" #include "ScheduleDAGSDNodes.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LatencyPriorityQueue.h" +#include "llvm/CodeGen/ResourcePriorityQueue.h" #include "llvm/CodeGen/ScheduleHazardRecognizer.h" -#include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAGISel.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/IR/DataLayout.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/CodeGen/ResourcePriorityQueue.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <climits> using namespace llvm; @@ -123,6 +123,8 @@ void ScheduleDAGVLIW::releaseSucc(SUnit *SU, const SDep &D) { llvm_unreachable(0); } #endif + assert(!D.isWeak() && "unexpected artificial DAG edge"); + --SuccSU->NumPredsLeft; SuccSU->setDepthToAtLeast(SU->getDepth() + D.getLatency()); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index f000ce3..6424431 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -12,42 +12,43 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/SelectionDAG.h" -#include "SDNodeOrdering.h" #include "SDNodeDbgValue.h" -#include "llvm/CallingConv.h" -#include "llvm/Constants.h" -#include "llvm/DebugInfo.h" -#include "llvm/DerivedTypes.h" -#include "llvm/Function.h" -#include "llvm/GlobalAlias.h" -#include "llvm/GlobalVariable.h" -#include "llvm/Intrinsics.h" +#include "SDNodeOrdering.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Assembly/Writer.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetSelectionDAGInfo.h" -#include "llvm/Target/TargetOptions.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetIntrinsicInfo.h" -#include "llvm/Target/TargetMachine.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/CallingConv.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MathExtras.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Support/Mutex.h" -#include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetIntrinsicInfo.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSelectionDAGInfo.h" #include <algorithm> #include <cmath> using namespace llvm; @@ -59,18 +60,6 @@ static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { return Res; } -static const fltSemantics *EVTToAPFloatSemantics(EVT VT) { - switch (VT.getSimpleVT().SimpleTy) { - default: llvm_unreachable("Unknown FP format"); - case MVT::f16: return &APFloat::IEEEhalf; - case MVT::f32: return &APFloat::IEEEsingle; - case MVT::f64: return &APFloat::IEEEdouble; - case MVT::f80: return &APFloat::x87DoubleExtended; - case MVT::f128: return &APFloat::IEEEquad; - case MVT::ppcf128: return &APFloat::PPCDoubleDouble; - } -} - // Default null implementations of the callbacks. void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} @@ -94,7 +83,8 @@ bool ConstantFPSDNode::isValueValidForType(EVT VT, // convert modifies in place, so make a copy. APFloat Val2 = APFloat(Val); bool losesInfo; - (void) Val2.convert(*EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven, + (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), + APFloat::rmNearestTiesToEven, &losesInfo); return !losesInfo; } @@ -884,15 +874,17 @@ unsigned SelectionDAG::getEVTAlignment(EVT VT) const { // EntryNode could meaningfully have debug info if we can find it... SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) : TM(tm), TLI(*tm.getTargetLowering()), TSI(*tm.getSelectionDAGInfo()), - OptLevel(OL), EntryNode(ISD::EntryToken, DebugLoc(), getVTList(MVT::Other)), + TTI(0), OptLevel(OL), EntryNode(ISD::EntryToken, DebugLoc(), + getVTList(MVT::Other)), Root(getEntryNode()), Ordering(0), UpdateListeners(0) { AllNodes.push_back(&EntryNode); Ordering = new SDNodeOrdering(); DbgInfo = new SDDbgInfo(); } -void SelectionDAG::init(MachineFunction &mf) { +void SelectionDAG::init(MachineFunction &mf, const TargetTransformInfo *tti) { MF = &mf; + TTI = tti; Context = &mf.getFunction()->getContext(); } @@ -1074,10 +1066,11 @@ SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { return getConstantFP(APFloat((float)Val), VT, isTarget); else if (EltVT==MVT::f64) return getConstantFP(APFloat(Val), VT, isTarget); - else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::f16) { + else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 || + EltVT==MVT::f16) { bool ignored; APFloat apf = APFloat(Val); - apf.convert(*EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, + apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, &ignored); return getConstantFP(apf, VT, isTarget); } else @@ -1525,7 +1518,7 @@ SDValue SelectionDAG::getMDNode(const MDNode *MD) { /// the target's desired shift amount type. SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { EVT OpTy = Op.getValueType(); - MVT ShTy = TLI.getShiftAmountTy(LHSTy); + EVT ShTy = TLI.getShiftAmountTy(LHSTy); if (OpTy == ShTy || OpTy.isVector()) return Op; ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; @@ -1924,7 +1917,8 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, APInt &KnownZero, } case ISD::LOAD: { LoadSDNode *LD = cast<LoadSDNode>(Op); - if (ISD::isZEXTLoad(Op.getNode())) { + // If this is a ZEXTLoad and we are looking at the loaded value. + if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { EVT VT = LD->getMemoryVT(); unsigned MemBits = VT.getScalarType().getSizeInBits(); KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); @@ -2294,17 +2288,20 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ break; } - // Handle LOADX separately here. EXTLOAD case will fallthrough. - if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { - unsigned ExtType = LD->getExtensionType(); - switch (ExtType) { - default: break; - case ISD::SEXTLOAD: // '17' bits known - Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); - return VTBits-Tmp+1; - case ISD::ZEXTLOAD: // '16' bits known - Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); - return VTBits-Tmp; + // If we are looking at the loaded value of the SDNode. + if (Op.getResNo() == 0) { + // Handle LOADX separately here. EXTLOAD case will fallthrough. + if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { + unsigned ExtType = LD->getExtensionType(); + switch (ExtType) { + default: break; + case ISD::SEXTLOAD: // '17' bits known + Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); + return VTBits-Tmp+1; + case ISD::ZEXTLOAD: // '16' bits known + Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); + return VTBits-Tmp; + } } } @@ -2438,7 +2435,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); case ISD::UINT_TO_FP: case ISD::SINT_TO_FP: { - APFloat apf(APInt::getNullValue(VT.getSizeInBits())); + APFloat apf(EVTToAPFloatSemantics(VT), + APInt::getNullValue(VT.getSizeInBits())); (void)apf.convertFromAPInt(Val, Opcode==ISD::SINT_TO_FP, APFloat::rmNearestTiesToEven); @@ -2446,9 +2444,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, } case ISD::BITCAST: if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) - return getConstantFP(APFloat(Val), VT); + return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT); else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) - return getConstantFP(APFloat(Val), VT); + return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT); break; case ISD::BSWAP: return getConstant(Val.byteSwap(), VT); @@ -2495,7 +2493,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, bool ignored; // This can return overflow, underflow, or inexact; we don't care. // FIXME need to be more flexible about rounding mode. - (void)V.convert(*EVTToAPFloatSemantics(VT), + (void)V.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven, &ignored); return getConstantFP(V, VT); } @@ -2686,44 +2684,117 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, return SDValue(N, 0); } -SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, - EVT VT, - ConstantSDNode *Cst1, - ConstantSDNode *Cst2) { - const APInt &C1 = Cst1->getAPIntValue(), &C2 = Cst2->getAPIntValue(); +SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT, + SDNode *Cst1, SDNode *Cst2) { + SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs; + SmallVector<SDValue, 4> Outputs; + EVT SVT = VT.getScalarType(); - switch (Opcode) { - case ISD::ADD: return getConstant(C1 + C2, VT); - case ISD::SUB: return getConstant(C1 - C2, VT); - case ISD::MUL: return getConstant(C1 * C2, VT); - case ISD::UDIV: - if (C2.getBoolValue()) return getConstant(C1.udiv(C2), VT); - break; - case ISD::UREM: - if (C2.getBoolValue()) return getConstant(C1.urem(C2), VT); - break; - case ISD::SDIV: - if (C2.getBoolValue()) return getConstant(C1.sdiv(C2), VT); - break; - case ISD::SREM: - if (C2.getBoolValue()) return getConstant(C1.srem(C2), VT); - break; - case ISD::AND: return getConstant(C1 & C2, VT); - case ISD::OR: return getConstant(C1 | C2, VT); - case ISD::XOR: return getConstant(C1 ^ C2, VT); - case ISD::SHL: return getConstant(C1 << C2, VT); - case ISD::SRL: return getConstant(C1.lshr(C2), VT); - case ISD::SRA: return getConstant(C1.ashr(C2), VT); - case ISD::ROTL: return getConstant(C1.rotl(C2), VT); - case ISD::ROTR: return getConstant(C1.rotr(C2), VT); - default: break; + ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1); + ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2); + if (Scalar1 && Scalar2) { + // Scalar instruction. + Inputs.push_back(std::make_pair(Scalar1, Scalar2)); + } else { + // For vectors extract each constant element into Inputs so we can constant + // fold them individually. + BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1); + BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2); + if (!BV1 || !BV2) + return SDValue(); + + assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!"); + + for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) { + ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I)); + ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I)); + if (!V1 || !V2) // Not a constant, bail. + return SDValue(); + + // Avoid BUILD_VECTOR nodes that perform implicit truncation. + // FIXME: This is valid and could be handled by truncating the APInts. + if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT) + return SDValue(); + + Inputs.push_back(std::make_pair(V1, V2)); + } } - return SDValue(); + // We have a number of constant values, constant fold them element by element. + for (unsigned I = 0, E = Inputs.size(); I != E; ++I) { + const APInt &C1 = Inputs[I].first->getAPIntValue(); + const APInt &C2 = Inputs[I].second->getAPIntValue(); + + switch (Opcode) { + case ISD::ADD: + Outputs.push_back(getConstant(C1 + C2, SVT)); + break; + case ISD::SUB: + Outputs.push_back(getConstant(C1 - C2, SVT)); + break; + case ISD::MUL: + Outputs.push_back(getConstant(C1 * C2, SVT)); + break; + case ISD::UDIV: + if (!C2.getBoolValue()) + return SDValue(); + Outputs.push_back(getConstant(C1.udiv(C2), SVT)); + break; + case ISD::UREM: + if (!C2.getBoolValue()) + return SDValue(); + Outputs.push_back(getConstant(C1.urem(C2), SVT)); + break; + case ISD::SDIV: + if (!C2.getBoolValue()) + return SDValue(); + Outputs.push_back(getConstant(C1.sdiv(C2), SVT)); + break; + case ISD::SREM: + if (!C2.getBoolValue()) + return SDValue(); + Outputs.push_back(getConstant(C1.srem(C2), SVT)); + break; + case ISD::AND: + Outputs.push_back(getConstant(C1 & C2, SVT)); + break; + case ISD::OR: + Outputs.push_back(getConstant(C1 | C2, SVT)); + break; + case ISD::XOR: + Outputs.push_back(getConstant(C1 ^ C2, SVT)); + break; + case ISD::SHL: + Outputs.push_back(getConstant(C1 << C2, SVT)); + break; + case ISD::SRL: + Outputs.push_back(getConstant(C1.lshr(C2), SVT)); + break; + case ISD::SRA: + Outputs.push_back(getConstant(C1.ashr(C2), SVT)); + break; + case ISD::ROTL: + Outputs.push_back(getConstant(C1.rotl(C2), SVT)); + break; + case ISD::ROTR: + Outputs.push_back(getConstant(C1.rotr(C2), SVT)); + break; + default: + return SDValue(); + } + } + + // Handle the scalar case first. + if (Outputs.size() == 1) + return Outputs.back(); + + // Otherwise build a big vector out of the scalar elements we generated. + return getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, Outputs.data(), + Outputs.size()); } -SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, - SDValue N1, SDValue N2) { +SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, SDValue N1, + SDValue N2) { ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); switch (Opcode) { @@ -2845,6 +2916,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, "Shift operators return type must be the same as their first arg"); assert(VT.isInteger() && N2.getValueType().isInteger() && "Shifts only work on integers"); + assert((!VT.isVector() || VT == N2.getValueType()) && + "Vector shift amounts must be in the same as their first arg"); // Verify that the shift amount VT is bit enough to hold valid shift // amounts. This catches things like trying to shift an i1024 value by an // i8, which is easy to fall into in generic code that uses @@ -3019,16 +3092,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, } } - if (N1C) { - if (N2C) { - SDValue SV = FoldConstantArithmetic(Opcode, VT, N1C, N2C); - if (SV.getNode()) return SV; - } else { // Cannonicalize constant to RHS if commutative - if (isCommutativeBinOp(Opcode)) { - std::swap(N1C, N2C); - std::swap(N1, N2); - } - } + // Perform trivial constant folding. + SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()); + if (SV.getNode()) return SV; + + // Canonicalize constant to RHS if commutative. + if (N1C && !N2C && isCommutativeBinOp(Opcode)) { + std::swap(N1C, N2C); + std::swap(N1, N2); } // Constant fold FP operations. @@ -3036,7 +3107,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); if (N1CFP) { if (!N2CFP && isCommutativeBinOp(Opcode)) { - // Cannonicalize constant to RHS if commutative + // Canonicalize constant to RHS if commutative. std::swap(N1CFP, N2CFP); std::swap(N1, N2); } else if (N2CFP) { @@ -3080,7 +3151,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, bool ignored; // This can return overflow, underflow, or inexact; we don't care. // FIXME need to be more flexible about rounding mode. - (void)V.convert(*EVTToAPFloatSemantics(VT), + (void)V.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven, &ignored); return getConstantFP(V, VT); } @@ -3312,17 +3383,6 @@ SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { &ArgChains[0], ArgChains.size()); } -/// SplatByte - Distribute ByteVal over NumBits bits. -static APInt SplatByte(unsigned NumBits, uint8_t ByteVal) { - APInt Val = APInt(NumBits, ByteVal); - unsigned Shift = 8; - for (unsigned i = NumBits; i > 8; i >>= 1) { - Val = (Val << Shift) | Val; - Shift <<= 1; - } - return Val; -} - /// getMemsetValue - Vectorized representation of the memset value /// operand. static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, @@ -3331,17 +3391,18 @@ static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, unsigned NumBits = VT.getScalarType().getSizeInBits(); if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { - APInt Val = SplatByte(NumBits, C->getZExtValue() & 255); + assert(C->getAPIntValue().getBitWidth() == 8); + APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); if (VT.isInteger()) return DAG.getConstant(Val, VT); - return DAG.getConstantFP(APFloat(Val), VT); + return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT); } Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); if (NumBits > 8) { // Use a multiplication with 0x010101... to extend the input to the // required length. - APInt Magic = SplatByte(NumBits, 0x01); + APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); } @@ -3370,10 +3431,11 @@ static SDValue getMemsetStringVal(EVT VT, DebugLoc dl, SelectionDAG &DAG, } assert(!VT.isVector() && "Can't handle vector type here!"); - unsigned NumVTBytes = VT.getSizeInBits() / 8; + unsigned NumVTBits = VT.getSizeInBits(); + unsigned NumVTBytes = NumVTBits / 8; unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size())); - uint64_t Val = 0; + APInt Val(NumVTBits, 0); if (TLI.isLittleEndian()) { for (unsigned i = 0; i != NumBytes; ++i) Val |= (uint64_t)(unsigned char)Str[i] << i*8; @@ -3382,7 +3444,12 @@ static SDValue getMemsetStringVal(EVT VT, DebugLoc dl, SelectionDAG &DAG, Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8; } - return DAG.getConstant(Val, VT); + // If the "cost" of materializing the integer immediate is 1 or free, then + // it is cost effective to turn the load into the immediate. + const TargetTransformInfo *TTI = DAG.getTargetTransformInfo(); + if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) < 2) + return DAG.getConstant(Val, VT); + return SDValue(0, 0); } /// getMemBasePlusOffset - Returns base and offset node for the @@ -3420,8 +3487,10 @@ static bool isMemSrcFromString(SDValue Src, StringRef &Str) { static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, unsigned Limit, uint64_t Size, unsigned DstAlign, unsigned SrcAlign, - bool IsZeroVal, + bool IsMemset, + bool ZeroMemset, bool MemcpyStrSrc, + bool AllowOverlap, SelectionDAG &DAG, const TargetLowering &TLI) { assert((SrcAlign == 0 || SrcAlign >= DstAlign) && @@ -3434,7 +3503,7 @@ static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does // not need to be loaded. EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, - IsZeroVal, MemcpyStrSrc, + IsMemset, ZeroMemset, MemcpyStrSrc, DAG.getMachineFunction()); if (VT == MVT::Other) { @@ -3464,21 +3533,51 @@ static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, unsigned VTSize = VT.getSizeInBits() / 8; while (VTSize > Size) { // For now, only use non-vector load / store's for the left-over pieces. + EVT NewVT = VT; + unsigned NewVTSize; + + bool Found = false; if (VT.isVector() || VT.isFloatingPoint()) { - VT = MVT::i64; - while (!TLI.isTypeLegal(VT)) - VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); - VTSize = VT.getSizeInBits() / 8; - } else { - // This can result in a type that is not legal on the target, e.g. - // 1 or 2 bytes on PPC. - VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); - VTSize >>= 1; + NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; + if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) && + TLI.isSafeMemOpType(NewVT.getSimpleVT())) + Found = true; + else if (NewVT == MVT::i64 && + TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) && + TLI.isSafeMemOpType(MVT::f64)) { + // i64 is usually not legal on 32-bit targets, but f64 may be. + NewVT = MVT::f64; + Found = true; + } + } + + if (!Found) { + do { + NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); + if (NewVT == MVT::i8) + break; + } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT())); + } + NewVTSize = NewVT.getSizeInBits() / 8; + + // If the new VT cannot cover all of the remaining bits, then consider + // issuing a (or a pair of) unaligned and overlapping load / store. + // FIXME: Only does this for 64-bit or more since we don't have proper + // cost model for unaligned load / store. + bool Fast; + if (NumMemOps && AllowOverlap && + VTSize >= 8 && NewVTSize < Size && + TLI.allowsUnalignedMemoryAccesses(VT, &Fast) && Fast) + VTSize = Size; + else { + VT = NewVT; + VTSize = NewVTSize; } } if (++NumMemOps > Limit) return false; + MemOps.push_back(VT); Size -= VTSize; } @@ -3507,8 +3606,8 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo *MFI = MF.getFrameInfo(); bool OptSize = - MF.getFunction()->getFnAttributes(). - hasAttribute(Attributes::OptimizeForSize); + MF.getFunction()->getAttributes(). + hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) DstAlignCanChange = true; @@ -3523,12 +3622,21 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, if (!FindOptimalMemOpLowering(MemOps, Limit, Size, (DstAlignCanChange ? 0 : Align), (isZeroStr ? 0 : SrcAlign), - true, CopyFromStr, DAG, TLI)) + false, false, CopyFromStr, true, DAG, TLI)) return SDValue(); if (DstAlignCanChange) { Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); + + // Don't promote to an alignment that would require dynamic stack + // realignment. + const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); + if (!TRI->needsStackRealignment(MF)) + while (NewAlign > Align && + TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign)) + NewAlign /= 2; + if (NewAlign > Align) { // Give the stack frame object a larger alignment if needed. if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) @@ -3545,6 +3653,14 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, unsigned VTSize = VT.getSizeInBits() / 8; SDValue Value, Store; + if (VTSize > Size) { + // Issuing an unaligned load / store pair that overlaps with the previous + // pair. Adjust the offset accordingly. + assert(i == NumMemOps-1 && i != 0); + SrcOff -= VTSize - Size; + DstOff -= VTSize - Size; + } + if (CopyFromStr && (isZeroStr || (VT.isInteger() && !VT.isVector()))) { // It's unlikely a store of a vector immediate can be done in a single @@ -3553,11 +3669,14 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, // FIXME: Handle other cases where store of vector immediate is done in // a single instruction. Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff)); - Store = DAG.getStore(Chain, dl, Value, - getMemBasePlusOffset(Dst, DstOff, DAG), - DstPtrInfo.getWithOffset(DstOff), isVol, - false, Align); - } else { + if (Value.getNode()) + Store = DAG.getStore(Chain, dl, Value, + getMemBasePlusOffset(Dst, DstOff, DAG), + DstPtrInfo.getWithOffset(DstOff), isVol, + false, Align); + } + + if (!Store.getNode()) { // The type might not be legal for the target. This should only happen // if the type is smaller than a legal type, as on PPC, so the right // thing to do is generate a LoadExt/StoreTrunc pair. These simplify @@ -3577,6 +3696,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, OutChains.push_back(Store); SrcOff += VTSize; DstOff += VTSize; + Size -= VTSize; } return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, @@ -3601,8 +3721,8 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, bool DstAlignCanChange = false; MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo *MFI = MF.getFrameInfo(); - bool OptSize = MF.getFunction()->getFnAttributes(). - hasAttribute(Attributes::OptimizeForSize); + bool OptSize = MF.getFunction()->getAttributes(). + hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) DstAlignCanChange = true; @@ -3612,8 +3732,8 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); if (!FindOptimalMemOpLowering(MemOps, Limit, Size, - (DstAlignCanChange ? 0 : Align), - SrcAlign, true, false, DAG, TLI)) + (DstAlignCanChange ? 0 : Align), SrcAlign, + false, false, false, false, DAG, TLI)) return SDValue(); if (DstAlignCanChange) { @@ -3680,8 +3800,8 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, bool DstAlignCanChange = false; MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo *MFI = MF.getFrameInfo(); - bool OptSize = MF.getFunction()->getFnAttributes(). - hasAttribute(Attributes::OptimizeForSize); + bool OptSize = MF.getFunction()->getAttributes(). + hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) DstAlignCanChange = true; @@ -3689,7 +3809,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), Size, (DstAlignCanChange ? 0 : Align), 0, - IsZeroVal, false, DAG, TLI)) + true, IsZeroVal, false, true, DAG, TLI)) return SDValue(); if (DstAlignCanChange) { @@ -3716,6 +3836,13 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, for (unsigned i = 0; i < NumMemOps; i++) { EVT VT = MemOps[i]; + unsigned VTSize = VT.getSizeInBits() / 8; + if (VTSize > Size) { + // Issuing an unaligned load / store pair that overlaps with the previous + // pair. Adjust the offset accordingly. + assert(i == NumMemOps-1 && i != 0); + DstOff -= VTSize - Size; + } // If this store is smaller than the largest store see whether we can get // the smaller value for free with a truncate. @@ -3734,6 +3861,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, isVol, false, Align); OutChains.push_back(Store); DstOff += VT.getSizeInBits() / 8; + Size -= VTSize; } return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, @@ -3745,6 +3873,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, unsigned Align, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) { + assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); // Check to see if we should lower the memcpy to loads and stores first. // For cases within the target-specified limits, this is the best choice. @@ -3812,6 +3941,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, unsigned Align, bool isVol, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) { + assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); // Check to see if we should lower the memmove to loads and stores first. // For cases within the target-specified limits, this is the best choice. @@ -3866,6 +3996,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, MachinePointerInfo DstPtrInfo) { + assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); // Check to see if we should lower the memset to stores first. // For cases within the target-specified limits, this is the best choice. @@ -4577,7 +4708,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, } SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, - const std::vector<EVT> &ResultTys, + ArrayRef<EVT> ResultTys, const SDValue *Ops, unsigned NumOps) { return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()), Ops, NumOps); @@ -5229,7 +5360,7 @@ SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, MachineSDNode * SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, - const std::vector<EVT> &ResultTys, + ArrayRef<EVT> ResultTys, const SDValue *Ops, unsigned NumOps) { SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); return getMachineNode(Opcode, dl, VTs, Ops, NumOps); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 3fbf7c2..ce40cd6 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -12,51 +12,51 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "isel" -#include "SDNodeDbgValue.h" #include "SelectionDAGBuilder.h" +#include "SDNodeDbgValue.h" #include "llvm/ADT/BitVector.h" -#include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/Constants.h" -#include "llvm/CallingConv.h" -#include "llvm/DebugInfo.h" -#include "llvm/DerivedTypes.h" -#include "llvm/Function.h" -#include "llvm/GlobalVariable.h" -#include "llvm/InlineAsm.h" -#include "llvm/Instructions.h" -#include "llvm/Intrinsics.h" -#include "llvm/IntrinsicInst.h" -#include "llvm/LLVMContext.h" -#include "llvm/Module.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/FastISel.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" -#include "llvm/CodeGen/GCStrategy.h" #include "llvm/CodeGen/GCMetadata.h" -#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/GCStrategy.h" #include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/DataLayout.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/CallingConv.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/IntegersSubsetMapping.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetIntrinsicInfo.h" #include "llvm/Target/TargetLibraryInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetOptions.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/IntegersSubsetMapping.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MathExtras.h" -#include "llvm/Support/raw_ostream.h" #include <algorithm> using namespace llvm; @@ -89,7 +89,7 @@ static const unsigned MaxParallelChains = 64; static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL, const SDValue *Parts, unsigned NumParts, - EVT PartVT, EVT ValueVT, const Value *V); + MVT PartVT, EVT ValueVT, const Value *V); /// getCopyFromParts - Create a value that contains the specified legal parts /// combined into the value they represent. If the parts combine to a type @@ -98,7 +98,7 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL, /// (ISD::AssertSext). static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc DL, const SDValue *Parts, - unsigned NumParts, EVT PartVT, EVT ValueVT, + unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, ISD::NodeType AssertOp = ISD::DELETED_NODE) { if (ValueVT.isVector()) @@ -161,7 +161,7 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc DL, } } else if (PartVT.isFloatingPoint()) { // FP split into multiple FP parts (for ppcf128) - assert(ValueVT == EVT(MVT::ppcf128) && PartVT == EVT(MVT::f64) && + assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && "Unexpected split"); SDValue Lo, Hi; Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]); @@ -179,25 +179,25 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc DL, } // There is now one part, held in Val. Correct it to match ValueVT. - PartVT = Val.getValueType(); + EVT PartEVT = Val.getValueType(); - if (PartVT == ValueVT) + if (PartEVT == ValueVT) return Val; - if (PartVT.isInteger() && ValueVT.isInteger()) { - if (ValueVT.bitsLT(PartVT)) { + if (PartEVT.isInteger() && ValueVT.isInteger()) { + if (ValueVT.bitsLT(PartEVT)) { // For a truncate, see if we have any information to // indicate whether the truncated bits will always be // zero or sign-extension. if (AssertOp != ISD::DELETED_NODE) - Val = DAG.getNode(AssertOp, DL, PartVT, Val, + Val = DAG.getNode(AssertOp, DL, PartEVT, Val, DAG.getValueType(ValueVT)); return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); } return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); } - if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { + if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { // FP_ROUND's are always exact here. if (ValueVT.bitsLT(Val.getValueType())) return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, @@ -206,7 +206,7 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc DL, return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val); } - if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) + if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits()) return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); llvm_unreachable("Unknown mismatch!"); @@ -219,7 +219,7 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc DL, /// ValueVT (ISD::AssertSext). static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL, const SDValue *Parts, unsigned NumParts, - EVT PartVT, EVT ValueVT, const Value *V) { + MVT PartVT, EVT ValueVT, const Value *V) { assert(ValueVT.isVector() && "Not a vector value"); assert(NumParts > 0 && "No parts to assemble!"); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); @@ -227,7 +227,8 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL, // Handle a multi-element vector. if (NumParts > 1) { - EVT IntermediateVT, RegisterVT; + EVT IntermediateVT; + MVT RegisterVT; unsigned NumIntermediates; unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, @@ -235,7 +236,7 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL, assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); NumParts = NumRegs; // Silence a compiler warning. assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); - assert(RegisterVT == Parts[0].getValueType() && + assert(RegisterVT == Parts[0].getSimpleValueType() && "Part type doesn't match part!"); // Assemble the parts into intermediate operands. @@ -265,31 +266,31 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL, } // There is now one part, held in Val. Correct it to match ValueVT. - PartVT = Val.getValueType(); + EVT PartEVT = Val.getValueType(); - if (PartVT == ValueVT) + if (PartEVT == ValueVT) return Val; - if (PartVT.isVector()) { + if (PartEVT.isVector()) { // If the element type of the source/dest vectors are the same, but the // parts vector has more elements than the value vector, then we have a // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the // elements we want. - if (PartVT.getVectorElementType() == ValueVT.getVectorElementType()) { - assert(PartVT.getVectorNumElements() > ValueVT.getVectorNumElements() && + if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) { + assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && "Cannot narrow, it would be a lossy transformation"); return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val, DAG.getIntPtrConstant(0)); } // Vector/Vector bitcast. - if (ValueVT.getSizeInBits() == PartVT.getSizeInBits()) + if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); - assert(PartVT.getVectorNumElements() == ValueVT.getVectorNumElements() && + assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && "Cannot handle this kind of promotion"); // Promoted vector extract - bool Smaller = ValueVT.bitsLE(PartVT); + bool Smaller = ValueVT.bitsLE(PartEVT); return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), DL, ValueVT, Val); @@ -297,7 +298,7 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL, // Trivial bitcast if the types are the same size and the destination // vector type is legal. - if (PartVT.getSizeInBits() == ValueVT.getSizeInBits() && + if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() && TLI.isTypeLegal(ValueVT)) return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); @@ -317,8 +318,8 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL, } if (ValueVT.getVectorNumElements() == 1 && - ValueVT.getVectorElementType() != PartVT) { - bool Smaller = ValueVT.bitsLE(PartVT); + ValueVT.getVectorElementType() != PartEVT) { + bool Smaller = ValueVT.bitsLE(PartEVT); Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), DL, ValueVT.getScalarType(), Val); } @@ -328,14 +329,14 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL, static void getCopyToPartsVector(SelectionDAG &DAG, DebugLoc dl, SDValue Val, SDValue *Parts, unsigned NumParts, - EVT PartVT, const Value *V); + MVT PartVT, const Value *V); /// getCopyToParts - Create a series of nodes that contain the specified value /// split into legal parts. If the parts contain more bits than Val, then, for /// integers, ExtendKind can be used to specify how to generate the extra bits. static void getCopyToParts(SelectionDAG &DAG, DebugLoc DL, SDValue Val, SDValue *Parts, unsigned NumParts, - EVT PartVT, const Value *V, + MVT PartVT, const Value *V, ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { EVT ValueVT = Val.getValueType(); @@ -352,7 +353,8 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc DL, return; assert(!ValueVT.isVector() && "Vector case handled elsewhere"); - if (PartVT == ValueVT) { + EVT PartEVT = PartVT; + if (PartEVT == ValueVT) { assert(NumParts == 1 && "No-op copy with multiple parts!"); Parts[0] = Val; return; @@ -374,7 +376,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc DL, } } else if (PartBits == ValueVT.getSizeInBits()) { // Different types of the same size. - assert(NumParts == 1 && PartVT != ValueVT); + assert(NumParts == 1 && PartEVT != ValueVT); Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { // If the parts cover less bits than value has, truncate the value. @@ -393,7 +395,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc DL, "Failed to tile the value with PartVT!"); if (NumParts == 1) { - if (PartVT != ValueVT) { + if (PartEVT != ValueVT) { LLVMContext &Ctx = *DAG.getContext(); Twine ErrMsg("scalar-to-vector conversion failed"); if (const Instruction *I = dyn_cast_or_null<Instruction>(V)) { @@ -466,20 +468,21 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc DL, /// value split into legal parts. static void getCopyToPartsVector(SelectionDAG &DAG, DebugLoc DL, SDValue Val, SDValue *Parts, unsigned NumParts, - EVT PartVT, const Value *V) { + MVT PartVT, const Value *V) { EVT ValueVT = Val.getValueType(); assert(ValueVT.isVector() && "Not a vector"); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (NumParts == 1) { - if (PartVT == ValueVT) { + EVT PartEVT = PartVT; + if (PartEVT == ValueVT) { // Nothing to do. } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { // Bitconvert vector->vector case. Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); } else if (PartVT.isVector() && - PartVT.getVectorElementType() == ValueVT.getVectorElementType() && - PartVT.getVectorNumElements() > ValueVT.getVectorNumElements()) { + PartEVT.getVectorElementType() == ValueVT.getVectorElementType() && + PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) { EVT ElementVT = PartVT.getVectorElementType(); // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in // undef elements. @@ -499,12 +502,12 @@ static void getCopyToPartsVector(SelectionDAG &DAG, DebugLoc DL, //SDValue UndefElts = DAG.getUNDEF(VectorTy); //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts); } else if (PartVT.isVector() && - PartVT.getVectorElementType().bitsGE( + PartEVT.getVectorElementType().bitsGE( ValueVT.getVectorElementType()) && - PartVT.getVectorNumElements() == ValueVT.getVectorNumElements()) { + PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) { // Promoted vector extract - bool Smaller = PartVT.bitsLE(ValueVT); + bool Smaller = PartEVT.bitsLE(ValueVT); Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND), DL, PartVT, Val); } else{ @@ -524,7 +527,8 @@ static void getCopyToPartsVector(SelectionDAG &DAG, DebugLoc DL, } // Handle a multi-element vector. - EVT IntermediateVT, RegisterVT; + EVT IntermediateVT; + MVT RegisterVT; unsigned NumIntermediates; unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, @@ -589,7 +593,7 @@ namespace { /// getRegisterType member function, however when with physical registers /// it is necessary to have a separate record of the types. /// - SmallVector<EVT, 4> RegVTs; + SmallVector<MVT, 4> RegVTs; /// Regs - This list holds the registers assigned to the values. /// Each legal or promoted value requires one register, and each @@ -600,7 +604,7 @@ namespace { RegsForValue() {} RegsForValue(const SmallVector<unsigned, 4> ®s, - EVT regvt, EVT valuevt) + MVT regvt, EVT valuevt) : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {} RegsForValue(LLVMContext &Context, const TargetLowering &tli, @@ -610,7 +614,7 @@ namespace { for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { EVT ValueVT = ValueVTs[Value]; unsigned NumRegs = tli.getNumRegisters(Context, ValueVT); - EVT RegisterVT = tli.getRegisterType(Context, ValueVT); + MVT RegisterVT = tli.getRegisterType(Context, ValueVT); for (unsigned i = 0; i != NumRegs; ++i) Regs.push_back(Reg + i); RegVTs.push_back(RegisterVT); @@ -621,7 +625,7 @@ namespace { /// areValueTypesLegal - Return true if types of all the values are legal. bool areValueTypesLegal(const TargetLowering &TLI) { for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { - EVT RegisterVT = RegVTs[Value]; + MVT RegisterVT = RegVTs[Value]; if (!TLI.isTypeLegal(RegisterVT)) return false; } @@ -683,7 +687,7 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, // Copy the legal parts from the registers. EVT ValueVT = ValueVTs[Value]; unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT); - EVT RegisterVT = RegVTs[Value]; + MVT RegisterVT = RegVTs[Value]; Parts.resize(NumRegs); for (unsigned i = 0; i != NumRegs; ++i) { @@ -768,10 +772,12 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { EVT ValueVT = ValueVTs[Value]; unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT); - EVT RegisterVT = RegVTs[Value]; + MVT RegisterVT = RegVTs[Value]; + ISD::NodeType ExtendKind = + TLI.isZExtFree(Val, RegisterVT)? ISD::ZERO_EXTEND: ISD::ANY_EXTEND; getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), - &Parts[Part], NumParts, RegisterVT, V); + &Parts[Part], NumParts, RegisterVT, V, ExtendKind); Part += NumParts; } @@ -834,7 +840,7 @@ void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]); - EVT RegisterVT = RegVTs[Value]; + MVT RegisterVT = RegVTs[Value]; for (unsigned i = 0; i != NumRegs; ++i) { assert(Reg < Regs.size() && "Mismatch in # registers expected"); Ops.push_back(DAG.getRegister(Regs[Reg++], RegisterVT)); @@ -967,7 +973,7 @@ void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) { // Build the switch statement using the Instruction.def file. #define HANDLE_INST(NUM, OPCODE, CLASS) \ case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; -#include "llvm/Instruction.def" +#include "llvm/IR/Instruction.def" } // Assign the ordering to the freshly created DAG nodes. @@ -1227,16 +1233,18 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { ISD::NodeType ExtendKind = ISD::ANY_EXTEND; const Function *F = I.getParent()->getParent(); - if (F->getRetAttributes().hasAttribute(Attributes::SExt)) + if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, + Attribute::SExt)) ExtendKind = ISD::SIGN_EXTEND; - else if (F->getRetAttributes().hasAttribute(Attributes::ZExt)) + else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, + Attribute::ZExt)) ExtendKind = ISD::ZERO_EXTEND; if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) - VT = TLI.getTypeForExtArgOrReturn(*DAG.getContext(), VT, ExtendKind); + VT = TLI.getTypeForExtArgOrReturn(VT.getSimpleVT(), ExtendKind); unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), VT); - EVT PartVT = TLI.getRegisterType(*DAG.getContext(), VT); + MVT PartVT = TLI.getRegisterType(*DAG.getContext(), VT); SmallVector<SDValue, 4> Parts(NumParts); getCopyToParts(DAG, getCurDebugLoc(), SDValue(RetOp.getNode(), RetOp.getResNo() + j), @@ -1244,7 +1252,8 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { // 'inreg' on function refers to return value ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); - if (F->getRetAttributes().hasAttribute(Attributes::InReg)) + if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex, + Attribute::InReg)) Flags.setInReg(); // Propagate extension type if any @@ -1758,8 +1767,8 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, Sub = DAG.getZExtOrTrunc(Sub, getCurDebugLoc(), VT); } - B.RegVT = VT; - B.Reg = FuncInfo.CreateReg(VT); + B.RegVT = VT.getSimpleVT(); + B.Reg = FuncInfo.CreateReg(B.RegVT); SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurDebugLoc(), B.Reg, Sub); @@ -1793,7 +1802,7 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, unsigned Reg, BitTestCase &B, MachineBasicBlock *SwitchBB) { - EVT VT = BB.RegVT; + MVT VT = BB.RegVT; SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurDebugLoc(), Reg, VT); SDValue Cmp; @@ -2645,7 +2654,7 @@ void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { SDValue Op1 = getValue(I.getOperand(0)); SDValue Op2 = getValue(I.getOperand(1)); - MVT ShiftTy = TLI.getShiftAmountTy(Op2.getValueType()); + EVT ShiftTy = TLI.getShiftAmountTy(Op2.getValueType()); // Coerce the shift amount to the right type if we can. if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) { @@ -3137,12 +3146,12 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { OI != E; ++OI) { const Value *Idx = *OI; if (StructType *StTy = dyn_cast<StructType>(Ty)) { - unsigned Field = cast<ConstantInt>(Idx)->getZExtValue(); + unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue(); if (Field) { // N = N + Offset uint64_t Offset = TD->getStructLayout(StTy)->getElementOffset(Field); N = DAG.getNode(ISD::ADD, getCurDebugLoc(), N.getValueType(), N, - DAG.getIntPtrConstant(Offset)); + DAG.getConstant(Offset, N.getValueType())); } Ty = StTy->getElementType(Field); @@ -3187,7 +3196,7 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { N.getValueType(), IdxN, DAG.getConstant(Amt, IdxN.getValueType())); } else { - SDValue Scale = DAG.getConstant(ElementSize, TLI.getPointerTy()); + SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType()); IdxN = DAG.getNode(ISD::MUL, getCurDebugLoc(), N.getValueType(), IdxN, Scale); } @@ -3510,7 +3519,7 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { EVT VT = TLI.getValueType(I.getType()); - if (I.getAlignment() * 8 < VT.getSizeInBits()) + if (I.getAlignment() < VT.getSizeInBits() / 8) report_fatal_error("Cannot generate unaligned atomic load"); SDValue L = @@ -3540,7 +3549,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { EVT VT = TLI.getValueType(I.getValueOperand()->getType()); - if (I.getAlignment() * 8 < VT.getSizeInBits()) + if (I.getAlignment() < VT.getSizeInBits() / 8) report_fatal_error("Cannot generate unaligned atomic store"); if (TLI.getInsertFencesForAtomic()) @@ -3654,7 +3663,7 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, /// /// Op = (Op & 0x007fffff) | 0x3f800000; /// -/// where Op is the hexidecimal representation of floating point value. +/// where Op is the hexadecimal representation of floating point value. static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, DebugLoc dl) { SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, @@ -3668,7 +3677,7 @@ GetSignificand(SelectionDAG &DAG, SDValue Op, DebugLoc dl) { /// /// (float)(int)(((Op & 0x7f800000) >> 23) - 127); /// -/// where Op is the hexidecimal representation of floating point value. +/// where Op is the hexadecimal representation of floating point value. static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, DebugLoc dl) { @@ -3684,19 +3693,16 @@ GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, /// getF32Constant - Get 32-bit floating point constant. static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt) { - return DAG.getConstantFP(APFloat(APInt(32, Flt)), MVT::f32); + return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)), + MVT::f32); } -/// visitExp - Lower an exp intrinsic. Handles the special sequences for +/// expandExp - Lower an exp intrinsic. Handles the special sequences for /// limited-precision mode. -void -SelectionDAGBuilder::visitExp(const CallInst &I) { - SDValue result; - DebugLoc dl = getCurDebugLoc(); - - if (getValue(I.getArgOperand(0)).getValueType() == MVT::f32 && +static SDValue expandExp(DebugLoc dl, SDValue Op, SelectionDAG &DAG, + const TargetLowering &TLI) { + if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { - SDValue Op = getValue(I.getArgOperand(0)); // Put the exponent in the right bit position for later addition to the // final result: @@ -3715,6 +3721,7 @@ SelectionDAGBuilder::visitExp(const CallInst &I) { IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, DAG.getConstant(23, TLI.getPointerTy())); + SDValue TwoToFracPartOfX; if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -3728,16 +3735,9 @@ SelectionDAGBuilder::visitExp(const CallInst &I) { SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, getF32Constant(DAG, 0x3f3c50c8)); SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3f7f5e7e)); - SDValue TwoToFracPartOfX = DAG.getNode(ISD::BITCAST, dl,MVT::i32, t5); - - // Add the exponent into the result in integer domain. - SDValue t6 = DAG.getNode(ISD::ADD, dl, MVT::i32, - TwoToFracPartOfX, IntegerPartOfX); - - result = DAG.getNode(ISD::BITCAST, dl, MVT::f32, t6); - } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { + TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, + getF32Constant(DAG, 0x3f7f5e7e)); + } else if (LimitFloatPrecision <= 12) { // For floating-point precision of 12: // // TwoToFractionalPartOfX = @@ -3754,16 +3754,9 @@ SelectionDAGBuilder::visitExp(const CallInst &I) { SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, getF32Constant(DAG, 0x3f324b07)); SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); - SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, - getF32Constant(DAG, 0x3f7ff8fd)); - SDValue TwoToFracPartOfX = DAG.getNode(ISD::BITCAST, dl,MVT::i32, t7); - - // Add the exponent into the result in integer domain. - SDValue t8 = DAG.getNode(ISD::ADD, dl, MVT::i32, - TwoToFracPartOfX, IntegerPartOfX); - - result = DAG.getNode(ISD::BITCAST, dl, MVT::f32, t8); - } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 + TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, + getF32Constant(DAG, 0x3f7ff8fd)); + } else { // LimitFloatPrecision <= 18 // For floating-point precision of 18: // // TwoToFractionalPartOfX = @@ -3792,37 +3785,27 @@ SelectionDAGBuilder::visitExp(const CallInst &I) { SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, getF32Constant(DAG, 0x3f317234)); SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); - SDValue t13 = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, - getF32Constant(DAG, 0x3f800000)); - SDValue TwoToFracPartOfX = DAG.getNode(ISD::BITCAST, dl, - MVT::i32, t13); - - // Add the exponent into the result in integer domain. - SDValue t14 = DAG.getNode(ISD::ADD, dl, MVT::i32, - TwoToFracPartOfX, IntegerPartOfX); - - result = DAG.getNode(ISD::BITCAST, dl, MVT::f32, t14); + TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, + getF32Constant(DAG, 0x3f800000)); } - } else { - // No special expansion. - result = DAG.getNode(ISD::FEXP, dl, - getValue(I.getArgOperand(0)).getValueType(), - getValue(I.getArgOperand(0))); + + // Add the exponent into the result in integer domain. + SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX); + return DAG.getNode(ISD::BITCAST, dl, MVT::f32, + DAG.getNode(ISD::ADD, dl, MVT::i32, + t13, IntegerPartOfX)); } - setValue(&I, result); + // No special expansion. + return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op); } -/// visitLog - Lower a log intrinsic. Handles the special sequences for +/// expandLog - Lower a log intrinsic. Handles the special sequences for /// limited-precision mode. -void -SelectionDAGBuilder::visitLog(const CallInst &I) { - SDValue result; - DebugLoc dl = getCurDebugLoc(); - - if (getValue(I.getArgOperand(0)).getValueType() == MVT::f32 && +static SDValue expandLog(DebugLoc dl, SDValue Op, SelectionDAG &DAG, + const TargetLowering &TLI) { + if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { - SDValue Op = getValue(I.getArgOperand(0)); SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); // Scale the exponent by log(2) [0.69314718f]. @@ -3834,6 +3817,7 @@ SelectionDAGBuilder::visitLog(const CallInst &I) { // exponent of 1. SDValue X = GetSignificand(DAG, Op1, dl); + SDValue LogOfMantissa; if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -3847,12 +3831,9 @@ SelectionDAGBuilder::visitLog(const CallInst &I) { SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, getF32Constant(DAG, 0x3fb3a2b1)); SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); - SDValue LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3f949a29)); - - result = DAG.getNode(ISD::FADD, dl, - MVT::f32, LogOfExponent, LogOfMantissa); - } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { + LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, + getF32Constant(DAG, 0x3f949a29)); + } else if (LimitFloatPrecision <= 12) { // For floating-point precision of 12: // // LogOfMantissa = @@ -3873,12 +3854,9 @@ SelectionDAGBuilder::visitLog(const CallInst &I) { SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, getF32Constant(DAG, 0x40348e95)); SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); - SDValue LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, - getF32Constant(DAG, 0x3fdef31a)); - - result = DAG.getNode(ISD::FADD, dl, - MVT::f32, LogOfExponent, LogOfMantissa); - } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 + LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, + getF32Constant(DAG, 0x3fdef31a)); + } else { // LimitFloatPrecision <= 18 // For floating-point precision of 18: // // LogOfMantissa = @@ -3907,32 +3885,23 @@ SelectionDAGBuilder::visitLog(const CallInst &I) { SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, getF32Constant(DAG, 0x408797cb)); SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); - SDValue LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, - getF32Constant(DAG, 0x4006dcab)); - - result = DAG.getNode(ISD::FADD, dl, - MVT::f32, LogOfExponent, LogOfMantissa); + LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, + getF32Constant(DAG, 0x4006dcab)); } - } else { - // No special expansion. - result = DAG.getNode(ISD::FLOG, dl, - getValue(I.getArgOperand(0)).getValueType(), - getValue(I.getArgOperand(0))); + + return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); } - setValue(&I, result); + // No special expansion. + return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op); } -/// visitLog2 - Lower a log2 intrinsic. Handles the special sequences for +/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for /// limited-precision mode. -void -SelectionDAGBuilder::visitLog2(const CallInst &I) { - SDValue result; - DebugLoc dl = getCurDebugLoc(); - - if (getValue(I.getArgOperand(0)).getValueType() == MVT::f32 && +static SDValue expandLog2(DebugLoc dl, SDValue Op, SelectionDAG &DAG, + const TargetLowering &TLI) { + if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { - SDValue Op = getValue(I.getArgOperand(0)); SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); // Get the exponent. @@ -3944,6 +3913,7 @@ SelectionDAGBuilder::visitLog2(const CallInst &I) { // Different possible minimax approximations of significand in // floating-point for various degrees of accuracy over [1,2]. + SDValue Log2ofMantissa; if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -3955,12 +3925,9 @@ SelectionDAGBuilder::visitLog2(const CallInst &I) { SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, getF32Constant(DAG, 0x40019463)); SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); - SDValue Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3fd6633d)); - - result = DAG.getNode(ISD::FADD, dl, - MVT::f32, LogOfExponent, Log2ofMantissa); - } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { + Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, + getF32Constant(DAG, 0x3fd6633d)); + } else if (LimitFloatPrecision <= 12) { // For floating-point precision of 12: // // Log2ofMantissa = @@ -3981,12 +3948,9 @@ SelectionDAGBuilder::visitLog2(const CallInst &I) { SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, getF32Constant(DAG, 0x40823e2f)); SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); - SDValue Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, - getF32Constant(DAG, 0x4020d29c)); - - result = DAG.getNode(ISD::FADD, dl, - MVT::f32, LogOfExponent, Log2ofMantissa); - } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 + Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, + getF32Constant(DAG, 0x4020d29c)); + } else { // LimitFloatPrecision <= 18 // For floating-point precision of 18: // // Log2ofMantissa = @@ -4016,32 +3980,23 @@ SelectionDAGBuilder::visitLog2(const CallInst &I) { SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, getF32Constant(DAG, 0x40c39dad)); SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); - SDValue Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, - getF32Constant(DAG, 0x4042902c)); - - result = DAG.getNode(ISD::FADD, dl, - MVT::f32, LogOfExponent, Log2ofMantissa); + Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, + getF32Constant(DAG, 0x4042902c)); } - } else { - // No special expansion. - result = DAG.getNode(ISD::FLOG2, dl, - getValue(I.getArgOperand(0)).getValueType(), - getValue(I.getArgOperand(0))); + + return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); } - setValue(&I, result); + // No special expansion. + return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op); } -/// visitLog10 - Lower a log10 intrinsic. Handles the special sequences for +/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for /// limited-precision mode. -void -SelectionDAGBuilder::visitLog10(const CallInst &I) { - SDValue result; - DebugLoc dl = getCurDebugLoc(); - - if (getValue(I.getArgOperand(0)).getValueType() == MVT::f32 && +static SDValue expandLog10(DebugLoc dl, SDValue Op, SelectionDAG &DAG, + const TargetLowering &TLI) { + if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { - SDValue Op = getValue(I.getArgOperand(0)); SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); // Scale the exponent by log10(2) [0.30102999f]. @@ -4053,6 +4008,7 @@ SelectionDAGBuilder::visitLog10(const CallInst &I) { // exponent of 1. SDValue X = GetSignificand(DAG, Op1, dl); + SDValue Log10ofMantissa; if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -4066,12 +4022,9 @@ SelectionDAGBuilder::visitLog10(const CallInst &I) { SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, getF32Constant(DAG, 0x3f1c0789)); SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); - SDValue Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, - getF32Constant(DAG, 0x3f011300)); - - result = DAG.getNode(ISD::FADD, dl, - MVT::f32, LogOfExponent, Log10ofMantissa); - } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { + Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, + getF32Constant(DAG, 0x3f011300)); + } else if (LimitFloatPrecision <= 12) { // For floating-point precision of 12: // // Log10ofMantissa = @@ -4088,12 +4041,9 @@ SelectionDAGBuilder::visitLog10(const CallInst &I) { SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, getF32Constant(DAG, 0x3f6ae232)); SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - SDValue Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3f25f7c3)); - - result = DAG.getNode(ISD::FADD, dl, - MVT::f32, LogOfExponent, Log10ofMantissa); - } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 + Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, + getF32Constant(DAG, 0x3f25f7c3)); + } else { // LimitFloatPrecision <= 18 // For floating-point precision of 18: // // Log10ofMantissa = @@ -4118,33 +4068,23 @@ SelectionDAGBuilder::visitLog10(const CallInst &I) { SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, getF32Constant(DAG, 0x3fc4316c)); SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); - SDValue Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8, - getF32Constant(DAG, 0x3f57ce70)); - - result = DAG.getNode(ISD::FADD, dl, - MVT::f32, LogOfExponent, Log10ofMantissa); + Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8, + getF32Constant(DAG, 0x3f57ce70)); } - } else { - // No special expansion. - result = DAG.getNode(ISD::FLOG10, dl, - getValue(I.getArgOperand(0)).getValueType(), - getValue(I.getArgOperand(0))); + + return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); } - setValue(&I, result); + // No special expansion. + return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op); } -/// visitExp2 - Lower an exp2 intrinsic. Handles the special sequences for +/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for /// limited-precision mode. -void -SelectionDAGBuilder::visitExp2(const CallInst &I) { - SDValue result; - DebugLoc dl = getCurDebugLoc(); - - if (getValue(I.getArgOperand(0)).getValueType() == MVT::f32 && +static SDValue expandExp2(DebugLoc dl, SDValue Op, SelectionDAG &DAG, + const TargetLowering &TLI) { + if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { - SDValue Op = getValue(I.getArgOperand(0)); - SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op); // FractionalPartOfX = x - (float)IntegerPartOfX; @@ -4155,6 +4095,7 @@ SelectionDAGBuilder::visitExp2(const CallInst &I) { IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, DAG.getConstant(23, TLI.getPointerTy())); + SDValue TwoToFractionalPartOfX; if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -4168,15 +4109,9 @@ SelectionDAGBuilder::visitExp2(const CallInst &I) { SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, getF32Constant(DAG, 0x3f3c50c8)); SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3f7f5e7e)); - SDValue t6 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, t5); - SDValue TwoToFractionalPartOfX = - DAG.getNode(ISD::ADD, dl, MVT::i32, t6, IntegerPartOfX); - - result = DAG.getNode(ISD::BITCAST, dl, - MVT::f32, TwoToFractionalPartOfX); - } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { + TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, + getF32Constant(DAG, 0x3f7f5e7e)); + } else if (LimitFloatPrecision <= 12) { // For floating-point precision of 12: // // TwoToFractionalPartOfX = @@ -4193,15 +4128,9 @@ SelectionDAGBuilder::visitExp2(const CallInst &I) { SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, getF32Constant(DAG, 0x3f324b07)); SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); - SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, - getF32Constant(DAG, 0x3f7ff8fd)); - SDValue t8 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, t7); - SDValue TwoToFractionalPartOfX = - DAG.getNode(ISD::ADD, dl, MVT::i32, t8, IntegerPartOfX); - - result = DAG.getNode(ISD::BITCAST, dl, - MVT::f32, TwoToFractionalPartOfX); - } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 + TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, + getF32Constant(DAG, 0x3f7ff8fd)); + } else { // LimitFloatPrecision <= 18 // For floating-point precision of 18: // // TwoToFractionalPartOfX = @@ -4229,54 +4158,42 @@ SelectionDAGBuilder::visitExp2(const CallInst &I) { SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, getF32Constant(DAG, 0x3f317234)); SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); - SDValue t13 = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, - getF32Constant(DAG, 0x3f800000)); - SDValue t14 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, t13); - SDValue TwoToFractionalPartOfX = - DAG.getNode(ISD::ADD, dl, MVT::i32, t14, IntegerPartOfX); - - result = DAG.getNode(ISD::BITCAST, dl, - MVT::f32, TwoToFractionalPartOfX); + TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, + getF32Constant(DAG, 0x3f800000)); } - } else { - // No special expansion. - result = DAG.getNode(ISD::FEXP2, dl, - getValue(I.getArgOperand(0)).getValueType(), - getValue(I.getArgOperand(0))); + + // Add the exponent into the result in integer domain. + SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, + TwoToFractionalPartOfX); + return DAG.getNode(ISD::BITCAST, dl, MVT::f32, + DAG.getNode(ISD::ADD, dl, MVT::i32, + t13, IntegerPartOfX)); } - setValue(&I, result); + // No special expansion. + return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op); } /// visitPow - Lower a pow intrinsic. Handles the special sequences for /// limited-precision mode with x == 10.0f. -void -SelectionDAGBuilder::visitPow(const CallInst &I) { - SDValue result; - const Value *Val = I.getArgOperand(0); - DebugLoc dl = getCurDebugLoc(); +static SDValue expandPow(DebugLoc dl, SDValue LHS, SDValue RHS, + SelectionDAG &DAG, const TargetLowering &TLI) { bool IsExp10 = false; - - if (getValue(Val).getValueType() == MVT::f32 && - getValue(I.getArgOperand(1)).getValueType() == MVT::f32 && + if (LHS.getValueType() == MVT::f32 && LHS.getValueType() == MVT::f32 && LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { - if (Constant *C = const_cast<Constant*>(dyn_cast<Constant>(Val))) { - if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) { - APFloat Ten(10.0f); - IsExp10 = CFP->getValueAPF().bitwiseIsEqual(Ten); - } + if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) { + APFloat Ten(10.0f); + IsExp10 = LHSC->isExactlyValue(Ten); } } - if (IsExp10 && LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { - SDValue Op = getValue(I.getArgOperand(1)); - + if (IsExp10) { // Put the exponent in the right bit position for later addition to the // final result: // // #define LOG2OF10 3.3219281f // IntegerPartOfX = (int32_t)(x * LOG2OF10); - SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op, + SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS, getF32Constant(DAG, 0x40549a78)); SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); @@ -4288,6 +4205,7 @@ SelectionDAGBuilder::visitPow(const CallInst &I) { IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, DAG.getConstant(23, TLI.getPointerTy())); + SDValue TwoToFractionalPartOfX; if (LimitFloatPrecision <= 6) { // For floating-point precision of 6: // @@ -4301,15 +4219,9 @@ SelectionDAGBuilder::visitPow(const CallInst &I) { SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, getF32Constant(DAG, 0x3f3c50c8)); SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); - SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, - getF32Constant(DAG, 0x3f7f5e7e)); - SDValue t6 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, t5); - SDValue TwoToFractionalPartOfX = - DAG.getNode(ISD::ADD, dl, MVT::i32, t6, IntegerPartOfX); - - result = DAG.getNode(ISD::BITCAST, dl, - MVT::f32, TwoToFractionalPartOfX); - } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) { + TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, + getF32Constant(DAG, 0x3f7f5e7e)); + } else if (LimitFloatPrecision <= 12) { // For floating-point precision of 12: // // TwoToFractionalPartOfX = @@ -4326,15 +4238,9 @@ SelectionDAGBuilder::visitPow(const CallInst &I) { SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, getF32Constant(DAG, 0x3f324b07)); SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); - SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, - getF32Constant(DAG, 0x3f7ff8fd)); - SDValue t8 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, t7); - SDValue TwoToFractionalPartOfX = - DAG.getNode(ISD::ADD, dl, MVT::i32, t8, IntegerPartOfX); - - result = DAG.getNode(ISD::BITCAST, dl, - MVT::f32, TwoToFractionalPartOfX); - } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18 + TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, + getF32Constant(DAG, 0x3f7ff8fd)); + } else { // LimitFloatPrecision <= 18 // For floating-point precision of 18: // // TwoToFractionalPartOfX = @@ -4362,24 +4268,18 @@ SelectionDAGBuilder::visitPow(const CallInst &I) { SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, getF32Constant(DAG, 0x3f317234)); SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); - SDValue t13 = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, - getF32Constant(DAG, 0x3f800000)); - SDValue t14 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, t13); - SDValue TwoToFractionalPartOfX = - DAG.getNode(ISD::ADD, dl, MVT::i32, t14, IntegerPartOfX); - - result = DAG.getNode(ISD::BITCAST, dl, - MVT::f32, TwoToFractionalPartOfX); + TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, + getF32Constant(DAG, 0x3f800000)); } - } else { - // No special expansion. - result = DAG.getNode(ISD::FPOW, dl, - getValue(I.getArgOperand(0)).getValueType(), - getValue(I.getArgOperand(0)), - getValue(I.getArgOperand(1))); + + SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX); + return DAG.getNode(ISD::BITCAST, dl, MVT::f32, + DAG.getNode(ISD::ADD, dl, MVT::i32, + t13, IntegerPartOfX)); } - setValue(&I, result); + // No special expansion. + return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS); } @@ -4400,7 +4300,8 @@ static SDValue ExpandPowI(DebugLoc DL, SDValue LHS, SDValue RHS, return DAG.getConstantFP(1.0, LHS.getValueType()); const Function *F = DAG.getMachineFunction().getFunction(); - if (!F->getFnAttributes().hasAttribute(Attributes::OptimizeForSize) || + if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, + Attribute::OptimizeForSize) || // If optimizing for size, don't insert too many multiplies. This // inserts up to 5 multiplies. CountPopulation_32(Val)+Log2_32(Val) < 7) { @@ -4566,6 +4467,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue Op2 = getValue(I.getArgOperand(1)); SDValue Op3 = getValue(I.getArgOperand(2)); unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); + if (!Align) + Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment. bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); DAG.setRoot(DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, isVol, false, MachinePointerInfo(I.getArgOperand(0)), @@ -4582,6 +4485,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue Op2 = getValue(I.getArgOperand(1)); SDValue Op3 = getValue(I.getArgOperand(2)); unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); + if (!Align) + Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment. bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); DAG.setRoot(DAG.getMemset(getRoot(), dl, Op1, Op2, Op3, Align, isVol, MachinePointerInfo(I.getArgOperand(0)))); @@ -4599,6 +4504,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue Op2 = getValue(I.getArgOperand(1)); SDValue Op3 = getValue(I.getArgOperand(2)); unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); + if (!Align) + Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment. bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue(); DAG.setRoot(DAG.getMemmove(getRoot(), dl, Op1, Op2, Op3, Align, isVol, MachinePointerInfo(I.getArgOperand(0)), @@ -4873,7 +4780,6 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits // to be zero. // We must do this early because v2i32 is not a legal type. - DebugLoc dl = getCurDebugLoc(); SDValue ShOps[2]; ShOps[0] = ShAmt; ShOps[1] = DAG.getConstant(0, MVT::i32); @@ -4890,7 +4796,6 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { case Intrinsic::x86_avx_vinsertf128_ps_256: case Intrinsic::x86_avx_vinsertf128_si_256: case Intrinsic::x86_avx2_vinserti128: { - DebugLoc dl = getCurDebugLoc(); EVT DestVT = TLI.getValueType(I.getType()); EVT ElVT = TLI.getValueType(I.getArgOperand(1)->getType()); uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) * @@ -4906,7 +4811,6 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { case Intrinsic::x86_avx_vextractf128_ps_256: case Intrinsic::x86_avx_vextractf128_si_256: case Intrinsic::x86_avx2_vextracti128: { - DebugLoc dl = getCurDebugLoc(); EVT DestVT = TLI.getValueType(I.getType()); uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) * DestVT.getVectorNumElements(); @@ -4940,7 +4844,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { } EVT DestVT = TLI.getValueType(I.getType()); const Value *Op1 = I.getArgOperand(0); - Res = DAG.getConvertRndSat(DestVT, getCurDebugLoc(), getValue(Op1), + Res = DAG.getConvertRndSat(DestVT, dl, getValue(Op1), DAG.getValueType(DestVT), DAG.getValueType(getValue(Op1).getValueType()), getValue(I.getArgOperand(1)), @@ -4949,53 +4853,57 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { setValue(&I, Res); return 0; } - case Intrinsic::sqrt: - setValue(&I, DAG.getNode(ISD::FSQRT, dl, - getValue(I.getArgOperand(0)).getValueType(), - getValue(I.getArgOperand(0)))); - return 0; case Intrinsic::powi: setValue(&I, ExpandPowI(dl, getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), DAG)); return 0; - case Intrinsic::sin: - setValue(&I, DAG.getNode(ISD::FSIN, dl, - getValue(I.getArgOperand(0)).getValueType(), - getValue(I.getArgOperand(0)))); - return 0; - case Intrinsic::cos: - setValue(&I, DAG.getNode(ISD::FCOS, dl, - getValue(I.getArgOperand(0)).getValueType(), - getValue(I.getArgOperand(0)))); - return 0; case Intrinsic::log: - visitLog(I); + setValue(&I, expandLog(dl, getValue(I.getArgOperand(0)), DAG, TLI)); return 0; case Intrinsic::log2: - visitLog2(I); + setValue(&I, expandLog2(dl, getValue(I.getArgOperand(0)), DAG, TLI)); return 0; case Intrinsic::log10: - visitLog10(I); + setValue(&I, expandLog10(dl, getValue(I.getArgOperand(0)), DAG, TLI)); return 0; case Intrinsic::exp: - visitExp(I); + setValue(&I, expandExp(dl, getValue(I.getArgOperand(0)), DAG, TLI)); return 0; case Intrinsic::exp2: - visitExp2(I); + setValue(&I, expandExp2(dl, getValue(I.getArgOperand(0)), DAG, TLI)); return 0; case Intrinsic::pow: - visitPow(I); + setValue(&I, expandPow(dl, getValue(I.getArgOperand(0)), + getValue(I.getArgOperand(1)), DAG, TLI)); return 0; + case Intrinsic::sqrt: case Intrinsic::fabs: - setValue(&I, DAG.getNode(ISD::FABS, dl, - getValue(I.getArgOperand(0)).getValueType(), - getValue(I.getArgOperand(0)))); - return 0; + case Intrinsic::sin: + case Intrinsic::cos: case Intrinsic::floor: - setValue(&I, DAG.getNode(ISD::FFLOOR, dl, + case Intrinsic::ceil: + case Intrinsic::trunc: + case Intrinsic::rint: + case Intrinsic::nearbyint: { + unsigned Opcode; + switch (Intrinsic) { + default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. + case Intrinsic::sqrt: Opcode = ISD::FSQRT; break; + case Intrinsic::fabs: Opcode = ISD::FABS; break; + case Intrinsic::sin: Opcode = ISD::FSIN; break; + case Intrinsic::cos: Opcode = ISD::FCOS; break; + case Intrinsic::floor: Opcode = ISD::FFLOOR; break; + case Intrinsic::ceil: Opcode = ISD::FCEIL; break; + case Intrinsic::trunc: Opcode = ISD::FTRUNC; break; + case Intrinsic::rint: Opcode = ISD::FRINT; break; + case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break; + } + + setValue(&I, DAG.getNode(Opcode, dl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)))); return 0; + } case Intrinsic::fma: setValue(&I, DAG.getNode(ISD::FMA, dl, getValue(I.getArgOperand(0)).getValueType(), @@ -5006,7 +4914,6 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { case Intrinsic::fmuladd: { EVT VT = TLI.getValueType(I.getType()); if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && - TLI.isOperationLegal(ISD::FMA, VT) && TLI.isFMAFasterThanMulAndAdd(VT)){ setValue(&I, DAG.getNode(ISD::FMA, dl, getValue(I.getArgOperand(0)).getValueType(), @@ -5103,7 +5010,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue FIN = DAG.getFrameIndex(FI, PtrTy); // Store the stack protector onto the stack. - Res = DAG.getStore(getRoot(), getCurDebugLoc(), Src, FIN, + Res = DAG.getStore(getRoot(), dl, Src, FIN, MachinePointerInfo::getFixedStack(FI), true, false, 0); setValue(&I, Res); @@ -5191,7 +5098,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { /*isTailCall=*/false, /*doesNotRet=*/false, /*isReturnValueUsed=*/true, DAG.getExternalSymbol(TrapFuncName.data(), TLI.getPointerTy()), - Args, DAG, getCurDebugLoc()); + Args, DAG, dl); std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); DAG.setRoot(Result.second); return 0; @@ -5217,7 +5124,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue Op2 = getValue(I.getArgOperand(1)); SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1); - setValue(&I, DAG.getNode(Op, getCurDebugLoc(), VTs, Op1, Op2)); + setValue(&I, DAG.getNode(Op, dl, VTs, Op1, Op2)); return 0; } case Intrinsic::prefetch: { @@ -5267,6 +5174,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { Res = DAG.getNode(Opcode, dl, MVT::Other, Ops, 2); DAG.setRoot(Res); } + return 0; } case Intrinsic::invariant_start: // Discard region information. @@ -5296,8 +5204,7 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, // Check whether the function can return without sret-demotion. SmallVector<ISD::OutputArg, 4> Outs; - GetReturnInfo(RetTy, CS.getAttributes().getRetAttributes(), - Outs, TLI); + GetReturnInfo(RetTy, CS.getAttributes(), Outs, TLI); bool CanLowerReturn = TLI.CanLowerReturn(CS.getCallingConv(), DAG.getMachineFunction(), @@ -5342,12 +5249,12 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, Entry.Node = ArgNode; Entry.Ty = V->getType(); unsigned attrInd = i - CS.arg_begin() + 1; - Entry.isSExt = CS.paramHasAttr(attrInd, Attributes::SExt); - Entry.isZExt = CS.paramHasAttr(attrInd, Attributes::ZExt); - Entry.isInReg = CS.paramHasAttr(attrInd, Attributes::InReg); - Entry.isSRet = CS.paramHasAttr(attrInd, Attributes::StructRet); - Entry.isNest = CS.paramHasAttr(attrInd, Attributes::Nest); - Entry.isByVal = CS.paramHasAttr(attrInd, Attributes::ByVal); + Entry.isSExt = CS.paramHasAttr(attrInd, Attribute::SExt); + Entry.isZExt = CS.paramHasAttr(attrInd, Attribute::ZExt); + Entry.isInReg = CS.paramHasAttr(attrInd, Attribute::InReg); + Entry.isSRet = CS.paramHasAttr(attrInd, Attribute::StructRet); + Entry.isNest = CS.paramHasAttr(attrInd, Attribute::Nest); + Entry.isByVal = CS.paramHasAttr(attrInd, Attribute::ByVal); Entry.Alignment = CS.getParamAlignment(attrInd); Args.push_back(Entry); } @@ -5376,13 +5283,7 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, // Check if target-independent constraints permit a tail call here. // Target-dependent constraints are checked within TLI.LowerCallTo. - if (isTailCall && - !isInTailCallPosition(CS, CS.getAttributes().getRetAttributes(), TLI)) - isTailCall = false; - - // If there's a possibility that fast-isel has already selected some amount - // of the current basic block, don't emit a tail call. - if (isTailCall && TM.Options.EnableFastISel) + if (isTailCall && !isInTailCallPosition(CS, TLI)) isTailCall = false; TargetLowering:: @@ -5856,7 +5757,7 @@ static void GetRegistersForValue(SelectionDAG &DAG, // Try to convert to the first EVT that the reg class contains. If the // types are identical size, use a bitcast to convert (e.g. two differing // vector types). - EVT RegVT = *PhysReg.second->vt_begin(); + MVT RegVT = *PhysReg.second->vt_begin(); if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) { OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand); @@ -5866,8 +5767,7 @@ static void GetRegistersForValue(SelectionDAG &DAG, // bitcast to the corresponding integer type. This turns an f64 value // into i64, which can be passed with two i32 values on a 32-bit // machine. - RegVT = EVT::getIntegerVT(Context, - OpInfo.ConstraintVT.getSizeInBits()); + RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits()); OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand); OpInfo.ConstraintVT = RegVT; @@ -5877,7 +5777,7 @@ static void GetRegistersForValue(SelectionDAG &DAG, NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT); } - EVT RegVT; + MVT RegVT; EVT ValueVT = OpInfo.ConstraintVT; // If this is a constraint for a specific physical register, like {r17}, @@ -5951,7 +5851,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i])); SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back(); - EVT OpVT = MVT::Other; + MVT OpVT = MVT::Other; // Compute the value type for each operand. switch (OpInfo.Type) { @@ -5966,10 +5866,10 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { // corresponding argument. assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); if (StructType *STy = dyn_cast<StructType>(CS.getType())) { - OpVT = TLI.getValueType(STy->getElementType(ResNo)); + OpVT = TLI.getSimpleValueType(STy->getElementType(ResNo)); } else { assert(ResNo == 0 && "Asm only has one result!"); - OpVT = TLI.getValueType(CS.getType()); + OpVT = TLI.getSimpleValueType(CS.getType()); } ++ResNo; break; @@ -5990,7 +5890,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); } - OpVT = OpInfo.getCallOperandValEVT(*DAG.getContext(), TLI, TD); + OpVT = OpInfo.getCallOperandValEVT(*DAG.getContext(), TLI, TD). + getSimpleVT(); } OpInfo.ConstraintVT = OpVT; @@ -6052,6 +5953,10 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { // Compute the constraint code and ConstraintType to use. TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); + if (OpInfo.ConstraintType == TargetLowering::C_Memory && + OpInfo.Type == InlineAsm::isClobber) + continue; + // If this is a memory input, and if the operand is not indirect, do what we // need to to provide an address for the memory input. if (OpInfo.ConstraintType == TargetLowering::C_Memory && @@ -6155,6 +6060,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { ExtraInfo |= InlineAsm::Extra_MayLoad; else if (OpInfo.Type == InlineAsm::isOutput) ExtraInfo |= InlineAsm::Extra_MayStore; + else if (OpInfo.Type == InlineAsm::isClobber) + ExtraInfo |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore); } } @@ -6253,11 +6160,12 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { Ctx.emitError(CS.getInstruction(), "inline asm not supported yet:" " don't know how to handle tied " "indirect register inputs"); + report_fatal_error("Cannot handle indirect register inputs!"); } RegsForValue MatchedRegs; MatchedRegs.ValueVTs.push_back(InOperandVal.getValueType()); - EVT RegVT = AsmNodeOperands[CurOp+1].getValueType(); + MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType(); MatchedRegs.RegVTs.push_back(RegVT); MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo(); for (unsigned i = 0, e = InlineAsm::getNumOperandRegisters(OpFlag); @@ -6524,7 +6432,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { Flags.setNest(); Flags.setOrigAlign(OriginalAlignment); - EVT PartVT = getRegisterType(CLI.RetTy->getContext(), VT); + MVT PartVT = getRegisterType(CLI.RetTy->getContext(), VT); unsigned NumParts = getNumRegisters(CLI.RetTy->getContext(), VT); SmallVector<SDValue, 4> Parts(NumParts); ISD::NodeType ExtendKind = ISD::ANY_EXTEND; @@ -6559,11 +6467,11 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { ComputeValueVTs(*this, CLI.RetTy, RetTys); for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { EVT VT = RetTys[I]; - EVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT); + MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT); unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT); for (unsigned i = 0; i != NumRegs; ++i) { ISD::InputArg MyFlags; - MyFlags.VT = RegisterVT.getSimpleVT(); + MyFlags.VT = RegisterVT; MyFlags.Used = CLI.IsReturnValueUsed; if (CLI.RetSExt) MyFlags.Flags.setSExt(); @@ -6613,7 +6521,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { unsigned CurReg = 0; for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { EVT VT = RetTys[I]; - EVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT); + MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT); unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT); ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg], @@ -6681,19 +6589,12 @@ static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) { return true; } -void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { - // If this is the entry block, emit arguments. - const Function &F = *LLVMBB->getParent(); +void SelectionDAGISel::LowerArguments(const Function &F) { SelectionDAG &DAG = SDB->DAG; DebugLoc dl = SDB->getCurDebugLoc(); const DataLayout *TD = TLI.getDataLayout(); SmallVector<ISD::InputArg, 16> Ins; - // Check whether the function can return without sret-demotion. - SmallVector<ISD::OutputArg, 4> Outs; - GetReturnInfo(F.getReturnType(), F.getAttributes().getRetAttributes(), - Outs, TLI); - if (!FuncInfo->CanLowerReturn) { // Put in an sret pointer parameter before all the other parameters. SmallVector<EVT, 1> ValueVTs; @@ -6703,7 +6604,7 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { // or one register. ISD::ArgFlagsTy Flags; Flags.setSRet(); - EVT RegisterVT = TLI.getRegisterType(*DAG.getContext(), ValueVTs[0]); + MVT RegisterVT = TLI.getRegisterType(*DAG.getContext(), ValueVTs[0]); ISD::InputArg RetArg(Flags, RegisterVT, true, 0, 0); Ins.push_back(RetArg); } @@ -6723,15 +6624,15 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { unsigned OriginalAlignment = TD->getABITypeAlignment(ArgTy); - if (F.getParamAttributes(Idx).hasAttribute(Attributes::ZExt)) + if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt)) Flags.setZExt(); - if (F.getParamAttributes(Idx).hasAttribute(Attributes::SExt)) + if (F.getAttributes().hasAttribute(Idx, Attribute::SExt)) Flags.setSExt(); - if (F.getParamAttributes(Idx).hasAttribute(Attributes::InReg)) + if (F.getAttributes().hasAttribute(Idx, Attribute::InReg)) Flags.setInReg(); - if (F.getParamAttributes(Idx).hasAttribute(Attributes::StructRet)) + if (F.getAttributes().hasAttribute(Idx, Attribute::StructRet)) Flags.setSRet(); - if (F.getParamAttributes(Idx).hasAttribute(Attributes::ByVal)) { + if (F.getAttributes().hasAttribute(Idx, Attribute::ByVal)) { Flags.setByVal(); PointerType *Ty = cast<PointerType>(I->getType()); Type *ElementTy = Ty->getElementType(); @@ -6745,11 +6646,11 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { FrameAlign = TLI.getByValTypeAlignment(ElementTy); Flags.setByValAlign(FrameAlign); } - if (F.getParamAttributes(Idx).hasAttribute(Attributes::Nest)) + if (F.getAttributes().hasAttribute(Idx, Attribute::Nest)) Flags.setNest(); Flags.setOrigAlign(OriginalAlignment); - EVT RegisterVT = TLI.getRegisterType(*CurDAG->getContext(), VT); + MVT RegisterVT = TLI.getRegisterType(*CurDAG->getContext(), VT); unsigned NumRegs = TLI.getNumRegisters(*CurDAG->getContext(), VT); for (unsigned i = 0; i != NumRegs; ++i) { ISD::InputArg MyFlags(Flags, RegisterVT, isArgValueUsed, @@ -6795,8 +6696,8 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { // from the sret argument into it. SmallVector<EVT, 1> ValueVTs; ComputeValueVTs(TLI, PointerType::getUnqual(F.getReturnType()), ValueVTs); - EVT VT = ValueVTs[0]; - EVT RegVT = TLI.getRegisterType(*CurDAG->getContext(), VT); + MVT VT = ValueVTs[0].getSimpleVT(); + MVT RegVT = TLI.getRegisterType(*CurDAG->getContext(), VT); ISD::NodeType AssertOp = ISD::DELETED_NODE; SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, NULL, AssertOp); @@ -6828,14 +6729,14 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { for (unsigned Val = 0; Val != NumValues; ++Val) { EVT VT = ValueVTs[Val]; - EVT PartVT = TLI.getRegisterType(*CurDAG->getContext(), VT); + MVT PartVT = TLI.getRegisterType(*CurDAG->getContext(), VT); unsigned NumParts = TLI.getNumRegisters(*CurDAG->getContext(), VT); if (!I->use_empty()) { ISD::NodeType AssertOp = ISD::DELETED_NODE; - if (F.getParamAttributes(Idx).hasAttribute(Attributes::SExt)) + if (F.getAttributes().hasAttribute(Idx, Attribute::SExt)) AssertOp = ISD::AssertSext; - else if (F.getParamAttributes(Idx).hasAttribute(Attributes::ZExt)) + else if (F.getAttributes().hasAttribute(Idx, Attribute::ZExt)) AssertOp = ISD::AssertZext; ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 9e46d96..9188945 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -14,12 +14,12 @@ #ifndef SELECTIONDAGBUILDER_H #define SELECTIONDAGBUILDER_H -#include "llvm/Constants.h" -#include "llvm/CodeGen/SelectionDAG.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/ValueTypes.h" +#include "llvm/IR/Constants.h" #include "llvm/Support/CallSite.h" #include "llvm/Support/ErrorHandling.h" #include <vector> @@ -262,7 +262,7 @@ private: struct BitTestBlock { BitTestBlock(APInt F, APInt R, const Value* SV, - unsigned Rg, EVT RgVT, bool E, + unsigned Rg, MVT RgVT, bool E, MachineBasicBlock* P, MachineBasicBlock* D, const BitTestInfo& C): First(F), Range(R), SValue(SV), Reg(Rg), RegVT(RgVT), Emitted(E), @@ -271,7 +271,7 @@ private: APInt Range; const Value *SValue; unsigned Reg; - EVT RegVT; + MVT RegVT; bool Emitted; MachineBasicBlock *Parent; MachineBasicBlock *Default; @@ -533,13 +533,6 @@ private: const char *visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic); - void visitPow(const CallInst &I); - void visitExp2(const CallInst &I); - void visitExp(const CallInst &I); - void visitLog(const CallInst &I); - void visitLog2(const CallInst &I); - void visitLog10(const CallInst &I); - void visitVAStart(const CallInst &I); void visitVAArg(const VAArgInst &I); void visitVAEnd(const CallInst &I); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index 6f3ce7a..3b5823b 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -11,23 +11,23 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CodeGen/SelectionDAG.h" #include "ScheduleDAGSDNodes.h" -#include "llvm/DebugInfo.h" -#include "llvm/Function.h" -#include "llvm/Intrinsics.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/Assembly/Writer.h" -#include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/GraphWriter.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetIntrinsicInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/GraphWriter.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/StringExtras.h" using namespace llvm; std::string SDNode::getOperationName(const SelectionDAG *G) const { @@ -140,6 +140,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::FSQRT: return "fsqrt"; case ISD::FSIN: return "fsin"; case ISD::FCOS: return "fcos"; + case ISD::FSINCOS: return "fsincos"; case ISD::FTRUNC: return "ftrunc"; case ISD::FFLOOR: return "ffloor"; case ISD::FCEIL: return "fceil"; diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index c314fa5..eeea9e4 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -12,23 +12,18 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "isel" +#include "llvm/CodeGen/SelectionDAGISel.h" #include "ScheduleDAGSDNodes.h" #include "SelectionDAGBuilder.h" -#include "llvm/Constants.h" -#include "llvm/DebugInfo.h" -#include "llvm/Function.h" -#include "llvm/InlineAsm.h" -#include "llvm/Instructions.h" -#include "llvm/Intrinsics.h" -#include "llvm/IntrinsicInst.h" -#include "llvm/LLVMContext.h" -#include "llvm/Module.h" +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/FastISel.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" -#include "llvm/CodeGen/GCStrategy.h" #include "llvm/CodeGen/GCMetadata.h" +#include "llvm/CodeGen/GCStrategy.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" @@ -37,22 +32,29 @@ #include "llvm/CodeGen/ScheduleHazardRecognizer.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/CodeGen/SelectionDAGISel.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetIntrinsicInfo.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetIntrinsicInfo.h" #include "llvm/Target/TargetLibraryInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/PostOrderIterator.h" -#include "llvm/ADT/Statistic.h" #include <algorithm> using namespace llvm; @@ -142,7 +144,12 @@ EnableFastISelVerbose("fast-isel-verbose", cl::Hidden, "instruction selector")); static cl::opt<bool> EnableFastISelAbort("fast-isel-abort", cl::Hidden, - cl::desc("Enable abort calls when \"fast\" instruction fails")); + cl::desc("Enable abort calls when \"fast\" instruction selection " + "fails to lower an instruction")); +static cl::opt<bool> +EnableFastISelAbortArgs("fast-isel-abort-args", cl::Hidden, + cl::desc("Enable abort calls when \"fast\" instruction selection " + "fails to lower a formal argument")); static cl::opt<bool> UseMBPI("use-mbpi", @@ -216,8 +223,9 @@ namespace llvm { ScheduleDAGSDNodes* createDefaultScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel) { const TargetLowering &TLI = IS->getTargetLowering(); + const TargetSubtargetInfo &ST = IS->TM.getSubtarget<TargetSubtargetInfo>(); - if (OptLevel == CodeGenOpt::None || + if (OptLevel == CodeGenOpt::None || ST.enableMachineScheduler() || TLI.getSchedulingPreference() == Sched::Source) return createSourceListDAGScheduler(IS, OptLevel); if (TLI.getSchedulingPreference() == Sched::RegPressure) @@ -348,13 +356,19 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { RegInfo = &MF->getRegInfo(); AA = &getAnalysis<AliasAnalysis>(); LibInfo = &getAnalysis<TargetLibraryInfo>(); + TTI = getAnalysisIfAvailable<TargetTransformInfo>(); GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : 0; + TargetSubtargetInfo &ST = + const_cast<TargetSubtargetInfo&>(TM.getSubtarget<TargetSubtargetInfo>()); + ST.resetSubtargetFeatures(MF); + TM.resetTargetOptions(MF); + DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n"); SplitCriticalSideEffectEdges(const_cast<Function&>(Fn), this); - CurDAG->init(*MF); + CurDAG->init(*MF, TTI); FuncInfo->set(Fn, *MF); if (UseMBPI && OptLevel != CodeGenOpt::None) @@ -364,6 +378,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { SDB->init(GFI, *AA, LibInfo); + MF->setHasMSInlineAsm(false); SelectAllBasicBlocks(Fn); // If the first basic block in the function has live ins that need to be @@ -434,24 +449,26 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // Determine if there are any calls in this machine function. MachineFrameInfo *MFI = MF->getFrameInfo(); - if (!MFI->hasCalls()) { - for (MachineFunction::const_iterator - I = MF->begin(), E = MF->end(); I != E; ++I) { - const MachineBasicBlock *MBB = I; - for (MachineBasicBlock::const_iterator - II = MBB->begin(), IE = MBB->end(); II != IE; ++II) { - const MCInstrDesc &MCID = TM.getInstrInfo()->get(II->getOpcode()); - - if ((MCID.isCall() && !MCID.isReturn()) || - II->isStackAligningInlineAsm()) { - MFI->setHasCalls(true); - goto done; - } + for (MachineFunction::const_iterator I = MF->begin(), E = MF->end(); I != E; + ++I) { + + if (MFI->hasCalls() && MF->hasMSInlineAsm()) + break; + + const MachineBasicBlock *MBB = I; + for (MachineBasicBlock::const_iterator II = MBB->begin(), IE = MBB->end(); + II != IE; ++II) { + const MCInstrDesc &MCID = TM.getInstrInfo()->get(II->getOpcode()); + if ((MCID.isCall() && !MCID.isReturn()) || + II->isStackAligningInlineAsm()) { + MFI->setHasCalls(true); + } + if (II->isMSInlineAsm()) { + MF->setHasMSInlineAsm(true); } } } - done: // Determine if there is a call to setjmp in the machine function. MF->setExposesReturnsTwice(Fn.callsFunctionThatReturnsTwice()); @@ -768,8 +785,12 @@ void SelectionDAGISel::DoInstructionSelection() { if (ResNode == Node || Node->getOpcode() == ISD::DELETED_NODE) continue; // Replace node. - if (ResNode) + if (ResNode) { + // Propagate ordering + CurDAG->AssignOrdering(ResNode, CurDAG->GetOrdering(Node)); + ReplaceUses(Node, ResNode); + } // If after the replacement this node is not used any more, // remove this dead node. @@ -1004,34 +1025,28 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { if (AllPredsVisited) { for (BasicBlock::const_iterator I = LLVMBB->begin(); - isa<PHINode>(I); ++I) - FuncInfo->ComputePHILiveOutRegInfo(cast<PHINode>(I)); + const PHINode *PN = dyn_cast<PHINode>(I); ++I) + FuncInfo->ComputePHILiveOutRegInfo(PN); } else { for (BasicBlock::const_iterator I = LLVMBB->begin(); - isa<PHINode>(I); ++I) - FuncInfo->InvalidatePHILiveOutRegInfo(cast<PHINode>(I)); + const PHINode *PN = dyn_cast<PHINode>(I); ++I) + FuncInfo->InvalidatePHILiveOutRegInfo(PN); } FuncInfo->VisitedBBs.insert(LLVMBB); } - FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB]; - FuncInfo->InsertPt = FuncInfo->MBB->getFirstNonPHI(); - BasicBlock::const_iterator const Begin = LLVMBB->getFirstNonPHI(); BasicBlock::const_iterator const End = LLVMBB->end(); BasicBlock::const_iterator BI = End; + FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB]; FuncInfo->InsertPt = FuncInfo->MBB->getFirstNonPHI(); // Setup an EH landing-pad block. if (FuncInfo->MBB->isLandingPad()) PrepareEHLandingPad(); - // Lower any arguments needed in this block if this is the entry block. - if (LLVMBB == &Fn.getEntryBlock()) - LowerArguments(LLVMBB); - // Before doing SelectionDAG ISel, see if FastISel has been requested. if (FastIS) { FastIS->startNewBlock(); @@ -1039,9 +1054,18 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { // Emit code for any incoming arguments. This must happen before // beginning FastISel on the entry block. if (LLVMBB == &Fn.getEntryBlock()) { - CurDAG->setRoot(SDB->getControlRoot()); - SDB->clear(); - CodeGenAndEmitDAG(); + // Lower any arguments needed in this block if this is the entry block. + if (!FastIS->LowerArguments()) { + // Fast isel failed to lower these arguments + if (EnableFastISelAbortArgs) + llvm_unreachable("FastISel didn't lower all arguments"); + + // Use SelectionDAG argument lowering + LowerArguments(Fn); + CurDAG->setRoot(SDB->getControlRoot()); + SDB->clear(); + CodeGenAndEmitDAG(); + } // If we inserted any instructions at the beginning, make a note of // where they are, so we can be sure to emit subsequent instructions @@ -1111,19 +1135,21 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { } bool HadTailCall = false; + MachineBasicBlock::iterator SavedInsertPt = FuncInfo->InsertPt; SelectBasicBlock(Inst, BI, HadTailCall); - // Recompute NumFastIselRemaining as Selection DAG instruction - // selection may have handled the call, input args, etc. - unsigned RemainingNow = std::distance(Begin, BI); - NumFastIselFailures += NumFastIselRemaining - RemainingNow; - // If the call was emitted as a tail call, we're done with the block. + // We also need to delete any previously emitted instructions. if (HadTailCall) { + FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end()); --BI; break; } + // Recompute NumFastIselRemaining as Selection DAG instruction + // selection may have handled the call, input args, etc. + unsigned RemainingNow = std::distance(Begin, BI); + NumFastIselFailures += NumFastIselRemaining - RemainingNow; NumFastIselRemaining = RemainingNow; continue; } @@ -1150,6 +1176,10 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { } FastIS->recomputeInsertPt(); + } else { + // Lower any arguments needed in this block if this is the entry block. + if (LLVMBB == &Fn.getEntryBlock()) + LowerArguments(Fn); } if (Begin != BI) @@ -1189,14 +1219,12 @@ SelectionDAGISel::FinishBasicBlock() { SDB->JTCases.empty() && SDB->BitTestCases.empty()) { for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) { - MachineInstr *PHI = FuncInfo->PHINodesToUpdate[i].first; + MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first); assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); if (!FuncInfo->MBB->isSuccessor(PHI->getParent())) continue; - PHI->addOperand( - MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[i].second, false)); - PHI->addOperand(MachineOperand::CreateMBB(FuncInfo->MBB)); + PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB); } return; } @@ -1248,33 +1276,23 @@ SelectionDAGISel::FinishBasicBlock() { // Update PHI Nodes for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size(); pi != pe; ++pi) { - MachineInstr *PHI = FuncInfo->PHINodesToUpdate[pi].first; + MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first); MachineBasicBlock *PHIBB = PHI->getParent(); assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); // This is "default" BB. We have two jumps to it. From "header" BB and // from last "case" BB. - if (PHIBB == SDB->BitTestCases[i].Default) { - PHI->addOperand(MachineOperand:: - CreateReg(FuncInfo->PHINodesToUpdate[pi].second, - false)); - PHI->addOperand(MachineOperand::CreateMBB(SDB->BitTestCases[i].Parent)); - PHI->addOperand(MachineOperand:: - CreateReg(FuncInfo->PHINodesToUpdate[pi].second, - false)); - PHI->addOperand(MachineOperand::CreateMBB(SDB->BitTestCases[i].Cases. - back().ThisBB)); - } + if (PHIBB == SDB->BitTestCases[i].Default) + PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second) + .addMBB(SDB->BitTestCases[i].Parent) + .addReg(FuncInfo->PHINodesToUpdate[pi].second) + .addMBB(SDB->BitTestCases[i].Cases.back().ThisBB); // One of "cases" BB. for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) { MachineBasicBlock* cBB = SDB->BitTestCases[i].Cases[j].ThisBB; - if (cBB->isSuccessor(PHIBB)) { - PHI->addOperand(MachineOperand:: - CreateReg(FuncInfo->PHINodesToUpdate[pi].second, - false)); - PHI->addOperand(MachineOperand::CreateMBB(cBB)); - } + if (cBB->isSuccessor(PHIBB)) + PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(cBB); } } } @@ -1309,25 +1327,17 @@ SelectionDAGISel::FinishBasicBlock() { // Update PHI Nodes for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size(); pi != pe; ++pi) { - MachineInstr *PHI = FuncInfo->PHINodesToUpdate[pi].first; + MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first); MachineBasicBlock *PHIBB = PHI->getParent(); assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); // "default" BB. We can go there only from header BB. - if (PHIBB == SDB->JTCases[i].second.Default) { - PHI->addOperand - (MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[pi].second, - false)); - PHI->addOperand - (MachineOperand::CreateMBB(SDB->JTCases[i].first.HeaderBB)); - } + if (PHIBB == SDB->JTCases[i].second.Default) + PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second) + .addMBB(SDB->JTCases[i].first.HeaderBB); // JT BB. Just iterate over successors here - if (FuncInfo->MBB->isSuccessor(PHIBB)) { - PHI->addOperand - (MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[pi].second, - false)); - PHI->addOperand(MachineOperand::CreateMBB(FuncInfo->MBB)); - } + if (FuncInfo->MBB->isSuccessor(PHIBB)) + PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB); } } SDB->JTCases.clear(); @@ -1335,14 +1345,11 @@ SelectionDAGISel::FinishBasicBlock() { // If the switch block involved a branch to one of the actual successors, we // need to update PHI nodes in that block. for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) { - MachineInstr *PHI = FuncInfo->PHINodesToUpdate[i].first; + MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first); assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); - if (FuncInfo->MBB->isSuccessor(PHI->getParent())) { - PHI->addOperand( - MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[i].second, false)); - PHI->addOperand(MachineOperand::CreateMBB(FuncInfo->MBB)); - } + if (FuncInfo->MBB->isSuccessor(PHI->getParent())) + PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB); } // If we generated any switch lowering information, build and codegen any @@ -1378,18 +1385,16 @@ SelectionDAGISel::FinishBasicBlock() { // FuncInfo->MBB may have been removed from the CFG if a branch was // constant folded. if (ThisBB->isSuccessor(FuncInfo->MBB)) { - for (MachineBasicBlock::iterator Phi = FuncInfo->MBB->begin(); - Phi != FuncInfo->MBB->end() && Phi->isPHI(); - ++Phi) { + for (MachineBasicBlock::iterator + MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end(); + MBBI != MBBE && MBBI->isPHI(); ++MBBI) { + MachineInstrBuilder PHI(*MF, MBBI); // This value for this PHI node is recorded in PHINodesToUpdate. for (unsigned pn = 0; ; ++pn) { assert(pn != FuncInfo->PHINodesToUpdate.size() && "Didn't find PHI entry!"); - if (FuncInfo->PHINodesToUpdate[pn].first == Phi) { - Phi->addOperand(MachineOperand:: - CreateReg(FuncInfo->PHINodesToUpdate[pn].second, - false)); - Phi->addOperand(MachineOperand::CreateMBB(ThisBB)); + if (FuncInfo->PHINodesToUpdate[pn].first == PHI) { + PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB); break; } } @@ -1669,9 +1674,7 @@ SDNode *SelectionDAGISel::Select_INLINEASM(SDNode *N) { std::vector<SDValue> Ops(N->op_begin(), N->op_end()); SelectInlineAsmMemoryOperands(Ops); - std::vector<EVT> VTs; - VTs.push_back(MVT::Other); - VTs.push_back(MVT::Glue); + EVT VTs[] = { MVT::Other, MVT::Glue }; SDValue New = CurDAG->getNode(ISD::INLINEASM, N->getDebugLoc(), VTs, &Ops[0], Ops.size()); New->setNodeId(-1); @@ -2605,11 +2608,11 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, SDValue Imm = RecordedNodes[RecNo].first; if (Imm->getOpcode() == ISD::Constant) { - int64_t Val = cast<ConstantSDNode>(Imm)->getZExtValue(); - Imm = CurDAG->getTargetConstant(Val, Imm.getValueType()); + const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue(); + Imm = CurDAG->getConstant(*Val, Imm.getValueType(), true); } else if (Imm->getOpcode() == ISD::ConstantFP) { const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue(); - Imm = CurDAG->getTargetConstantFP(*Val, Imm.getValueType()); + Imm = CurDAG->getConstantFP(*Val, Imm.getValueType(), true); } RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second)); diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp index 3921635..b752b482 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp @@ -11,21 +11,21 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CodeGen/SelectionDAG.h" #include "ScheduleDAGSDNodes.h" -#include "llvm/Constants.h" -#include "llvm/DebugInfo.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/Assembly/Writer.h" -#include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetMachine.h" +#include "llvm/DebugInfo.h" +#include "llvm/IR/Constants.h" #include "llvm/Support/Debug.h" #include "llvm/Support/GraphWriter.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/StringExtras.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; namespace llvm { diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 49f55e2..f5fc66c 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -12,1037 +12,191 @@ //===----------------------------------------------------------------------===// #include "llvm/Target/TargetLowering.h" -#include "llvm/MC/MCAsmInfo.h" -#include "llvm/MC/MCExpr.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetLoweringObjectFile.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/GlobalVariable.h" -#include "llvm/DerivedTypes.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/ADT/BitVector.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCExpr.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Target/TargetLoweringObjectFile.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <cctype> using namespace llvm; -/// InitLibcallNames - Set default libcall names. -/// -static void InitLibcallNames(const char **Names) { - Names[RTLIB::SHL_I16] = "__ashlhi3"; - Names[RTLIB::SHL_I32] = "__ashlsi3"; - Names[RTLIB::SHL_I64] = "__ashldi3"; - Names[RTLIB::SHL_I128] = "__ashlti3"; - Names[RTLIB::SRL_I16] = "__lshrhi3"; - Names[RTLIB::SRL_I32] = "__lshrsi3"; - Names[RTLIB::SRL_I64] = "__lshrdi3"; - Names[RTLIB::SRL_I128] = "__lshrti3"; - Names[RTLIB::SRA_I16] = "__ashrhi3"; - Names[RTLIB::SRA_I32] = "__ashrsi3"; - Names[RTLIB::SRA_I64] = "__ashrdi3"; - Names[RTLIB::SRA_I128] = "__ashrti3"; - Names[RTLIB::MUL_I8] = "__mulqi3"; - Names[RTLIB::MUL_I16] = "__mulhi3"; - Names[RTLIB::MUL_I32] = "__mulsi3"; - Names[RTLIB::MUL_I64] = "__muldi3"; - Names[RTLIB::MUL_I128] = "__multi3"; - Names[RTLIB::MULO_I32] = "__mulosi4"; - Names[RTLIB::MULO_I64] = "__mulodi4"; - Names[RTLIB::MULO_I128] = "__muloti4"; - Names[RTLIB::SDIV_I8] = "__divqi3"; - Names[RTLIB::SDIV_I16] = "__divhi3"; - Names[RTLIB::SDIV_I32] = "__divsi3"; - Names[RTLIB::SDIV_I64] = "__divdi3"; - Names[RTLIB::SDIV_I128] = "__divti3"; - Names[RTLIB::UDIV_I8] = "__udivqi3"; - Names[RTLIB::UDIV_I16] = "__udivhi3"; - Names[RTLIB::UDIV_I32] = "__udivsi3"; - Names[RTLIB::UDIV_I64] = "__udivdi3"; - Names[RTLIB::UDIV_I128] = "__udivti3"; - Names[RTLIB::SREM_I8] = "__modqi3"; - Names[RTLIB::SREM_I16] = "__modhi3"; - Names[RTLIB::SREM_I32] = "__modsi3"; - Names[RTLIB::SREM_I64] = "__moddi3"; - Names[RTLIB::SREM_I128] = "__modti3"; - Names[RTLIB::UREM_I8] = "__umodqi3"; - Names[RTLIB::UREM_I16] = "__umodhi3"; - Names[RTLIB::UREM_I32] = "__umodsi3"; - Names[RTLIB::UREM_I64] = "__umoddi3"; - Names[RTLIB::UREM_I128] = "__umodti3"; - - // These are generally not available. - Names[RTLIB::SDIVREM_I8] = 0; - Names[RTLIB::SDIVREM_I16] = 0; - Names[RTLIB::SDIVREM_I32] = 0; - Names[RTLIB::SDIVREM_I64] = 0; - Names[RTLIB::SDIVREM_I128] = 0; - Names[RTLIB::UDIVREM_I8] = 0; - Names[RTLIB::UDIVREM_I16] = 0; - Names[RTLIB::UDIVREM_I32] = 0; - Names[RTLIB::UDIVREM_I64] = 0; - Names[RTLIB::UDIVREM_I128] = 0; - - Names[RTLIB::NEG_I32] = "__negsi2"; - Names[RTLIB::NEG_I64] = "__negdi2"; - Names[RTLIB::ADD_F32] = "__addsf3"; - Names[RTLIB::ADD_F64] = "__adddf3"; - Names[RTLIB::ADD_F80] = "__addxf3"; - Names[RTLIB::ADD_PPCF128] = "__gcc_qadd"; - Names[RTLIB::SUB_F32] = "__subsf3"; - Names[RTLIB::SUB_F64] = "__subdf3"; - Names[RTLIB::SUB_F80] = "__subxf3"; - Names[RTLIB::SUB_PPCF128] = "__gcc_qsub"; - Names[RTLIB::MUL_F32] = "__mulsf3"; - Names[RTLIB::MUL_F64] = "__muldf3"; - Names[RTLIB::MUL_F80] = "__mulxf3"; - Names[RTLIB::MUL_PPCF128] = "__gcc_qmul"; - Names[RTLIB::DIV_F32] = "__divsf3"; - Names[RTLIB::DIV_F64] = "__divdf3"; - Names[RTLIB::DIV_F80] = "__divxf3"; - Names[RTLIB::DIV_PPCF128] = "__gcc_qdiv"; - Names[RTLIB::REM_F32] = "fmodf"; - Names[RTLIB::REM_F64] = "fmod"; - Names[RTLIB::REM_F80] = "fmodl"; - Names[RTLIB::REM_PPCF128] = "fmodl"; - Names[RTLIB::FMA_F32] = "fmaf"; - Names[RTLIB::FMA_F64] = "fma"; - Names[RTLIB::FMA_F80] = "fmal"; - Names[RTLIB::FMA_PPCF128] = "fmal"; - Names[RTLIB::POWI_F32] = "__powisf2"; - Names[RTLIB::POWI_F64] = "__powidf2"; - Names[RTLIB::POWI_F80] = "__powixf2"; - Names[RTLIB::POWI_PPCF128] = "__powitf2"; - Names[RTLIB::SQRT_F32] = "sqrtf"; - Names[RTLIB::SQRT_F64] = "sqrt"; - Names[RTLIB::SQRT_F80] = "sqrtl"; - Names[RTLIB::SQRT_PPCF128] = "sqrtl"; - Names[RTLIB::LOG_F32] = "logf"; - Names[RTLIB::LOG_F64] = "log"; - Names[RTLIB::LOG_F80] = "logl"; - Names[RTLIB::LOG_PPCF128] = "logl"; - Names[RTLIB::LOG2_F32] = "log2f"; - Names[RTLIB::LOG2_F64] = "log2"; - Names[RTLIB::LOG2_F80] = "log2l"; - Names[RTLIB::LOG2_PPCF128] = "log2l"; - Names[RTLIB::LOG10_F32] = "log10f"; - Names[RTLIB::LOG10_F64] = "log10"; - Names[RTLIB::LOG10_F80] = "log10l"; - Names[RTLIB::LOG10_PPCF128] = "log10l"; - Names[RTLIB::EXP_F32] = "expf"; - Names[RTLIB::EXP_F64] = "exp"; - Names[RTLIB::EXP_F80] = "expl"; - Names[RTLIB::EXP_PPCF128] = "expl"; - Names[RTLIB::EXP2_F32] = "exp2f"; - Names[RTLIB::EXP2_F64] = "exp2"; - Names[RTLIB::EXP2_F80] = "exp2l"; - Names[RTLIB::EXP2_PPCF128] = "exp2l"; - Names[RTLIB::SIN_F32] = "sinf"; - Names[RTLIB::SIN_F64] = "sin"; - Names[RTLIB::SIN_F80] = "sinl"; - Names[RTLIB::SIN_PPCF128] = "sinl"; - Names[RTLIB::COS_F32] = "cosf"; - Names[RTLIB::COS_F64] = "cos"; - Names[RTLIB::COS_F80] = "cosl"; - Names[RTLIB::COS_PPCF128] = "cosl"; - Names[RTLIB::POW_F32] = "powf"; - Names[RTLIB::POW_F64] = "pow"; - Names[RTLIB::POW_F80] = "powl"; - Names[RTLIB::POW_PPCF128] = "powl"; - Names[RTLIB::CEIL_F32] = "ceilf"; - Names[RTLIB::CEIL_F64] = "ceil"; - Names[RTLIB::CEIL_F80] = "ceill"; - Names[RTLIB::CEIL_PPCF128] = "ceill"; - Names[RTLIB::TRUNC_F32] = "truncf"; - Names[RTLIB::TRUNC_F64] = "trunc"; - Names[RTLIB::TRUNC_F80] = "truncl"; - Names[RTLIB::TRUNC_PPCF128] = "truncl"; - Names[RTLIB::RINT_F32] = "rintf"; - Names[RTLIB::RINT_F64] = "rint"; - Names[RTLIB::RINT_F80] = "rintl"; - Names[RTLIB::RINT_PPCF128] = "rintl"; - Names[RTLIB::NEARBYINT_F32] = "nearbyintf"; - Names[RTLIB::NEARBYINT_F64] = "nearbyint"; - Names[RTLIB::NEARBYINT_F80] = "nearbyintl"; - Names[RTLIB::NEARBYINT_PPCF128] = "nearbyintl"; - Names[RTLIB::FLOOR_F32] = "floorf"; - Names[RTLIB::FLOOR_F64] = "floor"; - Names[RTLIB::FLOOR_F80] = "floorl"; - Names[RTLIB::FLOOR_PPCF128] = "floorl"; - Names[RTLIB::COPYSIGN_F32] = "copysignf"; - Names[RTLIB::COPYSIGN_F64] = "copysign"; - Names[RTLIB::COPYSIGN_F80] = "copysignl"; - Names[RTLIB::COPYSIGN_PPCF128] = "copysignl"; - Names[RTLIB::FPEXT_F32_F64] = "__extendsfdf2"; - Names[RTLIB::FPEXT_F16_F32] = "__gnu_h2f_ieee"; - Names[RTLIB::FPROUND_F32_F16] = "__gnu_f2h_ieee"; - Names[RTLIB::FPROUND_F64_F32] = "__truncdfsf2"; - Names[RTLIB::FPROUND_F80_F32] = "__truncxfsf2"; - Names[RTLIB::FPROUND_PPCF128_F32] = "__trunctfsf2"; - Names[RTLIB::FPROUND_F80_F64] = "__truncxfdf2"; - Names[RTLIB::FPROUND_PPCF128_F64] = "__trunctfdf2"; - Names[RTLIB::FPTOSINT_F32_I8] = "__fixsfqi"; - Names[RTLIB::FPTOSINT_F32_I16] = "__fixsfhi"; - Names[RTLIB::FPTOSINT_F32_I32] = "__fixsfsi"; - Names[RTLIB::FPTOSINT_F32_I64] = "__fixsfdi"; - Names[RTLIB::FPTOSINT_F32_I128] = "__fixsfti"; - Names[RTLIB::FPTOSINT_F64_I8] = "__fixdfqi"; - Names[RTLIB::FPTOSINT_F64_I16] = "__fixdfhi"; - Names[RTLIB::FPTOSINT_F64_I32] = "__fixdfsi"; - Names[RTLIB::FPTOSINT_F64_I64] = "__fixdfdi"; - Names[RTLIB::FPTOSINT_F64_I128] = "__fixdfti"; - Names[RTLIB::FPTOSINT_F80_I32] = "__fixxfsi"; - Names[RTLIB::FPTOSINT_F80_I64] = "__fixxfdi"; - Names[RTLIB::FPTOSINT_F80_I128] = "__fixxfti"; - Names[RTLIB::FPTOSINT_PPCF128_I32] = "__fixtfsi"; - Names[RTLIB::FPTOSINT_PPCF128_I64] = "__fixtfdi"; - Names[RTLIB::FPTOSINT_PPCF128_I128] = "__fixtfti"; - Names[RTLIB::FPTOUINT_F32_I8] = "__fixunssfqi"; - Names[RTLIB::FPTOUINT_F32_I16] = "__fixunssfhi"; - Names[RTLIB::FPTOUINT_F32_I32] = "__fixunssfsi"; - Names[RTLIB::FPTOUINT_F32_I64] = "__fixunssfdi"; - Names[RTLIB::FPTOUINT_F32_I128] = "__fixunssfti"; - Names[RTLIB::FPTOUINT_F64_I8] = "__fixunsdfqi"; - Names[RTLIB::FPTOUINT_F64_I16] = "__fixunsdfhi"; - Names[RTLIB::FPTOUINT_F64_I32] = "__fixunsdfsi"; - Names[RTLIB::FPTOUINT_F64_I64] = "__fixunsdfdi"; - Names[RTLIB::FPTOUINT_F64_I128] = "__fixunsdfti"; - Names[RTLIB::FPTOUINT_F80_I32] = "__fixunsxfsi"; - Names[RTLIB::FPTOUINT_F80_I64] = "__fixunsxfdi"; - Names[RTLIB::FPTOUINT_F80_I128] = "__fixunsxfti"; - Names[RTLIB::FPTOUINT_PPCF128_I32] = "__fixunstfsi"; - Names[RTLIB::FPTOUINT_PPCF128_I64] = "__fixunstfdi"; - Names[RTLIB::FPTOUINT_PPCF128_I128] = "__fixunstfti"; - Names[RTLIB::SINTTOFP_I32_F32] = "__floatsisf"; - Names[RTLIB::SINTTOFP_I32_F64] = "__floatsidf"; - Names[RTLIB::SINTTOFP_I32_F80] = "__floatsixf"; - Names[RTLIB::SINTTOFP_I32_PPCF128] = "__floatsitf"; - Names[RTLIB::SINTTOFP_I64_F32] = "__floatdisf"; - Names[RTLIB::SINTTOFP_I64_F64] = "__floatdidf"; - Names[RTLIB::SINTTOFP_I64_F80] = "__floatdixf"; - Names[RTLIB::SINTTOFP_I64_PPCF128] = "__floatditf"; - Names[RTLIB::SINTTOFP_I128_F32] = "__floattisf"; - Names[RTLIB::SINTTOFP_I128_F64] = "__floattidf"; - Names[RTLIB::SINTTOFP_I128_F80] = "__floattixf"; - Names[RTLIB::SINTTOFP_I128_PPCF128] = "__floattitf"; - Names[RTLIB::UINTTOFP_I32_F32] = "__floatunsisf"; - Names[RTLIB::UINTTOFP_I32_F64] = "__floatunsidf"; - Names[RTLIB::UINTTOFP_I32_F80] = "__floatunsixf"; - Names[RTLIB::UINTTOFP_I32_PPCF128] = "__floatunsitf"; - Names[RTLIB::UINTTOFP_I64_F32] = "__floatundisf"; - Names[RTLIB::UINTTOFP_I64_F64] = "__floatundidf"; - Names[RTLIB::UINTTOFP_I64_F80] = "__floatundixf"; - Names[RTLIB::UINTTOFP_I64_PPCF128] = "__floatunditf"; - Names[RTLIB::UINTTOFP_I128_F32] = "__floatuntisf"; - Names[RTLIB::UINTTOFP_I128_F64] = "__floatuntidf"; - Names[RTLIB::UINTTOFP_I128_F80] = "__floatuntixf"; - Names[RTLIB::UINTTOFP_I128_PPCF128] = "__floatuntitf"; - Names[RTLIB::OEQ_F32] = "__eqsf2"; - Names[RTLIB::OEQ_F64] = "__eqdf2"; - Names[RTLIB::UNE_F32] = "__nesf2"; - Names[RTLIB::UNE_F64] = "__nedf2"; - Names[RTLIB::OGE_F32] = "__gesf2"; - Names[RTLIB::OGE_F64] = "__gedf2"; - Names[RTLIB::OLT_F32] = "__ltsf2"; - Names[RTLIB::OLT_F64] = "__ltdf2"; - Names[RTLIB::OLE_F32] = "__lesf2"; - Names[RTLIB::OLE_F64] = "__ledf2"; - Names[RTLIB::OGT_F32] = "__gtsf2"; - Names[RTLIB::OGT_F64] = "__gtdf2"; - Names[RTLIB::UO_F32] = "__unordsf2"; - Names[RTLIB::UO_F64] = "__unorddf2"; - Names[RTLIB::O_F32] = "__unordsf2"; - Names[RTLIB::O_F64] = "__unorddf2"; - Names[RTLIB::MEMCPY] = "memcpy"; - Names[RTLIB::MEMMOVE] = "memmove"; - Names[RTLIB::MEMSET] = "memset"; - Names[RTLIB::UNWIND_RESUME] = "_Unwind_Resume"; - Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1] = "__sync_val_compare_and_swap_1"; - Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2] = "__sync_val_compare_and_swap_2"; - Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4] = "__sync_val_compare_and_swap_4"; - Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8] = "__sync_val_compare_and_swap_8"; - Names[RTLIB::SYNC_LOCK_TEST_AND_SET_1] = "__sync_lock_test_and_set_1"; - Names[RTLIB::SYNC_LOCK_TEST_AND_SET_2] = "__sync_lock_test_and_set_2"; - Names[RTLIB::SYNC_LOCK_TEST_AND_SET_4] = "__sync_lock_test_and_set_4"; - Names[RTLIB::SYNC_LOCK_TEST_AND_SET_8] = "__sync_lock_test_and_set_8"; - Names[RTLIB::SYNC_FETCH_AND_ADD_1] = "__sync_fetch_and_add_1"; - Names[RTLIB::SYNC_FETCH_AND_ADD_2] = "__sync_fetch_and_add_2"; - Names[RTLIB::SYNC_FETCH_AND_ADD_4] = "__sync_fetch_and_add_4"; - Names[RTLIB::SYNC_FETCH_AND_ADD_8] = "__sync_fetch_and_add_8"; - Names[RTLIB::SYNC_FETCH_AND_SUB_1] = "__sync_fetch_and_sub_1"; - Names[RTLIB::SYNC_FETCH_AND_SUB_2] = "__sync_fetch_and_sub_2"; - Names[RTLIB::SYNC_FETCH_AND_SUB_4] = "__sync_fetch_and_sub_4"; - Names[RTLIB::SYNC_FETCH_AND_SUB_8] = "__sync_fetch_and_sub_8"; - Names[RTLIB::SYNC_FETCH_AND_AND_1] = "__sync_fetch_and_and_1"; - Names[RTLIB::SYNC_FETCH_AND_AND_2] = "__sync_fetch_and_and_2"; - Names[RTLIB::SYNC_FETCH_AND_AND_4] = "__sync_fetch_and_and_4"; - Names[RTLIB::SYNC_FETCH_AND_AND_8] = "__sync_fetch_and_and_8"; - Names[RTLIB::SYNC_FETCH_AND_OR_1] = "__sync_fetch_and_or_1"; - Names[RTLIB::SYNC_FETCH_AND_OR_2] = "__sync_fetch_and_or_2"; - Names[RTLIB::SYNC_FETCH_AND_OR_4] = "__sync_fetch_and_or_4"; - Names[RTLIB::SYNC_FETCH_AND_OR_8] = "__sync_fetch_and_or_8"; - Names[RTLIB::SYNC_FETCH_AND_XOR_1] = "__sync_fetch_and_xor_1"; - Names[RTLIB::SYNC_FETCH_AND_XOR_2] = "__sync_fetch_and_xor_2"; - Names[RTLIB::SYNC_FETCH_AND_XOR_4] = "__sync_fetch_and_xor_4"; - Names[RTLIB::SYNC_FETCH_AND_XOR_8] = "__sync_fetch_and_xor_8"; - Names[RTLIB::SYNC_FETCH_AND_NAND_1] = "__sync_fetch_and_nand_1"; - Names[RTLIB::SYNC_FETCH_AND_NAND_2] = "__sync_fetch_and_nand_2"; - Names[RTLIB::SYNC_FETCH_AND_NAND_4] = "__sync_fetch_and_nand_4"; - Names[RTLIB::SYNC_FETCH_AND_NAND_8] = "__sync_fetch_and_nand_8"; -} - -/// InitLibcallCallingConvs - Set default libcall CallingConvs. -/// -static void InitLibcallCallingConvs(CallingConv::ID *CCs) { - for (int i = 0; i < RTLIB::UNKNOWN_LIBCALL; ++i) { - CCs[i] = CallingConv::C; - } -} - -/// getFPEXT - Return the FPEXT_*_* value for the given types, or -/// UNKNOWN_LIBCALL if there is none. -RTLIB::Libcall RTLIB::getFPEXT(EVT OpVT, EVT RetVT) { - if (OpVT == MVT::f32) { - if (RetVT == MVT::f64) - return FPEXT_F32_F64; - } - - return UNKNOWN_LIBCALL; -} - -/// getFPROUND - Return the FPROUND_*_* value for the given types, or -/// UNKNOWN_LIBCALL if there is none. -RTLIB::Libcall RTLIB::getFPROUND(EVT OpVT, EVT RetVT) { - if (RetVT == MVT::f32) { - if (OpVT == MVT::f64) - return FPROUND_F64_F32; - if (OpVT == MVT::f80) - return FPROUND_F80_F32; - if (OpVT == MVT::ppcf128) - return FPROUND_PPCF128_F32; - } else if (RetVT == MVT::f64) { - if (OpVT == MVT::f80) - return FPROUND_F80_F64; - if (OpVT == MVT::ppcf128) - return FPROUND_PPCF128_F64; - } - - return UNKNOWN_LIBCALL; -} - -/// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or -/// UNKNOWN_LIBCALL if there is none. -RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) { - if (OpVT == MVT::f32) { - if (RetVT == MVT::i8) - return FPTOSINT_F32_I8; - if (RetVT == MVT::i16) - return FPTOSINT_F32_I16; - if (RetVT == MVT::i32) - return FPTOSINT_F32_I32; - if (RetVT == MVT::i64) - return FPTOSINT_F32_I64; - if (RetVT == MVT::i128) - return FPTOSINT_F32_I128; - } else if (OpVT == MVT::f64) { - if (RetVT == MVT::i8) - return FPTOSINT_F64_I8; - if (RetVT == MVT::i16) - return FPTOSINT_F64_I16; - if (RetVT == MVT::i32) - return FPTOSINT_F64_I32; - if (RetVT == MVT::i64) - return FPTOSINT_F64_I64; - if (RetVT == MVT::i128) - return FPTOSINT_F64_I128; - } else if (OpVT == MVT::f80) { - if (RetVT == MVT::i32) - return FPTOSINT_F80_I32; - if (RetVT == MVT::i64) - return FPTOSINT_F80_I64; - if (RetVT == MVT::i128) - return FPTOSINT_F80_I128; - } else if (OpVT == MVT::ppcf128) { - if (RetVT == MVT::i32) - return FPTOSINT_PPCF128_I32; - if (RetVT == MVT::i64) - return FPTOSINT_PPCF128_I64; - if (RetVT == MVT::i128) - return FPTOSINT_PPCF128_I128; - } - return UNKNOWN_LIBCALL; -} - -/// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or -/// UNKNOWN_LIBCALL if there is none. -RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) { - if (OpVT == MVT::f32) { - if (RetVT == MVT::i8) - return FPTOUINT_F32_I8; - if (RetVT == MVT::i16) - return FPTOUINT_F32_I16; - if (RetVT == MVT::i32) - return FPTOUINT_F32_I32; - if (RetVT == MVT::i64) - return FPTOUINT_F32_I64; - if (RetVT == MVT::i128) - return FPTOUINT_F32_I128; - } else if (OpVT == MVT::f64) { - if (RetVT == MVT::i8) - return FPTOUINT_F64_I8; - if (RetVT == MVT::i16) - return FPTOUINT_F64_I16; - if (RetVT == MVT::i32) - return FPTOUINT_F64_I32; - if (RetVT == MVT::i64) - return FPTOUINT_F64_I64; - if (RetVT == MVT::i128) - return FPTOUINT_F64_I128; - } else if (OpVT == MVT::f80) { - if (RetVT == MVT::i32) - return FPTOUINT_F80_I32; - if (RetVT == MVT::i64) - return FPTOUINT_F80_I64; - if (RetVT == MVT::i128) - return FPTOUINT_F80_I128; - } else if (OpVT == MVT::ppcf128) { - if (RetVT == MVT::i32) - return FPTOUINT_PPCF128_I32; - if (RetVT == MVT::i64) - return FPTOUINT_PPCF128_I64; - if (RetVT == MVT::i128) - return FPTOUINT_PPCF128_I128; - } - return UNKNOWN_LIBCALL; -} - -/// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or -/// UNKNOWN_LIBCALL if there is none. -RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) { - if (OpVT == MVT::i32) { - if (RetVT == MVT::f32) - return SINTTOFP_I32_F32; - else if (RetVT == MVT::f64) - return SINTTOFP_I32_F64; - else if (RetVT == MVT::f80) - return SINTTOFP_I32_F80; - else if (RetVT == MVT::ppcf128) - return SINTTOFP_I32_PPCF128; - } else if (OpVT == MVT::i64) { - if (RetVT == MVT::f32) - return SINTTOFP_I64_F32; - else if (RetVT == MVT::f64) - return SINTTOFP_I64_F64; - else if (RetVT == MVT::f80) - return SINTTOFP_I64_F80; - else if (RetVT == MVT::ppcf128) - return SINTTOFP_I64_PPCF128; - } else if (OpVT == MVT::i128) { - if (RetVT == MVT::f32) - return SINTTOFP_I128_F32; - else if (RetVT == MVT::f64) - return SINTTOFP_I128_F64; - else if (RetVT == MVT::f80) - return SINTTOFP_I128_F80; - else if (RetVT == MVT::ppcf128) - return SINTTOFP_I128_PPCF128; - } - return UNKNOWN_LIBCALL; -} - -/// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or -/// UNKNOWN_LIBCALL if there is none. -RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) { - if (OpVT == MVT::i32) { - if (RetVT == MVT::f32) - return UINTTOFP_I32_F32; - else if (RetVT == MVT::f64) - return UINTTOFP_I32_F64; - else if (RetVT == MVT::f80) - return UINTTOFP_I32_F80; - else if (RetVT == MVT::ppcf128) - return UINTTOFP_I32_PPCF128; - } else if (OpVT == MVT::i64) { - if (RetVT == MVT::f32) - return UINTTOFP_I64_F32; - else if (RetVT == MVT::f64) - return UINTTOFP_I64_F64; - else if (RetVT == MVT::f80) - return UINTTOFP_I64_F80; - else if (RetVT == MVT::ppcf128) - return UINTTOFP_I64_PPCF128; - } else if (OpVT == MVT::i128) { - if (RetVT == MVT::f32) - return UINTTOFP_I128_F32; - else if (RetVT == MVT::f64) - return UINTTOFP_I128_F64; - else if (RetVT == MVT::f80) - return UINTTOFP_I128_F80; - else if (RetVT == MVT::ppcf128) - return UINTTOFP_I128_PPCF128; - } - return UNKNOWN_LIBCALL; -} - -/// InitCmpLibcallCCs - Set default comparison libcall CC. -/// -static void InitCmpLibcallCCs(ISD::CondCode *CCs) { - memset(CCs, ISD::SETCC_INVALID, sizeof(ISD::CondCode)*RTLIB::UNKNOWN_LIBCALL); - CCs[RTLIB::OEQ_F32] = ISD::SETEQ; - CCs[RTLIB::OEQ_F64] = ISD::SETEQ; - CCs[RTLIB::UNE_F32] = ISD::SETNE; - CCs[RTLIB::UNE_F64] = ISD::SETNE; - CCs[RTLIB::OGE_F32] = ISD::SETGE; - CCs[RTLIB::OGE_F64] = ISD::SETGE; - CCs[RTLIB::OLT_F32] = ISD::SETLT; - CCs[RTLIB::OLT_F64] = ISD::SETLT; - CCs[RTLIB::OLE_F32] = ISD::SETLE; - CCs[RTLIB::OLE_F64] = ISD::SETLE; - CCs[RTLIB::OGT_F32] = ISD::SETGT; - CCs[RTLIB::OGT_F64] = ISD::SETGT; - CCs[RTLIB::UO_F32] = ISD::SETNE; - CCs[RTLIB::UO_F64] = ISD::SETNE; - CCs[RTLIB::O_F32] = ISD::SETEQ; - CCs[RTLIB::O_F64] = ISD::SETEQ; -} - /// NOTE: The constructor takes ownership of TLOF. TargetLowering::TargetLowering(const TargetMachine &tm, const TargetLoweringObjectFile *tlof) - : TM(tm), TD(TM.getDataLayout()), TLOF(*tlof) { - // All operations default to being supported. - memset(OpActions, 0, sizeof(OpActions)); - memset(LoadExtActions, 0, sizeof(LoadExtActions)); - memset(TruncStoreActions, 0, sizeof(TruncStoreActions)); - memset(IndexedModeActions, 0, sizeof(IndexedModeActions)); - memset(CondCodeActions, 0, sizeof(CondCodeActions)); - - // Set default actions for various operations. - for (unsigned VT = 0; VT != (unsigned)MVT::LAST_VALUETYPE; ++VT) { - // Default all indexed load / store to expand. - for (unsigned IM = (unsigned)ISD::PRE_INC; - IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) { - setIndexedLoadAction(IM, (MVT::SimpleValueType)VT, Expand); - setIndexedStoreAction(IM, (MVT::SimpleValueType)VT, Expand); - } - - // These operations default to expand. - setOperationAction(ISD::FGETSIGN, (MVT::SimpleValueType)VT, Expand); - setOperationAction(ISD::CONCAT_VECTORS, (MVT::SimpleValueType)VT, Expand); - } - - // Most targets ignore the @llvm.prefetch intrinsic. - setOperationAction(ISD::PREFETCH, MVT::Other, Expand); - - // ConstantFP nodes default to expand. Targets can either change this to - // Legal, in which case all fp constants are legal, or use isFPImmLegal() - // to optimize expansions for certain constants. - setOperationAction(ISD::ConstantFP, MVT::f16, Expand); - setOperationAction(ISD::ConstantFP, MVT::f32, Expand); - setOperationAction(ISD::ConstantFP, MVT::f64, Expand); - setOperationAction(ISD::ConstantFP, MVT::f80, Expand); - - // These library functions default to expand. - setOperationAction(ISD::FLOG , MVT::f16, Expand); - setOperationAction(ISD::FLOG2, MVT::f16, Expand); - setOperationAction(ISD::FLOG10, MVT::f16, Expand); - setOperationAction(ISD::FEXP , MVT::f16, Expand); - setOperationAction(ISD::FEXP2, MVT::f16, Expand); - setOperationAction(ISD::FFLOOR, MVT::f16, Expand); - setOperationAction(ISD::FNEARBYINT, MVT::f16, Expand); - setOperationAction(ISD::FCEIL, MVT::f16, Expand); - setOperationAction(ISD::FRINT, MVT::f16, Expand); - setOperationAction(ISD::FTRUNC, MVT::f16, Expand); - setOperationAction(ISD::FLOG , MVT::f32, Expand); - setOperationAction(ISD::FLOG2, MVT::f32, Expand); - setOperationAction(ISD::FLOG10, MVT::f32, Expand); - setOperationAction(ISD::FEXP , MVT::f32, Expand); - setOperationAction(ISD::FEXP2, MVT::f32, Expand); - setOperationAction(ISD::FFLOOR, MVT::f32, Expand); - setOperationAction(ISD::FNEARBYINT, MVT::f32, Expand); - setOperationAction(ISD::FCEIL, MVT::f32, Expand); - setOperationAction(ISD::FRINT, MVT::f32, Expand); - setOperationAction(ISD::FTRUNC, MVT::f32, Expand); - setOperationAction(ISD::FLOG , MVT::f64, Expand); - setOperationAction(ISD::FLOG2, MVT::f64, Expand); - setOperationAction(ISD::FLOG10, MVT::f64, Expand); - setOperationAction(ISD::FEXP , MVT::f64, Expand); - setOperationAction(ISD::FEXP2, MVT::f64, Expand); - setOperationAction(ISD::FFLOOR, MVT::f64, Expand); - setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand); - setOperationAction(ISD::FCEIL, MVT::f64, Expand); - setOperationAction(ISD::FRINT, MVT::f64, Expand); - setOperationAction(ISD::FTRUNC, MVT::f64, Expand); - - // Default ISD::TRAP to expand (which turns it into abort). - setOperationAction(ISD::TRAP, MVT::Other, Expand); - - // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand" - // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP. - // - setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand); - - IsLittleEndian = TD->isLittleEndian(); - PointerTy = MVT::getIntegerVT(8*TD->getPointerSize(0)); - memset(RegClassForVT, 0,MVT::LAST_VALUETYPE*sizeof(TargetRegisterClass*)); - memset(TargetDAGCombineArray, 0, array_lengthof(TargetDAGCombineArray)); - maxStoresPerMemset = maxStoresPerMemcpy = maxStoresPerMemmove = 8; - maxStoresPerMemsetOptSize = maxStoresPerMemcpyOptSize - = maxStoresPerMemmoveOptSize = 4; - benefitFromCodePlacementOpt = false; - UseUnderscoreSetJmp = false; - UseUnderscoreLongJmp = false; - SelectIsExpensive = false; - IntDivIsCheap = false; - Pow2DivIsCheap = false; - JumpIsExpensive = false; - predictableSelectIsExpensive = false; - StackPointerRegisterToSaveRestore = 0; - ExceptionPointerRegister = 0; - ExceptionSelectorRegister = 0; - BooleanContents = UndefinedBooleanContent; - BooleanVectorContents = UndefinedBooleanContent; - SchedPreferenceInfo = Sched::ILP; - JumpBufSize = 0; - JumpBufAlignment = 0; - MinFunctionAlignment = 0; - PrefFunctionAlignment = 0; - PrefLoopAlignment = 0; - MinStackArgumentAlignment = 1; - ShouldFoldAtomicFences = false; - InsertFencesForAtomic = false; - SupportJumpTables = true; - MinimumJumpTableEntries = 4; - - InitLibcallNames(LibcallRoutineNames); - InitCmpLibcallCCs(CmpLibcallCCs); - InitLibcallCallingConvs(LibcallCallingConvs); -} + : TargetLoweringBase(tm, tlof) {} -TargetLowering::~TargetLowering() { - delete &TLOF; -} - -MVT TargetLowering::getShiftAmountTy(EVT LHSTy) const { - return MVT::getIntegerVT(8*TD->getPointerSize(0)); +const char *TargetLowering::getTargetNodeName(unsigned Opcode) const { + return NULL; } -/// canOpTrap - Returns true if the operation can trap for the value type. -/// VT must be a legal type. -bool TargetLowering::canOpTrap(unsigned Op, EVT VT) const { - assert(isTypeLegal(VT)); - switch (Op) { - default: +/// Check whether a given call node is in tail position within its function. If +/// so, it sets Chain to the input chain of the tail call. +bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, + SDValue &Chain) const { + const Function *F = DAG.getMachineFunction().getFunction(); + + // Conservatively require the attributes of the call to match those of + // the return. Ignore noalias because it doesn't affect the call sequence. + AttributeSet CallerAttrs = F->getAttributes(); + if (AttrBuilder(CallerAttrs, AttributeSet::ReturnIndex) + .removeAttribute(Attribute::NoAlias).hasAttributes()) return false; - case ISD::FDIV: - case ISD::FREM: - case ISD::SDIV: - case ISD::UDIV: - case ISD::SREM: - case ISD::UREM: - return true; - } -} - - -static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT, - unsigned &NumIntermediates, - EVT &RegisterVT, - TargetLowering *TLI) { - // Figure out the right, legal destination reg to copy into. - unsigned NumElts = VT.getVectorNumElements(); - MVT EltTy = VT.getVectorElementType(); - - unsigned NumVectorRegs = 1; - - // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally we - // could break down into LHS/RHS like LegalizeDAG does. - if (!isPowerOf2_32(NumElts)) { - NumVectorRegs = NumElts; - NumElts = 1; - } - - // Divide the input until we get to a supported size. This will always - // end with a scalar if the target doesn't support vectors. - while (NumElts > 1 && !TLI->isTypeLegal(MVT::getVectorVT(EltTy, NumElts))) { - NumElts >>= 1; - NumVectorRegs <<= 1; - } - - NumIntermediates = NumVectorRegs; - MVT NewVT = MVT::getVectorVT(EltTy, NumElts); - if (!TLI->isTypeLegal(NewVT)) - NewVT = EltTy; - IntermediateVT = NewVT; - - unsigned NewVTSize = NewVT.getSizeInBits(); - - // Convert sizes such as i33 to i64. - if (!isPowerOf2_32(NewVTSize)) - NewVTSize = NextPowerOf2(NewVTSize); - - EVT DestVT = TLI->getRegisterType(NewVT); - RegisterVT = DestVT; - if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16. - return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits()); + // It's not safe to eliminate the sign / zero extension of the return value. + if (CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::ZExt) || + CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::SExt)) + return false; - // Otherwise, promotion or legal types use the same number of registers as - // the vector decimated to the appropriate level. - return NumVectorRegs; + // Check if the only use is a function return node. + return isUsedByReturnOnly(Node, Chain); } -/// isLegalRC - Return true if the value types that can be represented by the -/// specified register class are all legal. -bool TargetLowering::isLegalRC(const TargetRegisterClass *RC) const { - for (TargetRegisterClass::vt_iterator I = RC->vt_begin(), E = RC->vt_end(); - I != E; ++I) { - if (isTypeLegal(*I)) - return true; - } - return false; -} -/// findRepresentativeClass - Return the largest legal super-reg register class -/// of the register class for the specified type and its associated "cost". -std::pair<const TargetRegisterClass*, uint8_t> -TargetLowering::findRepresentativeClass(EVT VT) const { - const TargetRegisterInfo *TRI = getTargetMachine().getRegisterInfo(); - const TargetRegisterClass *RC = RegClassForVT[VT.getSimpleVT().SimpleTy]; - if (!RC) - return std::make_pair(RC, 0); - - // Compute the set of all super-register classes. - BitVector SuperRegRC(TRI->getNumRegClasses()); - for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI) - SuperRegRC.setBitsInMask(RCI.getMask()); - - // Find the first legal register class with the largest spill size. - const TargetRegisterClass *BestRC = RC; - for (int i = SuperRegRC.find_first(); i >= 0; i = SuperRegRC.find_next(i)) { - const TargetRegisterClass *SuperRC = TRI->getRegClass(i); - // We want the largest possible spill size. - if (SuperRC->getSize() <= BestRC->getSize()) - continue; - if (!isLegalRC(SuperRC)) - continue; - BestRC = SuperRC; - } - return std::make_pair(BestRC, 1); +/// Generate a libcall taking the given operands as arguments and returning a +/// result of type RetVT. +SDValue TargetLowering::makeLibCall(SelectionDAG &DAG, + RTLIB::Libcall LC, EVT RetVT, + const SDValue *Ops, unsigned NumOps, + bool isSigned, DebugLoc dl) const { + TargetLowering::ArgListTy Args; + Args.reserve(NumOps); + + TargetLowering::ArgListEntry Entry; + for (unsigned i = 0; i != NumOps; ++i) { + Entry.Node = Ops[i]; + Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext()); + Entry.isSExt = isSigned; + Entry.isZExt = !isSigned; + Args.push_back(Entry); + } + SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC), getPointerTy()); + + Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); + TargetLowering:: + CallLoweringInfo CLI(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false, + false, 0, getLibcallCallingConv(LC), + /*isTailCall=*/false, + /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, + Callee, Args, DAG, dl); + std::pair<SDValue,SDValue> CallInfo = LowerCallTo(CLI); + + return CallInfo.first; } -/// computeRegisterProperties - Once all of the register classes are added, -/// this allows us to compute derived properties we expose. -void TargetLowering::computeRegisterProperties() { - assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE && - "Too many value types for ValueTypeActions to hold!"); - // Everything defaults to needing one register. - for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i) { - NumRegistersForVT[i] = 1; - RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i; - } - // ...except isVoid, which doesn't need any registers. - NumRegistersForVT[MVT::isVoid] = 0; - - // Find the largest integer register class. - unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE; - for (; RegClassForVT[LargestIntReg] == 0; --LargestIntReg) - assert(LargestIntReg != MVT::i1 && "No integer registers defined!"); - - // Every integer value type larger than this largest register takes twice as - // many registers to represent as the previous ValueType. - for (unsigned ExpandedReg = LargestIntReg + 1; ; ++ExpandedReg) { - EVT ExpandedVT = (MVT::SimpleValueType)ExpandedReg; - if (!ExpandedVT.isInteger()) +/// SoftenSetCCOperands - Soften the operands of a comparison. This code is +/// shared among BR_CC, SELECT_CC, and SETCC handlers. +void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT, + SDValue &NewLHS, SDValue &NewRHS, + ISD::CondCode &CCCode, + DebugLoc dl) const { + assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128) + && "Unsupported setcc type!"); + + // Expand into one or more soft-fp libcall(s). + RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL; + switch (CCCode) { + case ISD::SETEQ: + case ISD::SETOEQ: + LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : + (VT == MVT::f64) ? RTLIB::OEQ_F64 : RTLIB::OEQ_F128; + break; + case ISD::SETNE: + case ISD::SETUNE: + LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : + (VT == MVT::f64) ? RTLIB::UNE_F64 : RTLIB::UNE_F128; + break; + case ISD::SETGE: + case ISD::SETOGE: + LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : + (VT == MVT::f64) ? RTLIB::OGE_F64 : RTLIB::OGE_F128; + break; + case ISD::SETLT: + case ISD::SETOLT: + LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : + (VT == MVT::f64) ? RTLIB::OLT_F64 : RTLIB::OLT_F128; + break; + case ISD::SETLE: + case ISD::SETOLE: + LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : + (VT == MVT::f64) ? RTLIB::OLE_F64 : RTLIB::OLE_F128; + break; + case ISD::SETGT: + case ISD::SETOGT: + LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : + (VT == MVT::f64) ? RTLIB::OGT_F64 : RTLIB::OGT_F128; + break; + case ISD::SETUO: + LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : + (VT == MVT::f64) ? RTLIB::UO_F64 : RTLIB::UO_F128; + break; + case ISD::SETO: + LC1 = (VT == MVT::f32) ? RTLIB::O_F32 : + (VT == MVT::f64) ? RTLIB::O_F64 : RTLIB::O_F128; + break; + default: + LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : + (VT == MVT::f64) ? RTLIB::UO_F64 : RTLIB::UO_F128; + switch (CCCode) { + case ISD::SETONE: + // SETONE = SETOLT | SETOGT + LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : + (VT == MVT::f64) ? RTLIB::OLT_F64 : RTLIB::OLT_F128; + // Fallthrough + case ISD::SETUGT: + LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 : + (VT == MVT::f64) ? RTLIB::OGT_F64 : RTLIB::OGT_F128; break; - NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1]; - RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg; - TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1); - ValueTypeActions.setTypeAction(ExpandedVT, TypeExpandInteger); - } - - // Inspect all of the ValueType's smaller than the largest integer - // register to see which ones need promotion. - unsigned LegalIntReg = LargestIntReg; - for (unsigned IntReg = LargestIntReg - 1; - IntReg >= (unsigned)MVT::i1; --IntReg) { - EVT IVT = (MVT::SimpleValueType)IntReg; - if (isTypeLegal(IVT)) { - LegalIntReg = IntReg; - } else { - RegisterTypeForVT[IntReg] = TransformToType[IntReg] = - (const MVT::SimpleValueType)LegalIntReg; - ValueTypeActions.setTypeAction(IVT, TypePromoteInteger); - } - } - - // ppcf128 type is really two f64's. - if (!isTypeLegal(MVT::ppcf128)) { - NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64]; - RegisterTypeForVT[MVT::ppcf128] = MVT::f64; - TransformToType[MVT::ppcf128] = MVT::f64; - ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat); - } - - // Decide how to handle f64. If the target does not have native f64 support, - // expand it to i64 and we will be generating soft float library calls. - if (!isTypeLegal(MVT::f64)) { - NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64]; - RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64]; - TransformToType[MVT::f64] = MVT::i64; - ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat); - } - - // Decide how to handle f32. If the target does not have native support for - // f32, promote it to f64 if it is legal. Otherwise, expand it to i32. - if (!isTypeLegal(MVT::f32)) { - if (isTypeLegal(MVT::f64)) { - NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::f64]; - RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::f64]; - TransformToType[MVT::f32] = MVT::f64; - ValueTypeActions.setTypeAction(MVT::f32, TypePromoteInteger); - } else { - NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32]; - RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32]; - TransformToType[MVT::f32] = MVT::i32; - ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat); - } - } - - // Loop over all of the vector value types to see which need transformations. - for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE; - i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) { - MVT VT = (MVT::SimpleValueType)i; - if (isTypeLegal(VT)) continue; - - // Determine if there is a legal wider type. If so, we should promote to - // that wider vector type. - EVT EltVT = VT.getVectorElementType(); - unsigned NElts = VT.getVectorNumElements(); - if (NElts != 1) { - bool IsLegalWiderType = false; - // First try to promote the elements of integer vectors. If no legal - // promotion was found, fallback to the widen-vector method. - for (unsigned nVT = i+1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) { - EVT SVT = (MVT::SimpleValueType)nVT; - // Promote vectors of integers to vectors with the same number - // of elements, with a wider element type. - if (SVT.getVectorElementType().getSizeInBits() > EltVT.getSizeInBits() - && SVT.getVectorNumElements() == NElts && - isTypeLegal(SVT) && SVT.getScalarType().isInteger()) { - TransformToType[i] = SVT; - RegisterTypeForVT[i] = SVT; - NumRegistersForVT[i] = 1; - ValueTypeActions.setTypeAction(VT, TypePromoteInteger); - IsLegalWiderType = true; - break; - } - } - - if (IsLegalWiderType) continue; - - // Try to widen the vector. - for (unsigned nVT = i+1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) { - EVT SVT = (MVT::SimpleValueType)nVT; - if (SVT.getVectorElementType() == EltVT && - SVT.getVectorNumElements() > NElts && - isTypeLegal(SVT)) { - TransformToType[i] = SVT; - RegisterTypeForVT[i] = SVT; - NumRegistersForVT[i] = 1; - ValueTypeActions.setTypeAction(VT, TypeWidenVector); - IsLegalWiderType = true; - break; - } - } - if (IsLegalWiderType) continue; - } - - MVT IntermediateVT; - EVT RegisterVT; - unsigned NumIntermediates; - NumRegistersForVT[i] = - getVectorTypeBreakdownMVT(VT, IntermediateVT, NumIntermediates, - RegisterVT, this); - RegisterTypeForVT[i] = RegisterVT; - - EVT NVT = VT.getPow2VectorType(); - if (NVT == VT) { - // Type is already a power of 2. The default action is to split. - TransformToType[i] = MVT::Other; - unsigned NumElts = VT.getVectorNumElements(); - ValueTypeActions.setTypeAction(VT, - NumElts > 1 ? TypeSplitVector : TypeScalarizeVector); - } else { - TransformToType[i] = NVT; - ValueTypeActions.setTypeAction(VT, TypeWidenVector); + case ISD::SETUGE: + LC2 = (VT == MVT::f32) ? RTLIB::OGE_F32 : + (VT == MVT::f64) ? RTLIB::OGE_F64 : RTLIB::OGE_F128; + break; + case ISD::SETULT: + LC2 = (VT == MVT::f32) ? RTLIB::OLT_F32 : + (VT == MVT::f64) ? RTLIB::OLT_F64 : RTLIB::OLT_F128; + break; + case ISD::SETULE: + LC2 = (VT == MVT::f32) ? RTLIB::OLE_F32 : + (VT == MVT::f64) ? RTLIB::OLE_F64 : RTLIB::OLE_F128; + break; + case ISD::SETUEQ: + LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : + (VT == MVT::f64) ? RTLIB::OEQ_F64 : RTLIB::OEQ_F128; + break; + default: llvm_unreachable("Do not know how to soften this setcc!"); } } - // Determine the 'representative' register class for each value type. - // An representative register class is the largest (meaning one which is - // not a sub-register class / subreg register class) legal register class for - // a group of value types. For example, on i386, i8, i16, and i32 - // representative would be GR32; while on x86_64 it's GR64. - for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i) { - const TargetRegisterClass* RRC; - uint8_t Cost; - tie(RRC, Cost) = findRepresentativeClass((MVT::SimpleValueType)i); - RepRegClassForVT[i] = RRC; - RepRegClassCostForVT[i] = Cost; + // Use the target specific return value for comparions lib calls. + EVT RetVT = getCmpLibcallReturnType(); + SDValue Ops[2] = { NewLHS, NewRHS }; + NewLHS = makeLibCall(DAG, LC1, RetVT, Ops, 2, false/*sign irrelevant*/, dl); + NewRHS = DAG.getConstant(0, RetVT); + CCCode = getCmpLibcallCC(LC1); + if (LC2 != RTLIB::UNKNOWN_LIBCALL) { + SDValue Tmp = DAG.getNode(ISD::SETCC, dl, getSetCCResultType(RetVT), + NewLHS, NewRHS, DAG.getCondCode(CCCode)); + NewLHS = makeLibCall(DAG, LC2, RetVT, Ops, 2, false/*sign irrelevant*/, dl); + NewLHS = DAG.getNode(ISD::SETCC, dl, getSetCCResultType(RetVT), NewLHS, + NewRHS, DAG.getCondCode(getCmpLibcallCC(LC2))); + NewLHS = DAG.getNode(ISD::OR, dl, Tmp.getValueType(), Tmp, NewLHS); + NewRHS = SDValue(); } } -const char *TargetLowering::getTargetNodeName(unsigned Opcode) const { - return NULL; -} - -EVT TargetLowering::getSetCCResultType(EVT VT) const { - assert(!VT.isVector() && "No default SetCC type for vectors!"); - return getPointerTy(0).SimpleTy; -} - -MVT::SimpleValueType TargetLowering::getCmpLibcallReturnType() const { - return MVT::i32; // return the default value -} - -/// getVectorTypeBreakdown - Vector types are broken down into some number of -/// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32 -/// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack. -/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86. -/// -/// This method returns the number of registers needed, and the VT for each -/// register. It also returns the VT and quantity of the intermediate values -/// before they are promoted/expanded. -/// -unsigned TargetLowering::getVectorTypeBreakdown(LLVMContext &Context, EVT VT, - EVT &IntermediateVT, - unsigned &NumIntermediates, - EVT &RegisterVT) const { - unsigned NumElts = VT.getVectorNumElements(); - - // If there is a wider vector type with the same element type as this one, - // or a promoted vector type that has the same number of elements which - // are wider, then we should convert to that legal vector type. - // This handles things like <2 x float> -> <4 x float> and - // <4 x i1> -> <4 x i32>. - LegalizeTypeAction TA = getTypeAction(Context, VT); - if (NumElts != 1 && (TA == TypeWidenVector || TA == TypePromoteInteger)) { - RegisterVT = getTypeToTransformTo(Context, VT); - if (isTypeLegal(RegisterVT)) { - IntermediateVT = RegisterVT; - NumIntermediates = 1; - return 1; - } - } - - // Figure out the right, legal destination reg to copy into. - EVT EltTy = VT.getVectorElementType(); - - unsigned NumVectorRegs = 1; - - // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally we - // could break down into LHS/RHS like LegalizeDAG does. - if (!isPowerOf2_32(NumElts)) { - NumVectorRegs = NumElts; - NumElts = 1; - } - - // Divide the input until we get to a supported size. This will always - // end with a scalar if the target doesn't support vectors. - while (NumElts > 1 && !isTypeLegal( - EVT::getVectorVT(Context, EltTy, NumElts))) { - NumElts >>= 1; - NumVectorRegs <<= 1; - } - - NumIntermediates = NumVectorRegs; - - EVT NewVT = EVT::getVectorVT(Context, EltTy, NumElts); - if (!isTypeLegal(NewVT)) - NewVT = EltTy; - IntermediateVT = NewVT; - - EVT DestVT = getRegisterType(Context, NewVT); - RegisterVT = DestVT; - unsigned NewVTSize = NewVT.getSizeInBits(); - - // Convert sizes such as i33 to i64. - if (!isPowerOf2_32(NewVTSize)) - NewVTSize = NextPowerOf2(NewVTSize); - - if (DestVT.bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16. - return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits()); - - // Otherwise, promotion or legal types use the same number of registers as - // the vector decimated to the appropriate level. - return NumVectorRegs; -} - -/// Get the EVTs and ArgFlags collections that represent the legalized return -/// type of the given function. This does not require a DAG or a return value, -/// and is suitable for use before any DAGs for the function are constructed. -/// TODO: Move this out of TargetLowering.cpp. -void llvm::GetReturnInfo(Type* ReturnType, Attributes attr, - SmallVectorImpl<ISD::OutputArg> &Outs, - const TargetLowering &TLI) { - SmallVector<EVT, 4> ValueVTs; - ComputeValueVTs(TLI, ReturnType, ValueVTs); - unsigned NumValues = ValueVTs.size(); - if (NumValues == 0) return; - - for (unsigned j = 0, f = NumValues; j != f; ++j) { - EVT VT = ValueVTs[j]; - ISD::NodeType ExtendKind = ISD::ANY_EXTEND; - - if (attr.hasAttribute(Attributes::SExt)) - ExtendKind = ISD::SIGN_EXTEND; - else if (attr.hasAttribute(Attributes::ZExt)) - ExtendKind = ISD::ZERO_EXTEND; - - // FIXME: C calling convention requires the return type to be promoted to - // at least 32-bit. But this is not necessary for non-C calling - // conventions. The frontend should mark functions whose return values - // require promoting with signext or zeroext attributes. - if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) { - EVT MinVT = TLI.getRegisterType(ReturnType->getContext(), MVT::i32); - if (VT.bitsLT(MinVT)) - VT = MinVT; - } - - unsigned NumParts = TLI.getNumRegisters(ReturnType->getContext(), VT); - EVT PartVT = TLI.getRegisterType(ReturnType->getContext(), VT); - - // 'inreg' on function refers to return value - ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); - if (attr.hasAttribute(Attributes::InReg)) - Flags.setInReg(); - - // Propagate extension type if any - if (attr.hasAttribute(Attributes::SExt)) - Flags.setSExt(); - else if (attr.hasAttribute(Attributes::ZExt)) - Flags.setZExt(); - - for (unsigned i = 0; i < NumParts; ++i) - Outs.push_back(ISD::OutputArg(Flags, PartVT, /*isFixed=*/true, 0, 0)); - } -} - -/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate -/// function arguments in the caller parameter area. This is the actual -/// alignment, not its logarithm. -unsigned TargetLowering::getByValTypeAlignment(Type *Ty) const { - return TD->getCallFrameTypeAlignment(Ty); -} - /// getJumpTableEncoding - Return the entry encoding for a jump table in the /// current function. The returned value is a member of the /// MachineJumpTableInfo::JTEntryKind enum. @@ -1162,7 +316,8 @@ TargetLowering::TargetLoweringOpt::ShrinkDemandedOp(SDValue Op, // Search for the smallest integer type with free casts to and from // Op's type. For expedience, just check power-of-2 integer types. const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - unsigned SmallVTBits = BitWidth - Demanded.countLeadingZeros(); + unsigned DemandedSize = BitWidth - Demanded.countLeadingZeros(); + unsigned SmallVTBits = DemandedSize; if (!isPowerOf2_32(SmallVTBits)) SmallVTBits = NextPowerOf2(SmallVTBits); for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) { @@ -1175,7 +330,9 @@ TargetLowering::TargetLoweringOpt::ShrinkDemandedOp(SDValue Op, Op.getNode()->getOperand(0)), DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getNode()->getOperand(1))); - SDValue Z = DAG.getNode(ISD::ZERO_EXTEND, dl, Op.getValueType(), X); + bool NeedZext = DemandedSize > SmallVTBits; + SDValue Z = DAG.getNode(NeedZext ? ISD::ZERO_EXTEND : ISD::ANY_EXTEND, + dl, Op.getValueType(), X); return CombineTo(Op, Z); } } @@ -2039,7 +1196,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, APInt newMask = APInt::getLowBitsSet(maskWidth, width); for (unsigned offset=0; offset<origWidth/width; offset++) { if ((newMask & Mask) == Mask) { - if (!TD->isLittleEndian()) + if (!getDataLayout()->isLittleEndian()) bestOffset = (origWidth/width - offset - 1) * (width/8); else bestOffset = (uint64_t)offset * (width/8); @@ -2111,7 +1268,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, EVT newVT = N0.getOperand(0).getValueType(); if (DCI.isBeforeLegalizeOps() || (isOperationLegal(ISD::SETCC, newVT) && - getCondCodeAction(Cond, newVT)==Legal)) + getCondCodeAction(Cond, newVT.getSimpleVT())==Legal)) return DAG.getSetCC(dl, VT, N0.getOperand(0), DAG.getConstant(C1.trunc(InSize), newVT), Cond); @@ -2207,9 +1364,10 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ; return DAG.getSetCC(dl, VT, Op0.getOperand(0), Op0.getOperand(1), Cond); - } else if (Op0.getOpcode() == ISD::AND && - isa<ConstantSDNode>(Op0.getOperand(1)) && - cast<ConstantSDNode>(Op0.getOperand(1))->getAPIntValue() == 1) { + } + if (Op0.getOpcode() == ISD::AND && + isa<ConstantSDNode>(Op0.getOperand(1)) && + cast<ConstantSDNode>(Op0.getOperand(1))->getAPIntValue() == 1) { // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0. if (Op0.getValueType().bitsGT(VT)) Op0 = DAG.getNode(ISD::AND, dl, VT, @@ -2224,6 +1382,11 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, DAG.getConstant(0, Op0.getValueType()), Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); } + if (Op0.getOpcode() == ISD::AssertZext && + cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1) + return DAG.getSetCC(dl, VT, Op0, + DAG.getConstant(0, Op0.getValueType()), + Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); } } @@ -2276,7 +1439,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, DAG.getConstant(MinVal, N0.getValueType()), ISD::SETEQ); // If we have setugt X, Max-1, turn it into seteq X, Max - else if ((Cond == ISD::SETGT || Cond == ISD::SETUGT) && C1 == MaxVal-1) + if ((Cond == ISD::SETGT || Cond == ISD::SETUGT) && C1 == MaxVal-1) return DAG.getSetCC(dl, VT, N0, DAG.getConstant(MaxVal, N0.getValueType()), ISD::SETEQ); @@ -2406,36 +1569,36 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // If the condition is not legal, see if we can find an equivalent one // which is legal. - if (!isCondCodeLegal(Cond, N0.getValueType())) { + if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) { // If the comparison was an awkward floating-point == or != and one of // the comparison operands is infinity or negative infinity, convert the // condition to a less-awkward <= or >=. if (CFP->getValueAPF().isInfinity()) { if (CFP->getValueAPF().isNegative()) { if (Cond == ISD::SETOEQ && - isCondCodeLegal(ISD::SETOLE, N0.getValueType())) + isCondCodeLegal(ISD::SETOLE, N0.getSimpleValueType())) return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLE); if (Cond == ISD::SETUEQ && - isCondCodeLegal(ISD::SETOLE, N0.getValueType())) + isCondCodeLegal(ISD::SETOLE, N0.getSimpleValueType())) return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULE); if (Cond == ISD::SETUNE && - isCondCodeLegal(ISD::SETUGT, N0.getValueType())) + isCondCodeLegal(ISD::SETUGT, N0.getSimpleValueType())) return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGT); if (Cond == ISD::SETONE && - isCondCodeLegal(ISD::SETUGT, N0.getValueType())) + isCondCodeLegal(ISD::SETUGT, N0.getSimpleValueType())) return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGT); } else { if (Cond == ISD::SETOEQ && - isCondCodeLegal(ISD::SETOGE, N0.getValueType())) + isCondCodeLegal(ISD::SETOGE, N0.getSimpleValueType())) return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGE); if (Cond == ISD::SETUEQ && - isCondCodeLegal(ISD::SETOGE, N0.getValueType())) + isCondCodeLegal(ISD::SETOGE, N0.getSimpleValueType())) return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGE); if (Cond == ISD::SETUNE && - isCondCodeLegal(ISD::SETULT, N0.getValueType())) + isCondCodeLegal(ISD::SETULT, N0.getSimpleValueType())) return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULT); if (Cond == ISD::SETONE && - isCondCodeLegal(ISD::SETULT, N0.getValueType())) + isCondCodeLegal(ISD::SETULT, N0.getSimpleValueType())) return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLT); } } @@ -2469,7 +1632,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // if it is not already. ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO; if (NewCond != Cond && (DCI.isBeforeLegalizeOps() || - getCondCodeAction(NewCond, N0.getValueType()) == Legal)) + getCondCodeAction(NewCond, N0.getSimpleValueType()) == Legal)) return DAG.getSetCC(dl, VT, N0, N1, NewCond); } @@ -2550,7 +1713,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, if (DAG.isCommutativeBinOp(N0.getOpcode())) return DAG.getSetCC(dl, VT, N0.getOperand(0), DAG.getConstant(0, N0.getValueType()), Cond); - else if (N0.getNode()->hasOneUse()) { + if (N0.getNode()->hasOneUse()) { assert(N0.getOpcode() == ISD::SUB && "Unexpected operation!"); // (Z-X) == X --> Z == X<<1 SDValue SH = DAG.getNode(ISD::SHL, dl, N1.getValueType(), N1, @@ -2566,14 +1729,14 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::XOR) { // Simplify X == (X+Z) --> Z == 0 - if (N1.getOperand(0) == N0) { + if (N1.getOperand(0) == N0) return DAG.getSetCC(dl, VT, N1.getOperand(1), DAG.getConstant(0, N1.getValueType()), Cond); - } else if (N1.getOperand(1) == N0) { - if (DAG.isCommutativeBinOp(N1.getOpcode())) { + if (N1.getOperand(1) == N0) { + if (DAG.isCommutativeBinOp(N1.getOpcode())) return DAG.getSetCC(dl, VT, N1.getOperand(0), DAG.getConstant(0, N1.getValueType()), Cond); - } else if (N1.getNode()->hasOneUse()) { + if (N1.getNode()->hasOneUse()) { assert(N1.getOpcode() == ISD::SUB && "Unexpected operation!"); // X == (Z-X) --> X<<1 == Z SDValue SH = DAG.getNode(ISD::SHL, dl, N1.getValueType(), N0, @@ -2707,7 +1870,9 @@ PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { TargetLowering::ConstraintType TargetLowering::getConstraintType(const std::string &Constraint) const { - if (Constraint.size() == 1) { + unsigned S = Constraint.size(); + + if (S == 1) { switch (Constraint[0]) { default: break; case 'r': return C_RegisterClass; @@ -2736,9 +1901,11 @@ TargetLowering::getConstraintType(const std::string &Constraint) const { } } - if (Constraint.size() > 1 && Constraint[0] == '{' && - Constraint[Constraint.size()-1] == '}') + if (S > 1 && Constraint[0] == '{' && Constraint[S-1] == '}') { + if (S == 8 && !Constraint.compare(1, 6, "memory", 6)) // "{memory}" + return C_Memory; return C_Register; + } return C_Unknown; } @@ -2830,8 +1997,11 @@ getRegForInlineAsmConstraint(const std::string &Constraint, // Remove the braces from around the name. StringRef RegName(Constraint.data()+1, Constraint.size()-2); + std::pair<unsigned, const TargetRegisterClass*> R = + std::make_pair(0u, static_cast<const TargetRegisterClass*>(0)); + // Figure out which register class contains this reg. - const TargetRegisterInfo *RI = TM.getRegisterInfo(); + const TargetRegisterInfo *RI = getTargetMachine().getRegisterInfo(); for (TargetRegisterInfo::regclass_iterator RCI = RI->regclass_begin(), E = RI->regclass_end(); RCI != E; ++RCI) { const TargetRegisterClass *RC = *RCI; @@ -2843,12 +2013,22 @@ getRegForInlineAsmConstraint(const std::string &Constraint, for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end(); I != E; ++I) { - if (RegName.equals_lower(RI->getName(*I))) - return std::make_pair(*I, RC); + if (RegName.equals_lower(RI->getName(*I))) { + std::pair<unsigned, const TargetRegisterClass*> S = + std::make_pair(*I, RC); + + // If this register class has the requested value type, return it, + // otherwise keep searching and return the first class found + // if no other is found which explicitly has the requested type. + if (RC->hasType(VT)) + return S; + else if (!R.second) + R = S; + } } } - return std::make_pair(0u, static_cast<const TargetRegisterClass*>(0)); + return R; } //===----------------------------------------------------------------------===// @@ -2858,7 +2038,7 @@ getRegForInlineAsmConstraint(const std::string &Constraint, /// a matching constraint like "4". bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const { assert(!ConstraintCode.empty() && "No known constraint!"); - return isdigit(ConstraintCode[0]); + return isdigit(static_cast<unsigned char>(ConstraintCode[0])); } /// getMatchedOperand - If this is an input matching constraint, this method @@ -2913,10 +2093,10 @@ TargetLowering::AsmOperandInfoVector TargetLowering::ParseConstraints( assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); if (StructType *STy = dyn_cast<StructType>(CS.getType())) { - OpInfo.ConstraintVT = getValueType(STy->getElementType(ResNo)); + OpInfo.ConstraintVT = getSimpleValueType(STy->getElementType(ResNo)); } else { assert(ResNo == 0 && "Asm only has one result!"); - OpInfo.ConstraintVT = getValueType(CS.getType()); + OpInfo.ConstraintVT = getSimpleValueType(CS.getType()); } ++ResNo; break; @@ -2945,7 +2125,7 @@ TargetLowering::AsmOperandInfoVector TargetLowering::ParseConstraints( // If OpTy is not a single value, it may be a struct/union that we // can tile with integers. if (!OpTy->isSingleValueType() && OpTy->isSized()) { - unsigned BitSize = TD->getTypeSizeInBits(OpTy); + unsigned BitSize = getDataLayout()->getTypeSizeInBits(OpTy); switch (BitSize) { default: break; case 1: @@ -2955,14 +2135,14 @@ TargetLowering::AsmOperandInfoVector TargetLowering::ParseConstraints( case 64: case 128: OpInfo.ConstraintVT = - EVT::getEVT(IntegerType::get(OpTy->getContext(), BitSize), true); + MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true); break; } } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) { OpInfo.ConstraintVT = MVT::getIntegerVT( - 8*TD->getPointerSize(PT->getAddressSpace())); + 8*getDataLayout()->getPointerSize(PT->getAddressSpace())); } else { - OpInfo.ConstraintVT = EVT::getEVT(OpTy, true); + OpInfo.ConstraintVT = MVT::getVT(OpTy, true); } } } @@ -3255,44 +2435,6 @@ void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo, } } -//===----------------------------------------------------------------------===// -// Loop Strength Reduction hooks -//===----------------------------------------------------------------------===// - -/// isLegalAddressingMode - Return true if the addressing mode represented -/// by AM is legal for this target, for a load/store of the specified type. -bool TargetLowering::isLegalAddressingMode(const AddrMode &AM, - Type *Ty) const { - // The default implementation of this implements a conservative RISCy, r+r and - // r+i addr mode. - - // Allows a sign-extended 16-bit immediate field. - if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1) - return false; - - // No global is ever allowed as a base. - if (AM.BaseGV) - return false; - - // Only support r+r, - switch (AM.Scale) { - case 0: // "r+i" or just "i", depending on HasBaseReg. - break; - case 1: - if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. - return false; - // Otherwise we have r+r or r+i. - break; - case 2: - if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed. - return false; - // Allow 2*r as r+r. - break; - } - - return true; -} - /// BuildExactDiv - Given an exact SDIV by a constant, create a multiplication /// with the multiplicative inverse of the constant. SDValue TargetLowering::BuildExactSDIV(SDValue Op1, SDValue Op2, DebugLoc dl, @@ -3325,7 +2467,7 @@ SDValue TargetLowering::BuildExactSDIV(SDValue Op1, SDValue Op2, DebugLoc dl, /// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html> SDValue TargetLowering:: BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, - std::vector<SDNode*>* Created) const { + std::vector<SDNode*> *Created) const { EVT VT = N->getValueType(0); DebugLoc dl= N->getDebugLoc(); @@ -3385,7 +2527,7 @@ BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, /// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html> SDValue TargetLowering:: BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, - std::vector<SDNode*>* Created) const { + std::vector<SDNode*> *Created) const { EVT VT = N->getValueType(0); DebugLoc dl = N->getDebugLoc(); diff --git a/contrib/llvm/lib/CodeGen/ShadowStackGC.cpp b/contrib/llvm/lib/CodeGen/ShadowStackGC.cpp index 8a6b120..10f64c7 100644 --- a/contrib/llvm/lib/CodeGen/ShadowStackGC.cpp +++ b/contrib/llvm/lib/CodeGen/ShadowStackGC.cpp @@ -26,12 +26,12 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "shadowstackgc" -#include "llvm/IRBuilder.h" -#include "llvm/IntrinsicInst.h" -#include "llvm/Module.h" +#include "llvm/CodeGen/GCs.h" #include "llvm/ADT/StringExtras.h" #include "llvm/CodeGen/GCStrategy.h" -#include "llvm/CodeGen/GCs.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Module.h" #include "llvm/Support/CallSite.h" using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/ShrinkWrapping.cpp b/contrib/llvm/lib/CodeGen/ShrinkWrapping.cpp index 4fbe1b3..9ab4918 100644 --- a/contrib/llvm/lib/CodeGen/ShrinkWrapping.cpp +++ b/contrib/llvm/lib/CodeGen/ShrinkWrapping.cpp @@ -35,22 +35,21 @@ #define DEBUG_TYPE "shrink-wrap" #include "PrologEpilogInserter.h" -#include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/ADT/SparseBitVector.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SparseBitVector.h" #include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/Statistic.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <sstream> using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp b/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp index 4b566fc..3903743 100644 --- a/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp +++ b/contrib/llvm/lib/CodeGen/SjLjEHPrepare.cpp @@ -13,24 +13,24 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "sjljehprepare" -#include "llvm/Constants.h" -#include "llvm/DerivedTypes.h" -#include "llvm/IRBuilder.h" -#include "llvm/Instructions.h" -#include "llvm/Intrinsics.h" -#include "llvm/LLVMContext.h" -#include "llvm/Module.h" -#include "llvm/Pass.h" +#include "llvm/CodeGen/Passes.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" -#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/DataLayout.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" @@ -43,7 +43,7 @@ STATISTIC(NumSpilled, "Number of registers live across unwind edges"); namespace { class SjLjEHPrepare : public FunctionPass { - const TargetLowering *TLI; + const TargetLoweringBase *TLI; Type *FunctionContextTy; Constant *RegisterFn; Constant *UnregisterFn; @@ -58,7 +58,7 @@ namespace { AllocaInst *FuncCtx; public: static char ID; // Pass identification, replacement for typeid - explicit SjLjEHPrepare(const TargetLowering *tli = NULL) + explicit SjLjEHPrepare(const TargetLoweringBase *tli = NULL) : FunctionPass(ID), TLI(tli) { } bool doInitialization(Module &M); bool runOnFunction(Function &F); @@ -82,7 +82,7 @@ namespace { char SjLjEHPrepare::ID = 0; // Public Interface To the SjLjEHPrepare pass. -FunctionPass *llvm::createSjLjEHPreparePass(const TargetLowering *TLI) { +FunctionPass *llvm::createSjLjEHPreparePass(const TargetLoweringBase *TLI) { return new SjLjEHPrepare(TLI); } // doInitialization - Set up decalarations and types needed to process @@ -379,13 +379,22 @@ void SjLjEHPrepare::lowerAcrossUnwindEdges(Function &F, /// the function context and marking the call sites with the appropriate /// values. These values are used by the DWARF EH emitter. bool SjLjEHPrepare::setupEntryBlockAndCallSites(Function &F) { - SmallVector<ReturnInst*, 16> Returns; - SmallVector<InvokeInst*, 16> Invokes; + SmallVector<ReturnInst*, 16> Returns; + SmallVector<InvokeInst*, 16> Invokes; SmallSetVector<LandingPadInst*, 16> LPads; // Look through the terminators of the basic blocks to find invokes. for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) if (InvokeInst *II = dyn_cast<InvokeInst>(BB->getTerminator())) { + if (Function *Callee = II->getCalledFunction()) + if (Callee->isIntrinsic() && + Callee->getIntrinsicID() == Intrinsic::donothing) { + // Remove the NOP invoke. + BranchInst::Create(II->getNormalDest(), II); + II->eraseFromParent(); + continue; + } + Invokes.push_back(II); LPads.insert(II->getUnwindDest()->getLandingPadInst()); } else if (ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator())) { diff --git a/contrib/llvm/lib/CodeGen/SlotIndexes.cpp b/contrib/llvm/lib/CodeGen/SlotIndexes.cpp index 95faafab..20049a8 100644 --- a/contrib/llvm/lib/CodeGen/SlotIndexes.cpp +++ b/contrib/llvm/lib/CodeGen/SlotIndexes.cpp @@ -142,6 +142,76 @@ void SlotIndexes::renumberIndexes(IndexList::iterator curItr) { ++NumLocalRenum; } +// Repair indexes after adding and removing instructions. +void SlotIndexes::repairIndexesInRange(MachineBasicBlock *MBB, + MachineBasicBlock::iterator Begin, + MachineBasicBlock::iterator End) { + // FIXME: Is this really necessary? The only caller repairIntervalsForRange() + // does the same thing. + // Find anchor points, which are at the beginning/end of blocks or at + // instructions that already have indexes. + while (Begin != MBB->begin() && !hasIndex(Begin)) + --Begin; + while (End != MBB->end() && !hasIndex(End)) + ++End; + + bool includeStart = (Begin == MBB->begin()); + SlotIndex startIdx; + if (includeStart) + startIdx = getMBBStartIdx(MBB); + else + startIdx = getInstructionIndex(Begin); + + SlotIndex endIdx; + if (End == MBB->end()) + endIdx = getMBBEndIdx(MBB); + else + endIdx = getInstructionIndex(End); + + // FIXME: Conceptually, this code is implementing an iterator on MBB that + // optionally includes an additional position prior to MBB->begin(), indicated + // by the includeStart flag. This is done so that we can iterate MIs in a MBB + // in parallel with SlotIndexes, but there should be a better way to do this. + IndexList::iterator ListB = startIdx.listEntry(); + IndexList::iterator ListI = endIdx.listEntry(); + MachineBasicBlock::iterator MBBI = End; + bool pastStart = false; + while (ListI != ListB || MBBI != Begin || (includeStart && !pastStart)) { + assert(ListI->getIndex() >= startIdx.getIndex() && + (includeStart || !pastStart) && + "Decremented past the beginning of region to repair."); + + MachineInstr *SlotMI = ListI->getInstr(); + MachineInstr *MI = (MBBI != MBB->end() && !pastStart) ? MBBI : 0; + bool MBBIAtBegin = MBBI == Begin && (!includeStart || pastStart); + + if (SlotMI == MI && !MBBIAtBegin) { + --ListI; + if (MBBI != Begin) + --MBBI; + else + pastStart = true; + } else if (MI && mi2iMap.find(MI) == mi2iMap.end()) { + if (MBBI != Begin) + --MBBI; + else + pastStart = true; + } else { + --ListI; + if (SlotMI) + removeMachineInstrFromMaps(SlotMI); + } + } + + // In theory this could be combined with the previous loop, but it is tricky + // to update the IndexList while we are iterating it. + for (MachineBasicBlock::iterator I = End; I != Begin;) { + --I; + MachineInstr *MI = I; + if (!MI->isDebugValue() && mi2iMap.find(MI) == mi2iMap.end()) + insertMachineInstrInMaps(MI); + } +} #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void SlotIndexes::dump() const { diff --git a/contrib/llvm/lib/CodeGen/SpillPlacement.cpp b/contrib/llvm/lib/CodeGen/SpillPlacement.cpp index 320128a..c5bbba3 100644 --- a/contrib/llvm/lib/CodeGen/SpillPlacement.cpp +++ b/contrib/llvm/lib/CodeGen/SpillPlacement.cpp @@ -29,6 +29,7 @@ #define DEBUG_TYPE "spillplacement" #include "SpillPlacement.h" +#include "llvm/ADT/BitVector.h" #include "llvm/CodeGen/EdgeBundles.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/MachineBasicBlock.h" diff --git a/contrib/llvm/lib/CodeGen/Spiller.cpp b/contrib/llvm/lib/CodeGen/Spiller.cpp index 4cd22eb..209792f 100644 --- a/contrib/llvm/lib/CodeGen/Spiller.cpp +++ b/contrib/llvm/lib/CodeGen/Spiller.cpp @@ -10,7 +10,6 @@ #define DEBUG_TYPE "spiller" #include "Spiller.h" -#include "VirtRegMap.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveRangeEdit.h" #include "llvm/CodeGen/LiveStackAnalysis.h" @@ -19,12 +18,13 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/SplitKit.cpp b/contrib/llvm/lib/CodeGen/SplitKit.cpp index dca15ee..0a3818e 100644 --- a/contrib/llvm/lib/CodeGen/SplitKit.cpp +++ b/contrib/llvm/lib/CodeGen/SplitKit.cpp @@ -14,7 +14,6 @@ #define DEBUG_TYPE "regalloc" #include "SplitKit.h" -#include "VirtRegMap.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveRangeEdit.h" @@ -22,6 +21,7 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" diff --git a/contrib/llvm/lib/CodeGen/StackColoring.cpp b/contrib/llvm/lib/CodeGen/StackColoring.cpp index 1cbee84..a789a25 100644 --- a/contrib/llvm/lib/CodeGen/StackColoring.cpp +++ b/contrib/llvm/lib/CodeGen/StackColoring.cpp @@ -22,39 +22,37 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "stackcoloring" -#include "MachineTraceMetrics.h" -#include "llvm/Function.h" -#include "llvm/Module.h" +#include "llvm/CodeGen/Passes.h" #include "llvm/ADT/BitVector.h" -#include "llvm/Analysis/Dominators.h" -#include "llvm/Analysis/ValueTracking.h" #include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SparseSet.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/Dominators.h" +#include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/LiveInterval.h" -#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineMemOperand.h" -#include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/SlotIndexes.h" #include "llvm/DebugInfo.h" -#include "llvm/Instructions.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCInstrItineraries.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; @@ -69,14 +67,14 @@ DisableColoring("no-stack-coloring", /// code. If this flag is enabled, we try to save the user. static cl::opt<bool> ProtectFromEscapedAllocas("protect-from-escaped-allocas", - cl::init(false), cl::Hidden, - cl::desc("Do not optimize lifetime zones that are broken")); + cl::init(false), cl::Hidden, + cl::desc("Do not optimize lifetime zones that " + "are broken")); STATISTIC(NumMarkerSeen, "Number of lifetime markers found."); STATISTIC(StackSpaceSaved, "Number of bytes saved due to merging slots."); STATISTIC(StackSlotMerged, "Number of stack slot merged."); -STATISTIC(EscapedAllocas, - "Number of allocas that escaped the lifetime region"); +STATISTIC(EscapedAllocas, "Number of allocas that escaped the lifetime region"); //===----------------------------------------------------------------------===// // StackColoring Pass @@ -104,12 +102,13 @@ class StackColoring : public MachineFunctionPass { }; /// Maps active slots (per bit) for each basic block. - DenseMap<MachineBasicBlock*, BlockLifetimeInfo> BlockLiveness; + typedef DenseMap<const MachineBasicBlock*, BlockLifetimeInfo> LivenessMap; + LivenessMap BlockLiveness; /// Maps serial numbers to basic blocks. - DenseMap<MachineBasicBlock*, int> BasicBlocks; + DenseMap<const MachineBasicBlock*, int> BasicBlocks; /// Maps basic blocks to a serial number. - SmallVector<MachineBasicBlock*, 8> BasicBlockNumbering; + SmallVector<const MachineBasicBlock*, 8> BasicBlockNumbering; /// Maps liveness intervals for each slot. SmallVector<LiveInterval*, 16> Intervals; @@ -146,7 +145,7 @@ public: private: /// Debug. - void dump(); + void dump() const; /// Removes all of the lifetime marker instructions from the function. /// \returns true if any markers were removed. @@ -201,31 +200,35 @@ void StackColoring::getAnalysisUsage(AnalysisUsage &AU) const { MachineFunctionPass::getAnalysisUsage(AU); } -void StackColoring::dump() { +void StackColoring::dump() const { for (df_iterator<MachineFunction*> FI = df_begin(MF), FE = df_end(MF); FI != FE; ++FI) { - unsigned Num = BasicBlocks[*FI]; - DEBUG(dbgs()<<"Inspecting block #"<<Num<<" ["<<FI->getName()<<"]\n"); - Num = 0; + DEBUG(dbgs()<<"Inspecting block #"<<BasicBlocks.lookup(*FI)<< + " ["<<FI->getName()<<"]\n"); + + LivenessMap::const_iterator BI = BlockLiveness.find(*FI); + assert(BI != BlockLiveness.end() && "Block not found"); + const BlockLifetimeInfo &BlockInfo = BI->second; + DEBUG(dbgs()<<"BEGIN : {"); - for (unsigned i=0; i < BlockLiveness[*FI].Begin.size(); ++i) - DEBUG(dbgs()<<BlockLiveness[*FI].Begin.test(i)<<" "); + for (unsigned i=0; i < BlockInfo.Begin.size(); ++i) + DEBUG(dbgs()<<BlockInfo.Begin.test(i)<<" "); DEBUG(dbgs()<<"}\n"); DEBUG(dbgs()<<"END : {"); - for (unsigned i=0; i < BlockLiveness[*FI].End.size(); ++i) - DEBUG(dbgs()<<BlockLiveness[*FI].End.test(i)<<" "); + for (unsigned i=0; i < BlockInfo.End.size(); ++i) + DEBUG(dbgs()<<BlockInfo.End.test(i)<<" "); DEBUG(dbgs()<<"}\n"); DEBUG(dbgs()<<"LIVE_IN: {"); - for (unsigned i=0; i < BlockLiveness[*FI].LiveIn.size(); ++i) - DEBUG(dbgs()<<BlockLiveness[*FI].LiveIn.test(i)<<" "); + for (unsigned i=0; i < BlockInfo.LiveIn.size(); ++i) + DEBUG(dbgs()<<BlockInfo.LiveIn.test(i)<<" "); DEBUG(dbgs()<<"}\n"); DEBUG(dbgs()<<"LIVEOUT: {"); - for (unsigned i=0; i < BlockLiveness[*FI].LiveOut.size(); ++i) - DEBUG(dbgs()<<BlockLiveness[*FI].LiveOut.test(i)<<" "); + for (unsigned i=0; i < BlockInfo.LiveOut.size(); ++i) + DEBUG(dbgs()<<BlockInfo.LiveOut.test(i)<<" "); DEBUG(dbgs()<<"}\n"); } } @@ -243,8 +246,11 @@ unsigned StackColoring::collectMarkers(unsigned NumSlot) { BasicBlocks[*FI] = BasicBlockNumbering.size(); BasicBlockNumbering.push_back(*FI); - BlockLiveness[*FI].Begin.resize(NumSlot); - BlockLiveness[*FI].End.resize(NumSlot); + // Keep a reference to avoid repeated lookups. + BlockLifetimeInfo &BlockInfo = BlockLiveness[*FI]; + + BlockInfo.Begin.resize(NumSlot); + BlockInfo.End.resize(NumSlot); for (MachineBasicBlock::iterator BI = (*FI)->begin(), BE = (*FI)->end(); BI != BE; ++BI) { @@ -256,7 +262,7 @@ unsigned StackColoring::collectMarkers(unsigned NumSlot) { Markers.push_back(BI); bool IsStart = BI->getOpcode() == TargetOpcode::LIFETIME_START; - MachineOperand &MI = BI->getOperand(0); + const MachineOperand &MI = BI->getOperand(0); unsigned Slot = MI.getIndex(); MarkersFound++; @@ -268,15 +274,15 @@ unsigned StackColoring::collectMarkers(unsigned NumSlot) { } if (IsStart) { - BlockLiveness[*FI].Begin.set(Slot); + BlockInfo.Begin.set(Slot); } else { - if (BlockLiveness[*FI].Begin.test(Slot)) { + if (BlockInfo.Begin.test(Slot)) { // Allocas that start and end within a single block are handled // specially when computing the LiveIntervals to avoid pessimizing // the liveness propagation. - BlockLiveness[*FI].Begin.reset(Slot); + BlockInfo.Begin.reset(Slot); } else { - BlockLiveness[*FI].End.set(Slot); + BlockInfo.End.set(Slot); } } } @@ -293,47 +299,58 @@ void StackColoring::calculateLocalLiveness() { // formulation, and END is equivalent to GEN. The result of this computation // is a map from blocks to bitvectors where the bitvectors represent which // allocas are live in/out of that block. - SmallPtrSet<MachineBasicBlock*, 8> BBSet(BasicBlockNumbering.begin(), - BasicBlockNumbering.end()); + SmallPtrSet<const MachineBasicBlock*, 8> BBSet(BasicBlockNumbering.begin(), + BasicBlockNumbering.end()); unsigned NumSSMIters = 0; bool changed = true; while (changed) { changed = false; ++NumSSMIters; - SmallPtrSet<MachineBasicBlock*, 8> NextBBSet; + SmallPtrSet<const MachineBasicBlock*, 8> NextBBSet; - for (SmallVector<MachineBasicBlock*, 8>::iterator + for (SmallVector<const MachineBasicBlock*, 8>::iterator PI = BasicBlockNumbering.begin(), PE = BasicBlockNumbering.end(); PI != PE; ++PI) { - MachineBasicBlock *BB = *PI; + const MachineBasicBlock *BB = *PI; if (!BBSet.count(BB)) continue; + // Use an iterator to avoid repeated lookups. + LivenessMap::iterator BI = BlockLiveness.find(BB); + assert(BI != BlockLiveness.end() && "Block not found"); + BlockLifetimeInfo &BlockInfo = BI->second; + BitVector LocalLiveIn; BitVector LocalLiveOut; // Forward propagation from begins to ends. - for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(), - PE = BB->pred_end(); PI != PE; ++PI) - LocalLiveIn |= BlockLiveness[*PI].LiveOut; - LocalLiveIn |= BlockLiveness[BB].End; - LocalLiveIn.reset(BlockLiveness[BB].Begin); + for (MachineBasicBlock::const_pred_iterator PI = BB->pred_begin(), + PE = BB->pred_end(); PI != PE; ++PI) { + LivenessMap::const_iterator I = BlockLiveness.find(*PI); + assert(I != BlockLiveness.end() && "Predecessor not found"); + LocalLiveIn |= I->second.LiveOut; + } + LocalLiveIn |= BlockInfo.End; + LocalLiveIn.reset(BlockInfo.Begin); // Reverse propagation from ends to begins. - for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), - SE = BB->succ_end(); SI != SE; ++SI) - LocalLiveOut |= BlockLiveness[*SI].LiveIn; - LocalLiveOut |= BlockLiveness[BB].Begin; - LocalLiveOut.reset(BlockLiveness[BB].End); + for (MachineBasicBlock::const_succ_iterator SI = BB->succ_begin(), + SE = BB->succ_end(); SI != SE; ++SI) { + LivenessMap::const_iterator I = BlockLiveness.find(*SI); + assert(I != BlockLiveness.end() && "Successor not found"); + LocalLiveOut |= I->second.LiveIn; + } + LocalLiveOut |= BlockInfo.Begin; + LocalLiveOut.reset(BlockInfo.End); LocalLiveIn |= LocalLiveOut; LocalLiveOut |= LocalLiveIn; // After adopting the live bits, we need to turn-off the bits which // are de-activated in this block. - LocalLiveOut.reset(BlockLiveness[BB].End); - LocalLiveIn.reset(BlockLiveness[BB].Begin); + LocalLiveOut.reset(BlockInfo.End); + LocalLiveIn.reset(BlockInfo.Begin); // If we have both BEGIN and END markers in the same basic block then // we know that the BEGIN marker comes after the END, because we already @@ -342,25 +359,25 @@ void StackColoring::calculateLocalLiveness() { // Want to enable the LIVE_IN and LIVE_OUT of slots that have both // BEGIN and END because it means that the value lives before and after // this basic block. - BitVector LocalEndBegin = BlockLiveness[BB].End; - LocalEndBegin &= BlockLiveness[BB].Begin; + BitVector LocalEndBegin = BlockInfo.End; + LocalEndBegin &= BlockInfo.Begin; LocalLiveIn |= LocalEndBegin; LocalLiveOut |= LocalEndBegin; - if (LocalLiveIn.test(BlockLiveness[BB].LiveIn)) { + if (LocalLiveIn.test(BlockInfo.LiveIn)) { changed = true; - BlockLiveness[BB].LiveIn |= LocalLiveIn; + BlockInfo.LiveIn |= LocalLiveIn; - for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(), + for (MachineBasicBlock::const_pred_iterator PI = BB->pred_begin(), PE = BB->pred_end(); PI != PE; ++PI) NextBBSet.insert(*PI); } - if (LocalLiveOut.test(BlockLiveness[BB].LiveOut)) { + if (LocalLiveOut.test(BlockInfo.LiveOut)) { changed = true; - BlockLiveness[BB].LiveOut |= LocalLiveOut; + BlockInfo.LiveOut |= LocalLiveOut; - for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), + for (MachineBasicBlock::const_succ_iterator SI = BB->succ_begin(), SE = BB->succ_end(); SI != SE; ++SI) NextBBSet.insert(*SI); } @@ -384,9 +401,9 @@ void StackColoring::calculateLiveIntervals(unsigned NumSlots) { Finishes.resize(NumSlots); // Create the interval for the basic blocks with lifetime markers in them. - for (SmallVector<MachineInstr*, 8>::iterator it = Markers.begin(), + for (SmallVectorImpl<MachineInstr*>::const_iterator it = Markers.begin(), e = Markers.end(); it != e; ++it) { - MachineInstr *MI = *it; + const MachineInstr *MI = *it; if (MI->getParent() != MBB) continue; @@ -395,7 +412,7 @@ void StackColoring::calculateLiveIntervals(unsigned NumSlots) { "Invalid Lifetime marker"); bool IsStart = MI->getOpcode() == TargetOpcode::LIFETIME_START; - MachineOperand &Mo = MI->getOperand(0); + const MachineOperand &Mo = MI->getOperand(0); int Slot = Mo.getIndex(); assert(Slot >= 0 && "Invalid slot"); @@ -482,7 +499,7 @@ void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) { // Keep a list of *allocas* which need to be remapped. DenseMap<const AllocaInst*, const AllocaInst*> Allocas; - for (DenseMap<int, int>::iterator it = SlotRemap.begin(), + for (DenseMap<int, int>::const_iterator it = SlotRemap.begin(), e = SlotRemap.end(); it != e; ++it) { const AllocaInst *From = MFI->getObjectAllocation(it->first); const AllocaInst *To = MFI->getObjectAllocation(it->second); @@ -560,7 +577,7 @@ void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) { SlotIndex Index = Indexes->getInstructionIndex(I); LiveInterval *Interval = Intervals[FromSlot]; assert(Interval->find(Index) != Interval->end() && - "Found instruction usage outside of live range."); + "Found instruction usage outside of live range."); } #endif @@ -577,8 +594,8 @@ void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) { } void StackColoring::removeInvalidSlotRanges() { - MachineFunction::iterator BB, BBE; - MachineBasicBlock::iterator I, IE; + MachineFunction::const_iterator BB, BBE; + MachineBasicBlock::const_iterator I, IE; for (BB = MF->begin(), BBE = MF->end(); BB != BBE; ++BB) for (I = BB->begin(), IE = BB->end(); I != IE; ++I) { @@ -597,7 +614,7 @@ void StackColoring::removeInvalidSlotRanges() { // Check all of the machine operands. for (unsigned i = 0 ; i < I->getNumOperands(); ++i) { - MachineOperand &MO = I->getOperand(i); + const MachineOperand &MO = I->getOperand(i); if (!MO.isFI()) continue; @@ -720,11 +737,13 @@ bool StackColoring::runOnMachineFunction(MachineFunction &Func) { // and continue. // Sort the slots according to their size. Place unused slots at the end. - std::sort(SortedSlots.begin(), SortedSlots.end(), SlotSizeSorter(MFI)); + // Use stable sort to guarantee deterministic code generation. + std::stable_sort(SortedSlots.begin(), SortedSlots.end(), + SlotSizeSorter(MFI)); - bool Chanded = true; - while (Chanded) { - Chanded = false; + bool Changed = true; + while (Changed) { + Changed = false; for (unsigned I = 0; I < NumSlots; ++I) { if (SortedSlots[I] == -1) continue; @@ -741,7 +760,7 @@ bool StackColoring::runOnMachineFunction(MachineFunction &Func) { // Merge disjoint slots. if (!First->overlaps(*Second)) { - Chanded = true; + Changed = true; First->MergeRangesInAsValue(*Second, First->getValNumInfo(0)); SlotRemap[SecondSlot] = FirstSlot; SortedSlots[J] = -1; diff --git a/contrib/llvm/lib/CodeGen/StackProtector.cpp b/contrib/llvm/lib/CodeGen/StackProtector.cpp index 31e9ec0..fbef347 100644 --- a/contrib/llvm/lib/CodeGen/StackProtector.cpp +++ b/contrib/llvm/lib/CodeGen/StackProtector.cpp @@ -16,33 +16,44 @@ #define DEBUG_TYPE "stack-protector" #include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/Triple.h" #include "llvm/Analysis/Dominators.h" -#include "llvm/Attributes.h" -#include "llvm/Constants.h" -#include "llvm/DerivedTypes.h" -#include "llvm/Function.h" -#include "llvm/Instructions.h" -#include "llvm/Intrinsics.h" -#include "llvm/Module.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Module.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" -#include "llvm/DataLayout.h" #include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetOptions.h" -#include "llvm/ADT/Triple.h" using namespace llvm; +STATISTIC(NumFunProtected, "Number of functions protected"); +STATISTIC(NumAddrTaken, "Number of local variables that have their address" + " taken."); + namespace { class StackProtector : public FunctionPass { /// TLI - Keep a pointer of a TargetLowering to consult for determining /// target type sizes. - const TargetLowering *TLI; + const TargetLoweringBase *TLI; Function *F; Module *M; DominatorTree *DT; + /// VisitedPHIs - The set of PHI nodes visited when determining + /// if a variable's reference has been taken. This set + /// is maintained to ensure we don't visit the same PHI node multiple + /// times. + SmallPtrSet<const PHINode*, 16> VisitedPHIs; + /// InsertStackProtectors - Insert code into the prologue and epilogue of /// the function. /// @@ -58,17 +69,21 @@ namespace { /// ContainsProtectableArray - Check whether the type either is an array or /// contains an array of sufficient size so that we need stack protectors /// for it. - bool ContainsProtectableArray(Type *Ty, bool InStruct = false) const; + bool ContainsProtectableArray(Type *Ty, bool Strong = false, + bool InStruct = false) const; + + /// \brief Check whether a stack allocation has its address taken. + bool HasAddressTaken(const Instruction *AI); /// RequiresStackProtector - Check whether or not this function needs a /// stack protector based upon the stack protector level. - bool RequiresStackProtector() const; + bool RequiresStackProtector(); public: static char ID; // Pass identification, replacement for typeid. StackProtector() : FunctionPass(ID), TLI(0) { initializeStackProtectorPass(*PassRegistry::getPassRegistry()); } - StackProtector(const TargetLowering *tli) + StackProtector(const TargetLoweringBase *tli) : FunctionPass(ID), TLI(tli) { initializeStackProtectorPass(*PassRegistry::getPassRegistry()); } @@ -85,7 +100,7 @@ char StackProtector::ID = 0; INITIALIZE_PASS(StackProtector, "stack-protector", "Insert stack protectors", false, false) -FunctionPass *llvm::createStackProtectorPass(const TargetLowering *tli) { +FunctionPass *llvm::createStackProtectorPass(const TargetLoweringBase *tli) { return new StackProtector(tli); } @@ -96,15 +111,21 @@ bool StackProtector::runOnFunction(Function &Fn) { if (!RequiresStackProtector()) return false; + ++NumFunProtected; return InsertStackProtectors(); } /// ContainsProtectableArray - Check whether the type either is an array or /// contains a char array of sufficient size so that we need stack protectors /// for it. -bool StackProtector::ContainsProtectableArray(Type *Ty, bool InStruct) const { +bool StackProtector::ContainsProtectableArray(Type *Ty, bool Strong, + bool InStruct) const { if (!Ty) return false; if (ArrayType *AT = dyn_cast<ArrayType>(Ty)) { + // In strong mode any array, regardless of type and size, triggers a + // protector + if (Strong) + return true; const TargetMachine &TM = TLI->getTargetMachine(); if (!AT->getElementType()->isIntegerTy(8)) { Triple Trip(TM.getTargetTriple()); @@ -126,37 +147,103 @@ bool StackProtector::ContainsProtectableArray(Type *Ty, bool InStruct) const { for (StructType::element_iterator I = ST->element_begin(), E = ST->element_end(); I != E; ++I) - if (ContainsProtectableArray(*I, true)) + if (ContainsProtectableArray(*I, Strong, true)) return true; return false; } -/// RequiresStackProtector - Check whether or not this function needs a stack -/// protector based upon the stack protector level. The heuristic we use is to -/// add a guard variable to functions that call alloca, and functions with -/// buffers larger than SSPBufferSize bytes. -bool StackProtector::RequiresStackProtector() const { - if (F->getFnAttributes().hasAttribute(Attributes::StackProtectReq)) - return true; +bool StackProtector::HasAddressTaken(const Instruction *AI) { + for (Value::const_use_iterator UI = AI->use_begin(), UE = AI->use_end(); + UI != UE; ++UI) { + const User *U = *UI; + if (const StoreInst *SI = dyn_cast<StoreInst>(U)) { + if (AI == SI->getValueOperand()) + return true; + } else if (const PtrToIntInst *SI = dyn_cast<PtrToIntInst>(U)) { + if (AI == SI->getOperand(0)) + return true; + } else if (isa<CallInst>(U)) { + return true; + } else if (isa<InvokeInst>(U)) { + return true; + } else if (const SelectInst *SI = dyn_cast<SelectInst>(U)) { + if (HasAddressTaken(SI)) + return true; + } else if (const PHINode *PN = dyn_cast<PHINode>(U)) { + // Keep track of what PHI nodes we have already visited to ensure + // they are only visited once. + if (VisitedPHIs.insert(PN)) + if (HasAddressTaken(PN)) + return true; + } else if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(U)) { + if (HasAddressTaken(GEP)) + return true; + } else if (const BitCastInst *BI = dyn_cast<BitCastInst>(U)) { + if (HasAddressTaken(BI)) + return true; + } + } + return false; +} - if (!F->getFnAttributes().hasAttribute(Attributes::StackProtect)) +/// \brief Check whether or not this function needs a stack protector based +/// upon the stack protector level. +/// +/// We use two heuristics: a standard (ssp) and strong (sspstrong). +/// The standard heuristic which will add a guard variable to functions that +/// call alloca with a either a variable size or a size >= SSPBufferSize, +/// functions with character buffers larger than SSPBufferSize, and functions +/// with aggregates containing character buffers larger than SSPBufferSize. The +/// strong heuristic will add a guard variables to functions that call alloca +/// regardless of size, functions with any buffer regardless of type and size, +/// functions with aggregates that contain any buffer regardless of type and +/// size, and functions that contain stack-based variables that have had their +/// address taken. +bool StackProtector::RequiresStackProtector() { + bool Strong = false; + if (F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, + Attribute::StackProtectReq)) + return true; + else if (F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, + Attribute::StackProtectStrong)) + Strong = true; + else if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex, + Attribute::StackProtect)) return false; for (Function::iterator I = F->begin(), E = F->end(); I != E; ++I) { BasicBlock *BB = I; for (BasicBlock::iterator - II = BB->begin(), IE = BB->end(); II != IE; ++II) + II = BB->begin(), IE = BB->end(); II != IE; ++II) { if (AllocaInst *AI = dyn_cast<AllocaInst>(II)) { - if (AI->isArrayAllocation()) - // This is a call to alloca with a variable size. Emit stack - // protectors. + if (AI->isArrayAllocation()) { + // SSP-Strong: Enable protectors for any call to alloca, regardless + // of size. + if (Strong) + return true; + + if (const ConstantInt *CI = + dyn_cast<ConstantInt>(AI->getArraySize())) { + unsigned BufferSize = TLI->getTargetMachine().Options.SSPBufferSize; + if (CI->getLimitedValue(BufferSize) >= BufferSize) + // A call to alloca with size >= SSPBufferSize requires + // stack protectors. + return true; + } else // A call to alloca with a variable size requires protectors. + return true; + } + + if (ContainsProtectableArray(AI->getAllocatedType(), Strong)) return true; - if (ContainsProtectableArray(AI->getAllocatedType())) + if (Strong && HasAddressTaken(AI)) { + ++NumAddrTaken; return true; + } } + } } return false; diff --git a/contrib/llvm/lib/CodeGen/StackSlotColoring.cpp b/contrib/llvm/lib/CodeGen/StackSlotColoring.cpp index d349abc..f951561 100644 --- a/contrib/llvm/lib/CodeGen/StackSlotColoring.cpp +++ b/contrib/llvm/lib/CodeGen/StackSlotColoring.cpp @@ -12,8 +12,11 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "stackslotcoloring" -#include "llvm/Module.h" #include "llvm/CodeGen/Passes.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveStackAnalysis.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -22,14 +25,11 @@ #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/IR/Module.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/ADT/BitVector.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" #include <vector> using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/StrongPHIElimination.cpp b/contrib/llvm/lib/CodeGen/StrongPHIElimination.cpp index 39fd600..b337c53 100644 --- a/contrib/llvm/lib/CodeGen/StrongPHIElimination.cpp +++ b/contrib/llvm/lib/CodeGen/StrongPHIElimination.cpp @@ -39,17 +39,17 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "strongphielim" -#include "PHIEliminationUtils.h" #include "llvm/CodeGen/Passes.h" +#include "PHIEliminationUtils.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/Statistic.h" #include "llvm/Support/Debug.h" +#include "llvm/Target/TargetInstrInfo.h" using namespace llvm; namespace { diff --git a/contrib/llvm/lib/CodeGen/TailDuplication.cpp b/contrib/llvm/lib/CodeGen/TailDuplication.cpp index 1497d1b..1ec8817 100644 --- a/contrib/llvm/lib/CodeGen/TailDuplication.cpp +++ b/contrib/llvm/lib/CodeGen/TailDuplication.cpp @@ -13,25 +13,25 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "tailduplication" -#include "llvm/Function.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/OwningPtr.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/MachineSSAUpdater.h" #include "llvm/CodeGen/RegisterScavenging.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/IR/Function.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/OwningPtr.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SetVector.h" -#include "llvm/ADT/Statistic.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; STATISTIC(NumTails , "Number of tails duplicated"); @@ -461,6 +461,7 @@ TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, II != EE; ++II) { if (!II->isPHI()) break; + MachineInstrBuilder MIB(*FromBB->getParent(), II); unsigned Idx = 0; for (unsigned i = 1, e = II->getNumOperands(); i != e; i += 2) { MachineOperand &MO = II->getOperand(i+1); @@ -508,8 +509,7 @@ TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, II->getOperand(Idx+1).setMBB(SrcBB); Idx = 0; } else { - II->addOperand(MachineOperand::CreateReg(SrcReg, false)); - II->addOperand(MachineOperand::CreateMBB(SrcBB)); + MIB.addReg(SrcReg).addMBB(SrcBB); } } } else { @@ -521,8 +521,7 @@ TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, II->getOperand(Idx+1).setMBB(SrcBB); Idx = 0; } else { - II->addOperand(MachineOperand::CreateReg(Reg, false)); - II->addOperand(MachineOperand::CreateMBB(SrcBB)); + MIB.addReg(Reg).addMBB(SrcBB); } } } @@ -552,8 +551,8 @@ TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, // compensate for the duplication. unsigned MaxDuplicateCount; if (TailDuplicateSize.getNumOccurrences() == 0 && - MF.getFunction()->getFnAttributes(). - hasAttribute(Attributes::OptimizeForSize)) + MF.getFunction()->getAttributes(). + hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize)) MaxDuplicateCount = 1; else MaxDuplicateCount = TailDuplicateSize; diff --git a/contrib/llvm/lib/CodeGen/TargetFrameLoweringImpl.cpp b/contrib/llvm/lib/CodeGen/TargetFrameLoweringImpl.cpp index cadb878..883e9d1 100644 --- a/contrib/llvm/lib/CodeGen/TargetFrameLoweringImpl.cpp +++ b/contrib/llvm/lib/CodeGen/TargetFrameLoweringImpl.cpp @@ -11,12 +11,11 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Target/TargetFrameLowering.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" - #include <cstdlib> using namespace llvm; diff --git a/contrib/llvm/lib/CodeGen/TargetInstrInfoImpl.cpp b/contrib/llvm/lib/CodeGen/TargetInstrInfo.cpp index 4439192..20eb918 100644 --- a/contrib/llvm/lib/CodeGen/TargetInstrInfoImpl.cpp +++ b/contrib/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -1,4 +1,4 @@ -//===-- TargetInstrInfoImpl.cpp - Target Instruction Information ----------===// +//===-- TargetInstrInfo.cpp - Target Instruction Information --------------===// // // The LLVM Compiler Infrastructure // @@ -7,39 +7,96 @@ // //===----------------------------------------------------------------------===// // -// This file implements the TargetInstrInfoImpl class, it just provides default -// implementations of various methods. +// This file implements the TargetInstrInfo class. // //===----------------------------------------------------------------------===// #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/ScoreboardHazardRecognizer.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/CodeGen/ScoreboardHazardRecognizer.h" +#include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCInstrItineraries.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include <cctype> using namespace llvm; static cl::opt<bool> DisableHazardRecognizer( "disable-sched-hazard", cl::Hidden, cl::init(false), cl::desc("Disable hazard detection during preRA scheduling")); +TargetInstrInfo::~TargetInstrInfo() { +} + +const TargetRegisterClass* +TargetInstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum, + const TargetRegisterInfo *TRI, + const MachineFunction &MF) const { + if (OpNum >= MCID.getNumOperands()) + return 0; + + short RegClass = MCID.OpInfo[OpNum].RegClass; + if (MCID.OpInfo[OpNum].isLookupPtrRegClass()) + return TRI->getPointerRegClass(MF, RegClass); + + // Instructions like INSERT_SUBREG do not have fixed register classes. + if (RegClass < 0) + return 0; + + // Otherwise just look it up normally. + return TRI->getRegClass(RegClass); +} + +/// insertNoop - Insert a noop into the instruction stream at the specified +/// point. +void TargetInstrInfo::insertNoop(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MI) const { + llvm_unreachable("Target didn't implement insertNoop!"); +} + +/// Measure the specified inline asm to determine an approximation of its +/// length. +/// Comments (which run till the next SeparatorString or newline) do not +/// count as an instruction. +/// Any other non-whitespace text is considered an instruction, with +/// multiple instructions separated by SeparatorString or newlines. +/// Variable-length instructions are not handled here; this function +/// may be overloaded in the target code to do that. +unsigned TargetInstrInfo::getInlineAsmLength(const char *Str, + const MCAsmInfo &MAI) const { + + + // Count the number of instructions in the asm. + bool atInsnStart = true; + unsigned Length = 0; + for (; *Str; ++Str) { + if (*Str == '\n' || strncmp(Str, MAI.getSeparatorString(), + strlen(MAI.getSeparatorString())) == 0) + atInsnStart = true; + if (atInsnStart && !std::isspace(static_cast<unsigned char>(*Str))) { + Length += MAI.getMaxInstLength(); + atInsnStart = false; + } + if (atInsnStart && strncmp(Str, MAI.getCommentString(), + strlen(MAI.getCommentString())) == 0) + atInsnStart = false; + } + + return Length; +} + /// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything /// after it, replacing it with an unconditional branch to NewDest. void -TargetInstrInfoImpl::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, - MachineBasicBlock *NewDest) const { +TargetInstrInfo::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, + MachineBasicBlock *NewDest) const { MachineBasicBlock *MBB = Tail->getParent(); // Remove all the old successors of MBB from the CFG. @@ -58,8 +115,8 @@ TargetInstrInfoImpl::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, // commuteInstruction - The default implementation of this method just exchanges // the two operands returned by findCommutedOpIndices. -MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI, - bool NewMI) const { +MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr *MI, + bool NewMI) const { const MCInstrDesc &MCID = MI->getDesc(); bool HasDef = MCID.getNumDefs(); if (HasDef && !MI->getOperand(0).isReg()) @@ -119,11 +176,11 @@ MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI, /// findCommutedOpIndices - If specified MI is commutable, return the two /// operand indices that would swap value. Return true if the instruction /// is not in a form which this routine understands. -bool TargetInstrInfoImpl::findCommutedOpIndices(MachineInstr *MI, - unsigned &SrcOpIdx1, - unsigned &SrcOpIdx2) const { +bool TargetInstrInfo::findCommutedOpIndices(MachineInstr *MI, + unsigned &SrcOpIdx1, + unsigned &SrcOpIdx2) const { assert(!MI->isBundle() && - "TargetInstrInfoImpl::findCommutedOpIndices() can't handle bundles"); + "TargetInstrInfo::findCommutedOpIndices() can't handle bundles"); const MCInstrDesc &MCID = MI->getDesc(); if (!MCID.isCommutable()) @@ -141,7 +198,7 @@ bool TargetInstrInfoImpl::findCommutedOpIndices(MachineInstr *MI, bool -TargetInstrInfoImpl::isUnpredicatedTerminator(const MachineInstr *MI) const { +TargetInstrInfo::isUnpredicatedTerminator(const MachineInstr *MI) const { if (!MI->isTerminator()) return false; // Conditional branch is a special case. @@ -153,12 +210,12 @@ TargetInstrInfoImpl::isUnpredicatedTerminator(const MachineInstr *MI) const { } -bool TargetInstrInfoImpl::PredicateInstruction(MachineInstr *MI, +bool TargetInstrInfo::PredicateInstruction(MachineInstr *MI, const SmallVectorImpl<MachineOperand> &Pred) const { bool MadeChange = false; assert(!MI->isBundle() && - "TargetInstrInfoImpl::PredicateInstruction() can't handle bundles"); + "TargetInstrInfo::PredicateInstruction() can't handle bundles"); const MCInstrDesc &MCID = MI->getDesc(); if (!MI->isPredicable()) @@ -183,9 +240,9 @@ bool TargetInstrInfoImpl::PredicateInstruction(MachineInstr *MI, return MadeChange; } -bool TargetInstrInfoImpl::hasLoadFromStackSlot(const MachineInstr *MI, - const MachineMemOperand *&MMO, - int &FrameIndex) const { +bool TargetInstrInfo::hasLoadFromStackSlot(const MachineInstr *MI, + const MachineMemOperand *&MMO, + int &FrameIndex) const { for (MachineInstr::mmo_iterator o = MI->memoperands_begin(), oe = MI->memoperands_end(); o != oe; @@ -201,9 +258,9 @@ bool TargetInstrInfoImpl::hasLoadFromStackSlot(const MachineInstr *MI, return false; } -bool TargetInstrInfoImpl::hasStoreToStackSlot(const MachineInstr *MI, - const MachineMemOperand *&MMO, - int &FrameIndex) const { +bool TargetInstrInfo::hasStoreToStackSlot(const MachineInstr *MI, + const MachineMemOperand *&MMO, + int &FrameIndex) const { for (MachineInstr::mmo_iterator o = MI->memoperands_begin(), oe = MI->memoperands_end(); o != oe; @@ -219,26 +276,26 @@ bool TargetInstrInfoImpl::hasStoreToStackSlot(const MachineInstr *MI, return false; } -void TargetInstrInfoImpl::reMaterialize(MachineBasicBlock &MBB, - MachineBasicBlock::iterator I, - unsigned DestReg, - unsigned SubIdx, - const MachineInstr *Orig, - const TargetRegisterInfo &TRI) const { +void TargetInstrInfo::reMaterialize(MachineBasicBlock &MBB, + MachineBasicBlock::iterator I, + unsigned DestReg, + unsigned SubIdx, + const MachineInstr *Orig, + const TargetRegisterInfo &TRI) const { MachineInstr *MI = MBB.getParent()->CloneMachineInstr(Orig); MI->substituteRegister(MI->getOperand(0).getReg(), DestReg, SubIdx, TRI); MBB.insert(I, MI); } bool -TargetInstrInfoImpl::produceSameValue(const MachineInstr *MI0, - const MachineInstr *MI1, - const MachineRegisterInfo *MRI) const { +TargetInstrInfo::produceSameValue(const MachineInstr *MI0, + const MachineInstr *MI1, + const MachineRegisterInfo *MRI) const { return MI0->isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs); } -MachineInstr *TargetInstrInfoImpl::duplicate(MachineInstr *Orig, - MachineFunction &MF) const { +MachineInstr *TargetInstrInfo::duplicate(MachineInstr *Orig, + MachineFunction &MF) const { assert(!Orig->isNotDuplicable() && "Instruction cannot be duplicated"); return MF.CloneMachineInstr(Orig); @@ -278,7 +335,7 @@ static const TargetRegisterClass *canFoldCopy(const MachineInstr *MI, return 0; } -bool TargetInstrInfoImpl:: +bool TargetInstrInfo:: canFoldMemoryOperand(const MachineInstr *MI, const SmallVectorImpl<unsigned> &Ops) const { return MI->isCopy() && Ops.size() == 1 && canFoldCopy(MI, Ops[0]); @@ -459,9 +516,9 @@ isReallyTriviallyReMaterializableGeneric(const MachineInstr *MI, /// isSchedulingBoundary - Test if the given instruction should be /// considered a scheduling boundary. This primarily includes labels /// and terminators. -bool TargetInstrInfoImpl::isSchedulingBoundary(const MachineInstr *MI, - const MachineBasicBlock *MBB, - const MachineFunction &MF) const{ +bool TargetInstrInfo::isSchedulingBoundary(const MachineInstr *MI, + const MachineBasicBlock *MBB, + const MachineFunction &MF) const { // Terminators and labels can't be scheduled around. if (MI->isTerminator() || MI->isLabel()) return true; @@ -472,7 +529,8 @@ bool TargetInstrInfoImpl::isSchedulingBoundary(const MachineInstr *MI, // stack slot reference to depend on the instruction that does the // modification. const TargetLowering &TLI = *MF.getTarget().getTargetLowering(); - if (MI->definesRegister(TLI.getStackPointerRegisterToSaveRestore())) + const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); + if (MI->modifiesRegister(TLI.getStackPointerRegisterToSaveRestore(), TRI)) return true; return false; @@ -480,12 +538,12 @@ bool TargetInstrInfoImpl::isSchedulingBoundary(const MachineInstr *MI, // Provide a global flag for disabling the PreRA hazard recognizer that targets // may choose to honor. -bool TargetInstrInfoImpl::usePreRAHazardRecognizer() const { +bool TargetInstrInfo::usePreRAHazardRecognizer() const { return !DisableHazardRecognizer; } // Default implementation of CreateTargetRAHazardRecognizer. -ScheduleHazardRecognizer *TargetInstrInfoImpl:: +ScheduleHazardRecognizer *TargetInstrInfo:: CreateTargetHazardRecognizer(const TargetMachine *TM, const ScheduleDAG *DAG) const { // Dummy hazard recognizer allows all instructions to issue. @@ -493,7 +551,7 @@ CreateTargetHazardRecognizer(const TargetMachine *TM, } // Default implementation of CreateTargetMIHazardRecognizer. -ScheduleHazardRecognizer *TargetInstrInfoImpl:: +ScheduleHazardRecognizer *TargetInstrInfo:: CreateTargetMIHazardRecognizer(const InstrItineraryData *II, const ScheduleDAG *DAG) const { return (ScheduleHazardRecognizer *) @@ -501,7 +559,7 @@ CreateTargetMIHazardRecognizer(const InstrItineraryData *II, } // Default implementation of CreateTargetPostRAHazardRecognizer. -ScheduleHazardRecognizer *TargetInstrInfoImpl:: +ScheduleHazardRecognizer *TargetInstrInfo:: CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, const ScheduleDAG *DAG) const { return (ScheduleHazardRecognizer *) @@ -513,9 +571,9 @@ CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, //===----------------------------------------------------------------------===// int -TargetInstrInfoImpl::getOperandLatency(const InstrItineraryData *ItinData, - SDNode *DefNode, unsigned DefIdx, - SDNode *UseNode, unsigned UseIdx) const { +TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, + SDNode *DefNode, unsigned DefIdx, + SDNode *UseNode, unsigned UseIdx) const { if (!ItinData || ItinData->isEmpty()) return -1; @@ -529,8 +587,8 @@ TargetInstrInfoImpl::getOperandLatency(const InstrItineraryData *ItinData, return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); } -int TargetInstrInfoImpl::getInstrLatency(const InstrItineraryData *ItinData, - SDNode *N) const { +int TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, + SDNode *N) const { if (!ItinData || ItinData->isEmpty()) return 1; @@ -545,8 +603,8 @@ int TargetInstrInfoImpl::getInstrLatency(const InstrItineraryData *ItinData, //===----------------------------------------------------------------------===// unsigned -TargetInstrInfoImpl::getNumMicroOps(const InstrItineraryData *ItinData, - const MachineInstr *MI) const { +TargetInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData, + const MachineInstr *MI) const { if (!ItinData || ItinData->isEmpty()) return 1; @@ -572,7 +630,7 @@ unsigned TargetInstrInfo::defaultDefLatency(const MCSchedModel *SchedModel, return 1; } -unsigned TargetInstrInfoImpl:: +unsigned TargetInstrInfo:: getInstrLatency(const InstrItineraryData *ItinData, const MachineInstr *MI, unsigned *PredCost) const { @@ -584,9 +642,9 @@ getInstrLatency(const InstrItineraryData *ItinData, return ItinData->getStageLatency(MI->getDesc().getSchedClass()); } -bool TargetInstrInfoImpl::hasLowDefLatency(const InstrItineraryData *ItinData, - const MachineInstr *DefMI, - unsigned DefIdx) const { +bool TargetInstrInfo::hasLowDefLatency(const InstrItineraryData *ItinData, + const MachineInstr *DefMI, + unsigned DefIdx) const { if (!ItinData || ItinData->isEmpty()) return false; @@ -597,7 +655,7 @@ bool TargetInstrInfoImpl::hasLowDefLatency(const InstrItineraryData *ItinData, /// Both DefMI and UseMI must be valid. By default, call directly to the /// itinerary. This may be overriden by the target. -int TargetInstrInfoImpl:: +int TargetInstrInfo:: getOperandLatency(const InstrItineraryData *ItinData, const MachineInstr *DefMI, unsigned DefIdx, const MachineInstr *UseMI, unsigned UseIdx) const { diff --git a/contrib/llvm/lib/CodeGen/TargetLoweringBase.cpp b/contrib/llvm/lib/CodeGen/TargetLoweringBase.cpp new file mode 100644 index 0000000..f42bdbd --- /dev/null +++ b/contrib/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -0,0 +1,1305 @@ +//===-- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This implements the TargetLoweringBase class. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Target/TargetLowering.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Triple.h" +#include "llvm/CodeGen/Analysis.h" +#include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineJumpTableInfo.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCExpr.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Target/TargetLoweringObjectFile.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include <cctype> +using namespace llvm; + +/// InitLibcallNames - Set default libcall names. +/// +static void InitLibcallNames(const char **Names, const TargetMachine &TM) { + Names[RTLIB::SHL_I16] = "__ashlhi3"; + Names[RTLIB::SHL_I32] = "__ashlsi3"; + Names[RTLIB::SHL_I64] = "__ashldi3"; + Names[RTLIB::SHL_I128] = "__ashlti3"; + Names[RTLIB::SRL_I16] = "__lshrhi3"; + Names[RTLIB::SRL_I32] = "__lshrsi3"; + Names[RTLIB::SRL_I64] = "__lshrdi3"; + Names[RTLIB::SRL_I128] = "__lshrti3"; + Names[RTLIB::SRA_I16] = "__ashrhi3"; + Names[RTLIB::SRA_I32] = "__ashrsi3"; + Names[RTLIB::SRA_I64] = "__ashrdi3"; + Names[RTLIB::SRA_I128] = "__ashrti3"; + Names[RTLIB::MUL_I8] = "__mulqi3"; + Names[RTLIB::MUL_I16] = "__mulhi3"; + Names[RTLIB::MUL_I32] = "__mulsi3"; + Names[RTLIB::MUL_I64] = "__muldi3"; + Names[RTLIB::MUL_I128] = "__multi3"; + Names[RTLIB::MULO_I32] = "__mulosi4"; + Names[RTLIB::MULO_I64] = "__mulodi4"; + Names[RTLIB::MULO_I128] = "__muloti4"; + Names[RTLIB::SDIV_I8] = "__divqi3"; + Names[RTLIB::SDIV_I16] = "__divhi3"; + Names[RTLIB::SDIV_I32] = "__divsi3"; + Names[RTLIB::SDIV_I64] = "__divdi3"; + Names[RTLIB::SDIV_I128] = "__divti3"; + Names[RTLIB::UDIV_I8] = "__udivqi3"; + Names[RTLIB::UDIV_I16] = "__udivhi3"; + Names[RTLIB::UDIV_I32] = "__udivsi3"; + Names[RTLIB::UDIV_I64] = "__udivdi3"; + Names[RTLIB::UDIV_I128] = "__udivti3"; + Names[RTLIB::SREM_I8] = "__modqi3"; + Names[RTLIB::SREM_I16] = "__modhi3"; + Names[RTLIB::SREM_I32] = "__modsi3"; + Names[RTLIB::SREM_I64] = "__moddi3"; + Names[RTLIB::SREM_I128] = "__modti3"; + Names[RTLIB::UREM_I8] = "__umodqi3"; + Names[RTLIB::UREM_I16] = "__umodhi3"; + Names[RTLIB::UREM_I32] = "__umodsi3"; + Names[RTLIB::UREM_I64] = "__umoddi3"; + Names[RTLIB::UREM_I128] = "__umodti3"; + + // These are generally not available. + Names[RTLIB::SDIVREM_I8] = 0; + Names[RTLIB::SDIVREM_I16] = 0; + Names[RTLIB::SDIVREM_I32] = 0; + Names[RTLIB::SDIVREM_I64] = 0; + Names[RTLIB::SDIVREM_I128] = 0; + Names[RTLIB::UDIVREM_I8] = 0; + Names[RTLIB::UDIVREM_I16] = 0; + Names[RTLIB::UDIVREM_I32] = 0; + Names[RTLIB::UDIVREM_I64] = 0; + Names[RTLIB::UDIVREM_I128] = 0; + + Names[RTLIB::NEG_I32] = "__negsi2"; + Names[RTLIB::NEG_I64] = "__negdi2"; + Names[RTLIB::ADD_F32] = "__addsf3"; + Names[RTLIB::ADD_F64] = "__adddf3"; + Names[RTLIB::ADD_F80] = "__addxf3"; + Names[RTLIB::ADD_F128] = "__addtf3"; + Names[RTLIB::ADD_PPCF128] = "__gcc_qadd"; + Names[RTLIB::SUB_F32] = "__subsf3"; + Names[RTLIB::SUB_F64] = "__subdf3"; + Names[RTLIB::SUB_F80] = "__subxf3"; + Names[RTLIB::SUB_F128] = "__subtf3"; + Names[RTLIB::SUB_PPCF128] = "__gcc_qsub"; + Names[RTLIB::MUL_F32] = "__mulsf3"; + Names[RTLIB::MUL_F64] = "__muldf3"; + Names[RTLIB::MUL_F80] = "__mulxf3"; + Names[RTLIB::MUL_F128] = "__multf3"; + Names[RTLIB::MUL_PPCF128] = "__gcc_qmul"; + Names[RTLIB::DIV_F32] = "__divsf3"; + Names[RTLIB::DIV_F64] = "__divdf3"; + Names[RTLIB::DIV_F80] = "__divxf3"; + Names[RTLIB::DIV_F128] = "__divtf3"; + Names[RTLIB::DIV_PPCF128] = "__gcc_qdiv"; + Names[RTLIB::REM_F32] = "fmodf"; + Names[RTLIB::REM_F64] = "fmod"; + Names[RTLIB::REM_F80] = "fmodl"; + Names[RTLIB::REM_F128] = "fmodl"; + Names[RTLIB::REM_PPCF128] = "fmodl"; + Names[RTLIB::FMA_F32] = "fmaf"; + Names[RTLIB::FMA_F64] = "fma"; + Names[RTLIB::FMA_F80] = "fmal"; + Names[RTLIB::FMA_F128] = "fmal"; + Names[RTLIB::FMA_PPCF128] = "fmal"; + Names[RTLIB::POWI_F32] = "__powisf2"; + Names[RTLIB::POWI_F64] = "__powidf2"; + Names[RTLIB::POWI_F80] = "__powixf2"; + Names[RTLIB::POWI_F128] = "__powitf2"; + Names[RTLIB::POWI_PPCF128] = "__powitf2"; + Names[RTLIB::SQRT_F32] = "sqrtf"; + Names[RTLIB::SQRT_F64] = "sqrt"; + Names[RTLIB::SQRT_F80] = "sqrtl"; + Names[RTLIB::SQRT_F128] = "sqrtl"; + Names[RTLIB::SQRT_PPCF128] = "sqrtl"; + Names[RTLIB::LOG_F32] = "logf"; + Names[RTLIB::LOG_F64] = "log"; + Names[RTLIB::LOG_F80] = "logl"; + Names[RTLIB::LOG_F128] = "logl"; + Names[RTLIB::LOG_PPCF128] = "logl"; + Names[RTLIB::LOG2_F32] = "log2f"; + Names[RTLIB::LOG2_F64] = "log2"; + Names[RTLIB::LOG2_F80] = "log2l"; + Names[RTLIB::LOG2_F128] = "log2l"; + Names[RTLIB::LOG2_PPCF128] = "log2l"; + Names[RTLIB::LOG10_F32] = "log10f"; + Names[RTLIB::LOG10_F64] = "log10"; + Names[RTLIB::LOG10_F80] = "log10l"; + Names[RTLIB::LOG10_F128] = "log10l"; + Names[RTLIB::LOG10_PPCF128] = "log10l"; + Names[RTLIB::EXP_F32] = "expf"; + Names[RTLIB::EXP_F64] = "exp"; + Names[RTLIB::EXP_F80] = "expl"; + Names[RTLIB::EXP_F128] = "expl"; + Names[RTLIB::EXP_PPCF128] = "expl"; + Names[RTLIB::EXP2_F32] = "exp2f"; + Names[RTLIB::EXP2_F64] = "exp2"; + Names[RTLIB::EXP2_F80] = "exp2l"; + Names[RTLIB::EXP2_F128] = "exp2l"; + Names[RTLIB::EXP2_PPCF128] = "exp2l"; + Names[RTLIB::SIN_F32] = "sinf"; + Names[RTLIB::SIN_F64] = "sin"; + Names[RTLIB::SIN_F80] = "sinl"; + Names[RTLIB::SIN_F128] = "sinl"; + Names[RTLIB::SIN_PPCF128] = "sinl"; + Names[RTLIB::COS_F32] = "cosf"; + Names[RTLIB::COS_F64] = "cos"; + Names[RTLIB::COS_F80] = "cosl"; + Names[RTLIB::COS_F128] = "cosl"; + Names[RTLIB::COS_PPCF128] = "cosl"; + Names[RTLIB::POW_F32] = "powf"; + Names[RTLIB::POW_F64] = "pow"; + Names[RTLIB::POW_F80] = "powl"; + Names[RTLIB::POW_F128] = "powl"; + Names[RTLIB::POW_PPCF128] = "powl"; + Names[RTLIB::CEIL_F32] = "ceilf"; + Names[RTLIB::CEIL_F64] = "ceil"; + Names[RTLIB::CEIL_F80] = "ceill"; + Names[RTLIB::CEIL_F128] = "ceill"; + Names[RTLIB::CEIL_PPCF128] = "ceill"; + Names[RTLIB::TRUNC_F32] = "truncf"; + Names[RTLIB::TRUNC_F64] = "trunc"; + Names[RTLIB::TRUNC_F80] = "truncl"; + Names[RTLIB::TRUNC_F128] = "truncl"; + Names[RTLIB::TRUNC_PPCF128] = "truncl"; + Names[RTLIB::RINT_F32] = "rintf"; + Names[RTLIB::RINT_F64] = "rint"; + Names[RTLIB::RINT_F80] = "rintl"; + Names[RTLIB::RINT_F128] = "rintl"; + Names[RTLIB::RINT_PPCF128] = "rintl"; + Names[RTLIB::NEARBYINT_F32] = "nearbyintf"; + Names[RTLIB::NEARBYINT_F64] = "nearbyint"; + Names[RTLIB::NEARBYINT_F80] = "nearbyintl"; + Names[RTLIB::NEARBYINT_F128] = "nearbyintl"; + Names[RTLIB::NEARBYINT_PPCF128] = "nearbyintl"; + Names[RTLIB::FLOOR_F32] = "floorf"; + Names[RTLIB::FLOOR_F64] = "floor"; + Names[RTLIB::FLOOR_F80] = "floorl"; + Names[RTLIB::FLOOR_F128] = "floorl"; + Names[RTLIB::FLOOR_PPCF128] = "floorl"; + Names[RTLIB::COPYSIGN_F32] = "copysignf"; + Names[RTLIB::COPYSIGN_F64] = "copysign"; + Names[RTLIB::COPYSIGN_F80] = "copysignl"; + Names[RTLIB::COPYSIGN_F128] = "copysignl"; + Names[RTLIB::COPYSIGN_PPCF128] = "copysignl"; + Names[RTLIB::FPEXT_F64_F128] = "__extenddftf2"; + Names[RTLIB::FPEXT_F32_F128] = "__extendsftf2"; + Names[RTLIB::FPEXT_F32_F64] = "__extendsfdf2"; + Names[RTLIB::FPEXT_F16_F32] = "__gnu_h2f_ieee"; + Names[RTLIB::FPROUND_F32_F16] = "__gnu_f2h_ieee"; + Names[RTLIB::FPROUND_F64_F32] = "__truncdfsf2"; + Names[RTLIB::FPROUND_F80_F32] = "__truncxfsf2"; + Names[RTLIB::FPROUND_F128_F32] = "__trunctfsf2"; + Names[RTLIB::FPROUND_PPCF128_F32] = "__trunctfsf2"; + Names[RTLIB::FPROUND_F80_F64] = "__truncxfdf2"; + Names[RTLIB::FPROUND_F128_F64] = "__trunctfdf2"; + Names[RTLIB::FPROUND_PPCF128_F64] = "__trunctfdf2"; + Names[RTLIB::FPTOSINT_F32_I8] = "__fixsfqi"; + Names[RTLIB::FPTOSINT_F32_I16] = "__fixsfhi"; + Names[RTLIB::FPTOSINT_F32_I32] = "__fixsfsi"; + Names[RTLIB::FPTOSINT_F32_I64] = "__fixsfdi"; + Names[RTLIB::FPTOSINT_F32_I128] = "__fixsfti"; + Names[RTLIB::FPTOSINT_F64_I8] = "__fixdfqi"; + Names[RTLIB::FPTOSINT_F64_I16] = "__fixdfhi"; + Names[RTLIB::FPTOSINT_F64_I32] = "__fixdfsi"; + Names[RTLIB::FPTOSINT_F64_I64] = "__fixdfdi"; + Names[RTLIB::FPTOSINT_F64_I128] = "__fixdfti"; + Names[RTLIB::FPTOSINT_F80_I32] = "__fixxfsi"; + Names[RTLIB::FPTOSINT_F80_I64] = "__fixxfdi"; + Names[RTLIB::FPTOSINT_F80_I128] = "__fixxfti"; + Names[RTLIB::FPTOSINT_F128_I32] = "__fixtfsi"; + Names[RTLIB::FPTOSINT_F128_I64] = "__fixtfdi"; + Names[RTLIB::FPTOSINT_F128_I128] = "__fixtfti"; + Names[RTLIB::FPTOSINT_PPCF128_I32] = "__fixtfsi"; + Names[RTLIB::FPTOSINT_PPCF128_I64] = "__fixtfdi"; + Names[RTLIB::FPTOSINT_PPCF128_I128] = "__fixtfti"; + Names[RTLIB::FPTOUINT_F32_I8] = "__fixunssfqi"; + Names[RTLIB::FPTOUINT_F32_I16] = "__fixunssfhi"; + Names[RTLIB::FPTOUINT_F32_I32] = "__fixunssfsi"; + Names[RTLIB::FPTOUINT_F32_I64] = "__fixunssfdi"; + Names[RTLIB::FPTOUINT_F32_I128] = "__fixunssfti"; + Names[RTLIB::FPTOUINT_F64_I8] = "__fixunsdfqi"; + Names[RTLIB::FPTOUINT_F64_I16] = "__fixunsdfhi"; + Names[RTLIB::FPTOUINT_F64_I32] = "__fixunsdfsi"; + Names[RTLIB::FPTOUINT_F64_I64] = "__fixunsdfdi"; + Names[RTLIB::FPTOUINT_F64_I128] = "__fixunsdfti"; + Names[RTLIB::FPTOUINT_F80_I32] = "__fixunsxfsi"; + Names[RTLIB::FPTOUINT_F80_I64] = "__fixunsxfdi"; + Names[RTLIB::FPTOUINT_F80_I128] = "__fixunsxfti"; + Names[RTLIB::FPTOUINT_F128_I32] = "__fixunstfsi"; + Names[RTLIB::FPTOUINT_F128_I64] = "__fixunstfdi"; + Names[RTLIB::FPTOUINT_F128_I128] = "__fixunstfti"; + Names[RTLIB::FPTOUINT_PPCF128_I32] = "__fixunstfsi"; + Names[RTLIB::FPTOUINT_PPCF128_I64] = "__fixunstfdi"; + Names[RTLIB::FPTOUINT_PPCF128_I128] = "__fixunstfti"; + Names[RTLIB::SINTTOFP_I32_F32] = "__floatsisf"; + Names[RTLIB::SINTTOFP_I32_F64] = "__floatsidf"; + Names[RTLIB::SINTTOFP_I32_F80] = "__floatsixf"; + Names[RTLIB::SINTTOFP_I32_F128] = "__floatsitf"; + Names[RTLIB::SINTTOFP_I32_PPCF128] = "__floatsitf"; + Names[RTLIB::SINTTOFP_I64_F32] = "__floatdisf"; + Names[RTLIB::SINTTOFP_I64_F64] = "__floatdidf"; + Names[RTLIB::SINTTOFP_I64_F80] = "__floatdixf"; + Names[RTLIB::SINTTOFP_I64_F128] = "__floatditf"; + Names[RTLIB::SINTTOFP_I64_PPCF128] = "__floatditf"; + Names[RTLIB::SINTTOFP_I128_F32] = "__floattisf"; + Names[RTLIB::SINTTOFP_I128_F64] = "__floattidf"; + Names[RTLIB::SINTTOFP_I128_F80] = "__floattixf"; + Names[RTLIB::SINTTOFP_I128_F128] = "__floattitf"; + Names[RTLIB::SINTTOFP_I128_PPCF128] = "__floattitf"; + Names[RTLIB::UINTTOFP_I32_F32] = "__floatunsisf"; + Names[RTLIB::UINTTOFP_I32_F64] = "__floatunsidf"; + Names[RTLIB::UINTTOFP_I32_F80] = "__floatunsixf"; + Names[RTLIB::UINTTOFP_I32_F128] = "__floatunsitf"; + Names[RTLIB::UINTTOFP_I32_PPCF128] = "__floatunsitf"; + Names[RTLIB::UINTTOFP_I64_F32] = "__floatundisf"; + Names[RTLIB::UINTTOFP_I64_F64] = "__floatundidf"; + Names[RTLIB::UINTTOFP_I64_F80] = "__floatundixf"; + Names[RTLIB::UINTTOFP_I64_F128] = "__floatunditf"; + Names[RTLIB::UINTTOFP_I64_PPCF128] = "__floatunditf"; + Names[RTLIB::UINTTOFP_I128_F32] = "__floatuntisf"; + Names[RTLIB::UINTTOFP_I128_F64] = "__floatuntidf"; + Names[RTLIB::UINTTOFP_I128_F80] = "__floatuntixf"; + Names[RTLIB::UINTTOFP_I128_F128] = "__floatuntitf"; + Names[RTLIB::UINTTOFP_I128_PPCF128] = "__floatuntitf"; + Names[RTLIB::OEQ_F32] = "__eqsf2"; + Names[RTLIB::OEQ_F64] = "__eqdf2"; + Names[RTLIB::OEQ_F128] = "__eqtf2"; + Names[RTLIB::UNE_F32] = "__nesf2"; + Names[RTLIB::UNE_F64] = "__nedf2"; + Names[RTLIB::UNE_F128] = "__netf2"; + Names[RTLIB::OGE_F32] = "__gesf2"; + Names[RTLIB::OGE_F64] = "__gedf2"; + Names[RTLIB::OGE_F128] = "__getf2"; + Names[RTLIB::OLT_F32] = "__ltsf2"; + Names[RTLIB::OLT_F64] = "__ltdf2"; + Names[RTLIB::OLT_F128] = "__lttf2"; + Names[RTLIB::OLE_F32] = "__lesf2"; + Names[RTLIB::OLE_F64] = "__ledf2"; + Names[RTLIB::OLE_F128] = "__letf2"; + Names[RTLIB::OGT_F32] = "__gtsf2"; + Names[RTLIB::OGT_F64] = "__gtdf2"; + Names[RTLIB::OGT_F128] = "__gttf2"; + Names[RTLIB::UO_F32] = "__unordsf2"; + Names[RTLIB::UO_F64] = "__unorddf2"; + Names[RTLIB::UO_F128] = "__unordtf2"; + Names[RTLIB::O_F32] = "__unordsf2"; + Names[RTLIB::O_F64] = "__unorddf2"; + Names[RTLIB::O_F128] = "__unordtf2"; + Names[RTLIB::MEMCPY] = "memcpy"; + Names[RTLIB::MEMMOVE] = "memmove"; + Names[RTLIB::MEMSET] = "memset"; + Names[RTLIB::UNWIND_RESUME] = "_Unwind_Resume"; + Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1] = "__sync_val_compare_and_swap_1"; + Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2] = "__sync_val_compare_and_swap_2"; + Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4] = "__sync_val_compare_and_swap_4"; + Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8] = "__sync_val_compare_and_swap_8"; + Names[RTLIB::SYNC_LOCK_TEST_AND_SET_1] = "__sync_lock_test_and_set_1"; + Names[RTLIB::SYNC_LOCK_TEST_AND_SET_2] = "__sync_lock_test_and_set_2"; + Names[RTLIB::SYNC_LOCK_TEST_AND_SET_4] = "__sync_lock_test_and_set_4"; + Names[RTLIB::SYNC_LOCK_TEST_AND_SET_8] = "__sync_lock_test_and_set_8"; + Names[RTLIB::SYNC_FETCH_AND_ADD_1] = "__sync_fetch_and_add_1"; + Names[RTLIB::SYNC_FETCH_AND_ADD_2] = "__sync_fetch_and_add_2"; + Names[RTLIB::SYNC_FETCH_AND_ADD_4] = "__sync_fetch_and_add_4"; + Names[RTLIB::SYNC_FETCH_AND_ADD_8] = "__sync_fetch_and_add_8"; + Names[RTLIB::SYNC_FETCH_AND_SUB_1] = "__sync_fetch_and_sub_1"; + Names[RTLIB::SYNC_FETCH_AND_SUB_2] = "__sync_fetch_and_sub_2"; + Names[RTLIB::SYNC_FETCH_AND_SUB_4] = "__sync_fetch_and_sub_4"; + Names[RTLIB::SYNC_FETCH_AND_SUB_8] = "__sync_fetch_and_sub_8"; + Names[RTLIB::SYNC_FETCH_AND_AND_1] = "__sync_fetch_and_and_1"; + Names[RTLIB::SYNC_FETCH_AND_AND_2] = "__sync_fetch_and_and_2"; + Names[RTLIB::SYNC_FETCH_AND_AND_4] = "__sync_fetch_and_and_4"; + Names[RTLIB::SYNC_FETCH_AND_AND_8] = "__sync_fetch_and_and_8"; + Names[RTLIB::SYNC_FETCH_AND_OR_1] = "__sync_fetch_and_or_1"; + Names[RTLIB::SYNC_FETCH_AND_OR_2] = "__sync_fetch_and_or_2"; + Names[RTLIB::SYNC_FETCH_AND_OR_4] = "__sync_fetch_and_or_4"; + Names[RTLIB::SYNC_FETCH_AND_OR_8] = "__sync_fetch_and_or_8"; + Names[RTLIB::SYNC_FETCH_AND_XOR_1] = "__sync_fetch_and_xor_1"; + Names[RTLIB::SYNC_FETCH_AND_XOR_2] = "__sync_fetch_and_xor_2"; + Names[RTLIB::SYNC_FETCH_AND_XOR_4] = "__sync_fetch_and_xor_4"; + Names[RTLIB::SYNC_FETCH_AND_XOR_8] = "__sync_fetch_and_xor_8"; + Names[RTLIB::SYNC_FETCH_AND_NAND_1] = "__sync_fetch_and_nand_1"; + Names[RTLIB::SYNC_FETCH_AND_NAND_2] = "__sync_fetch_and_nand_2"; + Names[RTLIB::SYNC_FETCH_AND_NAND_4] = "__sync_fetch_and_nand_4"; + Names[RTLIB::SYNC_FETCH_AND_NAND_8] = "__sync_fetch_and_nand_8"; + + if (Triple(TM.getTargetTriple()).getEnvironment() == Triple::GNU) { + Names[RTLIB::SINCOS_F32] = "sincosf"; + Names[RTLIB::SINCOS_F64] = "sincos"; + Names[RTLIB::SINCOS_F80] = "sincosl"; + Names[RTLIB::SINCOS_F128] = "sincosl"; + Names[RTLIB::SINCOS_PPCF128] = "sincosl"; + } else { + // These are generally not available. + Names[RTLIB::SINCOS_F32] = 0; + Names[RTLIB::SINCOS_F64] = 0; + Names[RTLIB::SINCOS_F80] = 0; + Names[RTLIB::SINCOS_F128] = 0; + Names[RTLIB::SINCOS_PPCF128] = 0; + } +} + +/// InitLibcallCallingConvs - Set default libcall CallingConvs. +/// +static void InitLibcallCallingConvs(CallingConv::ID *CCs) { + for (int i = 0; i < RTLIB::UNKNOWN_LIBCALL; ++i) { + CCs[i] = CallingConv::C; + } +} + +/// getFPEXT - Return the FPEXT_*_* value for the given types, or +/// UNKNOWN_LIBCALL if there is none. +RTLIB::Libcall RTLIB::getFPEXT(EVT OpVT, EVT RetVT) { + if (OpVT == MVT::f32) { + if (RetVT == MVT::f64) + return FPEXT_F32_F64; + if (RetVT == MVT::f128) + return FPEXT_F32_F128; + } else if (OpVT == MVT::f64) { + if (RetVT == MVT::f128) + return FPEXT_F64_F128; + } + + return UNKNOWN_LIBCALL; +} + +/// getFPROUND - Return the FPROUND_*_* value for the given types, or +/// UNKNOWN_LIBCALL if there is none. +RTLIB::Libcall RTLIB::getFPROUND(EVT OpVT, EVT RetVT) { + if (RetVT == MVT::f32) { + if (OpVT == MVT::f64) + return FPROUND_F64_F32; + if (OpVT == MVT::f80) + return FPROUND_F80_F32; + if (OpVT == MVT::f128) + return FPROUND_F128_F32; + if (OpVT == MVT::ppcf128) + return FPROUND_PPCF128_F32; + } else if (RetVT == MVT::f64) { + if (OpVT == MVT::f80) + return FPROUND_F80_F64; + if (OpVT == MVT::f128) + return FPROUND_F128_F64; + if (OpVT == MVT::ppcf128) + return FPROUND_PPCF128_F64; + } + + return UNKNOWN_LIBCALL; +} + +/// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or +/// UNKNOWN_LIBCALL if there is none. +RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) { + if (OpVT == MVT::f32) { + if (RetVT == MVT::i8) + return FPTOSINT_F32_I8; + if (RetVT == MVT::i16) + return FPTOSINT_F32_I16; + if (RetVT == MVT::i32) + return FPTOSINT_F32_I32; + if (RetVT == MVT::i64) + return FPTOSINT_F32_I64; + if (RetVT == MVT::i128) + return FPTOSINT_F32_I128; + } else if (OpVT == MVT::f64) { + if (RetVT == MVT::i8) + return FPTOSINT_F64_I8; + if (RetVT == MVT::i16) + return FPTOSINT_F64_I16; + if (RetVT == MVT::i32) + return FPTOSINT_F64_I32; + if (RetVT == MVT::i64) + return FPTOSINT_F64_I64; + if (RetVT == MVT::i128) + return FPTOSINT_F64_I128; + } else if (OpVT == MVT::f80) { + if (RetVT == MVT::i32) + return FPTOSINT_F80_I32; + if (RetVT == MVT::i64) + return FPTOSINT_F80_I64; + if (RetVT == MVT::i128) + return FPTOSINT_F80_I128; + } else if (OpVT == MVT::f128) { + if (RetVT == MVT::i32) + return FPTOSINT_F128_I32; + if (RetVT == MVT::i64) + return FPTOSINT_F128_I64; + if (RetVT == MVT::i128) + return FPTOSINT_F128_I128; + } else if (OpVT == MVT::ppcf128) { + if (RetVT == MVT::i32) + return FPTOSINT_PPCF128_I32; + if (RetVT == MVT::i64) + return FPTOSINT_PPCF128_I64; + if (RetVT == MVT::i128) + return FPTOSINT_PPCF128_I128; + } + return UNKNOWN_LIBCALL; +} + +/// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or +/// UNKNOWN_LIBCALL if there is none. +RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) { + if (OpVT == MVT::f32) { + if (RetVT == MVT::i8) + return FPTOUINT_F32_I8; + if (RetVT == MVT::i16) + return FPTOUINT_F32_I16; + if (RetVT == MVT::i32) + return FPTOUINT_F32_I32; + if (RetVT == MVT::i64) + return FPTOUINT_F32_I64; + if (RetVT == MVT::i128) + return FPTOUINT_F32_I128; + } else if (OpVT == MVT::f64) { + if (RetVT == MVT::i8) + return FPTOUINT_F64_I8; + if (RetVT == MVT::i16) + return FPTOUINT_F64_I16; + if (RetVT == MVT::i32) + return FPTOUINT_F64_I32; + if (RetVT == MVT::i64) + return FPTOUINT_F64_I64; + if (RetVT == MVT::i128) + return FPTOUINT_F64_I128; + } else if (OpVT == MVT::f80) { + if (RetVT == MVT::i32) + return FPTOUINT_F80_I32; + if (RetVT == MVT::i64) + return FPTOUINT_F80_I64; + if (RetVT == MVT::i128) + return FPTOUINT_F80_I128; + } else if (OpVT == MVT::f128) { + if (RetVT == MVT::i32) + return FPTOUINT_F128_I32; + if (RetVT == MVT::i64) + return FPTOUINT_F128_I64; + if (RetVT == MVT::i128) + return FPTOUINT_F128_I128; + } else if (OpVT == MVT::ppcf128) { + if (RetVT == MVT::i32) + return FPTOUINT_PPCF128_I32; + if (RetVT == MVT::i64) + return FPTOUINT_PPCF128_I64; + if (RetVT == MVT::i128) + return FPTOUINT_PPCF128_I128; + } + return UNKNOWN_LIBCALL; +} + +/// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or +/// UNKNOWN_LIBCALL if there is none. +RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) { + if (OpVT == MVT::i32) { + if (RetVT == MVT::f32) + return SINTTOFP_I32_F32; + if (RetVT == MVT::f64) + return SINTTOFP_I32_F64; + if (RetVT == MVT::f80) + return SINTTOFP_I32_F80; + if (RetVT == MVT::f128) + return SINTTOFP_I32_F128; + if (RetVT == MVT::ppcf128) + return SINTTOFP_I32_PPCF128; + } else if (OpVT == MVT::i64) { + if (RetVT == MVT::f32) + return SINTTOFP_I64_F32; + if (RetVT == MVT::f64) + return SINTTOFP_I64_F64; + if (RetVT == MVT::f80) + return SINTTOFP_I64_F80; + if (RetVT == MVT::f128) + return SINTTOFP_I64_F128; + if (RetVT == MVT::ppcf128) + return SINTTOFP_I64_PPCF128; + } else if (OpVT == MVT::i128) { + if (RetVT == MVT::f32) + return SINTTOFP_I128_F32; + if (RetVT == MVT::f64) + return SINTTOFP_I128_F64; + if (RetVT == MVT::f80) + return SINTTOFP_I128_F80; + if (RetVT == MVT::f128) + return SINTTOFP_I128_F128; + if (RetVT == MVT::ppcf128) + return SINTTOFP_I128_PPCF128; + } + return UNKNOWN_LIBCALL; +} + +/// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or +/// UNKNOWN_LIBCALL if there is none. +RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) { + if (OpVT == MVT::i32) { + if (RetVT == MVT::f32) + return UINTTOFP_I32_F32; + if (RetVT == MVT::f64) + return UINTTOFP_I32_F64; + if (RetVT == MVT::f80) + return UINTTOFP_I32_F80; + if (RetVT == MVT::f128) + return UINTTOFP_I32_F128; + if (RetVT == MVT::ppcf128) + return UINTTOFP_I32_PPCF128; + } else if (OpVT == MVT::i64) { + if (RetVT == MVT::f32) + return UINTTOFP_I64_F32; + if (RetVT == MVT::f64) + return UINTTOFP_I64_F64; + if (RetVT == MVT::f80) + return UINTTOFP_I64_F80; + if (RetVT == MVT::f128) + return UINTTOFP_I64_F128; + if (RetVT == MVT::ppcf128) + return UINTTOFP_I64_PPCF128; + } else if (OpVT == MVT::i128) { + if (RetVT == MVT::f32) + return UINTTOFP_I128_F32; + if (RetVT == MVT::f64) + return UINTTOFP_I128_F64; + if (RetVT == MVT::f80) + return UINTTOFP_I128_F80; + if (RetVT == MVT::f128) + return UINTTOFP_I128_F128; + if (RetVT == MVT::ppcf128) + return UINTTOFP_I128_PPCF128; + } + return UNKNOWN_LIBCALL; +} + +/// InitCmpLibcallCCs - Set default comparison libcall CC. +/// +static void InitCmpLibcallCCs(ISD::CondCode *CCs) { + memset(CCs, ISD::SETCC_INVALID, sizeof(ISD::CondCode)*RTLIB::UNKNOWN_LIBCALL); + CCs[RTLIB::OEQ_F32] = ISD::SETEQ; + CCs[RTLIB::OEQ_F64] = ISD::SETEQ; + CCs[RTLIB::OEQ_F128] = ISD::SETEQ; + CCs[RTLIB::UNE_F32] = ISD::SETNE; + CCs[RTLIB::UNE_F64] = ISD::SETNE; + CCs[RTLIB::UNE_F128] = ISD::SETNE; + CCs[RTLIB::OGE_F32] = ISD::SETGE; + CCs[RTLIB::OGE_F64] = ISD::SETGE; + CCs[RTLIB::OGE_F128] = ISD::SETGE; + CCs[RTLIB::OLT_F32] = ISD::SETLT; + CCs[RTLIB::OLT_F64] = ISD::SETLT; + CCs[RTLIB::OLT_F128] = ISD::SETLT; + CCs[RTLIB::OLE_F32] = ISD::SETLE; + CCs[RTLIB::OLE_F64] = ISD::SETLE; + CCs[RTLIB::OLE_F128] = ISD::SETLE; + CCs[RTLIB::OGT_F32] = ISD::SETGT; + CCs[RTLIB::OGT_F64] = ISD::SETGT; + CCs[RTLIB::OGT_F128] = ISD::SETGT; + CCs[RTLIB::UO_F32] = ISD::SETNE; + CCs[RTLIB::UO_F64] = ISD::SETNE; + CCs[RTLIB::UO_F128] = ISD::SETNE; + CCs[RTLIB::O_F32] = ISD::SETEQ; + CCs[RTLIB::O_F64] = ISD::SETEQ; + CCs[RTLIB::O_F128] = ISD::SETEQ; +} + +/// NOTE: The constructor takes ownership of TLOF. +TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm, + const TargetLoweringObjectFile *tlof) + : TM(tm), TD(TM.getDataLayout()), TLOF(*tlof) { + // All operations default to being supported. + memset(OpActions, 0, sizeof(OpActions)); + memset(LoadExtActions, 0, sizeof(LoadExtActions)); + memset(TruncStoreActions, 0, sizeof(TruncStoreActions)); + memset(IndexedModeActions, 0, sizeof(IndexedModeActions)); + memset(CondCodeActions, 0, sizeof(CondCodeActions)); + + // Set default actions for various operations. + for (unsigned VT = 0; VT != (unsigned)MVT::LAST_VALUETYPE; ++VT) { + // Default all indexed load / store to expand. + for (unsigned IM = (unsigned)ISD::PRE_INC; + IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) { + setIndexedLoadAction(IM, (MVT::SimpleValueType)VT, Expand); + setIndexedStoreAction(IM, (MVT::SimpleValueType)VT, Expand); + } + + // These operations default to expand. + setOperationAction(ISD::FGETSIGN, (MVT::SimpleValueType)VT, Expand); + setOperationAction(ISD::CONCAT_VECTORS, (MVT::SimpleValueType)VT, Expand); + } + + // Most targets ignore the @llvm.prefetch intrinsic. + setOperationAction(ISD::PREFETCH, MVT::Other, Expand); + + // ConstantFP nodes default to expand. Targets can either change this to + // Legal, in which case all fp constants are legal, or use isFPImmLegal() + // to optimize expansions for certain constants. + setOperationAction(ISD::ConstantFP, MVT::f16, Expand); + setOperationAction(ISD::ConstantFP, MVT::f32, Expand); + setOperationAction(ISD::ConstantFP, MVT::f64, Expand); + setOperationAction(ISD::ConstantFP, MVT::f80, Expand); + setOperationAction(ISD::ConstantFP, MVT::f128, Expand); + + // These library functions default to expand. + setOperationAction(ISD::FLOG , MVT::f16, Expand); + setOperationAction(ISD::FLOG2, MVT::f16, Expand); + setOperationAction(ISD::FLOG10, MVT::f16, Expand); + setOperationAction(ISD::FEXP , MVT::f16, Expand); + setOperationAction(ISD::FEXP2, MVT::f16, Expand); + setOperationAction(ISD::FFLOOR, MVT::f16, Expand); + setOperationAction(ISD::FNEARBYINT, MVT::f16, Expand); + setOperationAction(ISD::FCEIL, MVT::f16, Expand); + setOperationAction(ISD::FRINT, MVT::f16, Expand); + setOperationAction(ISD::FTRUNC, MVT::f16, Expand); + setOperationAction(ISD::FLOG , MVT::f32, Expand); + setOperationAction(ISD::FLOG2, MVT::f32, Expand); + setOperationAction(ISD::FLOG10, MVT::f32, Expand); + setOperationAction(ISD::FEXP , MVT::f32, Expand); + setOperationAction(ISD::FEXP2, MVT::f32, Expand); + setOperationAction(ISD::FFLOOR, MVT::f32, Expand); + setOperationAction(ISD::FNEARBYINT, MVT::f32, Expand); + setOperationAction(ISD::FCEIL, MVT::f32, Expand); + setOperationAction(ISD::FRINT, MVT::f32, Expand); + setOperationAction(ISD::FTRUNC, MVT::f32, Expand); + setOperationAction(ISD::FLOG , MVT::f64, Expand); + setOperationAction(ISD::FLOG2, MVT::f64, Expand); + setOperationAction(ISD::FLOG10, MVT::f64, Expand); + setOperationAction(ISD::FEXP , MVT::f64, Expand); + setOperationAction(ISD::FEXP2, MVT::f64, Expand); + setOperationAction(ISD::FFLOOR, MVT::f64, Expand); + setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand); + setOperationAction(ISD::FCEIL, MVT::f64, Expand); + setOperationAction(ISD::FRINT, MVT::f64, Expand); + setOperationAction(ISD::FTRUNC, MVT::f64, Expand); + setOperationAction(ISD::FLOG , MVT::f128, Expand); + setOperationAction(ISD::FLOG2, MVT::f128, Expand); + setOperationAction(ISD::FLOG10, MVT::f128, Expand); + setOperationAction(ISD::FEXP , MVT::f128, Expand); + setOperationAction(ISD::FEXP2, MVT::f128, Expand); + setOperationAction(ISD::FFLOOR, MVT::f128, Expand); + setOperationAction(ISD::FNEARBYINT, MVT::f128, Expand); + setOperationAction(ISD::FCEIL, MVT::f128, Expand); + setOperationAction(ISD::FRINT, MVT::f128, Expand); + setOperationAction(ISD::FTRUNC, MVT::f128, Expand); + + // Default ISD::TRAP to expand (which turns it into abort). + setOperationAction(ISD::TRAP, MVT::Other, Expand); + + // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand" + // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP. + // + setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand); + + IsLittleEndian = TD->isLittleEndian(); + PointerTy = MVT::getIntegerVT(8*TD->getPointerSize(0)); + memset(RegClassForVT, 0,MVT::LAST_VALUETYPE*sizeof(TargetRegisterClass*)); + memset(TargetDAGCombineArray, 0, array_lengthof(TargetDAGCombineArray)); + MaxStoresPerMemset = MaxStoresPerMemcpy = MaxStoresPerMemmove = 8; + MaxStoresPerMemsetOptSize = MaxStoresPerMemcpyOptSize + = MaxStoresPerMemmoveOptSize = 4; + UseUnderscoreSetJmp = false; + UseUnderscoreLongJmp = false; + SelectIsExpensive = false; + IntDivIsCheap = false; + Pow2DivIsCheap = false; + JumpIsExpensive = false; + PredictableSelectIsExpensive = false; + StackPointerRegisterToSaveRestore = 0; + ExceptionPointerRegister = 0; + ExceptionSelectorRegister = 0; + BooleanContents = UndefinedBooleanContent; + BooleanVectorContents = UndefinedBooleanContent; + SchedPreferenceInfo = Sched::ILP; + JumpBufSize = 0; + JumpBufAlignment = 0; + MinFunctionAlignment = 0; + PrefFunctionAlignment = 0; + PrefLoopAlignment = 0; + MinStackArgumentAlignment = 1; + ShouldFoldAtomicFences = false; + InsertFencesForAtomic = false; + SupportJumpTables = true; + MinimumJumpTableEntries = 4; + + InitLibcallNames(LibcallRoutineNames, TM); + InitCmpLibcallCCs(CmpLibcallCCs); + InitLibcallCallingConvs(LibcallCallingConvs); +} + +TargetLoweringBase::~TargetLoweringBase() { + delete &TLOF; +} + +MVT TargetLoweringBase::getScalarShiftAmountTy(EVT LHSTy) const { + return MVT::getIntegerVT(8*TD->getPointerSize(0)); +} + +EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy) const { + assert(LHSTy.isInteger() && "Shift amount is not an integer type!"); + if (LHSTy.isVector()) + return LHSTy; + return getScalarShiftAmountTy(LHSTy); +} + +/// canOpTrap - Returns true if the operation can trap for the value type. +/// VT must be a legal type. +bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const { + assert(isTypeLegal(VT)); + switch (Op) { + default: + return false; + case ISD::FDIV: + case ISD::FREM: + case ISD::SDIV: + case ISD::UDIV: + case ISD::SREM: + case ISD::UREM: + return true; + } +} + + +static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT, + unsigned &NumIntermediates, + MVT &RegisterVT, + TargetLoweringBase *TLI) { + // Figure out the right, legal destination reg to copy into. + unsigned NumElts = VT.getVectorNumElements(); + MVT EltTy = VT.getVectorElementType(); + + unsigned NumVectorRegs = 1; + + // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally we + // could break down into LHS/RHS like LegalizeDAG does. + if (!isPowerOf2_32(NumElts)) { + NumVectorRegs = NumElts; + NumElts = 1; + } + + // Divide the input until we get to a supported size. This will always + // end with a scalar if the target doesn't support vectors. + while (NumElts > 1 && !TLI->isTypeLegal(MVT::getVectorVT(EltTy, NumElts))) { + NumElts >>= 1; + NumVectorRegs <<= 1; + } + + NumIntermediates = NumVectorRegs; + + MVT NewVT = MVT::getVectorVT(EltTy, NumElts); + if (!TLI->isTypeLegal(NewVT)) + NewVT = EltTy; + IntermediateVT = NewVT; + + unsigned NewVTSize = NewVT.getSizeInBits(); + + // Convert sizes such as i33 to i64. + if (!isPowerOf2_32(NewVTSize)) + NewVTSize = NextPowerOf2(NewVTSize); + + MVT DestVT = TLI->getRegisterType(NewVT); + RegisterVT = DestVT; + if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16. + return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits()); + + // Otherwise, promotion or legal types use the same number of registers as + // the vector decimated to the appropriate level. + return NumVectorRegs; +} + +/// isLegalRC - Return true if the value types that can be represented by the +/// specified register class are all legal. +bool TargetLoweringBase::isLegalRC(const TargetRegisterClass *RC) const { + for (TargetRegisterClass::vt_iterator I = RC->vt_begin(), E = RC->vt_end(); + I != E; ++I) { + if (isTypeLegal(*I)) + return true; + } + return false; +} + +/// findRepresentativeClass - Return the largest legal super-reg register class +/// of the register class for the specified type and its associated "cost". +std::pair<const TargetRegisterClass*, uint8_t> +TargetLoweringBase::findRepresentativeClass(MVT VT) const { + const TargetRegisterInfo *TRI = getTargetMachine().getRegisterInfo(); + const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy]; + if (!RC) + return std::make_pair(RC, 0); + + // Compute the set of all super-register classes. + BitVector SuperRegRC(TRI->getNumRegClasses()); + for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI) + SuperRegRC.setBitsInMask(RCI.getMask()); + + // Find the first legal register class with the largest spill size. + const TargetRegisterClass *BestRC = RC; + for (int i = SuperRegRC.find_first(); i >= 0; i = SuperRegRC.find_next(i)) { + const TargetRegisterClass *SuperRC = TRI->getRegClass(i); + // We want the largest possible spill size. + if (SuperRC->getSize() <= BestRC->getSize()) + continue; + if (!isLegalRC(SuperRC)) + continue; + BestRC = SuperRC; + } + return std::make_pair(BestRC, 1); +} + +/// computeRegisterProperties - Once all of the register classes are added, +/// this allows us to compute derived properties we expose. +void TargetLoweringBase::computeRegisterProperties() { + assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE && + "Too many value types for ValueTypeActions to hold!"); + + // Everything defaults to needing one register. + for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i) { + NumRegistersForVT[i] = 1; + RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i; + } + // ...except isVoid, which doesn't need any registers. + NumRegistersForVT[MVT::isVoid] = 0; + + // Find the largest integer register class. + unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE; + for (; RegClassForVT[LargestIntReg] == 0; --LargestIntReg) + assert(LargestIntReg != MVT::i1 && "No integer registers defined!"); + + // Every integer value type larger than this largest register takes twice as + // many registers to represent as the previous ValueType. + for (unsigned ExpandedReg = LargestIntReg + 1; + ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg) { + NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1]; + RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg; + TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1); + ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg, + TypeExpandInteger); + } + + // Inspect all of the ValueType's smaller than the largest integer + // register to see which ones need promotion. + unsigned LegalIntReg = LargestIntReg; + for (unsigned IntReg = LargestIntReg - 1; + IntReg >= (unsigned)MVT::i1; --IntReg) { + MVT IVT = (MVT::SimpleValueType)IntReg; + if (isTypeLegal(IVT)) { + LegalIntReg = IntReg; + } else { + RegisterTypeForVT[IntReg] = TransformToType[IntReg] = + (const MVT::SimpleValueType)LegalIntReg; + ValueTypeActions.setTypeAction(IVT, TypePromoteInteger); + } + } + + // ppcf128 type is really two f64's. + if (!isTypeLegal(MVT::ppcf128)) { + NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64]; + RegisterTypeForVT[MVT::ppcf128] = MVT::f64; + TransformToType[MVT::ppcf128] = MVT::f64; + ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat); + } + + // Decide how to handle f128. If the target does not have native f128 support, + // expand it to i128 and we will be generating soft float library calls. + if (!isTypeLegal(MVT::f128)) { + NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128]; + RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128]; + TransformToType[MVT::f128] = MVT::i128; + ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat); + } + + // Decide how to handle f64. If the target does not have native f64 support, + // expand it to i64 and we will be generating soft float library calls. + if (!isTypeLegal(MVT::f64)) { + NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64]; + RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64]; + TransformToType[MVT::f64] = MVT::i64; + ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat); + } + + // Decide how to handle f32. If the target does not have native support for + // f32, promote it to f64 if it is legal. Otherwise, expand it to i32. + if (!isTypeLegal(MVT::f32)) { + if (isTypeLegal(MVT::f64)) { + NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::f64]; + RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::f64]; + TransformToType[MVT::f32] = MVT::f64; + ValueTypeActions.setTypeAction(MVT::f32, TypePromoteInteger); + } else { + NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32]; + RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32]; + TransformToType[MVT::f32] = MVT::i32; + ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat); + } + } + + // Loop over all of the vector value types to see which need transformations. + for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE; + i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) { + MVT VT = (MVT::SimpleValueType)i; + if (isTypeLegal(VT)) continue; + + // Determine if there is a legal wider type. If so, we should promote to + // that wider vector type. + MVT EltVT = VT.getVectorElementType(); + unsigned NElts = VT.getVectorNumElements(); + if (NElts != 1 && !shouldSplitVectorElementType(EltVT)) { + bool IsLegalWiderType = false; + // First try to promote the elements of integer vectors. If no legal + // promotion was found, fallback to the widen-vector method. + for (unsigned nVT = i+1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) { + MVT SVT = (MVT::SimpleValueType)nVT; + // Promote vectors of integers to vectors with the same number + // of elements, with a wider element type. + if (SVT.getVectorElementType().getSizeInBits() > EltVT.getSizeInBits() + && SVT.getVectorNumElements() == NElts && + isTypeLegal(SVT) && SVT.getScalarType().isInteger()) { + TransformToType[i] = SVT; + RegisterTypeForVT[i] = SVT; + NumRegistersForVT[i] = 1; + ValueTypeActions.setTypeAction(VT, TypePromoteInteger); + IsLegalWiderType = true; + break; + } + } + + if (IsLegalWiderType) continue; + + // Try to widen the vector. + for (unsigned nVT = i+1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) { + MVT SVT = (MVT::SimpleValueType)nVT; + if (SVT.getVectorElementType() == EltVT && + SVT.getVectorNumElements() > NElts && + isTypeLegal(SVT)) { + TransformToType[i] = SVT; + RegisterTypeForVT[i] = SVT; + NumRegistersForVT[i] = 1; + ValueTypeActions.setTypeAction(VT, TypeWidenVector); + IsLegalWiderType = true; + break; + } + } + if (IsLegalWiderType) continue; + } + + MVT IntermediateVT; + MVT RegisterVT; + unsigned NumIntermediates; + NumRegistersForVT[i] = + getVectorTypeBreakdownMVT(VT, IntermediateVT, NumIntermediates, + RegisterVT, this); + RegisterTypeForVT[i] = RegisterVT; + + MVT NVT = VT.getPow2VectorType(); + if (NVT == VT) { + // Type is already a power of 2. The default action is to split. + TransformToType[i] = MVT::Other; + unsigned NumElts = VT.getVectorNumElements(); + ValueTypeActions.setTypeAction(VT, + NumElts > 1 ? TypeSplitVector : TypeScalarizeVector); + } else { + TransformToType[i] = NVT; + ValueTypeActions.setTypeAction(VT, TypeWidenVector); + } + } + + // Determine the 'representative' register class for each value type. + // An representative register class is the largest (meaning one which is + // not a sub-register class / subreg register class) legal register class for + // a group of value types. For example, on i386, i8, i16, and i32 + // representative would be GR32; while on x86_64 it's GR64. + for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i) { + const TargetRegisterClass* RRC; + uint8_t Cost; + tie(RRC, Cost) = findRepresentativeClass((MVT::SimpleValueType)i); + RepRegClassForVT[i] = RRC; + RepRegClassCostForVT[i] = Cost; + } +} + +EVT TargetLoweringBase::getSetCCResultType(EVT VT) const { + assert(!VT.isVector() && "No default SetCC type for vectors!"); + return getPointerTy(0).SimpleTy; +} + +MVT::SimpleValueType TargetLoweringBase::getCmpLibcallReturnType() const { + return MVT::i32; // return the default value +} + +/// getVectorTypeBreakdown - Vector types are broken down into some number of +/// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32 +/// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack. +/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86. +/// +/// This method returns the number of registers needed, and the VT for each +/// register. It also returns the VT and quantity of the intermediate values +/// before they are promoted/expanded. +/// +unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context, EVT VT, + EVT &IntermediateVT, + unsigned &NumIntermediates, + MVT &RegisterVT) const { + unsigned NumElts = VT.getVectorNumElements(); + + // If there is a wider vector type with the same element type as this one, + // or a promoted vector type that has the same number of elements which + // are wider, then we should convert to that legal vector type. + // This handles things like <2 x float> -> <4 x float> and + // <4 x i1> -> <4 x i32>. + LegalizeTypeAction TA = getTypeAction(Context, VT); + if (NumElts != 1 && (TA == TypeWidenVector || TA == TypePromoteInteger)) { + EVT RegisterEVT = getTypeToTransformTo(Context, VT); + if (isTypeLegal(RegisterEVT)) { + IntermediateVT = RegisterEVT; + RegisterVT = RegisterEVT.getSimpleVT(); + NumIntermediates = 1; + return 1; + } + } + + // Figure out the right, legal destination reg to copy into. + EVT EltTy = VT.getVectorElementType(); + + unsigned NumVectorRegs = 1; + + // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally we + // could break down into LHS/RHS like LegalizeDAG does. + if (!isPowerOf2_32(NumElts)) { + NumVectorRegs = NumElts; + NumElts = 1; + } + + // Divide the input until we get to a supported size. This will always + // end with a scalar if the target doesn't support vectors. + while (NumElts > 1 && !isTypeLegal( + EVT::getVectorVT(Context, EltTy, NumElts))) { + NumElts >>= 1; + NumVectorRegs <<= 1; + } + + NumIntermediates = NumVectorRegs; + + EVT NewVT = EVT::getVectorVT(Context, EltTy, NumElts); + if (!isTypeLegal(NewVT)) + NewVT = EltTy; + IntermediateVT = NewVT; + + MVT DestVT = getRegisterType(Context, NewVT); + RegisterVT = DestVT; + unsigned NewVTSize = NewVT.getSizeInBits(); + + // Convert sizes such as i33 to i64. + if (!isPowerOf2_32(NewVTSize)) + NewVTSize = NextPowerOf2(NewVTSize); + + if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16. + return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits()); + + // Otherwise, promotion or legal types use the same number of registers as + // the vector decimated to the appropriate level. + return NumVectorRegs; +} + +/// Get the EVTs and ArgFlags collections that represent the legalized return +/// type of the given function. This does not require a DAG or a return value, +/// and is suitable for use before any DAGs for the function are constructed. +/// TODO: Move this out of TargetLowering.cpp. +void llvm::GetReturnInfo(Type* ReturnType, AttributeSet attr, + SmallVectorImpl<ISD::OutputArg> &Outs, + const TargetLowering &TLI) { + SmallVector<EVT, 4> ValueVTs; + ComputeValueVTs(TLI, ReturnType, ValueVTs); + unsigned NumValues = ValueVTs.size(); + if (NumValues == 0) return; + + for (unsigned j = 0, f = NumValues; j != f; ++j) { + EVT VT = ValueVTs[j]; + ISD::NodeType ExtendKind = ISD::ANY_EXTEND; + + if (attr.hasAttribute(AttributeSet::ReturnIndex, Attribute::SExt)) + ExtendKind = ISD::SIGN_EXTEND; + else if (attr.hasAttribute(AttributeSet::ReturnIndex, Attribute::ZExt)) + ExtendKind = ISD::ZERO_EXTEND; + + // FIXME: C calling convention requires the return type to be promoted to + // at least 32-bit. But this is not necessary for non-C calling + // conventions. The frontend should mark functions whose return values + // require promoting with signext or zeroext attributes. + if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) { + MVT MinVT = TLI.getRegisterType(ReturnType->getContext(), MVT::i32); + if (VT.bitsLT(MinVT)) + VT = MinVT; + } + + unsigned NumParts = TLI.getNumRegisters(ReturnType->getContext(), VT); + MVT PartVT = TLI.getRegisterType(ReturnType->getContext(), VT); + + // 'inreg' on function refers to return value + ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); + if (attr.hasAttribute(AttributeSet::ReturnIndex, Attribute::InReg)) + Flags.setInReg(); + + // Propagate extension type if any + if (attr.hasAttribute(AttributeSet::ReturnIndex, Attribute::SExt)) + Flags.setSExt(); + else if (attr.hasAttribute(AttributeSet::ReturnIndex, Attribute::ZExt)) + Flags.setZExt(); + + for (unsigned i = 0; i < NumParts; ++i) + Outs.push_back(ISD::OutputArg(Flags, PartVT, /*isFixed=*/true, 0, 0)); + } +} + +/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate +/// function arguments in the caller parameter area. This is the actual +/// alignment, not its logarithm. +unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty) const { + return TD->getCallFrameTypeAlignment(Ty); +} + +//===----------------------------------------------------------------------===// +// TargetTransformInfo Helpers +//===----------------------------------------------------------------------===// + +int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const { + enum InstructionOpcodes { +#define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM, +#define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM +#include "llvm/IR/Instruction.def" + }; + switch (static_cast<InstructionOpcodes>(Opcode)) { + case Ret: return 0; + case Br: return 0; + case Switch: return 0; + case IndirectBr: return 0; + case Invoke: return 0; + case Resume: return 0; + case Unreachable: return 0; + case Add: return ISD::ADD; + case FAdd: return ISD::FADD; + case Sub: return ISD::SUB; + case FSub: return ISD::FSUB; + case Mul: return ISD::MUL; + case FMul: return ISD::FMUL; + case UDiv: return ISD::UDIV; + case SDiv: return ISD::UDIV; + case FDiv: return ISD::FDIV; + case URem: return ISD::UREM; + case SRem: return ISD::SREM; + case FRem: return ISD::FREM; + case Shl: return ISD::SHL; + case LShr: return ISD::SRL; + case AShr: return ISD::SRA; + case And: return ISD::AND; + case Or: return ISD::OR; + case Xor: return ISD::XOR; + case Alloca: return 0; + case Load: return ISD::LOAD; + case Store: return ISD::STORE; + case GetElementPtr: return 0; + case Fence: return 0; + case AtomicCmpXchg: return 0; + case AtomicRMW: return 0; + case Trunc: return ISD::TRUNCATE; + case ZExt: return ISD::ZERO_EXTEND; + case SExt: return ISD::SIGN_EXTEND; + case FPToUI: return ISD::FP_TO_UINT; + case FPToSI: return ISD::FP_TO_SINT; + case UIToFP: return ISD::UINT_TO_FP; + case SIToFP: return ISD::SINT_TO_FP; + case FPTrunc: return ISD::FP_ROUND; + case FPExt: return ISD::FP_EXTEND; + case PtrToInt: return ISD::BITCAST; + case IntToPtr: return ISD::BITCAST; + case BitCast: return ISD::BITCAST; + case ICmp: return ISD::SETCC; + case FCmp: return ISD::SETCC; + case PHI: return 0; + case Call: return 0; + case Select: return ISD::SELECT; + case UserOp1: return 0; + case UserOp2: return 0; + case VAArg: return 0; + case ExtractElement: return ISD::EXTRACT_VECTOR_ELT; + case InsertElement: return ISD::INSERT_VECTOR_ELT; + case ShuffleVector: return ISD::VECTOR_SHUFFLE; + case ExtractValue: return ISD::MERGE_VALUES; + case InsertValue: return ISD::MERGE_VALUES; + case LandingPad: return 0; + } + + llvm_unreachable("Unknown instruction type encountered!"); +} + +std::pair<unsigned, MVT> +TargetLoweringBase::getTypeLegalizationCost(Type *Ty) const { + LLVMContext &C = Ty->getContext(); + EVT MTy = getValueType(Ty); + + unsigned Cost = 1; + // We keep legalizing the type until we find a legal kind. We assume that + // the only operation that costs anything is the split. After splitting + // we need to handle two types. + while (true) { + LegalizeKind LK = getTypeConversion(C, MTy); + + if (LK.first == TypeLegal) + return std::make_pair(Cost, MTy.getSimpleVT()); + + if (LK.first == TypeSplitVector || LK.first == TypeExpandInteger) + Cost *= 2; + + // Keep legalizing the type. + MTy = LK.second; + } +} + +//===----------------------------------------------------------------------===// +// Loop Strength Reduction hooks +//===----------------------------------------------------------------------===// + +/// isLegalAddressingMode - Return true if the addressing mode represented +/// by AM is legal for this target, for a load/store of the specified type. +bool TargetLoweringBase::isLegalAddressingMode(const AddrMode &AM, + Type *Ty) const { + // The default implementation of this implements a conservative RISCy, r+r and + // r+i addr mode. + + // Allows a sign-extended 16-bit immediate field. + if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1) + return false; + + // No global is ever allowed as a base. + if (AM.BaseGV) + return false; + + // Only support r+r, + switch (AM.Scale) { + case 0: // "r+i" or just "i", depending on HasBaseReg. + break; + case 1: + if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. + return false; + // Otherwise we have r+r or r+i. + break; + case 2: + if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed. + return false; + // Allow 2*r as r+r. + break; + } + + return true; +} diff --git a/contrib/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/contrib/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 8f5d770..3bdca4c 100644 --- a/contrib/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/contrib/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -13,30 +13,29 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" -#include "llvm/Constants.h" -#include "llvm/DerivedTypes.h" -#include "llvm/Function.h" -#include "llvm/GlobalVariable.h" -#include "llvm/Module.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/Triple.h" #include "llvm/CodeGen/MachineModuleInfoImpls.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Module.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" -#include "llvm/MC/MCSectionMachO.h" -#include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCSectionCOFF.h" +#include "llvm/MC/MCSectionELF.h" +#include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" -#include "llvm/Target/Mangler.h" -#include "llvm/DataLayout.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" #include "llvm/Support/Dwarf.h" #include "llvm/Support/ELF.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/Triple.h" +#include "llvm/Target/Mangler.h" +#include "llvm/Target/TargetMachine.h" using namespace llvm; using namespace dwarf; @@ -88,6 +87,36 @@ void TargetLoweringObjectFileELF::emitPersonalityValue(MCStreamer &Streamer, Streamer.EmitSymbolValue(Sym, Size); } +const MCExpr *TargetLoweringObjectFileELF:: +getTTypeGlobalReference(const GlobalValue *GV, Mangler *Mang, + MachineModuleInfo *MMI, unsigned Encoding, + MCStreamer &Streamer) const { + + if (Encoding & dwarf::DW_EH_PE_indirect) { + MachineModuleInfoELF &ELFMMI = MMI->getObjFileInfo<MachineModuleInfoELF>(); + + SmallString<128> Name; + Mang->getNameWithPrefix(Name, GV, true); + Name += ".DW.stub"; + + // Add information about the stub reference to ELFMMI so that the stub + // gets emitted by the asmprinter. + MCSymbol *SSym = getContext().GetOrCreateSymbol(Name.str()); + MachineModuleInfoImpl::StubValueTy &StubSym = ELFMMI.getGVStubEntry(SSym); + if (StubSym.getPointer() == 0) { + MCSymbol *Sym = Mang->getSymbol(GV); + StubSym = MachineModuleInfoImpl::StubValueTy(Sym, !GV->hasLocalLinkage()); + } + + return TargetLoweringObjectFile:: + getTTypeReference(MCSymbolRefExpr::Create(SSym, getContext()), + Encoding & ~dwarf::DW_EH_PE_indirect, Streamer); + } + + return TargetLoweringObjectFile:: + getTTypeGlobalReference(GV, Mang, MMI, Encoding, Streamer); +} + static SectionKind getELFKindForNamedSection(StringRef Name, SectionKind K) { // N.B.: The defaults used in here are no the same ones used in MC. @@ -314,35 +343,6 @@ getSectionForConstant(SectionKind Kind) const { return DataRelROSection; } -const MCExpr *TargetLoweringObjectFileELF:: -getExprForDwarfGlobalReference(const GlobalValue *GV, Mangler *Mang, - MachineModuleInfo *MMI, - unsigned Encoding, MCStreamer &Streamer) const { - - if (Encoding & dwarf::DW_EH_PE_indirect) { - MachineModuleInfoELF &ELFMMI = MMI->getObjFileInfo<MachineModuleInfoELF>(); - - SmallString<128> Name; - Mang->getNameWithPrefix(Name, GV, true); - Name += ".DW.stub"; - - // Add information about the stub reference to ELFMMI so that the stub - // gets emitted by the asmprinter. - MCSymbol *SSym = getContext().GetOrCreateSymbol(Name.str()); - MachineModuleInfoImpl::StubValueTy &StubSym = ELFMMI.getGVStubEntry(SSym); - if (StubSym.getPointer() == 0) { - MCSymbol *Sym = Mang->getSymbol(GV); - StubSym = MachineModuleInfoImpl::StubValueTy(Sym, !GV->hasLocalLinkage()); - } - - return TargetLoweringObjectFile:: - getExprForDwarfReference(SSym, Encoding & ~dwarf::DW_EH_PE_indirect, Streamer); - } - - return TargetLoweringObjectFile:: - getExprForDwarfGlobalReference(GV, Mang, MMI, Encoding, Streamer); -} - const MCSection * TargetLoweringObjectFileELF::getStaticCtorSection(unsigned Priority) const { // The default scheme is .ctor / .dtor, so we have to invert the priority @@ -405,14 +405,14 @@ TargetLoweringObjectFileELF::InitializeELF(bool UseInitArray_) { // MachO //===----------------------------------------------------------------------===// -/// emitModuleFlags - Emit the module flags that specify the garbage collection -/// information. +/// emitModuleFlags - Perform code emission for module flags. void TargetLoweringObjectFileMachO:: emitModuleFlags(MCStreamer &Streamer, ArrayRef<Module::ModuleFlagEntry> ModuleFlags, Mangler *Mang, const TargetMachine &TM) const { unsigned VersionVal = 0; unsigned ImageInfoFlags = 0; + MDNode *LinkerOptions = 0; StringRef SectionVal; for (ArrayRef<Module::ModuleFlagEntry>::iterator @@ -426,14 +426,33 @@ emitModuleFlags(MCStreamer &Streamer, StringRef Key = MFE.Key->getString(); Value *Val = MFE.Val; - if (Key == "Objective-C Image Info Version") + if (Key == "Objective-C Image Info Version") { VersionVal = cast<ConstantInt>(Val)->getZExtValue(); - else if (Key == "Objective-C Garbage Collection" || - Key == "Objective-C GC Only" || - Key == "Objective-C Is Simulated") + } else if (Key == "Objective-C Garbage Collection" || + Key == "Objective-C GC Only" || + Key == "Objective-C Is Simulated") { ImageInfoFlags |= cast<ConstantInt>(Val)->getZExtValue(); - else if (Key == "Objective-C Image Info Section") + } else if (Key == "Objective-C Image Info Section") { SectionVal = cast<MDString>(Val)->getString(); + } else if (Key == "Linker Options") { + LinkerOptions = cast<MDNode>(Val); + } + } + + // Emit the linker options if present. + if (LinkerOptions) { + for (unsigned i = 0, e = LinkerOptions->getNumOperands(); i != e; ++i) { + MDNode *MDOptions = cast<MDNode>(LinkerOptions->getOperand(i)); + SmallVector<std::string, 4> StrOptions; + + // Convert to strings. + for (unsigned ii = 0, ie = MDOptions->getNumOperands(); ii != ie; ++ii) { + MDString *MDOption = cast<MDString>(MDOptions->getOperand(ii)); + StrOptions.push_back(MDOption->getString()); + } + + Streamer.EmitLinkerOptions(StrOptions); + } } // The section is mandatory. If we don't have it, then we don't have GC info. @@ -604,9 +623,9 @@ shouldEmitUsedDirectiveFor(const GlobalValue *GV, Mangler *Mang) const { } const MCExpr *TargetLoweringObjectFileMachO:: -getExprForDwarfGlobalReference(const GlobalValue *GV, Mangler *Mang, - MachineModuleInfo *MMI, unsigned Encoding, - MCStreamer &Streamer) const { +getTTypeGlobalReference(const GlobalValue *GV, Mangler *Mang, + MachineModuleInfo *MMI, unsigned Encoding, + MCStreamer &Streamer) const { // The mach-o version of this method defaults to returning a stub reference. if (Encoding & DW_EH_PE_indirect) { @@ -629,11 +648,12 @@ getExprForDwarfGlobalReference(const GlobalValue *GV, Mangler *Mang, } return TargetLoweringObjectFile:: - getExprForDwarfReference(SSym, Encoding & ~dwarf::DW_EH_PE_indirect, Streamer); + getTTypeReference(MCSymbolRefExpr::Create(SSym, getContext()), + Encoding & ~dwarf::DW_EH_PE_indirect, Streamer); } return TargetLoweringObjectFile:: - getExprForDwarfGlobalReference(GV, Mang, MMI, Encoding, Streamer); + getTTypeGlobalReference(GV, Mang, MMI, Encoding, Streamer); } MCSymbol *TargetLoweringObjectFileMachO:: @@ -701,8 +721,19 @@ getCOFFSectionFlags(SectionKind K) { const MCSection *TargetLoweringObjectFileCOFF:: getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind, Mangler *Mang, const TargetMachine &TM) const { - return getContext().getCOFFSection(GV->getSection(), - getCOFFSectionFlags(Kind), + int Selection = 0; + unsigned Characteristics = getCOFFSectionFlags(Kind); + SmallString<128> Name(GV->getSection().c_str()); + if (GV->isWeakForLinker()) { + Selection = COFF::IMAGE_COMDAT_SELECT_ANY; + Characteristics |= COFF::IMAGE_SCN_LNK_COMDAT; + MCSymbol *Sym = Mang->getSymbol(GV); + Name.append("$"); + Name.append(Sym->getName().begin() + 1, Sym->getName().end()); + } + return getContext().getCOFFSection(Name, + Characteristics, + Selection, Kind); } @@ -711,8 +742,11 @@ static const char *getCOFFSectionPrefixForUniqueGlobal(SectionKind Kind) { return ".text$"; if (Kind.isBSS ()) return ".bss$"; - if (Kind.isThreadLocal()) - return ".tls$"; + if (Kind.isThreadLocal()) { + // 'LLVM' is just an arbitary string to ensure that the section name gets + // sorted in between '.tls$AAA' and '.tls$ZZZ' by the linker. + return ".tls$LLVM"; + } if (Kind.isWriteable()) return ".data$"; return ".rdata$"; diff --git a/contrib/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/contrib/llvm/lib/CodeGen/TargetRegisterInfo.cpp new file mode 100644 index 0000000..84b4bfc --- /dev/null +++ b/contrib/llvm/lib/CodeGen/TargetRegisterInfo.cpp @@ -0,0 +1,285 @@ +//===- TargetRegisterInfo.cpp - Target Register Information Implementation ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the TargetRegisterInfo interface. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +TargetRegisterInfo::TargetRegisterInfo(const TargetRegisterInfoDesc *ID, + regclass_iterator RCB, regclass_iterator RCE, + const char *const *SRINames, + const unsigned *SRILaneMasks) + : InfoDesc(ID), SubRegIndexNames(SRINames), + SubRegIndexLaneMasks(SRILaneMasks), + RegClassBegin(RCB), RegClassEnd(RCE) { +} + +TargetRegisterInfo::~TargetRegisterInfo() {} + +void PrintReg::print(raw_ostream &OS) const { + if (!Reg) + OS << "%noreg"; + else if (TargetRegisterInfo::isStackSlot(Reg)) + OS << "SS#" << TargetRegisterInfo::stackSlot2Index(Reg); + else if (TargetRegisterInfo::isVirtualRegister(Reg)) + OS << "%vreg" << TargetRegisterInfo::virtReg2Index(Reg); + else if (TRI && Reg < TRI->getNumRegs()) + OS << '%' << TRI->getName(Reg); + else + OS << "%physreg" << Reg; + if (SubIdx) { + if (TRI) + OS << ':' << TRI->getSubRegIndexName(SubIdx); + else + OS << ":sub(" << SubIdx << ')'; + } +} + +void PrintRegUnit::print(raw_ostream &OS) const { + // Generic printout when TRI is missing. + if (!TRI) { + OS << "Unit~" << Unit; + return; + } + + // Check for invalid register units. + if (Unit >= TRI->getNumRegUnits()) { + OS << "BadUnit~" << Unit; + return; + } + + // Normal units have at least one root. + MCRegUnitRootIterator Roots(Unit, TRI); + assert(Roots.isValid() && "Unit has no roots."); + OS << TRI->getName(*Roots); + for (++Roots; Roots.isValid(); ++Roots) + OS << '~' << TRI->getName(*Roots); +} + +/// getAllocatableClass - Return the maximal subclass of the given register +/// class that is alloctable, or NULL. +const TargetRegisterClass * +TargetRegisterInfo::getAllocatableClass(const TargetRegisterClass *RC) const { + if (!RC || RC->isAllocatable()) + return RC; + + const unsigned *SubClass = RC->getSubClassMask(); + for (unsigned Base = 0, BaseE = getNumRegClasses(); + Base < BaseE; Base += 32) { + unsigned Idx = Base; + for (unsigned Mask = *SubClass++; Mask; Mask >>= 1) { + unsigned Offset = CountTrailingZeros_32(Mask); + const TargetRegisterClass *SubRC = getRegClass(Idx + Offset); + if (SubRC->isAllocatable()) + return SubRC; + Mask >>= Offset; + Idx += Offset + 1; + } + } + return NULL; +} + +/// getMinimalPhysRegClass - Returns the Register Class of a physical +/// register of the given type, picking the most sub register class of +/// the right type that contains this physreg. +const TargetRegisterClass * +TargetRegisterInfo::getMinimalPhysRegClass(unsigned reg, EVT VT) const { + assert(isPhysicalRegister(reg) && "reg must be a physical register"); + + // Pick the most sub register class of the right type that contains + // this physreg. + const TargetRegisterClass* BestRC = 0; + for (regclass_iterator I = regclass_begin(), E = regclass_end(); I != E; ++I){ + const TargetRegisterClass* RC = *I; + if ((VT == MVT::Other || RC->hasType(VT)) && RC->contains(reg) && + (!BestRC || BestRC->hasSubClass(RC))) + BestRC = RC; + } + + assert(BestRC && "Couldn't find the register class"); + return BestRC; +} + +/// getAllocatableSetForRC - Toggle the bits that represent allocatable +/// registers for the specific register class. +static void getAllocatableSetForRC(const MachineFunction &MF, + const TargetRegisterClass *RC, BitVector &R){ + assert(RC->isAllocatable() && "invalid for nonallocatable sets"); + ArrayRef<uint16_t> Order = RC->getRawAllocationOrder(MF); + for (unsigned i = 0; i != Order.size(); ++i) + R.set(Order[i]); +} + +BitVector TargetRegisterInfo::getAllocatableSet(const MachineFunction &MF, + const TargetRegisterClass *RC) const { + BitVector Allocatable(getNumRegs()); + if (RC) { + // A register class with no allocatable subclass returns an empty set. + const TargetRegisterClass *SubClass = getAllocatableClass(RC); + if (SubClass) + getAllocatableSetForRC(MF, SubClass, Allocatable); + } else { + for (TargetRegisterInfo::regclass_iterator I = regclass_begin(), + E = regclass_end(); I != E; ++I) + if ((*I)->isAllocatable()) + getAllocatableSetForRC(MF, *I, Allocatable); + } + + // Mask out the reserved registers + BitVector Reserved = getReservedRegs(MF); + Allocatable &= Reserved.flip(); + + return Allocatable; +} + +static inline +const TargetRegisterClass *firstCommonClass(const uint32_t *A, + const uint32_t *B, + const TargetRegisterInfo *TRI) { + for (unsigned I = 0, E = TRI->getNumRegClasses(); I < E; I += 32) + if (unsigned Common = *A++ & *B++) + return TRI->getRegClass(I + CountTrailingZeros_32(Common)); + return 0; +} + +const TargetRegisterClass * +TargetRegisterInfo::getCommonSubClass(const TargetRegisterClass *A, + const TargetRegisterClass *B) const { + // First take care of the trivial cases. + if (A == B) + return A; + if (!A || !B) + return 0; + + // Register classes are ordered topologically, so the largest common + // sub-class it the common sub-class with the smallest ID. + return firstCommonClass(A->getSubClassMask(), B->getSubClassMask(), this); +} + +const TargetRegisterClass * +TargetRegisterInfo::getMatchingSuperRegClass(const TargetRegisterClass *A, + const TargetRegisterClass *B, + unsigned Idx) const { + assert(A && B && "Missing register class"); + assert(Idx && "Bad sub-register index"); + + // Find Idx in the list of super-register indices. + for (SuperRegClassIterator RCI(B, this); RCI.isValid(); ++RCI) + if (RCI.getSubReg() == Idx) + // The bit mask contains all register classes that are projected into B + // by Idx. Find a class that is also a sub-class of A. + return firstCommonClass(RCI.getMask(), A->getSubClassMask(), this); + return 0; +} + +const TargetRegisterClass *TargetRegisterInfo:: +getCommonSuperRegClass(const TargetRegisterClass *RCA, unsigned SubA, + const TargetRegisterClass *RCB, unsigned SubB, + unsigned &PreA, unsigned &PreB) const { + assert(RCA && SubA && RCB && SubB && "Invalid arguments"); + + // Search all pairs of sub-register indices that project into RCA and RCB + // respectively. This is quadratic, but usually the sets are very small. On + // most targets like X86, there will only be a single sub-register index + // (e.g., sub_16bit projecting into GR16). + // + // The worst case is a register class like DPR on ARM. + // We have indices dsub_0..dsub_7 projecting into that class. + // + // It is very common that one register class is a sub-register of the other. + // Arrange for RCA to be the larger register so the answer will be found in + // the first iteration. This makes the search linear for the most common + // case. + const TargetRegisterClass *BestRC = 0; + unsigned *BestPreA = &PreA; + unsigned *BestPreB = &PreB; + if (RCA->getSize() < RCB->getSize()) { + std::swap(RCA, RCB); + std::swap(SubA, SubB); + std::swap(BestPreA, BestPreB); + } + + // Also terminate the search one we have found a register class as small as + // RCA. + unsigned MinSize = RCA->getSize(); + + for (SuperRegClassIterator IA(RCA, this, true); IA.isValid(); ++IA) { + unsigned FinalA = composeSubRegIndices(IA.getSubReg(), SubA); + for (SuperRegClassIterator IB(RCB, this, true); IB.isValid(); ++IB) { + // Check if a common super-register class exists for this index pair. + const TargetRegisterClass *RC = + firstCommonClass(IA.getMask(), IB.getMask(), this); + if (!RC || RC->getSize() < MinSize) + continue; + + // The indexes must compose identically: PreA+SubA == PreB+SubB. + unsigned FinalB = composeSubRegIndices(IB.getSubReg(), SubB); + if (FinalA != FinalB) + continue; + + // Is RC a better candidate than BestRC? + if (BestRC && RC->getSize() >= BestRC->getSize()) + continue; + + // Yes, RC is the smallest super-register seen so far. + BestRC = RC; + *BestPreA = IA.getSubReg(); + *BestPreB = IB.getSubReg(); + + // Bail early if we reached MinSize. We won't find a better candidate. + if (BestRC->getSize() == MinSize) + return BestRC; + } + } + return BestRC; +} + +// Compute target-independent register allocator hints to help eliminate copies. +void +TargetRegisterInfo::getRegAllocationHints(unsigned VirtReg, + ArrayRef<MCPhysReg> Order, + SmallVectorImpl<MCPhysReg> &Hints, + const MachineFunction &MF, + const VirtRegMap *VRM) const { + const MachineRegisterInfo &MRI = MF.getRegInfo(); + std::pair<unsigned, unsigned> Hint = MRI.getRegAllocationHint(VirtReg); + + // Hints with HintType != 0 were set by target-dependent code. + // Such targets must provide their own implementation of + // TRI::getRegAllocationHints to interpret those hint types. + assert(Hint.first == 0 && "Target must implement TRI::getRegAllocationHints"); + + // Target-independent hints are either a physical or a virtual register. + unsigned Phys = Hint.second; + if (VRM && isVirtualRegister(Phys)) + Phys = VRM->getPhys(Phys); + + // Check that Phys is a valid hint in VirtReg's register class. + if (!isPhysicalRegister(Phys)) + return; + if (MRI.isReserved(Phys)) + return; + // Check that Phys is in the allocation order. We shouldn't heed hints + // from VirtReg's register class if they aren't in the allocation order. The + // target probably has a reason for removing the register. + if (std::find(Order.begin(), Order.end(), Phys) == Order.end()) + return; + + // All clear, tell the register allocator to prefer this register. + Hints.push_back(Phys); +} diff --git a/contrib/llvm/lib/CodeGen/TargetSchedule.cpp b/contrib/llvm/lib/CodeGen/TargetSchedule.cpp index ca3b0e0..783bfa1 100644 --- a/contrib/llvm/lib/CodeGen/TargetSchedule.cpp +++ b/contrib/llvm/lib/CodeGen/TargetSchedule.cpp @@ -13,12 +13,12 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/TargetSchedule.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -240,7 +240,10 @@ unsigned TargetSchedModel::computeOperandLatency( report_fatal_error(ss.str()); } #endif - return DefMI->isTransient() ? 0 : 1; + // FIXME: Automatically giving all implicit defs defaultDefLatency is + // undesirable. We should only do it for defs that are known to the MC + // desc like flags. Truly implicit defs should get 1 cycle latency. + return DefMI->isTransient() ? 0 : TII->defaultDefLatency(&SchedModel, DefMI); } unsigned TargetSchedModel::computeInstrLatency(const MachineInstr *MI) const { diff --git a/contrib/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/contrib/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp index a9058bc..e6dfe10 100644 --- a/contrib/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp +++ b/contrib/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -29,26 +29,25 @@ #define DEBUG_TYPE "twoaddrinstr" #include "llvm/CodeGen/Passes.h" -#include "llvm/Function.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/IR/Function.h" #include "llvm/MC/MCInstrItineraries.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/ADT/BitVector.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" using namespace llvm; STATISTIC(NumTwoAddressInstrs, "Number of two-address instructions"); @@ -67,7 +66,6 @@ class TwoAddressInstructionPass : public MachineFunctionPass { const InstrItineraryData *InstrItins; MachineRegisterInfo *MRI; LiveVariables *LV; - SlotIndexes *Indexes; LiveIntervals *LIS; AliasAnalysis *AA; CodeGenOpt::Level OptLevel; @@ -92,10 +90,6 @@ class TwoAddressInstructionPass : public MachineFunctionPass { // virtual registers. e.g. r1 = move v1024. DenseMap<unsigned, unsigned> DstRegMap; - /// RegSequences - Keep track the list of REG_SEQUENCE instructions seen - /// during the initial walk of the machine function. - SmallVector<MachineInstr*, 16> RegSequences; - bool sink3AddrInstruction(MachineInstr *MI, unsigned Reg, MachineBasicBlock::iterator OldPos); @@ -125,7 +119,7 @@ class TwoAddressInstructionPass : public MachineFunctionPass { bool tryInstructionTransform(MachineBasicBlock::iterator &mi, MachineBasicBlock::iterator &nmi, unsigned SrcIdx, unsigned DstIdx, - unsigned Dist); + unsigned Dist, bool shouldOnlyCommute); void scanUses(unsigned DstReg); @@ -135,11 +129,7 @@ class TwoAddressInstructionPass : public MachineFunctionPass { typedef SmallDenseMap<unsigned, TiedPairList> TiedOperandMap; bool collectTiedOperands(MachineInstr *MI, TiedOperandMap&); void processTiedPairs(MachineInstr *MI, TiedPairList&, unsigned &Dist); - - /// eliminateRegSequences - Eliminate REG_SEQUENCE instructions as part of - /// the de-ssa process. This replaces sources of REG_SEQUENCE as sub-register - /// references of the register defined by REG_SEQUENCE. - bool eliminateRegSequences(); + void eliminateRegSequence(MachineBasicBlock::iterator&); public: static char ID; // Pass identification, replacement for typeid @@ -172,6 +162,8 @@ INITIALIZE_PASS_END(TwoAddressInstructionPass, "twoaddressinstruction", char &llvm::TwoAddressInstructionPassID = TwoAddressInstructionPass::ID; +static bool isPlainlyKilled(MachineInstr *MI, unsigned Reg, LiveIntervals *LIS); + /// sink3AddrInstruction - A two-address instruction has been converted to a /// three-address instruction to avoid clobbering a register. Try to sink it /// past the instruction that would kill the above mentioned register to reduce @@ -213,14 +205,29 @@ sink3AddrInstruction(MachineInstr *MI, unsigned SavedReg, // Find the instruction that kills SavedReg. MachineInstr *KillMI = NULL; - for (MachineRegisterInfo::use_nodbg_iterator - UI = MRI->use_nodbg_begin(SavedReg), - UE = MRI->use_nodbg_end(); UI != UE; ++UI) { - MachineOperand &UseMO = UI.getOperand(); - if (!UseMO.isKill()) - continue; - KillMI = UseMO.getParent(); - break; + if (LIS) { + LiveInterval &LI = LIS->getInterval(SavedReg); + assert(LI.end() != LI.begin() && + "Reg should not have empty live interval."); + + SlotIndex MBBEndIdx = LIS->getMBBEndIdx(MBB).getPrevSlot(); + LiveInterval::const_iterator I = LI.find(MBBEndIdx); + if (I != LI.end() && I->start < MBBEndIdx) + return false; + + --I; + KillMI = LIS->getInstructionFromIndex(I->end); + } + if (!KillMI) { + for (MachineRegisterInfo::use_nodbg_iterator + UI = MRI->use_nodbg_begin(SavedReg), + UE = MRI->use_nodbg_end(); UI != UE; ++UI) { + MachineOperand &UseMO = UI.getOperand(); + if (!UseMO.isKill()) + continue; + KillMI = UseMO.getParent(); + break; + } } // If we find the instruction that kills SavedReg, and it is in an @@ -259,7 +266,7 @@ sink3AddrInstruction(MachineInstr *MI, unsigned SavedReg, if (DefReg == MOReg) return false; - if (MO.isKill()) { + if (MO.isKill() || (LIS && isPlainlyKilled(OtherMI, MOReg, LIS))) { if (OtherMI == KillMI && MOReg == SavedReg) // Save the operand that kills the register. We want to unset the kill // marker if we can sink MI past it. @@ -272,13 +279,15 @@ sink3AddrInstruction(MachineInstr *MI, unsigned SavedReg, } assert(KillMO && "Didn't find kill"); - // Update kill and LV information. - KillMO->setIsKill(false); - KillMO = MI->findRegisterUseOperand(SavedReg, false, TRI); - KillMO->setIsKill(true); + if (!LIS) { + // Update kill and LV information. + KillMO->setIsKill(false); + KillMO = MI->findRegisterUseOperand(SavedReg, false, TRI); + KillMO->setIsKill(true); - if (LV) - LV->replaceKillInstruction(SavedReg, KillMI, MI); + if (LV) + LV->replaceKillInstruction(SavedReg, KillMI, MI); + } // Move instruction to its destination. MBB->remove(MI); @@ -339,6 +348,33 @@ static bool isCopyToReg(MachineInstr &MI, const TargetInstrInfo *TII, return true; } +/// isPLainlyKilled - Test if the given register value, which is used by the +// given instruction, is killed by the given instruction. +static bool isPlainlyKilled(MachineInstr *MI, unsigned Reg, + LiveIntervals *LIS) { + if (LIS && TargetRegisterInfo::isVirtualRegister(Reg) && + !LIS->isNotInMIMap(MI)) { + // FIXME: Sometimes tryInstructionTransform() will add instructions and + // test whether they can be folded before keeping them. In this case it + // sets a kill before recursively calling tryInstructionTransform() again. + // If there is no interval available, we assume that this instruction is + // one of those. A kill flag is manually inserted on the operand so the + // check below will handle it. + LiveInterval &LI = LIS->getInterval(Reg); + // This is to match the kill flag version where undefs don't have kill + // flags. + if (!LI.hasAtLeastOneValue()) + return false; + + SlotIndex useIdx = LIS->getInstructionIndex(MI); + LiveInterval::const_iterator I = LI.find(useIdx); + assert(I != LI.end() && "Reg must be live-in to use."); + return !I->end.isBlock() && SlotIndex::isSameInstr(I->end, useIdx); + } + + return MI->killsRegister(Reg); +} + /// isKilled - Test if the given register value, which is used by the given /// instruction, is killed by the given instruction. This looks through /// coalescable copies to see if the original value is potentially not killed. @@ -354,12 +390,20 @@ static bool isCopyToReg(MachineInstr &MI, const TargetInstrInfo *TII, /// normal heuristics commute the (two-address) add, which lets /// coalescing eliminate the extra copy. /// +/// If allowFalsePositives is true then likely kills are treated as kills even +/// if it can't be proven that they are kills. static bool isKilled(MachineInstr &MI, unsigned Reg, const MachineRegisterInfo *MRI, - const TargetInstrInfo *TII) { + const TargetInstrInfo *TII, + LiveIntervals *LIS, + bool allowFalsePositives) { MachineInstr *DefMI = &MI; for (;;) { - if (!DefMI->killsRegister(Reg)) + // All uses of physical registers are likely to be kills. + if (TargetRegisterInfo::isPhysicalRegister(Reg) && + (allowFalsePositives || MRI->hasOneUse(Reg))) + return true; + if (!isPlainlyKilled(DefMI, Reg, LIS)) return false; if (TargetRegisterInfo::isPhysicalRegister(Reg)) return true; @@ -480,7 +524,7 @@ isProfitableToCommute(unsigned regA, unsigned regB, unsigned regC, // insert => %reg1030<def> = MOV8rr %reg1029 // %reg1030<def> = ADD8rr %reg1029<kill>, %reg1028<kill>, %EFLAGS<imp-def,dead> - if (!MI->killsRegister(regC)) + if (!isPlainlyKilled(MI, regC, LIS)) return false; // Ok, we have something like: @@ -536,19 +580,9 @@ commuteInstruction(MachineBasicBlock::iterator &mi, } DEBUG(dbgs() << "2addr: COMMUTED TO: " << *NewMI); - // If the instruction changed to commute it, update livevar. - if (NewMI != MI) { - if (LV) - // Update live variables - LV->replaceKillInstruction(RegC, MI, NewMI); - if (Indexes) - Indexes->replaceMachineInstrInMaps(MI, NewMI); - - MBB->insert(mi, NewMI); // Insert the new inst - MBB->erase(mi); // Nuke the old inst. - mi = NewMI; - DistanceMap.insert(std::make_pair(NewMI, Dist)); - } + assert(NewMI == MI && + "TargetInstrInfo::commuteInstruction() should not return a new " + "instruction unless it was requested."); // Update source register map. unsigned FromRegC = getMappedReg(RegC, SrcRegMap); @@ -595,8 +629,8 @@ TwoAddressInstructionPass::convertInstTo3Addr(MachineBasicBlock::iterator &mi, DEBUG(dbgs() << "2addr: TO 3-ADDR: " << *NewMI); bool Sunk = false; - if (Indexes) - Indexes->replaceMachineInstrInMaps(mi, NewMI); + if (LIS) + LIS->ReplaceMachineInstrInMaps(mi, NewMI); if (NewMI->findRegisterUseOperand(RegB, false, TRI)) // FIXME: Temporary workaround. If the new instruction doesn't @@ -708,9 +742,9 @@ bool TwoAddressInstructionPass:: rescheduleMIBelowKill(MachineBasicBlock::iterator &mi, MachineBasicBlock::iterator &nmi, unsigned Reg) { - // Bail immediately if we don't have LV available. We use it to find kills - // efficiently. - if (!LV) + // Bail immediately if we don't have LV or LIS available. We use them to find + // kills efficiently. + if (!LV && !LIS) return false; MachineInstr *MI = &*mi; @@ -719,7 +753,22 @@ rescheduleMIBelowKill(MachineBasicBlock::iterator &mi, // Must be created from unfolded load. Don't waste time trying this. return false; - MachineInstr *KillMI = LV->getVarInfo(Reg).findKill(MBB); + MachineInstr *KillMI = 0; + if (LIS) { + LiveInterval &LI = LIS->getInterval(Reg); + assert(LI.end() != LI.begin() && + "Reg should not have empty live interval."); + + SlotIndex MBBEndIdx = LIS->getMBBEndIdx(MBB).getPrevSlot(); + LiveInterval::const_iterator I = LI.find(MBBEndIdx); + if (I != LI.end() && I->start < MBBEndIdx) + return false; + + --I; + KillMI = LIS->getInstructionFromIndex(I->end); + } else { + KillMI = LV->getVarInfo(Reg).findKill(MBB); + } if (!KillMI || MI == KillMI || KillMI->isCopy() || KillMI->isCopyLike()) // Don't mess with copies, they may be coalesced later. return false; @@ -755,24 +804,27 @@ rescheduleMIBelowKill(MachineBasicBlock::iterator &mi, Defs.insert(MOReg); else { Uses.insert(MOReg); - if (MO.isKill() && MOReg != Reg) + if (MOReg != Reg && (MO.isKill() || + (LIS && isPlainlyKilled(MI, MOReg, LIS)))) Kills.insert(MOReg); } } // Move the copies connected to MI down as well. - MachineBasicBlock::iterator From = MI; - MachineBasicBlock::iterator To = llvm::next(From); - while (To->isCopy() && Defs.count(To->getOperand(1).getReg())) { - Defs.insert(To->getOperand(0).getReg()); - ++To; + MachineBasicBlock::iterator Begin = MI; + MachineBasicBlock::iterator AfterMI = llvm::next(Begin); + + MachineBasicBlock::iterator End = AfterMI; + while (End->isCopy() && Defs.count(End->getOperand(1).getReg())) { + Defs.insert(End->getOperand(0).getReg()); + ++End; } // Check if the reschedule will not break depedencies. unsigned NumVisited = 0; MachineBasicBlock::iterator KillPos = KillMI; ++KillPos; - for (MachineBasicBlock::iterator I = To; I != KillPos; ++I) { + for (MachineBasicBlock::iterator I = End; I != KillPos; ++I) { MachineInstr *OtherMI = I; // DBG_VALUE cannot be counted against the limit. if (OtherMI->isDebugValue()) @@ -803,11 +855,13 @@ rescheduleMIBelowKill(MachineBasicBlock::iterator &mi, } else { if (Defs.count(MOReg)) return false; + bool isKill = MO.isKill() || + (LIS && isPlainlyKilled(OtherMI, MOReg, LIS)); if (MOReg != Reg && - ((MO.isKill() && Uses.count(MOReg)) || Kills.count(MOReg))) + ((isKill && Uses.count(MOReg)) || Kills.count(MOReg))) // Don't want to extend other live ranges and update kills. return false; - if (MOReg == Reg && !MO.isKill()) + if (MOReg == Reg && !isKill) // We can't schedule across a use of the register in question. return false; // Ensure that if this is register in question, its the kill we expect. @@ -818,19 +872,35 @@ rescheduleMIBelowKill(MachineBasicBlock::iterator &mi, } // Move debug info as well. - while (From != MBB->begin() && llvm::prior(From)->isDebugValue()) - --From; + while (Begin != MBB->begin() && llvm::prior(Begin)->isDebugValue()) + --Begin; + + nmi = End; + MachineBasicBlock::iterator InsertPos = KillPos; + if (LIS) { + // We have to move the copies first so that the MBB is still well-formed + // when calling handleMove(). + for (MachineBasicBlock::iterator MBBI = AfterMI; MBBI != End;) { + MachineInstr *CopyMI = MBBI; + ++MBBI; + MBB->splice(InsertPos, MBB, CopyMI); + LIS->handleMove(CopyMI); + InsertPos = CopyMI; + } + End = llvm::next(MachineBasicBlock::iterator(MI)); + } // Copies following MI may have been moved as well. - nmi = To; - MBB->splice(KillPos, MBB, From, To); + MBB->splice(InsertPos, MBB, Begin, End); DistanceMap.erase(DI); // Update live variables - LV->removeVirtualRegisterKilled(Reg, KillMI); - LV->addVirtualRegisterKilled(Reg, MI); - if (LIS) + if (LIS) { LIS->handleMove(MI); + } else { + LV->removeVirtualRegisterKilled(Reg, KillMI); + LV->addVirtualRegisterKilled(Reg, MI); + } DEBUG(dbgs() << "\trescheduled below kill: " << *KillMI); return true; @@ -866,9 +936,9 @@ bool TwoAddressInstructionPass:: rescheduleKillAboveMI(MachineBasicBlock::iterator &mi, MachineBasicBlock::iterator &nmi, unsigned Reg) { - // Bail immediately if we don't have LV available. We use it to find kills - // efficiently. - if (!LV) + // Bail immediately if we don't have LV or LIS available. We use them to find + // kills efficiently. + if (!LV && !LIS) return false; MachineInstr *MI = &*mi; @@ -877,7 +947,22 @@ rescheduleKillAboveMI(MachineBasicBlock::iterator &mi, // Must be created from unfolded load. Don't waste time trying this. return false; - MachineInstr *KillMI = LV->getVarInfo(Reg).findKill(MBB); + MachineInstr *KillMI = 0; + if (LIS) { + LiveInterval &LI = LIS->getInterval(Reg); + assert(LI.end() != LI.begin() && + "Reg should not have empty live interval."); + + SlotIndex MBBEndIdx = LIS->getMBBEndIdx(MBB).getPrevSlot(); + LiveInterval::const_iterator I = LI.find(MBBEndIdx); + if (I != LI.end() && I->start < MBBEndIdx) + return false; + + --I; + KillMI = LIS->getInstructionFromIndex(I->end); + } else { + KillMI = LV->getVarInfo(Reg).findKill(MBB); + } if (!KillMI || MI == KillMI || KillMI->isCopy() || KillMI->isCopyLike()) // Don't mess with copies, they may be coalesced later. return false; @@ -904,10 +989,11 @@ rescheduleKillAboveMI(MachineBasicBlock::iterator &mi, continue; if (isDefTooClose(MOReg, DI->second, MI)) return false; - if (MOReg == Reg && !MO.isKill()) + bool isKill = MO.isKill() || (LIS && isPlainlyKilled(KillMI, MOReg, LIS)); + if (MOReg == Reg && !isKill) return false; Uses.insert(MOReg); - if (MO.isKill() && MOReg != Reg) + if (isKill && MOReg != Reg) Kills.insert(MOReg); } else if (TargetRegisterInfo::isPhysicalRegister(MOReg)) { Defs.insert(MOReg); @@ -947,7 +1033,8 @@ rescheduleKillAboveMI(MachineBasicBlock::iterator &mi, if (Kills.count(MOReg)) // Don't want to extend other live ranges and update kills. return false; - if (OtherMI != MI && MOReg == Reg && !MO.isKill()) + if (OtherMI != MI && MOReg == Reg && + !(MO.isKill() || (LIS && isPlainlyKilled(OtherMI, MOReg, LIS)))) // We can't schedule across a use of the register in question. return false; } else { @@ -981,10 +1068,12 @@ rescheduleKillAboveMI(MachineBasicBlock::iterator &mi, DistanceMap.erase(DI); // Update live variables - LV->removeVirtualRegisterKilled(Reg, KillMI); - LV->addVirtualRegisterKilled(Reg, MI); - if (LIS) + if (LIS) { LIS->handleMove(KillMI); + } else { + LV->removeVirtualRegisterKilled(Reg, KillMI); + LV->addVirtualRegisterKilled(Reg, MI); + } DEBUG(dbgs() << "\trescheduled kill: " << *KillMI); return true; @@ -995,11 +1084,13 @@ rescheduleKillAboveMI(MachineBasicBlock::iterator &mi, /// either eliminate the tied operands or improve the opportunities for /// coalescing away the register copy. Returns true if no copy needs to be /// inserted to untie mi's operands (either because they were untied, or -/// because mi was rescheduled, and will be visited again later). +/// because mi was rescheduled, and will be visited again later). If the +/// shouldOnlyCommute flag is true, only instruction commutation is attempted. bool TwoAddressInstructionPass:: tryInstructionTransform(MachineBasicBlock::iterator &mi, MachineBasicBlock::iterator &nmi, - unsigned SrcIdx, unsigned DstIdx, unsigned Dist) { + unsigned SrcIdx, unsigned DstIdx, + unsigned Dist, bool shouldOnlyCommute) { if (OptLevel == CodeGenOpt::None) return false; @@ -1009,7 +1100,7 @@ tryInstructionTransform(MachineBasicBlock::iterator &mi, assert(TargetRegisterInfo::isVirtualRegister(regB) && "cannot make instruction into two-address form"); - bool regBKilled = isKilled(MI, regB, MRI, TII); + bool regBKilled = isKilled(MI, regB, MRI, TII, LIS, true); if (TargetRegisterInfo::isVirtualRegister(regA)) scanUses(regA); @@ -1029,7 +1120,7 @@ tryInstructionTransform(MachineBasicBlock::iterator &mi, if (regCIdx != ~0U) { regC = MI.getOperand(regCIdx).getReg(); - if (!regBKilled && isKilled(MI, regC, MRI, TII)) + if (!regBKilled && isKilled(MI, regC, MRI, TII, LIS, false)) // If C dies but B does not, swap the B and C operands. // This makes the live ranges of A and C joinable. TryCommute = true; @@ -1048,6 +1139,9 @@ tryInstructionTransform(MachineBasicBlock::iterator &mi, return false; } + if (shouldOnlyCommute) + return false; + // If there is one more use of regB later in the same MBB, consider // re-schedule this MI below it. if (rescheduleMIBelowKill(mi, nmi, regB)) { @@ -1123,10 +1217,12 @@ tryInstructionTransform(MachineBasicBlock::iterator &mi, unsigned NewDstIdx = NewMIs[1]->findRegisterDefOperandIdx(regA); unsigned NewSrcIdx = NewMIs[1]->findRegisterUseOperandIdx(regB); MachineBasicBlock::iterator NewMI = NewMIs[1]; - bool TransformSuccess = - tryInstructionTransform(NewMI, mi, NewSrcIdx, NewDstIdx, Dist); - if (TransformSuccess || - NewMIs[1]->getOperand(NewSrcIdx).isKill()) { + bool TransformResult = + tryInstructionTransform(NewMI, mi, NewSrcIdx, NewDstIdx, Dist, true); + (void)TransformResult; + assert(!TransformResult && + "tryInstructionTransform() should return false."); + if (NewMIs[1]->getOperand(NewSrcIdx).isKill()) { // Success, or at least we made an improvement. Keep the unfolded // instructions and discard the original. if (LV) { @@ -1157,10 +1253,26 @@ tryInstructionTransform(MachineBasicBlock::iterator &mi, } LV->addVirtualRegisterKilled(Reg, NewMIs[1]); } + + SmallVector<unsigned, 4> OrigRegs; + if (LIS) { + for (MachineInstr::const_mop_iterator MOI = MI.operands_begin(), + MOE = MI.operands_end(); MOI != MOE; ++MOI) { + if (MOI->isReg()) + OrigRegs.push_back(MOI->getReg()); + } + } + MI.eraseFromParent(); + + // Update LiveIntervals. + if (LIS) { + MachineBasicBlock::iterator Begin(NewMIs[0]); + MachineBasicBlock::iterator End(NewMIs[1]); + LIS->repairIntervalsInRange(MBB, Begin, End, OrigRegs); + } + mi = NewMIs[1]; - if (TransformSuccess) - return true; } else { // Transforming didn't eliminate the tie and didn't lead to an // improvement. Clean up the unfolded instructions and keep the @@ -1223,9 +1335,15 @@ TwoAddressInstructionPass::processTiedPairs(MachineInstr *MI, TiedPairList &TiedPairs, unsigned &Dist) { bool IsEarlyClobber = false; + for (unsigned tpi = 0, tpe = TiedPairs.size(); tpi != tpe; ++tpi) { + const MachineOperand &DstMO = MI->getOperand(TiedPairs[tpi].second); + IsEarlyClobber |= DstMO.isEarlyClobber(); + } + bool RemovedKillFlag = false; bool AllUsesCopied = true; unsigned LastCopiedReg = 0; + SlotIndex LastCopyIdx; unsigned RegB = 0; for (unsigned tpi = 0, tpe = TiedPairs.size(); tpi != tpe; ++tpi) { unsigned SrcIdx = TiedPairs[tpi].first; @@ -1233,7 +1351,6 @@ TwoAddressInstructionPass::processTiedPairs(MachineInstr *MI, const MachineOperand &DstMO = MI->getOperand(DstIdx); unsigned RegA = DstMO.getReg(); - IsEarlyClobber |= DstMO.isEarlyClobber(); // Grab RegB from the instruction because it may have changed if the // instruction was commuted. @@ -1271,9 +1388,17 @@ TwoAddressInstructionPass::processTiedPairs(MachineInstr *MI, DistanceMap.insert(std::make_pair(PrevMI, Dist)); DistanceMap[MI] = ++Dist; - SlotIndex CopyIdx; - if (Indexes) - CopyIdx = Indexes->insertMachineInstrInMaps(PrevMI).getRegSlot(); + if (LIS) { + LastCopyIdx = LIS->InsertMachineInstrInMaps(PrevMI).getRegSlot(); + + if (TargetRegisterInfo::isVirtualRegister(RegA)) { + LiveInterval &LI = LIS->getInterval(RegA); + VNInfo *VNI = LI.getNextValue(LastCopyIdx, LIS->getVNInfoAllocator()); + SlotIndex endIdx = + LIS->getInstructionIndex(MI).getRegSlot(IsEarlyClobber); + LI.addRange(LiveRange(LastCopyIdx, endIdx, VNI)); + } + } DEBUG(dbgs() << "\t\tprepend:\t" << *PrevMI); @@ -1319,6 +1444,18 @@ TwoAddressInstructionPass::processTiedPairs(MachineInstr *MI, LV->addVirtualRegisterKilled(RegB, PrevMI); } + // Update LiveIntervals. + if (LIS) { + LiveInterval &LI = LIS->getInterval(RegB); + SlotIndex MIIdx = LIS->getInstructionIndex(MI); + LiveInterval::const_iterator I = LI.find(MIIdx); + assert(I != LI.end() && "RegB must be live-in to use."); + + SlotIndex UseIdx = MIIdx.getRegSlot(IsEarlyClobber); + if (I->end == UseIdx) + LI.removeRange(LastCopyIdx, UseIdx); + } + } else if (RemovedKillFlag) { // Some tied uses of regB matched their destination registers, so // regB is still used in this instruction, but a kill flag was @@ -1343,7 +1480,6 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &Func) { TII = TM.getInstrInfo(); TRI = TM.getRegisterInfo(); InstrItins = TM.getInstrItineraryData(); - Indexes = getAnalysisIfAvailable<SlotIndexes>(); LV = getAnalysisIfAvailable<LiveVariables>(); LIS = getAnalysisIfAvailable<LiveIntervals>(); AA = &getAnalysis<AliasAnalysis>(); @@ -1375,9 +1511,10 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &Func) { continue; } - // Remember REG_SEQUENCE instructions, we'll deal with them later. + // Expand REG_SEQUENCE instructions. This will position mi at the first + // expanded instruction. if (mi->isRegSequence()) - RegSequences.push_back(&*mi); + eliminateRegSequence(mi); DistanceMap.insert(std::make_pair(mi, ++Dist)); @@ -1406,7 +1543,7 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &Func) { unsigned SrcReg = mi->getOperand(SrcIdx).getReg(); unsigned DstReg = mi->getOperand(DstIdx).getReg(); if (SrcReg != DstReg && - tryInstructionTransform(mi, nmi, SrcIdx, DstIdx, Dist)) { + tryInstructionTransform(mi, nmi, SrcIdx, DstIdx, Dist, false)) { // The tied operands have been eliminated or shifted further down the // block to ease elimination. Continue processing with 'nmi'. TiedOperands.clear(); @@ -1444,192 +1581,98 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &Func) { } } - // Eliminate REG_SEQUENCE instructions. Their whole purpose was to preseve - // SSA form. It's now safe to de-SSA. - MadeChange |= eliminateRegSequences(); + if (LIS) + MF->verify(this, "After two-address instruction pass"); return MadeChange; } -static void UpdateRegSequenceSrcs(unsigned SrcReg, - unsigned DstReg, unsigned SubIdx, - MachineRegisterInfo *MRI, - const TargetRegisterInfo &TRI) { - for (MachineRegisterInfo::reg_iterator RI = MRI->reg_begin(SrcReg), - RE = MRI->reg_end(); RI != RE; ) { - MachineOperand &MO = RI.getOperand(); - ++RI; - MO.substVirtReg(DstReg, SubIdx, TRI); +/// Eliminate a REG_SEQUENCE instruction as part of the de-ssa process. +/// +/// The instruction is turned into a sequence of sub-register copies: +/// +/// %dst = REG_SEQUENCE %v1, ssub0, %v2, ssub1 +/// +/// Becomes: +/// +/// %dst:ssub0<def,undef> = COPY %v1 +/// %dst:ssub1<def> = COPY %v2 +/// +void TwoAddressInstructionPass:: +eliminateRegSequence(MachineBasicBlock::iterator &MBBI) { + MachineInstr *MI = MBBI; + unsigned DstReg = MI->getOperand(0).getReg(); + if (MI->getOperand(0).getSubReg() || + TargetRegisterInfo::isPhysicalRegister(DstReg) || + !(MI->getNumOperands() & 1)) { + DEBUG(dbgs() << "Illegal REG_SEQUENCE instruction:" << *MI); + llvm_unreachable(0); } -} - -// Find the first def of Reg, assuming they are all in the same basic block. -static MachineInstr *findFirstDef(unsigned Reg, MachineRegisterInfo *MRI) { - SmallPtrSet<MachineInstr*, 8> Defs; - MachineInstr *First = 0; - for (MachineRegisterInfo::def_iterator RI = MRI->def_begin(Reg); - MachineInstr *MI = RI.skipInstruction(); Defs.insert(MI)) - First = MI; - if (!First) - return 0; - - MachineBasicBlock *MBB = First->getParent(); - MachineBasicBlock::iterator A = First, B = First; - bool Moving; - do { - Moving = false; - if (A != MBB->begin()) { - Moving = true; - --A; - if (Defs.erase(A)) First = A; - } - if (B != MBB->end()) { - Defs.erase(B); - ++B; - Moving = true; - } - } while (Moving && !Defs.empty()); - assert(Defs.empty() && "Instructions outside basic block!"); - return First; -} -static bool HasOtherRegSequenceUses(unsigned Reg, MachineInstr *RegSeq, - MachineRegisterInfo *MRI) { - for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(Reg), - UE = MRI->use_end(); UI != UE; ++UI) { - MachineInstr *UseMI = &*UI; - if (UseMI != RegSeq && UseMI->isRegSequence()) - return true; + SmallVector<unsigned, 4> OrigRegs; + if (LIS) { + OrigRegs.push_back(MI->getOperand(0).getReg()); + for (unsigned i = 1, e = MI->getNumOperands(); i < e; i += 2) + OrigRegs.push_back(MI->getOperand(i).getReg()); } - return false; -} -/// eliminateRegSequences - Eliminate REG_SEQUENCE instructions as part -/// of the de-ssa process. This replaces sources of REG_SEQUENCE as -/// sub-register references of the register defined by REG_SEQUENCE. e.g. -/// -/// %reg1029<def>, %reg1030<def> = VLD1q16 %reg1024<kill>, ... -/// %reg1031<def> = REG_SEQUENCE %reg1029<kill>, 5, %reg1030<kill>, 6 -/// => -/// %reg1031:5<def>, %reg1031:6<def> = VLD1q16 %reg1024<kill>, ... -bool TwoAddressInstructionPass::eliminateRegSequences() { - if (RegSequences.empty()) - return false; - - for (unsigned i = 0, e = RegSequences.size(); i != e; ++i) { - MachineInstr *MI = RegSequences[i]; - unsigned DstReg = MI->getOperand(0).getReg(); - if (MI->getOperand(0).getSubReg() || - TargetRegisterInfo::isPhysicalRegister(DstReg) || - !(MI->getNumOperands() & 1)) { - DEBUG(dbgs() << "Illegal REG_SEQUENCE instruction:" << *MI); - llvm_unreachable(0); - } - - bool IsImpDef = true; - SmallVector<unsigned, 4> RealSrcs; - SmallSet<unsigned, 4> Seen; - for (unsigned i = 1, e = MI->getNumOperands(); i < e; i += 2) { - // Nothing needs to be inserted for <undef> operands. - if (MI->getOperand(i).isUndef()) { - MI->getOperand(i).setReg(0); - continue; - } - unsigned SrcReg = MI->getOperand(i).getReg(); - unsigned SrcSubIdx = MI->getOperand(i).getSubReg(); - unsigned SubIdx = MI->getOperand(i+1).getImm(); - // DefMI of NULL means the value does not have a vreg in this block - // i.e., its a physical register or a subreg. - // In either case we force a copy to be generated. - MachineInstr *DefMI = NULL; - if (!MI->getOperand(i).getSubReg() && - !TargetRegisterInfo::isPhysicalRegister(SrcReg)) { - DefMI = MRI->getUniqueVRegDef(SrcReg); - } + bool DefEmitted = false; + for (unsigned i = 1, e = MI->getNumOperands(); i < e; i += 2) { + MachineOperand &UseMO = MI->getOperand(i); + unsigned SrcReg = UseMO.getReg(); + unsigned SubIdx = MI->getOperand(i+1).getImm(); + // Nothing needs to be inserted for <undef> operands. + if (UseMO.isUndef()) + continue; - if (DefMI && DefMI->isImplicitDef()) { - DefMI->eraseFromParent(); - continue; - } - IsImpDef = false; - - // Remember COPY sources. These might be candidate for coalescing. - if (DefMI && DefMI->isCopy() && DefMI->getOperand(1).getSubReg()) - RealSrcs.push_back(DefMI->getOperand(1).getReg()); - - bool isKill = MI->getOperand(i).isKill(); - if (!DefMI || !Seen.insert(SrcReg) || - MI->getParent() != DefMI->getParent() || - !isKill || HasOtherRegSequenceUses(SrcReg, MI, MRI) || - !TRI->getMatchingSuperRegClass(MRI->getRegClass(DstReg), - MRI->getRegClass(SrcReg), SubIdx)) { - // REG_SEQUENCE cannot have duplicated operands, add a copy. - // Also add an copy if the source is live-in the block. We don't want - // to end up with a partial-redef of a livein, e.g. - // BB0: - // reg1051:10<def> = - // ... - // BB1: - // ... = reg1051:10 - // BB2: - // reg1051:9<def> = - // LiveIntervalAnalysis won't like it. - // - // If the REG_SEQUENCE doesn't kill its source, keeping live variables - // correctly up to date becomes very difficult. Insert a copy. - - // Defer any kill flag to the last operand using SrcReg. Otherwise, we - // might insert a COPY that uses SrcReg after is was killed. - if (isKill) - for (unsigned j = i + 2; j < e; j += 2) - if (MI->getOperand(j).getReg() == SrcReg) { - MI->getOperand(j).setIsKill(); - isKill = false; - break; - } + // Defer any kill flag to the last operand using SrcReg. Otherwise, we + // might insert a COPY that uses SrcReg after is was killed. + bool isKill = UseMO.isKill(); + if (isKill) + for (unsigned j = i + 2; j < e; j += 2) + if (MI->getOperand(j).getReg() == SrcReg) { + MI->getOperand(j).setIsKill(); + UseMO.setIsKill(false); + isKill = false; + break; + } - MachineBasicBlock::iterator InsertLoc = MI; - MachineInstr *CopyMI = BuildMI(*MI->getParent(), InsertLoc, - MI->getDebugLoc(), TII->get(TargetOpcode::COPY)) - .addReg(DstReg, RegState::Define, SubIdx) - .addReg(SrcReg, getKillRegState(isKill), SrcSubIdx); - MI->getOperand(i).setReg(0); - if (LV && isKill && !TargetRegisterInfo::isPhysicalRegister(SrcReg)) - LV->replaceKillInstruction(SrcReg, MI, CopyMI); - DEBUG(dbgs() << "Inserted: " << *CopyMI); - } + // Insert the sub-register copy. + MachineInstr *CopyMI = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), + TII->get(TargetOpcode::COPY)) + .addReg(DstReg, RegState::Define, SubIdx) + .addOperand(UseMO); + + // The first def needs an <undef> flag because there is no live register + // before it. + if (!DefEmitted) { + CopyMI->getOperand(0).setIsUndef(true); + // Return an iterator pointing to the first inserted instr. + MBBI = CopyMI; } + DefEmitted = true; - for (unsigned i = 1, e = MI->getNumOperands(); i < e; i += 2) { - unsigned SrcReg = MI->getOperand(i).getReg(); - if (!SrcReg) continue; - unsigned SubIdx = MI->getOperand(i+1).getImm(); - UpdateRegSequenceSrcs(SrcReg, DstReg, SubIdx, MRI, *TRI); - } + // Update LiveVariables' kill info. + if (LV && isKill && !TargetRegisterInfo::isPhysicalRegister(SrcReg)) + LV->replaceKillInstruction(SrcReg, MI, CopyMI); - // Set <def,undef> flags on the first DstReg def in the basic block. - // It marks the beginning of the live range. All the other defs are - // read-modify-write. - if (MachineInstr *Def = findFirstDef(DstReg, MRI)) { - for (unsigned i = 0, e = Def->getNumOperands(); i != e; ++i) { - MachineOperand &MO = Def->getOperand(i); - if (MO.isReg() && MO.isDef() && MO.getReg() == DstReg) - MO.setIsUndef(); - } - DEBUG(dbgs() << "First def: " << *Def); - } + DEBUG(dbgs() << "Inserted: " << *CopyMI); + } - if (IsImpDef) { - DEBUG(dbgs() << "Turned: " << *MI << " into an IMPLICIT_DEF"); - MI->setDesc(TII->get(TargetOpcode::IMPLICIT_DEF)); - for (int j = MI->getNumOperands() - 1, ee = 0; j > ee; --j) - MI->RemoveOperand(j); - } else { - DEBUG(dbgs() << "Eliminated: " << *MI); - MI->eraseFromParent(); - } + MachineBasicBlock::iterator EndMBBI = + llvm::next(MachineBasicBlock::iterator(MI)); + + if (!DefEmitted) { + DEBUG(dbgs() << "Turned: " << *MI << " into an IMPLICIT_DEF"); + MI->setDesc(TII->get(TargetOpcode::IMPLICIT_DEF)); + for (int j = MI->getNumOperands() - 1, ee = 0; j > ee; --j) + MI->RemoveOperand(j); + } else { + DEBUG(dbgs() << "Eliminated: " << *MI); + MI->eraseFromParent(); } - RegSequences.clear(); - return true; + // Udpate LiveIntervals. + if (LIS) + LIS->repairIntervalsInRange(MBB, MBBI, EndMBBI, OrigRegs); } diff --git a/contrib/llvm/lib/CodeGen/UnreachableBlockElim.cpp b/contrib/llvm/lib/CodeGen/UnreachableBlockElim.cpp index 52693f0..a95ebcd 100644 --- a/contrib/llvm/lib/CodeGen/UnreachableBlockElim.cpp +++ b/contrib/llvm/lib/CodeGen/UnreachableBlockElim.cpp @@ -21,22 +21,22 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/Passes.h" -#include "llvm/Constant.h" -#include "llvm/Instructions.h" -#include "llvm/Function.h" -#include "llvm/Pass.h" -#include "llvm/Type.h" +#include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/Dominators.h" #include "llvm/Analysis/ProfileInfo.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Type.h" +#include "llvm/Pass.h" #include "llvm/Support/CFG.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/ADT/DepthFirstIterator.h" -#include "llvm/ADT/SmallPtrSet.h" using namespace llvm; namespace { diff --git a/contrib/llvm/lib/CodeGen/VirtRegMap.cpp b/contrib/llvm/lib/CodeGen/VirtRegMap.cpp index bb93bdc..cd012d2 100644 --- a/contrib/llvm/lib/CodeGen/VirtRegMap.cpp +++ b/contrib/llvm/lib/CodeGen/VirtRegMap.cpp @@ -17,8 +17,10 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "regalloc" -#include "VirtRegMap.h" +#include "llvm/CodeGen/VirtRegMap.h" #include "LiveDebugVariables.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveStackAnalysis.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -26,15 +28,13 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" #include <algorithm> using namespace llvm; @@ -77,15 +77,22 @@ unsigned VirtRegMap::createSpillSlot(const TargetRegisterClass *RC) { return SS; } -unsigned VirtRegMap::getRegAllocPref(unsigned virtReg) { - std::pair<unsigned, unsigned> Hint = MRI->getRegAllocationHint(virtReg); - unsigned physReg = Hint.second; - if (TargetRegisterInfo::isVirtualRegister(physReg) && hasPhys(physReg)) - physReg = getPhys(physReg); - if (Hint.first == 0) - return (TargetRegisterInfo::isPhysicalRegister(physReg)) - ? physReg : 0; - return TRI->ResolveRegAllocHint(Hint.first, physReg, *MF); +bool VirtRegMap::hasPreferredPhys(unsigned VirtReg) { + unsigned Hint = MRI->getSimpleHint(VirtReg); + if (!Hint) + return 0; + if (TargetRegisterInfo::isVirtualRegister(Hint)) + Hint = getPhys(Hint); + return getPhys(VirtReg) == Hint; +} + +bool VirtRegMap::hasKnownPreference(unsigned VirtReg) { + std::pair<unsigned, unsigned> Hint = MRI->getRegAllocationHint(VirtReg); + if (TargetRegisterInfo::isPhysicalRegister(Hint.second)) + return true; + if (TargetRegisterInfo::isVirtualRegister(Hint.second)) + return hasPhys(Hint.second); + return false; } int VirtRegMap::assignVirt2StackSlot(unsigned virtReg) { diff --git a/contrib/llvm/lib/CodeGen/VirtRegMap.h b/contrib/llvm/lib/CodeGen/VirtRegMap.h deleted file mode 100644 index 7974dda..0000000 --- a/contrib/llvm/lib/CodeGen/VirtRegMap.h +++ /dev/null @@ -1,190 +0,0 @@ -//===-- llvm/CodeGen/VirtRegMap.h - Virtual Register Map -*- C++ -*--------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file implements a virtual register map. This maps virtual registers to -// physical registers and virtual registers to stack slots. It is created and -// updated by a register allocator and then used by a machine code rewriter that -// adds spill code and rewrites virtual into physical register references. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_VIRTREGMAP_H -#define LLVM_CODEGEN_VIRTREGMAP_H - -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/ADT/IndexedMap.h" - -namespace llvm { - class MachineInstr; - class MachineFunction; - class MachineRegisterInfo; - class TargetInstrInfo; - class raw_ostream; - class SlotIndexes; - - class VirtRegMap : public MachineFunctionPass { - public: - enum { - NO_PHYS_REG = 0, - NO_STACK_SLOT = (1L << 30)-1, - MAX_STACK_SLOT = (1L << 18)-1 - }; - - private: - MachineRegisterInfo *MRI; - const TargetInstrInfo *TII; - const TargetRegisterInfo *TRI; - MachineFunction *MF; - - /// Virt2PhysMap - This is a virtual to physical register - /// mapping. Each virtual register is required to have an entry in - /// it; even spilled virtual registers (the register mapped to a - /// spilled register is the temporary used to load it from the - /// stack). - IndexedMap<unsigned, VirtReg2IndexFunctor> Virt2PhysMap; - - /// Virt2StackSlotMap - This is virtual register to stack slot - /// mapping. Each spilled virtual register has an entry in it - /// which corresponds to the stack slot this register is spilled - /// at. - IndexedMap<int, VirtReg2IndexFunctor> Virt2StackSlotMap; - - /// Virt2SplitMap - This is virtual register to splitted virtual register - /// mapping. - IndexedMap<unsigned, VirtReg2IndexFunctor> Virt2SplitMap; - - /// createSpillSlot - Allocate a spill slot for RC from MFI. - unsigned createSpillSlot(const TargetRegisterClass *RC); - - VirtRegMap(const VirtRegMap&) LLVM_DELETED_FUNCTION; - void operator=(const VirtRegMap&) LLVM_DELETED_FUNCTION; - - public: - static char ID; - VirtRegMap() : MachineFunctionPass(ID), Virt2PhysMap(NO_PHYS_REG), - Virt2StackSlotMap(NO_STACK_SLOT), Virt2SplitMap(0) { } - virtual bool runOnMachineFunction(MachineFunction &MF); - - virtual void getAnalysisUsage(AnalysisUsage &AU) const { - AU.setPreservesAll(); - MachineFunctionPass::getAnalysisUsage(AU); - } - - MachineFunction &getMachineFunction() const { - assert(MF && "getMachineFunction called before runOnMachineFunction"); - return *MF; - } - - MachineRegisterInfo &getRegInfo() const { return *MRI; } - const TargetRegisterInfo &getTargetRegInfo() const { return *TRI; } - - void grow(); - - /// @brief returns true if the specified virtual register is - /// mapped to a physical register - bool hasPhys(unsigned virtReg) const { - return getPhys(virtReg) != NO_PHYS_REG; - } - - /// @brief returns the physical register mapped to the specified - /// virtual register - unsigned getPhys(unsigned virtReg) const { - assert(TargetRegisterInfo::isVirtualRegister(virtReg)); - return Virt2PhysMap[virtReg]; - } - - /// @brief creates a mapping for the specified virtual register to - /// the specified physical register - void assignVirt2Phys(unsigned virtReg, unsigned physReg) { - assert(TargetRegisterInfo::isVirtualRegister(virtReg) && - TargetRegisterInfo::isPhysicalRegister(physReg)); - assert(Virt2PhysMap[virtReg] == NO_PHYS_REG && - "attempt to assign physical register to already mapped " - "virtual register"); - Virt2PhysMap[virtReg] = physReg; - } - - /// @brief clears the specified virtual register's, physical - /// register mapping - void clearVirt(unsigned virtReg) { - assert(TargetRegisterInfo::isVirtualRegister(virtReg)); - assert(Virt2PhysMap[virtReg] != NO_PHYS_REG && - "attempt to clear a not assigned virtual register"); - Virt2PhysMap[virtReg] = NO_PHYS_REG; - } - - /// @brief clears all virtual to physical register mappings - void clearAllVirt() { - Virt2PhysMap.clear(); - grow(); - } - - /// @brief returns the register allocation preference. - unsigned getRegAllocPref(unsigned virtReg); - - /// @brief returns true if VirtReg is assigned to its preferred physreg. - bool hasPreferredPhys(unsigned VirtReg) { - return getPhys(VirtReg) == getRegAllocPref(VirtReg); - } - - /// @brief records virtReg is a split live interval from SReg. - void setIsSplitFromReg(unsigned virtReg, unsigned SReg) { - Virt2SplitMap[virtReg] = SReg; - } - - /// @brief returns the live interval virtReg is split from. - unsigned getPreSplitReg(unsigned virtReg) const { - return Virt2SplitMap[virtReg]; - } - - /// getOriginal - Return the original virtual register that VirtReg descends - /// from through splitting. - /// A register that was not created by splitting is its own original. - /// This operation is idempotent. - unsigned getOriginal(unsigned VirtReg) const { - unsigned Orig = getPreSplitReg(VirtReg); - return Orig ? Orig : VirtReg; - } - - /// @brief returns true if the specified virtual register is not - /// mapped to a stack slot or rematerialized. - bool isAssignedReg(unsigned virtReg) const { - if (getStackSlot(virtReg) == NO_STACK_SLOT) - return true; - // Split register can be assigned a physical register as well as a - // stack slot or remat id. - return (Virt2SplitMap[virtReg] && Virt2PhysMap[virtReg] != NO_PHYS_REG); - } - - /// @brief returns the stack slot mapped to the specified virtual - /// register - int getStackSlot(unsigned virtReg) const { - assert(TargetRegisterInfo::isVirtualRegister(virtReg)); - return Virt2StackSlotMap[virtReg]; - } - - /// @brief create a mapping for the specifed virtual register to - /// the next available stack slot - int assignVirt2StackSlot(unsigned virtReg); - /// @brief create a mapping for the specified virtual register to - /// the specified stack slot - void assignVirt2StackSlot(unsigned virtReg, int frameIndex); - - void print(raw_ostream &OS, const Module* M = 0) const; - void dump() const; - }; - - inline raw_ostream &operator<<(raw_ostream &OS, const VirtRegMap &VRM) { - VRM.print(OS); - return OS; - } -} // End llvm namespace - -#endif |
