diff options
Diffstat (limited to 'contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp')
| -rw-r--r-- | contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp | 524 |
1 files changed, 354 insertions, 170 deletions
diff --git a/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp index 939c500..20a9a39 100644 --- a/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp +++ b/contrib/llvm/lib/CodeGen/PrologEpilogInserter.cpp @@ -16,7 +16,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallSet.h" @@ -35,7 +34,6 @@ #include "llvm/IR/InlineAsm.h" #include "llvm/IR/LLVMContext.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetFrameLowering.h" @@ -49,48 +47,83 @@ using namespace llvm; #define DEBUG_TYPE "pei" +typedef SmallVector<MachineBasicBlock *, 4> MBBVector; +static void doSpillCalleeSavedRegs(MachineFunction &MF, RegScavenger *RS, + unsigned &MinCSFrameIndex, + unsigned &MaxCXFrameIndex, + const MBBVector &SaveBlocks, + const MBBVector &RestoreBlocks); + +static void doScavengeFrameVirtualRegs(MachineFunction &MF, RegScavenger *RS); + namespace { class PEI : public MachineFunctionPass { public: static char ID; - PEI() : MachineFunctionPass(ID) { + explicit PEI(const TargetMachine *TM = nullptr) : MachineFunctionPass(ID) { initializePEIPass(*PassRegistry::getPassRegistry()); + + if (TM && (!TM->usesPhysRegsForPEI())) { + SpillCalleeSavedRegisters = [](MachineFunction &, RegScavenger *, + unsigned &, unsigned &, const MBBVector &, + const MBBVector &) {}; + ScavengeFrameVirtualRegs = [](MachineFunction &, RegScavenger *) {}; + } else { + SpillCalleeSavedRegisters = doSpillCalleeSavedRegs; + ScavengeFrameVirtualRegs = doScavengeFrameVirtualRegs; + UsesCalleeSaves = true; + } } void getAnalysisUsage(AnalysisUsage &AU) const override; + MachineFunctionProperties getRequiredProperties() const override { + MachineFunctionProperties MFP; + if (UsesCalleeSaves) + MFP.set(MachineFunctionProperties::Property::AllVRegsAllocated); + return MFP; + } + /// runOnMachineFunction - Insert prolog/epilog code and replace abstract /// frame indexes with appropriate references. /// bool runOnMachineFunction(MachineFunction &Fn) override; private: + std::function<void(MachineFunction &MF, RegScavenger *RS, + unsigned &MinCSFrameIndex, unsigned &MaxCSFrameIndex, + const MBBVector &SaveBlocks, + const MBBVector &RestoreBlocks)> + SpillCalleeSavedRegisters; + std::function<void(MachineFunction &MF, RegScavenger *RS)> + ScavengeFrameVirtualRegs; + + bool UsesCalleeSaves = false; + RegScavenger *RS; // MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved // stack frame indexes. - unsigned MinCSFrameIndex, MaxCSFrameIndex; + unsigned MinCSFrameIndex = std::numeric_limits<unsigned>::max(); + unsigned MaxCSFrameIndex = 0; // Save and Restore blocks of the current function. Typically there is a // single save block, unless Windows EH funclets are involved. - SmallVector<MachineBasicBlock *, 1> SaveBlocks; - SmallVector<MachineBasicBlock *, 4> RestoreBlocks; + MBBVector SaveBlocks; + MBBVector RestoreBlocks; // Flag to control whether to use the register scavenger to resolve // frame index materialization registers. Set according to // TRI->requiresFrameIndexScavenging() for the current function. bool FrameIndexVirtualScavenging; - void calculateSets(MachineFunction &Fn); - void calculateCallsInformation(MachineFunction &Fn); - void assignCalleeSavedSpillSlots(MachineFunction &Fn, - const BitVector &SavedRegs); - void insertCSRSpillsAndRestores(MachineFunction &Fn); + void calculateCallFrameInfo(MachineFunction &Fn); + void calculateSaveRestoreBlocks(MachineFunction &Fn); + void calculateFrameObjectOffsets(MachineFunction &Fn); void replaceFrameIndices(MachineFunction &Fn); void replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, int &SPAdj); - void scavengeFrameVirtualRegs(MachineFunction &Fn); void insertPrologEpilogCode(MachineFunction &Fn); }; } // namespace @@ -103,15 +136,19 @@ WarnStackSize("warn-stack-size", cl::Hidden, cl::init((unsigned)-1), cl::desc("Warn for stack size bigger than the given" " number")); -INITIALIZE_PASS_BEGIN(PEI, "prologepilog", - "Prologue/Epilogue Insertion", false, false) +INITIALIZE_TM_PASS_BEGIN(PEI, "prologepilog", "Prologue/Epilogue Insertion", + false, false) INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) INITIALIZE_PASS_DEPENDENCY(StackProtector) -INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) -INITIALIZE_PASS_END(PEI, "prologepilog", - "Prologue/Epilogue Insertion & Frame Finalization", - false, false) +INITIALIZE_TM_PASS_END(PEI, "prologepilog", + "Prologue/Epilogue Insertion & Frame Finalization", + false, false) + +MachineFunctionPass * +llvm::createPrologEpilogInserterPass(const TargetMachine *TM) { + return new PEI(TM); +} STATISTIC(NumScavengedRegs, "Number of frame index regs scavenged"); STATISTIC(NumBytesStackSpace, @@ -122,40 +159,9 @@ void PEI::getAnalysisUsage(AnalysisUsage &AU) const { AU.addPreserved<MachineLoopInfo>(); AU.addPreserved<MachineDominatorTree>(); AU.addRequired<StackProtector>(); - AU.addRequired<TargetPassConfig>(); MachineFunctionPass::getAnalysisUsage(AU); } -/// Compute the set of return blocks -void PEI::calculateSets(MachineFunction &Fn) { - const MachineFrameInfo *MFI = Fn.getFrameInfo(); - - // Even when we do not change any CSR, we still want to insert the - // prologue and epilogue of the function. - // So set the save points for those. - - // Use the points found by shrink-wrapping, if any. - if (MFI->getSavePoint()) { - SaveBlocks.push_back(MFI->getSavePoint()); - assert(MFI->getRestorePoint() && "Both restore and save must be set"); - MachineBasicBlock *RestoreBlock = MFI->getRestorePoint(); - // If RestoreBlock does not have any successor and is not a return block - // then the end point is unreachable and we do not need to insert any - // epilogue. - if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock()) - RestoreBlocks.push_back(RestoreBlock); - return; - } - - // Save refs to entry and return blocks. - SaveBlocks.push_back(&Fn.front()); - for (MachineBasicBlock &MBB : Fn) { - if (MBB.isEHFuncletEntry()) - SaveBlocks.push_back(&MBB); - if (MBB.isReturnBlock()) - RestoreBlocks.push_back(&MBB); - } -} /// StackObjSet - A set of stack object indexes typedef SmallSetVector<int, 8> StackObjSet; @@ -168,30 +174,21 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) { const TargetRegisterInfo *TRI = Fn.getSubtarget().getRegisterInfo(); const TargetFrameLowering *TFI = Fn.getSubtarget().getFrameLowering(); - assert(!Fn.getRegInfo().getNumVirtRegs() && "Regalloc must assign all vregs"); - RS = TRI->requiresRegisterScavenging(Fn) ? new RegScavenger() : nullptr; FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(Fn); // Calculate the MaxCallFrameSize and AdjustsStack variables for the // function's frame information. Also eliminates call frame pseudo // instructions. - calculateCallsInformation(Fn); + calculateCallFrameInfo(Fn); - // Determine which of the registers in the callee save list should be saved. - BitVector SavedRegs; - TFI->determineCalleeSaves(Fn, SavedRegs, RS); - - // Insert spill code for any callee saved registers that are modified. - assignCalleeSavedSpillSlots(Fn, SavedRegs); - - // Determine placement of CSR spill/restore code: + // Determine placement of CSR spill/restore code and prolog/epilog code: // place all spills in the entry block, all restores in return blocks. - calculateSets(Fn); + calculateSaveRestoreBlocks(Fn); - // Add the code to save and restore the callee saved registers. - if (!F->hasFnAttribute(Attribute::Naked)) - insertCSRSpillsAndRestores(Fn); + // Handle CSR spilling and restoring, for targets that need it. + SpillCalleeSavedRegisters(Fn, RS, MinCSFrameIndex, MaxCSFrameIndex, + SaveBlocks, RestoreBlocks); // Allow the target machine to make final modifications to the function // before the frame layout is finalized. @@ -216,11 +213,12 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) { // If register scavenging is needed, as we've enabled doing it as a // post-pass, scavenge the virtual registers that frame index elimination // inserted. - if (TRI->requiresRegisterScavenging(Fn) && FrameIndexVirtualScavenging) - scavengeFrameVirtualRegs(Fn); + if (TRI->requiresRegisterScavenging(Fn) && FrameIndexVirtualScavenging) { + ScavengeFrameVirtualRegs(Fn, RS); - // Clear any vregs created by virtual scavenging. - Fn.getRegInfo().clearVirtRegs(); + // Clear any vregs created by virtual scavenging. + Fn.getRegInfo().clearVirtRegs(); + } // Warn on stack size when we exceeds the given limit. MachineFrameInfo *MFI = Fn.getFrameInfo(); @@ -233,13 +231,15 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) { delete RS; SaveBlocks.clear(); RestoreBlocks.clear(); + MFI->setSavePoint(nullptr); + MFI->setRestorePoint(nullptr); return true; } -/// calculateCallsInformation - Calculate the MaxCallFrameSize and AdjustsStack +/// Calculate the MaxCallFrameSize and AdjustsStack /// variables for the function's frame information and eliminate call frame /// pseudo instructions. -void PEI::calculateCallsInformation(MachineFunction &Fn) { +void PEI::calculateCallFrameInfo(MachineFunction &Fn) { const TargetInstrInfo &TII = *Fn.getSubtarget().getInstrInfo(); const TargetFrameLowering *TFI = Fn.getSubtarget().getFrameLowering(); MachineFrameInfo *MFI = Fn.getFrameInfo(); @@ -290,12 +290,42 @@ void PEI::calculateCallsInformation(MachineFunction &Fn) { } } -void PEI::assignCalleeSavedSpillSlots(MachineFunction &F, - const BitVector &SavedRegs) { - // These are used to keep track the callee-save area. Initialize them. - MinCSFrameIndex = INT_MAX; - MaxCSFrameIndex = 0; +/// Compute the sets of entry and return blocks for saving and restoring +/// callee-saved registers, and placing prolog and epilog code. +void PEI::calculateSaveRestoreBlocks(MachineFunction &Fn) { + const MachineFrameInfo *MFI = Fn.getFrameInfo(); + + // Even when we do not change any CSR, we still want to insert the + // prologue and epilogue of the function. + // So set the save points for those. + // Use the points found by shrink-wrapping, if any. + if (MFI->getSavePoint()) { + SaveBlocks.push_back(MFI->getSavePoint()); + assert(MFI->getRestorePoint() && "Both restore and save must be set"); + MachineBasicBlock *RestoreBlock = MFI->getRestorePoint(); + // If RestoreBlock does not have any successor and is not a return block + // then the end point is unreachable and we do not need to insert any + // epilogue. + if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock()) + RestoreBlocks.push_back(RestoreBlock); + return; + } + + // Save refs to entry and return blocks. + SaveBlocks.push_back(&Fn.front()); + for (MachineBasicBlock &MBB : Fn) { + if (MBB.isEHFuncletEntry()) + SaveBlocks.push_back(&MBB); + if (MBB.isReturnBlock()) + RestoreBlocks.push_back(&MBB); + } +} + +static void assignCalleeSavedSpillSlots(MachineFunction &F, + const BitVector &SavedRegs, + unsigned &MinCSFrameIndex, + unsigned &MaxCSFrameIndex) { if (SavedRegs.empty()) return; @@ -323,14 +353,13 @@ void PEI::assignCalleeSavedSpillSlots(MachineFunction &F, // Now that we know which registers need to be saved and restored, allocate // stack slots for them. - for (std::vector<CalleeSavedInfo>::iterator I = CSI.begin(), E = CSI.end(); - I != E; ++I) { - unsigned Reg = I->getReg(); + for (auto &CS : CSI) { + unsigned Reg = CS.getReg(); const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg); int FrameIdx; if (RegInfo->hasReservedSpillSlot(F, Reg, FrameIdx)) { - I->setFrameIdx(FrameIdx); + CS.setFrameIdx(FrameIdx); continue; } @@ -359,7 +388,7 @@ void PEI::assignCalleeSavedSpillSlots(MachineFunction &F, MFI->CreateFixedSpillStackObject(RC->getSize(), FixedSlot->Offset); } - I->setFrameIdx(FrameIdx); + CS.setFrameIdx(FrameIdx); } } @@ -427,7 +456,9 @@ static void updateLiveness(MachineFunction &MF) { /// insertCSRSpillsAndRestores - Insert spill and restore code for /// callee saved registers used in the function. /// -void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { +static void insertCSRSpillsAndRestores(MachineFunction &Fn, + const MBBVector &SaveBlocks, + const MBBVector &RestoreBlocks) { // Get callee saved register information. MachineFrameInfo *MFI = Fn.getFrameInfo(); const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo(); @@ -496,6 +527,28 @@ void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { } } +static void doSpillCalleeSavedRegs(MachineFunction &Fn, RegScavenger *RS, + unsigned &MinCSFrameIndex, + unsigned &MaxCSFrameIndex, + const MBBVector &SaveBlocks, + const MBBVector &RestoreBlocks) { + const Function *F = Fn.getFunction(); + const TargetFrameLowering *TFI = Fn.getSubtarget().getFrameLowering(); + MinCSFrameIndex = std::numeric_limits<unsigned>::max(); + MaxCSFrameIndex = 0; + + // Determine which of the registers in the callee save list should be saved. + BitVector SavedRegs; + TFI->determineCalleeSaves(Fn, SavedRegs, RS); + + // Assign stack slots for any callee-saved registers that must be spilled. + assignCalleeSavedSpillSlots(Fn, SavedRegs, MinCSFrameIndex, MaxCSFrameIndex); + + // Add the code to save and restore the callee saved registers. + if (!F->hasFnAttribute(Attribute::Naked)) + insertCSRSpillsAndRestores(Fn, SaveBlocks, RestoreBlocks); +} + /// AdjustStackOffset - Helper function used to adjust the stack frame offset. static inline void AdjustStackOffset(MachineFrameInfo *MFI, int FrameIdx, @@ -512,7 +565,7 @@ AdjustStackOffset(MachineFrameInfo *MFI, int FrameIdx, MaxAlign = std::max(MaxAlign, Align); // Adjust to alignment boundary. - Offset = RoundUpToAlignment(Offset, Align, Skew); + Offset = alignTo(Offset, Align, Skew); if (StackGrowsDown) { DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << -Offset << "]\n"); @@ -524,6 +577,108 @@ AdjustStackOffset(MachineFrameInfo *MFI, int FrameIdx, } } +/// Compute which bytes of fixed and callee-save stack area are unused and keep +/// track of them in StackBytesFree. +/// +static inline void +computeFreeStackSlots(MachineFrameInfo *MFI, bool StackGrowsDown, + unsigned MinCSFrameIndex, unsigned MaxCSFrameIndex, + int64_t FixedCSEnd, BitVector &StackBytesFree) { + // Avoid undefined int64_t -> int conversion below in extreme case. + if (FixedCSEnd > std::numeric_limits<int>::max()) + return; + + StackBytesFree.resize(FixedCSEnd, true); + + SmallVector<int, 16> AllocatedFrameSlots; + // Add fixed objects. + for (int i = MFI->getObjectIndexBegin(); i != 0; ++i) + AllocatedFrameSlots.push_back(i); + // Add callee-save objects. + for (int i = MinCSFrameIndex; i <= (int)MaxCSFrameIndex; ++i) + AllocatedFrameSlots.push_back(i); + + for (int i : AllocatedFrameSlots) { + // These are converted from int64_t, but they should always fit in int + // because of the FixedCSEnd check above. + int ObjOffset = MFI->getObjectOffset(i); + int ObjSize = MFI->getObjectSize(i); + int ObjStart, ObjEnd; + if (StackGrowsDown) { + // ObjOffset is negative when StackGrowsDown is true. + ObjStart = -ObjOffset - ObjSize; + ObjEnd = -ObjOffset; + } else { + ObjStart = ObjOffset; + ObjEnd = ObjOffset + ObjSize; + } + // Ignore fixed holes that are in the previous stack frame. + if (ObjEnd > 0) + StackBytesFree.reset(ObjStart, ObjEnd); + } +} + +/// Assign frame object to an unused portion of the stack in the fixed stack +/// object range. Return true if the allocation was successful. +/// +static inline bool scavengeStackSlot(MachineFrameInfo *MFI, int FrameIdx, + bool StackGrowsDown, unsigned MaxAlign, + BitVector &StackBytesFree) { + if (MFI->isVariableSizedObjectIndex(FrameIdx)) + return false; + + if (StackBytesFree.none()) { + // clear it to speed up later scavengeStackSlot calls to + // StackBytesFree.none() + StackBytesFree.clear(); + return false; + } + + unsigned ObjAlign = MFI->getObjectAlignment(FrameIdx); + if (ObjAlign > MaxAlign) + return false; + + int64_t ObjSize = MFI->getObjectSize(FrameIdx); + int FreeStart; + for (FreeStart = StackBytesFree.find_first(); FreeStart != -1; + FreeStart = StackBytesFree.find_next(FreeStart)) { + + // Check that free space has suitable alignment. + unsigned ObjStart = StackGrowsDown ? FreeStart + ObjSize : FreeStart; + if (alignTo(ObjStart, ObjAlign) != ObjStart) + continue; + + if (FreeStart + ObjSize > StackBytesFree.size()) + return false; + + bool AllBytesFree = true; + for (unsigned Byte = 0; Byte < ObjSize; ++Byte) + if (!StackBytesFree.test(FreeStart + Byte)) { + AllBytesFree = false; + break; + } + if (AllBytesFree) + break; + } + + if (FreeStart == -1) + return false; + + if (StackGrowsDown) { + int ObjStart = -(FreeStart + ObjSize); + DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP[" << ObjStart + << "]\n"); + MFI->setObjectOffset(FrameIdx, ObjStart); + } else { + DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP[" << FreeStart + << "]\n"); + MFI->setObjectOffset(FrameIdx, FreeStart); + } + + StackBytesFree.reset(FreeStart, FreeStart + ObjSize); + return true; +} + /// AssignProtectedObjSet - Helper function to assign large stack objects (i.e., /// those required to be close to the Stack Protector) to stack offsets. static void @@ -568,9 +723,8 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { // If there are fixed sized objects that are preallocated in the local area, // non-fixed objects can't be allocated right at the start of local area. - // We currently don't support filling in holes in between fixed sized - // objects, so we adjust 'Offset' to point to the end of last fixed sized - // preallocated object. + // Adjust 'Offset' to point to the end of last fixed sized preallocated + // object. for (int i = MFI->getObjectIndexBegin(); i != 0; ++i) { int64_t FixedOff; if (StackGrowsDown) { @@ -596,22 +750,27 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { unsigned Align = MFI->getObjectAlignment(i); // Adjust to alignment boundary - Offset = RoundUpToAlignment(Offset, Align, Skew); + Offset = alignTo(Offset, Align, Skew); + DEBUG(dbgs() << "alloc FI(" << i << ") at SP[" << -Offset << "]\n"); MFI->setObjectOffset(i, -Offset); // Set the computed offset } - } else { - int MaxCSFI = MaxCSFrameIndex, MinCSFI = MinCSFrameIndex; - for (int i = MaxCSFI; i >= MinCSFI ; --i) { + } else if (MaxCSFrameIndex >= MinCSFrameIndex) { + // Be careful about underflow in comparisons agains MinCSFrameIndex. + for (unsigned i = MaxCSFrameIndex; i != MinCSFrameIndex - 1; --i) { unsigned Align = MFI->getObjectAlignment(i); // Adjust to alignment boundary - Offset = RoundUpToAlignment(Offset, Align, Skew); + Offset = alignTo(Offset, Align, Skew); + DEBUG(dbgs() << "alloc FI(" << i << ") at SP[" << Offset << "]\n"); MFI->setObjectOffset(i, Offset); Offset += MFI->getObjectSize(i); } } + // FixedCSEnd is the stack offset to the end of the fixed and callee-save + // stack area. + int64_t FixedCSEnd = Offset; unsigned MaxAlign = MFI->getMaxAlignment(); // Make sure the special register scavenging spill slot is closest to the @@ -638,7 +797,7 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { unsigned Align = MFI->getLocalFrameMaxAlign(); // Adjust to alignment boundary. - Offset = RoundUpToAlignment(Offset, Align, Skew); + Offset = alignTo(Offset, Align, Skew); DEBUG(dbgs() << "Local frame base offset: " << Offset << "\n"); @@ -656,6 +815,11 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { MaxAlign = std::max(Align, MaxAlign); } + // Retrieve the Exception Handler registration node. + int EHRegNodeFrameIndex = INT_MAX; + if (const WinEHFuncInfo *FuncInfo = Fn.getWinEHFuncInfo()) + EHRegNodeFrameIndex = FuncInfo->EHRegNodeFrameIndex; + // Make sure that the stack protector comes before the local variables on the // stack. SmallSet<int, 16> ProtectedObjs; @@ -678,7 +842,8 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { continue; if (MFI->isDeadObjectIndex(i)) continue; - if (MFI->getStackProtectorIndex() == (int)i) + if (MFI->getStackProtectorIndex() == (int)i || + EHRegNodeFrameIndex == (int)i) continue; switch (SP->getSSPLayout(MFI->getObjectAllocation(i))) { @@ -705,8 +870,10 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { Offset, MaxAlign, Skew); } - // Then assign frame offsets to stack objects that are not used to spill - // callee saved registers. + SmallVector<int, 8> ObjectsToAllocate; + + // Then prepare to assign frame offsets to stack objects that are not used to + // spill callee saved registers. for (unsigned i = 0, e = MFI->getObjectIndexEnd(); i != e; ++i) { if (MFI->isObjectPreAllocated(i) && MFI->getUseLocalStackAllocationBlock()) @@ -717,14 +884,43 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { continue; if (MFI->isDeadObjectIndex(i)) continue; - if (MFI->getStackProtectorIndex() == (int)i) + if (MFI->getStackProtectorIndex() == (int)i || + EHRegNodeFrameIndex == (int)i) continue; if (ProtectedObjs.count(i)) continue; - AdjustStackOffset(MFI, i, StackGrowsDown, Offset, MaxAlign, Skew); + // Add the objects that we need to allocate to our working set. + ObjectsToAllocate.push_back(i); } + // Allocate the EH registration node first if one is present. + if (EHRegNodeFrameIndex != INT_MAX) + AdjustStackOffset(MFI, EHRegNodeFrameIndex, StackGrowsDown, Offset, + MaxAlign, Skew); + + // Give the targets a chance to order the objects the way they like it. + if (Fn.getTarget().getOptLevel() != CodeGenOpt::None && + Fn.getTarget().Options.StackSymbolOrdering) + TFI.orderFrameObjects(Fn, ObjectsToAllocate); + + // Keep track of which bytes in the fixed and callee-save range are used so we + // can use the holes when allocating later stack objects. Only do this if + // stack protector isn't being used and the target requests it and we're + // optimizing. + BitVector StackBytesFree; + if (!ObjectsToAllocate.empty() && + Fn.getTarget().getOptLevel() != CodeGenOpt::None && + MFI->getStackProtectorIndex() < 0 && TFI.enableStackSlotScavenging(Fn)) + computeFreeStackSlots(MFI, StackGrowsDown, MinCSFrameIndex, MaxCSFrameIndex, + FixedCSEnd, StackBytesFree); + + // Now walk the objects and actually assign base offsets to them. + for (auto &Object : ObjectsToAllocate) + if (!scavengeStackSlot(MFI, Object, StackGrowsDown, MaxAlign, + StackBytesFree)) + AdjustStackOffset(MFI, Object, StackGrowsDown, Offset, MaxAlign, Skew); + // Make sure the special register scavenging spill slot is closest to the // stack pointer. if (RS && !EarlyScavengingSlots) { @@ -757,7 +953,7 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) { // If the frame pointer is eliminated, all frame offsets will be relative to // SP not FP. Align to MaxAlign so this works. StackAlign = std::max(StackAlign, MaxAlign); - Offset = RoundUpToAlignment(Offset, StackAlign, Skew); + Offset = alignTo(Offset, StackAlign, Skew); } // Update frame info to pretend that this is part of the stack... @@ -851,7 +1047,7 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, unsigned FrameSetupOpcode = TII.getCallFrameSetupOpcode(); unsigned FrameDestroyOpcode = TII.getCallFrameDestroyOpcode(); - if (RS && !FrameIndexVirtualScavenging) RS->enterBasicBlock(BB); + if (RS && !FrameIndexVirtualScavenging) RS->enterBasicBlock(*BB); bool InsideCallSequence = false; @@ -860,38 +1056,31 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, if (I->getOpcode() == FrameSetupOpcode || I->getOpcode() == FrameDestroyOpcode) { InsideCallSequence = (I->getOpcode() == FrameSetupOpcode); - SPAdj += TII.getSPAdjust(I); - - MachineBasicBlock::iterator PrevI = BB->end(); - if (I != BB->begin()) PrevI = std::prev(I); - TFI->eliminateCallFramePseudoInstr(Fn, *BB, I); + SPAdj += TII.getSPAdjust(*I); - // Visit the instructions created by eliminateCallFramePseudoInstr(). - if (PrevI == BB->end()) - I = BB->begin(); // The replaced instr was the first in the block. - else - I = std::next(PrevI); + I = TFI->eliminateCallFramePseudoInstr(Fn, *BB, I); continue; } - MachineInstr *MI = I; + MachineInstr &MI = *I; bool DoIncr = true; - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - if (!MI->getOperand(i).isFI()) + bool DidFinishLoop = true; + for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { + if (!MI.getOperand(i).isFI()) continue; // Frame indices in debug values are encoded in a target independent // way with simply the frame index and offset rather than any // target-specific addressing mode. - if (MI->isDebugValue()) { + if (MI.isDebugValue()) { assert(i == 0 && "Frame indices can only appear as the first " "operand of a DBG_VALUE machine instruction"); unsigned Reg; - MachineOperand &Offset = MI->getOperand(1); - Offset.setImm(Offset.getImm() + - TFI->getFrameIndexReference( - Fn, MI->getOperand(0).getIndex(), Reg)); - MI->getOperand(0).ChangeToRegister(Reg, false /*isDef*/); + MachineOperand &Offset = MI.getOperand(1); + Offset.setImm( + Offset.getImm() + + TFI->getFrameIndexReference(Fn, MI.getOperand(0).getIndex(), Reg)); + MI.getOperand(0).ChangeToRegister(Reg, false /*isDef*/); continue; } @@ -900,18 +1089,16 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, // implementation other than historical accident. The only // remaining difference is the unconditional use of the stack // pointer as the base register. - if (MI->getOpcode() == TargetOpcode::STATEPOINT) { - assert((!MI->isDebugValue() || i == 0) && + if (MI.getOpcode() == TargetOpcode::STATEPOINT) { + assert((!MI.isDebugValue() || i == 0) && "Frame indicies can only appear as the first operand of a " "DBG_VALUE machine instruction"); unsigned Reg; - MachineOperand &Offset = MI->getOperand(i + 1); - const unsigned refOffset = - TFI->getFrameIndexReferenceFromSP(Fn, MI->getOperand(i).getIndex(), - Reg); - + MachineOperand &Offset = MI.getOperand(i + 1); + int refOffset = TFI->getFrameIndexReferencePreferSP( + Fn, MI.getOperand(i).getIndex(), Reg, /*IgnoreSPUpdates*/ false); Offset.setImm(Offset.getImm() + refOffset); - MI->getOperand(i).ChangeToRegister(Reg, false /*isDef*/); + MI.getOperand(i).ChangeToRegister(Reg, false /*isDef*/); continue; } @@ -937,7 +1124,7 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, DoIncr = false; } - MI = nullptr; + DidFinishLoop = false; break; } @@ -948,45 +1135,46 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, // Note that this must come after eliminateFrameIndex, because // if I itself referred to a frame index, we shouldn't count its own // adjustment. - if (MI && InsideCallSequence) + if (DidFinishLoop && InsideCallSequence) SPAdj += TII.getSPAdjust(MI); if (DoIncr && I != BB->end()) ++I; // Update register states. - if (RS && !FrameIndexVirtualScavenging && MI) RS->forward(MI); + if (RS && !FrameIndexVirtualScavenging && DidFinishLoop) + RS->forward(MI); } } -/// scavengeFrameVirtualRegs - Replace all frame index virtual registers +/// doScavengeFrameVirtualRegs - Replace all frame index virtual registers /// with physical registers. Use the register scavenger to find an /// appropriate register to use. /// /// FIXME: Iterating over the instruction stream is unnecessary. We can simply /// iterate over the vreg use list, which at this point only contains machine /// operands for which eliminateFrameIndex need a new scratch reg. -void -PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) { +static void +doScavengeFrameVirtualRegs(MachineFunction &MF, RegScavenger *RS) { // Run through the instructions and find any virtual registers. - for (MachineFunction::iterator BB = Fn.begin(), - E = Fn.end(); BB != E; ++BB) { - RS->enterBasicBlock(&*BB); + MachineRegisterInfo &MRI = MF.getRegInfo(); + for (MachineBasicBlock &MBB : MF) { + RS->enterBasicBlock(MBB); int SPAdj = 0; - // The instruction stream may change in the loop, so check BB->end() + // The instruction stream may change in the loop, so check MBB.end() // directly. - for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ) { + for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ) { // We might end up here again with a NULL iterator if we scavenged a // register for which we inserted spill code for definition by what was - // originally the first instruction in BB. + // originally the first instruction in MBB. if (I == MachineBasicBlock::iterator(nullptr)) - I = BB->begin(); + I = MBB.begin(); - MachineInstr *MI = I; + const MachineInstr &MI = *I; MachineBasicBlock::iterator J = std::next(I); MachineBasicBlock::iterator P = - I == BB->begin() ? MachineBasicBlock::iterator(nullptr) + I == MBB.begin() ? MachineBasicBlock::iterator(nullptr) : std::prev(I); // RS should process this instruction before we might scavenge at this @@ -995,35 +1183,31 @@ PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) { // instruction are available, and defined registers are not. RS->forward(I); - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - if (MI->getOperand(i).isReg()) { - MachineOperand &MO = MI->getOperand(i); - unsigned Reg = MO.getReg(); - if (Reg == 0) - continue; - if (!TargetRegisterInfo::isVirtualRegister(Reg)) - continue; - - // When we first encounter a new virtual register, it - // must be a definition. - assert(MI->getOperand(i).isDef() && - "frame index virtual missing def!"); - // Scavenge a new scratch register - const TargetRegisterClass *RC = Fn.getRegInfo().getRegClass(Reg); - unsigned ScratchReg = RS->scavengeRegister(RC, J, SPAdj); - - ++NumScavengedRegs; - - // Replace this reference to the virtual register with the - // scratch register. - assert (ScratchReg && "Missing scratch register!"); - Fn.getRegInfo().replaceRegWith(Reg, ScratchReg); - - // Because this instruction was processed by the RS before this - // register was allocated, make sure that the RS now records the - // register as being used. - RS->setRegUsed(ScratchReg); - } + for (const MachineOperand &MO : MI.operands()) { + if (!MO.isReg()) + continue; + unsigned Reg = MO.getReg(); + if (!TargetRegisterInfo::isVirtualRegister(Reg)) + continue; + + // When we first encounter a new virtual register, it + // must be a definition. + assert(MO.isDef() && "frame index virtual missing def!"); + // Scavenge a new scratch register + const TargetRegisterClass *RC = MRI.getRegClass(Reg); + unsigned ScratchReg = RS->scavengeRegister(RC, J, SPAdj); + + ++NumScavengedRegs; + + // Replace this reference to the virtual register with the + // scratch register. + assert(ScratchReg && "Missing scratch register!"); + MRI.replaceRegWith(Reg, ScratchReg); + + // Because this instruction was processed by the RS before this + // register was allocated, make sure that the RS now records the + // register as being used. + RS->setRegUsed(ScratchReg); } // If the scavenger needed to use one of its spill slots, the @@ -1031,7 +1215,7 @@ PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) { // problem because we need the spill code before I: Move I to just // prior to J. if (I != std::prev(J)) { - BB->splice(J, &*BB, I); + MBB.splice(J, &MBB, I); // Before we move I, we need to prepare the RS to visit I again. // Specifically, RS will assert if it sees uses of registers that |
