summaryrefslogtreecommitdiffstats
path: root/contrib/llvm/lib/Transforms/Utils/InlineFunction.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm/lib/Transforms/Utils/InlineFunction.cpp')
-rw-r--r--contrib/llvm/lib/Transforms/Utils/InlineFunction.cpp216
1 files changed, 145 insertions, 71 deletions
diff --git a/contrib/llvm/lib/Transforms/Utils/InlineFunction.cpp b/contrib/llvm/lib/Transforms/Utils/InlineFunction.cpp
index e82c07f..a40079c 100644
--- a/contrib/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/contrib/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -14,6 +14,7 @@
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
@@ -228,7 +229,7 @@ static Value *getUnwindDestTokenHelper(Instruction *EHPad,
Instruction *ChildPad = cast<Instruction>(Child);
auto Memo = MemoMap.find(ChildPad);
if (Memo == MemoMap.end()) {
- // Haven't figure out this child pad yet; queue it.
+ // Haven't figured out this child pad yet; queue it.
Worklist.push_back(ChildPad);
continue;
}
@@ -366,6 +367,10 @@ static Value *getUnwindDestToken(Instruction *EHPad,
// search up the chain to try to find a funclet with information. Put
// null entries in the memo map to avoid re-processing as we go up.
MemoMap[EHPad] = nullptr;
+#ifndef NDEBUG
+ SmallPtrSet<Instruction *, 4> TempMemos;
+ TempMemos.insert(EHPad);
+#endif
Instruction *LastUselessPad = EHPad;
Value *AncestorToken;
for (AncestorToken = getParentPad(EHPad);
@@ -374,6 +379,13 @@ static Value *getUnwindDestToken(Instruction *EHPad,
// Skip over catchpads since they just follow their catchswitches.
if (isa<CatchPadInst>(AncestorPad))
continue;
+ // If the MemoMap had an entry mapping AncestorPad to nullptr, since we
+ // haven't yet called getUnwindDestTokenHelper for AncestorPad in this
+ // call to getUnwindDestToken, that would mean that AncestorPad had no
+ // information in itself, its descendants, or its ancestors. If that
+ // were the case, then we should also have recorded the lack of information
+ // for the descendant that we're coming from. So assert that we don't
+ // find a null entry in the MemoMap for AncestorPad.
assert(!MemoMap.count(AncestorPad) || MemoMap[AncestorPad]);
auto AncestorMemo = MemoMap.find(AncestorPad);
if (AncestorMemo == MemoMap.end()) {
@@ -384,25 +396,85 @@ static Value *getUnwindDestToken(Instruction *EHPad,
if (UnwindDestToken)
break;
LastUselessPad = AncestorPad;
+ MemoMap[LastUselessPad] = nullptr;
+#ifndef NDEBUG
+ TempMemos.insert(LastUselessPad);
+#endif
}
- // Since the whole tree under LastUselessPad has no information, it all must
- // match UnwindDestToken; record that to avoid repeating the search.
+ // We know that getUnwindDestTokenHelper was called on LastUselessPad and
+ // returned nullptr (and likewise for EHPad and any of its ancestors up to
+ // LastUselessPad), so LastUselessPad has no information from below. Since
+ // getUnwindDestTokenHelper must investigate all downward paths through
+ // no-information nodes to prove that a node has no information like this,
+ // and since any time it finds information it records it in the MemoMap for
+ // not just the immediately-containing funclet but also any ancestors also
+ // exited, it must be the case that, walking downward from LastUselessPad,
+ // visiting just those nodes which have not been mapped to an unwind dest
+ // by getUnwindDestTokenHelper (the nullptr TempMemos notwithstanding, since
+ // they are just used to keep getUnwindDestTokenHelper from repeating work),
+ // any node visited must have been exhaustively searched with no information
+ // for it found.
SmallVector<Instruction *, 8> Worklist(1, LastUselessPad);
while (!Worklist.empty()) {
Instruction *UselessPad = Worklist.pop_back_val();
- assert(!MemoMap.count(UselessPad) || MemoMap[UselessPad] == nullptr);
+ auto Memo = MemoMap.find(UselessPad);
+ if (Memo != MemoMap.end() && Memo->second) {
+ // Here the name 'UselessPad' is a bit of a misnomer, because we've found
+ // that it is a funclet that does have information about unwinding to
+ // a particular destination; its parent was a useless pad.
+ // Since its parent has no information, the unwind edge must not escape
+ // the parent, and must target a sibling of this pad. This local unwind
+ // gives us no information about EHPad. Leave it and the subtree rooted
+ // at it alone.
+ assert(getParentPad(Memo->second) == getParentPad(UselessPad));
+ continue;
+ }
+ // We know we don't have information for UselesPad. If it has an entry in
+ // the MemoMap (mapping it to nullptr), it must be one of the TempMemos
+ // added on this invocation of getUnwindDestToken; if a previous invocation
+ // recorded nullptr, it would have had to prove that the ancestors of
+ // UselessPad, which include LastUselessPad, had no information, and that
+ // in turn would have required proving that the descendants of
+ // LastUselesPad, which include EHPad, have no information about
+ // LastUselessPad, which would imply that EHPad was mapped to nullptr in
+ // the MemoMap on that invocation, which isn't the case if we got here.
+ assert(!MemoMap.count(UselessPad) || TempMemos.count(UselessPad));
+ // Assert as we enumerate users that 'UselessPad' doesn't have any unwind
+ // information that we'd be contradicting by making a map entry for it
+ // (which is something that getUnwindDestTokenHelper must have proved for
+ // us to get here). Just assert on is direct users here; the checks in
+ // this downward walk at its descendants will verify that they don't have
+ // any unwind edges that exit 'UselessPad' either (i.e. they either have no
+ // unwind edges or unwind to a sibling).
MemoMap[UselessPad] = UnwindDestToken;
if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(UselessPad)) {
- for (BasicBlock *HandlerBlock : CatchSwitch->handlers())
- for (User *U : HandlerBlock->getFirstNonPHI()->users())
+ assert(CatchSwitch->getUnwindDest() == nullptr && "Expected useless pad");
+ for (BasicBlock *HandlerBlock : CatchSwitch->handlers()) {
+ auto *CatchPad = HandlerBlock->getFirstNonPHI();
+ for (User *U : CatchPad->users()) {
+ assert(
+ (!isa<InvokeInst>(U) ||
+ (getParentPad(
+ cast<InvokeInst>(U)->getUnwindDest()->getFirstNonPHI()) ==
+ CatchPad)) &&
+ "Expected useless pad");
if (isa<CatchSwitchInst>(U) || isa<CleanupPadInst>(U))
Worklist.push_back(cast<Instruction>(U));
+ }
+ }
} else {
assert(isa<CleanupPadInst>(UselessPad));
- for (User *U : UselessPad->users())
+ for (User *U : UselessPad->users()) {
+ assert(!isa<CleanupReturnInst>(U) && "Expected useless pad");
+ assert((!isa<InvokeInst>(U) ||
+ (getParentPad(
+ cast<InvokeInst>(U)->getUnwindDest()->getFirstNonPHI()) ==
+ UselessPad)) &&
+ "Expected useless pad");
if (isa<CatchSwitchInst>(U) || isa<CleanupPadInst>(U))
Worklist.push_back(cast<Instruction>(U));
+ }
}
}
@@ -463,37 +535,7 @@ static BasicBlock *HandleCallsInBlockInlinedThroughInvoke(
#endif // NDEBUG
}
- // Convert this function call into an invoke instruction. First, split the
- // basic block.
- BasicBlock *Split =
- BB->splitBasicBlock(CI->getIterator(), CI->getName() + ".noexc");
-
- // Delete the unconditional branch inserted by splitBasicBlock
- BB->getInstList().pop_back();
-
- // Create the new invoke instruction.
- SmallVector<Value*, 8> InvokeArgs(CI->arg_begin(), CI->arg_end());
- SmallVector<OperandBundleDef, 1> OpBundles;
-
- CI->getOperandBundlesAsDefs(OpBundles);
-
- // Note: we're round tripping operand bundles through memory here, and that
- // can potentially be avoided with a cleverer API design that we do not have
- // as of this time.
-
- InvokeInst *II =
- InvokeInst::Create(CI->getCalledValue(), Split, UnwindEdge, InvokeArgs,
- OpBundles, CI->getName(), BB);
- II->setDebugLoc(CI->getDebugLoc());
- II->setCallingConv(CI->getCallingConv());
- II->setAttributes(CI->getAttributes());
-
- // Make sure that anything using the call now uses the invoke! This also
- // updates the CallGraph if present, because it uses a WeakVH.
- CI->replaceAllUsesWith(II);
-
- // Delete the original call
- Split->getInstList().pop_front();
+ changeToInvokeAndSplitBasicBlock(CI, UnwindEdge);
return BB;
}
return nullptr;
@@ -718,7 +760,7 @@ static void PropagateParallelLoopAccessMetadata(CallSite CS,
/// When inlining a function that contains noalias scope metadata,
/// this metadata needs to be cloned so that the inlined blocks
-/// have different "unqiue scopes" at every call site. Were this not done, then
+/// have different "unique scopes" at every call site. Were this not done, then
/// aliasing scopes from a function inlined into a caller multiple times could
/// not be differentiated (and this would lead to miscompiles because the
/// non-aliasing property communicated by the metadata could have
@@ -1053,8 +1095,10 @@ static void AddAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap,
/// If the inlined function has non-byval align arguments, then
/// add @llvm.assume-based alignment assumptions to preserve this information.
static void AddAlignmentAssumptions(CallSite CS, InlineFunctionInfo &IFI) {
- if (!PreserveAlignmentAssumptions)
+ if (!PreserveAlignmentAssumptions || !IFI.GetAssumptionCache)
return;
+
+ AssumptionCache *AC = &(*IFI.GetAssumptionCache)(*CS.getCaller());
auto &DL = CS.getCaller()->getParent()->getDataLayout();
// To avoid inserting redundant assumptions, we should check for assumptions
@@ -1077,13 +1121,12 @@ static void AddAlignmentAssumptions(CallSite CS, InlineFunctionInfo &IFI) {
// If we can already prove the asserted alignment in the context of the
// caller, then don't bother inserting the assumption.
Value *Arg = CS.getArgument(I->getArgNo());
- if (getKnownAlignment(Arg, DL, CS.getInstruction(),
- &IFI.ACT->getAssumptionCache(*CS.getCaller()),
- &DT) >= Align)
+ if (getKnownAlignment(Arg, DL, CS.getInstruction(), AC, &DT) >= Align)
continue;
- IRBuilder<>(CS.getInstruction())
- .CreateAlignmentAssumption(DL, Arg, Align);
+ CallInst *NewAssumption = IRBuilder<>(CS.getInstruction())
+ .CreateAlignmentAssumption(DL, Arg, Align);
+ AC->registerAssumption(NewAssumption);
}
}
}
@@ -1194,12 +1237,13 @@ static Value *HandleByValArgument(Value *Arg, Instruction *TheCall,
if (ByValAlignment <= 1) // 0 = unspecified, 1 = no particular alignment.
return Arg;
+ AssumptionCache *AC =
+ IFI.GetAssumptionCache ? &(*IFI.GetAssumptionCache)(*Caller) : nullptr;
const DataLayout &DL = Caller->getParent()->getDataLayout();
// If the pointer is already known to be sufficiently aligned, or if we can
// round it up to a larger alignment, then we don't need a temporary.
- if (getOrEnforceKnownAlignment(Arg, ByValAlignment, DL, TheCall,
- &IFI.ACT->getAssumptionCache(*Caller)) >=
+ if (getOrEnforceKnownAlignment(Arg, ByValAlignment, DL, TheCall, AC) >=
ByValAlignment)
return Arg;
@@ -1304,7 +1348,7 @@ static bool allocaWouldBeStaticInEntry(const AllocaInst *AI ) {
/// Update inlined instructions' line numbers to
/// to encode location where these instructions are inlined.
static void fixupLineNumbers(Function *Fn, Function::iterator FI,
- Instruction *TheCall) {
+ Instruction *TheCall, bool CalleeHasDebugInfo) {
const DebugLoc &TheCallDL = TheCall->getDebugLoc();
if (!TheCallDL)
return;
@@ -1326,22 +1370,26 @@ static void fixupLineNumbers(Function *Fn, Function::iterator FI,
for (; FI != Fn->end(); ++FI) {
for (BasicBlock::iterator BI = FI->begin(), BE = FI->end();
BI != BE; ++BI) {
- DebugLoc DL = BI->getDebugLoc();
- if (!DL) {
- // If the inlined instruction has no line number, make it look as if it
- // originates from the call location. This is important for
- // ((__always_inline__, __nodebug__)) functions which must use caller
- // location for all instructions in their function body.
-
- // Don't update static allocas, as they may get moved later.
- if (auto *AI = dyn_cast<AllocaInst>(BI))
- if (allocaWouldBeStaticInEntry(AI))
- continue;
-
- BI->setDebugLoc(TheCallDL);
- } else {
- BI->setDebugLoc(updateInlinedAtInfo(DL, InlinedAtNode, BI->getContext(), IANodes));
+ if (DebugLoc DL = BI->getDebugLoc()) {
+ BI->setDebugLoc(
+ updateInlinedAtInfo(DL, InlinedAtNode, BI->getContext(), IANodes));
+ continue;
}
+
+ if (CalleeHasDebugInfo)
+ continue;
+
+ // If the inlined instruction has no line number, make it look as if it
+ // originates from the call location. This is important for
+ // ((__always_inline__, __nodebug__)) functions which must use caller
+ // location for all instructions in their function body.
+
+ // Don't update static allocas, as they may get moved later.
+ if (auto *AI = dyn_cast<AllocaInst>(BI))
+ if (allocaWouldBeStaticInEntry(AI))
+ continue;
+
+ BI->setDebugLoc(TheCallDL);
}
}
}
@@ -1597,8 +1645,11 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
if (IFI.CG)
UpdateCallGraphAfterInlining(CS, FirstNewBlock, VMap, IFI);
- // Update inlined instructions' line number information.
- fixupLineNumbers(Caller, FirstNewBlock, TheCall);
+ // For 'nodebug' functions, the associated DISubprogram is always null.
+ // Conservatively avoid propagating the callsite debug location to
+ // instructions inlined from a function whose DISubprogram is not null.
+ fixupLineNumbers(Caller, FirstNewBlock, TheCall,
+ CalledFunc->getSubprogram() != nullptr);
// Clone existing noalias metadata if necessary.
CloneAliasScopeMetadata(CS, VMap);
@@ -1609,10 +1660,15 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
// Propagate llvm.mem.parallel_loop_access if necessary.
PropagateParallelLoopAccessMetadata(CS, VMap);
- // FIXME: We could register any cloned assumptions instead of clearing the
- // whole function's cache.
- if (IFI.ACT)
- IFI.ACT->getAssumptionCache(*Caller).clear();
+ // Register any cloned assumptions.
+ if (IFI.GetAssumptionCache)
+ for (BasicBlock &NewBlock :
+ make_range(FirstNewBlock->getIterator(), Caller->end()))
+ for (Instruction &I : NewBlock) {
+ if (auto *II = dyn_cast<IntrinsicInst>(&I))
+ if (II->getIntrinsicID() == Intrinsic::assume)
+ (*IFI.GetAssumptionCache)(*Caller).registerAssumption(II);
+ }
}
// If there are any alloca instructions in the block that used to be the entry
@@ -1708,6 +1764,9 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
IRBuilder<> builder(&FirstNewBlock->front());
for (unsigned ai = 0, ae = IFI.StaticAllocas.size(); ai != ae; ++ai) {
AllocaInst *AI = IFI.StaticAllocas[ai];
+ // Don't mark swifterror allocas. They can't have bitcast uses.
+ if (AI->isSwiftError())
+ continue;
// If the alloca is already scoped to something smaller than the whole
// function then there's no need to add redundant, less accurate markers.
@@ -1949,6 +2008,20 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
std::swap(Returns, NormalReturns);
}
+ // Now that all of the transforms on the inlined code have taken place but
+ // before we splice the inlined code into the CFG and lose track of which
+ // blocks were actually inlined, collect the call sites. We only do this if
+ // call graph updates weren't requested, as those provide value handle based
+ // tracking of inlined call sites instead.
+ if (InlinedFunctionInfo.ContainsCalls && !IFI.CG) {
+ // Otherwise just collect the raw call sites that were inlined.
+ for (BasicBlock &NewBB :
+ make_range(FirstNewBlock->getIterator(), Caller->end()))
+ for (Instruction &I : NewBB)
+ if (auto CS = CallSite(&I))
+ IFI.InlinedCallSites.push_back(CS);
+ }
+
// If we cloned in _exactly one_ basic block, and if that block ends in a
// return instruction, we splice the body of the inlined callee directly into
// the calling basic block.
@@ -2130,9 +2203,10 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
// the entries are the same or undef). If so, remove the PHI so it doesn't
// block other optimizations.
if (PHI) {
+ AssumptionCache *AC =
+ IFI.GetAssumptionCache ? &(*IFI.GetAssumptionCache)(*Caller) : nullptr;
auto &DL = Caller->getParent()->getDataLayout();
- if (Value *V = SimplifyInstruction(PHI, DL, nullptr, nullptr,
- &IFI.ACT->getAssumptionCache(*Caller))) {
+ if (Value *V = SimplifyInstruction(PHI, DL, nullptr, nullptr, AC)) {
PHI->replaceAllUsesWith(V);
PHI->eraseFromParent();
}
OpenPOWER on IntegriCloud