summaryrefslogtreecommitdiffstats
path: root/contrib/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm/lib')
-rw-r--r--contrib/llvm/lib/CodeGen/CalcSpillWeights.cpp7
-rw-r--r--contrib/llvm/lib/CodeGen/LiveInterval.cpp34
-rw-r--r--contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp36
-rw-r--r--contrib/llvm/lib/CodeGen/WinEHPrepare.cpp20
-rw-r--r--contrib/llvm/lib/Support/Triple.cpp8
-rw-r--r--contrib/llvm/lib/Target/Sparc/SparcInstrAliases.td6
-rw-r--r--contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp46
-rw-r--r--contrib/llvm/lib/Target/X86/X86FrameLowering.cpp13
-rw-r--r--contrib/llvm/lib/Target/X86/X86InstrAVX512.td2
-rw-r--r--contrib/llvm/lib/Transforms/IPO/PruneEH.cpp63
-rw-r--r--contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp11
-rw-r--r--contrib/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp147
12 files changed, 192 insertions, 201 deletions
diff --git a/contrib/llvm/lib/CodeGen/CalcSpillWeights.cpp b/contrib/llvm/lib/CodeGen/CalcSpillWeights.cpp
index abc655a..26aa46f 100644
--- a/contrib/llvm/lib/CodeGen/CalcSpillWeights.cpp
+++ b/contrib/llvm/lib/CodeGen/CalcSpillWeights.cpp
@@ -213,8 +213,11 @@ VirtRegAuxInfo::calculateSpillWeightAndHint(LiveInterval &li) {
if (!Spillable)
return;
- // Mark li as unspillable if all live ranges are tiny.
- if (li.isZeroLength(LIS.getSlotIndexes())) {
+ // Mark li as unspillable if all live ranges are tiny and the interval
+ // is not live at any reg mask. If the interval is live at a reg mask
+ // spilling may be required.
+ if (li.isZeroLength(LIS.getSlotIndexes()) &&
+ !li.isLiveAtIndexes(LIS.getRegMaskSlots())) {
li.markNotSpillable();
return;
}
diff --git a/contrib/llvm/lib/CodeGen/LiveInterval.cpp b/contrib/llvm/lib/CodeGen/LiveInterval.cpp
index bb34883..5015800 100644
--- a/contrib/llvm/lib/CodeGen/LiveInterval.cpp
+++ b/contrib/llvm/lib/CodeGen/LiveInterval.cpp
@@ -748,6 +748,40 @@ void LiveRange::flushSegmentSet() {
verify();
}
+bool LiveRange::isLiveAtIndexes(ArrayRef<SlotIndex> Slots) const {
+ ArrayRef<SlotIndex>::iterator SlotI = Slots.begin();
+ ArrayRef<SlotIndex>::iterator SlotE = Slots.end();
+
+ // If there are no regmask slots, we have nothing to search.
+ if (SlotI == SlotE)
+ return false;
+
+ // Start our search at the first segment that ends after the first slot.
+ const_iterator SegmentI = find(*SlotI);
+ const_iterator SegmentE = end();
+
+ // If there are no segments that end after the first slot, we're done.
+ if (SegmentI == SegmentE)
+ return false;
+
+ // Look for each slot in the live range.
+ for ( ; SlotI != SlotE; ++SlotI) {
+ // Go to the next segment that ends after the current slot.
+ // The slot may be within a hole in the range.
+ SegmentI = advanceTo(SegmentI, *SlotI);
+ if (SegmentI == SegmentE)
+ return false;
+
+ // If this segment contains the slot, we're done.
+ if (SegmentI->contains(*SlotI))
+ return true;
+ // Otherwise, look for the next slot.
+ }
+
+ // We didn't find a segment containing any of the slots.
+ return false;
+}
+
void LiveInterval::freeSubRange(SubRange *S) {
S->~SubRange();
// Memory was allocated with BumpPtr allocator and is not freed here.
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 5d572c4..f783634 100644
--- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1637,6 +1637,7 @@ struct FloatSignAsInt {
MachinePointerInfo FloatPointerInfo;
SDValue IntValue;
APInt SignMask;
+ uint8_t SignBit;
};
}
@@ -1653,6 +1654,7 @@ void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
if (TLI.isTypeLegal(IVT)) {
State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
State.SignMask = APInt::getSignBit(NumBits);
+ State.SignBit = NumBits - 1;
return;
}
@@ -1689,6 +1691,7 @@ void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
IntPtr, State.IntPointerInfo, MVT::i8,
false, false, false, 0);
State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
+ State.SignBit = 7;
}
/// Replace the integer value produced by getSignAsIntValue() with a new value
@@ -1731,15 +1734,38 @@ SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
}
- // Transform values to integer, copy the sign bit and transform back.
+ // Transform Mag value to integer, and clear the sign bit.
FloatSignAsInt MagAsInt;
getSignAsIntValue(MagAsInt, DL, Mag);
- assert(SignAsInt.SignMask == MagAsInt.SignMask);
- SDValue ClearSignMask = DAG.getConstant(~SignAsInt.SignMask, DL, IntVT);
- SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, MagAsInt.IntValue,
+ EVT MagVT = MagAsInt.IntValue.getValueType();
+ SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
+ SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
ClearSignMask);
- SDValue CopiedSign = DAG.getNode(ISD::OR, DL, IntVT, ClearedSign, SignBit);
+ // Get the signbit at the right position for MagAsInt.
+ int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
+ if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
+ if (ShiftAmount > 0) {
+ SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT);
+ SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst);
+ } else if (ShiftAmount < 0) {
+ SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT);
+ SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst);
+ }
+ SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
+ } else if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
+ SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
+ if (ShiftAmount > 0) {
+ SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT);
+ SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst);
+ } else if (ShiftAmount < 0) {
+ SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT);
+ SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst);
+ }
+ }
+
+ // Store the part with the modified sign and convert back to float.
+ SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
return modifySignAsInt(MagAsInt, DL, CopiedSign);
}
diff --git a/contrib/llvm/lib/CodeGen/WinEHPrepare.cpp b/contrib/llvm/lib/CodeGen/WinEHPrepare.cpp
index 886c5f6..14ec911 100644
--- a/contrib/llvm/lib/CodeGen/WinEHPrepare.cpp
+++ b/contrib/llvm/lib/CodeGen/WinEHPrepare.cpp
@@ -257,10 +257,14 @@ static void calculateCXXStateNumbers(WinEHFuncInfo &FuncInfo,
if (auto *InnerCatchSwitch = dyn_cast<CatchSwitchInst>(UserI))
if (InnerCatchSwitch->getUnwindDest() == CatchSwitch->getUnwindDest())
calculateCXXStateNumbers(FuncInfo, UserI, CatchLow);
- if (auto *InnerCleanupPad = dyn_cast<CleanupPadInst>(UserI))
- if (getCleanupRetUnwindDest(InnerCleanupPad) ==
- CatchSwitch->getUnwindDest())
+ if (auto *InnerCleanupPad = dyn_cast<CleanupPadInst>(UserI)) {
+ BasicBlock *UnwindDest = getCleanupRetUnwindDest(InnerCleanupPad);
+ // If a nested cleanup pad reports a null unwind destination and the
+ // enclosing catch pad doesn't it must be post-dominated by an
+ // unreachable instruction.
+ if (!UnwindDest || UnwindDest == CatchSwitch->getUnwindDest())
calculateCXXStateNumbers(FuncInfo, UserI, CatchLow);
+ }
}
}
int CatchHigh = FuncInfo.getLastStateNumber();
@@ -360,10 +364,14 @@ static void calculateSEHStateNumbers(WinEHFuncInfo &FuncInfo,
if (auto *InnerCatchSwitch = dyn_cast<CatchSwitchInst>(UserI))
if (InnerCatchSwitch->getUnwindDest() == CatchSwitch->getUnwindDest())
calculateSEHStateNumbers(FuncInfo, UserI, ParentState);
- if (auto *InnerCleanupPad = dyn_cast<CleanupPadInst>(UserI))
- if (getCleanupRetUnwindDest(InnerCleanupPad) ==
- CatchSwitch->getUnwindDest())
+ if (auto *InnerCleanupPad = dyn_cast<CleanupPadInst>(UserI)) {
+ BasicBlock *UnwindDest = getCleanupRetUnwindDest(InnerCleanupPad);
+ // If a nested cleanup pad reports a null unwind destination and the
+ // enclosing catch pad doesn't it must be post-dominated by an
+ // unreachable instruction.
+ if (!UnwindDest || UnwindDest == CatchSwitch->getUnwindDest())
calculateSEHStateNumbers(FuncInfo, UserI, ParentState);
+ }
}
} else {
auto *CleanupPad = cast<CleanupPadInst>(FirstNonPHI);
diff --git a/contrib/llvm/lib/Support/Triple.cpp b/contrib/llvm/lib/Support/Triple.cpp
index 0e5d3ac..11afcf7 100644
--- a/contrib/llvm/lib/Support/Triple.cpp
+++ b/contrib/llvm/lib/Support/Triple.cpp
@@ -338,9 +338,9 @@ static Triple::ArchType parseArch(StringRef ArchName) {
// FIXME: Do we need to support these?
.Cases("i786", "i886", "i986", Triple::x86)
.Cases("amd64", "x86_64", "x86_64h", Triple::x86_64)
- .Case("powerpc", Triple::ppc)
- .Cases("powerpc64", "ppu", Triple::ppc64)
- .Case("powerpc64le", Triple::ppc64le)
+ .Cases("powerpc", "ppc32", Triple::ppc)
+ .Cases("powerpc64", "ppu", "ppc64", Triple::ppc64)
+ .Cases("powerpc64le", "ppc64le", Triple::ppc64le)
.Case("xscale", Triple::arm)
.Case("xscaleeb", Triple::armeb)
.Case("aarch64", Triple::aarch64)
@@ -359,7 +359,7 @@ static Triple::ArchType parseArch(StringRef ArchName) {
.Case("r600", Triple::r600)
.Case("amdgcn", Triple::amdgcn)
.Case("hexagon", Triple::hexagon)
- .Case("s390x", Triple::systemz)
+ .Cases("s390x", "systemz", Triple::systemz)
.Case("sparc", Triple::sparc)
.Case("sparcel", Triple::sparcel)
.Cases("sparcv9", "sparc64", Triple::sparcv9)
diff --git a/contrib/llvm/lib/Target/Sparc/SparcInstrAliases.td b/contrib/llvm/lib/Target/Sparc/SparcInstrAliases.td
index d51e2cc..361d214 100644
--- a/contrib/llvm/lib/Target/Sparc/SparcInstrAliases.td
+++ b/contrib/llvm/lib/Target/Sparc/SparcInstrAliases.td
@@ -267,9 +267,9 @@ defm : int_cond_alias<"neg", 0b0110>;
defm : int_cond_alias<"vc", 0b1111>;
defm : int_cond_alias<"vs", 0b0111>;
-defm : fp_cond_alias<"a", 0b0000>;
-defm : fp_cond_alias<"", 0b0000>; // same as a; gnu asm, not in manual
-defm : fp_cond_alias<"n", 0b1000>;
+defm : fp_cond_alias<"a", 0b1000>;
+defm : fp_cond_alias<"", 0b1000>; // same as a; gnu asm, not in manual
+defm : fp_cond_alias<"n", 0b0000>;
defm : fp_cond_alias<"u", 0b0111>;
defm : fp_cond_alias<"g", 0b0110>;
defm : fp_cond_alias<"ug", 0b0101>;
diff --git a/contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 133bd0e..135c32b 100644
--- a/contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/contrib/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -69,19 +69,15 @@ public:
class X86AsmBackend : public MCAsmBackend {
const StringRef CPU;
bool HasNopl;
- uint64_t MaxNopLength;
+ const uint64_t MaxNopLength;
public:
- X86AsmBackend(const Target &T, StringRef CPU) : MCAsmBackend(), CPU(CPU) {
+ X86AsmBackend(const Target &T, StringRef CPU)
+ : MCAsmBackend(), CPU(CPU), MaxNopLength(CPU == "slm" ? 7 : 15) {
HasNopl = CPU != "generic" && CPU != "i386" && CPU != "i486" &&
CPU != "i586" && CPU != "pentium" && CPU != "pentium-mmx" &&
CPU != "i686" && CPU != "k6" && CPU != "k6-2" && CPU != "k6-3" &&
CPU != "geode" && CPU != "winchip-c6" && CPU != "winchip2" &&
CPU != "c3" && CPU != "c3-2";
- // Max length of true long nop instruction is 15 bytes.
- // Max length of long nop replacement instruction is 7 bytes.
- // Taking into account SilverMont architecture features max length of nops
- // is reduced for it to achieve better performance.
- MaxNopLength = (!HasNopl || CPU == "slm") ? 7 : 15;
}
unsigned getNumFixupKinds() const override {
@@ -299,7 +295,7 @@ void X86AsmBackend::relaxInstruction(const MCInst &Inst, MCInst &Res) const {
/// bytes.
/// \return - true on success, false on failure
bool X86AsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
- static const uint8_t TrueNops[10][10] = {
+ static const uint8_t Nops[10][10] = {
// nop
{0x90},
// xchg %ax,%ax
@@ -322,31 +318,17 @@ bool X86AsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
{0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00},
};
- // Alternative nop instructions for CPUs which don't support long nops.
- static const uint8_t AltNops[7][10] = {
- // nop
- {0x90},
- // xchg %ax,%ax
- {0x66, 0x90},
- // lea 0x0(%esi),%esi
- {0x8d, 0x76, 0x00},
- // lea 0x0(%esi),%esi
- {0x8d, 0x74, 0x26, 0x00},
- // nop + lea 0x0(%esi),%esi
- {0x90, 0x8d, 0x74, 0x26, 0x00},
- // lea 0x0(%esi),%esi
- {0x8d, 0xb6, 0x00, 0x00, 0x00, 0x00 },
- // lea 0x0(%esi),%esi
- {0x8d, 0xb4, 0x26, 0x00, 0x00, 0x00, 0x00},
- };
-
- // Select the right NOP table.
- // FIXME: Can we get if CPU supports long nops from the subtarget somehow?
- const uint8_t (*Nops)[10] = HasNopl ? TrueNops : AltNops;
- assert(HasNopl || MaxNopLength <= 7);
+ // This CPU doesn't support long nops. If needed add more.
+ // FIXME: Can we get this from the subtarget somehow?
+ // FIXME: We could generated something better than plain 0x90.
+ if (!HasNopl) {
+ for (uint64_t i = 0; i < Count; ++i)
+ OW->write8(0x90);
+ return true;
+ }
- // Emit as many largest nops as needed, then emit a nop of the remaining
- // length.
+ // 15 is the longest single nop instruction. Emit as many 15-byte nops as
+ // needed, then emit a nop of the remaining length.
do {
const uint8_t ThisNopLength = (uint8_t) std::min(Count, MaxNopLength);
const uint8_t Prefixes = ThisNopLength <= 10 ? 0 : ThisNopLength - 10;
diff --git a/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp b/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp
index fad6ad9..62d88b7 100644
--- a/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/contrib/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -192,10 +192,9 @@ static unsigned findDeadCallerSavedReg(MachineBasicBlock &MBB,
return 0;
}
-static bool isEAXLiveIn(MachineFunction &MF) {
- for (MachineRegisterInfo::livein_iterator II = MF.getRegInfo().livein_begin(),
- EE = MF.getRegInfo().livein_end(); II != EE; ++II) {
- unsigned Reg = II->first;
+static bool isEAXLiveIn(MachineBasicBlock &MBB) {
+ for (MachineBasicBlock::RegisterMaskPair RegMask : MBB.liveins()) {
+ unsigned Reg = RegMask.PhysReg;
if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
Reg == X86::AH || Reg == X86::AL)
@@ -261,7 +260,7 @@ void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB,
// load the offset into a register and do one sub/add
unsigned Reg = 0;
- if (isSub && !isEAXLiveIn(*MBB.getParent()))
+ if (isSub && !isEAXLiveIn(MBB))
Reg = (unsigned)(Is64Bit ? X86::RAX : X86::EAX);
else
Reg = findDeadCallerSavedReg(MBB, MBBI, TRI, Is64Bit);
@@ -1133,8 +1132,8 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF,
if (IsWin64Prologue && !IsFunclet && TRI->needsStackRealignment(MF))
AlignedNumBytes = RoundUpToAlignment(AlignedNumBytes, MaxAlign);
if (AlignedNumBytes >= StackProbeSize && UseStackProbe) {
- // Check whether EAX is livein for this function.
- bool isEAXAlive = isEAXLiveIn(MF);
+ // Check whether EAX is livein for this block.
+ bool isEAXAlive = isEAXLiveIn(MBB);
if (isEAXAlive) {
// Sanity check that EAX is not livein for this function.
diff --git a/contrib/llvm/lib/Target/X86/X86InstrAVX512.td b/contrib/llvm/lib/Target/X86/X86InstrAVX512.td
index 49be648..6f0199b 100644
--- a/contrib/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/contrib/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -5896,7 +5896,7 @@ multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
def : Pat<(_.EltVT (OpNode (load addr:$src))),
(!cast<Instruction>(NAME#SUFF#Zm)
- (_.EltVT (IMPLICIT_DEF)), addr:$src)>, Requires<[OptForSize]>;
+ (_.EltVT (IMPLICIT_DEF)), addr:$src)>, Requires<[HasAVX512, OptForSize]>;
}
multiclass avx512_sqrt_scalar_all<bits<8> opc, string OpcodeStr> {
diff --git a/contrib/llvm/lib/Transforms/IPO/PruneEH.cpp b/contrib/llvm/lib/Transforms/IPO/PruneEH.cpp
index 3af4afb..22a95fa 100644
--- a/contrib/llvm/lib/Transforms/IPO/PruneEH.cpp
+++ b/contrib/llvm/lib/Transforms/IPO/PruneEH.cpp
@@ -29,6 +29,7 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
+#include "llvm/Transforms/Utils/Local.h"
#include <algorithm>
using namespace llvm;
@@ -186,32 +187,8 @@ bool PruneEH::SimplifyFunction(Function *F) {
for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
if (InvokeInst *II = dyn_cast<InvokeInst>(BB->getTerminator()))
if (II->doesNotThrow() && canSimplifyInvokeNoUnwind(F)) {
- SmallVector<Value*, 8> Args(II->arg_begin(), II->arg_end());
- SmallVector<OperandBundleDef, 1> OpBundles;
- II->getOperandBundlesAsDefs(OpBundles);
-
- // Insert a call instruction before the invoke.
- CallInst *Call = CallInst::Create(II->getCalledValue(), Args, OpBundles,
- "", II);
- Call->takeName(II);
- Call->setCallingConv(II->getCallingConv());
- Call->setAttributes(II->getAttributes());
- Call->setDebugLoc(II->getDebugLoc());
-
- // Anything that used the value produced by the invoke instruction
- // now uses the value produced by the call instruction. Note that we
- // do this even for void functions and calls with no uses so that the
- // callgraph edge is updated.
- II->replaceAllUsesWith(Call);
BasicBlock *UnwindBlock = II->getUnwindDest();
- UnwindBlock->removePredecessor(II->getParent());
-
- // Insert a branch to the normal destination right before the
- // invoke.
- BranchInst::Create(II->getNormalDest(), II);
-
- // Finally, delete the invoke instruction!
- BB->getInstList().pop_back();
+ removeUnwindEdge(&*BB);
// If the unwind block is now dead, nuke it.
if (pred_empty(UnwindBlock))
@@ -251,23 +228,39 @@ void PruneEH::DeleteBasicBlock(BasicBlock *BB) {
assert(pred_empty(BB) && "BB is not dead!");
CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
+ Instruction *TokenInst = nullptr;
+
CallGraphNode *CGN = CG[BB->getParent()];
for (BasicBlock::iterator I = BB->end(), E = BB->begin(); I != E; ) {
--I;
- if (CallInst *CI = dyn_cast<CallInst>(I)) {
- if (!isa<IntrinsicInst>(I))
- CGN->removeCallEdgeFor(CI);
- } else if (InvokeInst *II = dyn_cast<InvokeInst>(I))
- CGN->removeCallEdgeFor(II);
+
+ if (I->getType()->isTokenTy()) {
+ TokenInst = &*I;
+ break;
+ }
+
+ if (auto CS = CallSite (&*I)) {
+ const Function *Callee = CS.getCalledFunction();
+ if (!Callee || !Intrinsic::isLeaf(Callee->getIntrinsicID()))
+ CGN->removeCallEdgeFor(CS);
+ else if (!Callee->isIntrinsic())
+ CGN->removeCallEdgeFor(CS);
+ }
+
if (!I->use_empty())
I->replaceAllUsesWith(UndefValue::get(I->getType()));
}
- // Get the list of successors of this block.
- std::vector<BasicBlock*> Succs(succ_begin(BB), succ_end(BB));
+ if (TokenInst) {
+ if (!isa<TerminatorInst>(TokenInst))
+ changeToUnreachable(TokenInst->getNextNode(), /*UseLLVMTrap=*/false);
+ } else {
+ // Get the list of successors of this block.
+ std::vector<BasicBlock *> Succs(succ_begin(BB), succ_end(BB));
- for (unsigned i = 0, e = Succs.size(); i != e; ++i)
- Succs[i]->removePredecessor(BB);
+ for (unsigned i = 0, e = Succs.size(); i != e; ++i)
+ Succs[i]->removePredecessor(BB);
- BB->eraseFromParent();
+ BB->eraseFromParent();
+ }
}
diff --git a/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 2101225..acfdec4 100644
--- a/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/contrib/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -4799,6 +4799,17 @@ LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << U << "\n");
return;
}
+ // Bail out if we have a PHI on an EHPad that gets a value from a
+ // CatchSwitchInst. Because the CatchSwitchInst cannot be split, there is
+ // no good place to stick any instructions.
+ if (auto *PN = dyn_cast<PHINode>(U.getUser())) {
+ auto *FirstNonPHI = PN->getParent()->getFirstNonPHI();
+ if (isa<FuncletPadInst>(FirstNonPHI) ||
+ isa<CatchSwitchInst>(FirstNonPHI))
+ for (BasicBlock *PredBB : PN->blocks())
+ if (isa<CatchSwitchInst>(PredBB->getFirstNonPHI()))
+ return;
+ }
}
#ifndef NDEBUG
diff --git a/contrib/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/contrib/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2c0d317..17c25df 100644
--- a/contrib/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/contrib/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1409,14 +1409,15 @@ private:
/// different operations.
class LoopVectorizationCostModel {
public:
- LoopVectorizationCostModel(Loop *L, PredicatedScalarEvolution &PSE,
- LoopInfo *LI, LoopVectorizationLegality *Legal,
+ LoopVectorizationCostModel(Loop *L, ScalarEvolution *SE, LoopInfo *LI,
+ LoopVectorizationLegality *Legal,
const TargetTransformInfo &TTI,
const TargetLibraryInfo *TLI, DemandedBits *DB,
AssumptionCache *AC, const Function *F,
- const LoopVectorizeHints *Hints)
- : TheLoop(L), PSE(PSE), LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), DB(DB),
- AC(AC), TheFunction(F), Hints(Hints) {}
+ const LoopVectorizeHints *Hints,
+ SmallPtrSetImpl<const Value *> &ValuesToIgnore)
+ : TheLoop(L), SE(SE), LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), DB(DB),
+ TheFunction(F), Hints(Hints), ValuesToIgnore(ValuesToIgnore) {}
/// Information about vectorization costs
struct VectorizationFactor {
@@ -1464,9 +1465,6 @@ public:
SmallVector<RegisterUsage, 8>
calculateRegisterUsage(const SmallVector<unsigned, 8> &VFs);
- /// Collect values we want to ignore in the cost model.
- void collectValuesToIgnore();
-
private:
/// Returns the expected execution cost. The unit of the cost does
/// not matter because we use the 'cost' units to compare different
@@ -1498,8 +1496,8 @@ public:
/// The loop that we evaluate.
Loop *TheLoop;
- /// Predicated scalar evolution analysis.
- PredicatedScalarEvolution &PSE;
+ /// Scev analysis.
+ ScalarEvolution *SE;
/// Loop Info analysis.
LoopInfo *LI;
/// Vectorization legality.
@@ -1508,17 +1506,13 @@ public:
const TargetTransformInfo &TTI;
/// Target Library Info.
const TargetLibraryInfo *TLI;
- /// Demanded bits analysis.
+ /// Demanded bits analysis
DemandedBits *DB;
- /// Assumption cache.
- AssumptionCache *AC;
const Function *TheFunction;
- /// Loop Vectorize Hint.
+ // Loop Vectorize Hint.
const LoopVectorizeHints *Hints;
- /// Values to ignore in the cost model.
- SmallPtrSet<const Value *, 16> ValuesToIgnore;
- /// Values to ignore in the cost model when VF > 1.
- SmallPtrSet<const Value *, 16> VecValuesToIgnore;
+ // Values to ignore in the cost model.
+ const SmallPtrSetImpl<const Value *> &ValuesToIgnore;
};
/// \brief This holds vectorization requirements that must be verified late in
@@ -1763,10 +1757,19 @@ struct LoopVectorize : public FunctionPass {
return false;
}
+ // Collect values we want to ignore in the cost model. This includes
+ // type-promoting instructions we identified during reduction detection.
+ SmallPtrSet<const Value *, 32> ValuesToIgnore;
+ CodeMetrics::collectEphemeralValues(L, AC, ValuesToIgnore);
+ for (auto &Reduction : *LVL.getReductionVars()) {
+ RecurrenceDescriptor &RedDes = Reduction.second;
+ SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
+ ValuesToIgnore.insert(Casts.begin(), Casts.end());
+ }
+
// Use the cost model.
- LoopVectorizationCostModel CM(L, PSE, LI, &LVL, *TTI, TLI, DB, AC, F,
- &Hints);
- CM.collectValuesToIgnore();
+ LoopVectorizationCostModel CM(L, PSE.getSE(), LI, &LVL, *TTI, TLI, DB, AC,
+ F, &Hints, ValuesToIgnore);
// Check the function attributes to find out if this function should be
// optimized for size.
@@ -4636,6 +4639,8 @@ void InterleavedAccessInfo::analyzeInterleaving(
// Holds all interleaved store groups temporarily.
SmallSetVector<InterleaveGroup *, 4> StoreGroups;
+ // Holds all interleaved load groups temporarily.
+ SmallSetVector<InterleaveGroup *, 4> LoadGroups;
// Search the load-load/write-write pair B-A in bottom-up order and try to
// insert B into the interleave group of A according to 3 rules:
@@ -4663,6 +4668,8 @@ void InterleavedAccessInfo::analyzeInterleaving(
if (A->mayWriteToMemory())
StoreGroups.insert(Group);
+ else
+ LoadGroups.insert(Group);
for (auto II = std::next(I); II != E; ++II) {
Instruction *B = II->first;
@@ -4710,6 +4717,12 @@ void InterleavedAccessInfo::analyzeInterleaving(
for (InterleaveGroup *Group : StoreGroups)
if (Group->getNumMembers() != Group->getFactor())
releaseGroup(Group);
+
+ // Remove interleaved load groups that don't have the first and last member.
+ // This guarantees that we won't do speculative out of bounds loads.
+ for (InterleaveGroup *Group : LoadGroups)
+ if (!Group->getMember(0) || !Group->getMember(Group->getFactor() - 1))
+ releaseGroup(Group);
}
LoopVectorizationCostModel::VectorizationFactor
@@ -4734,7 +4747,7 @@ LoopVectorizationCostModel::selectVectorizationFactor(bool OptForSize) {
}
// Find the trip count.
- unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
+ unsigned TC = SE->getSmallConstantTripCount(TheLoop);
DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
@@ -4936,7 +4949,7 @@ unsigned LoopVectorizationCostModel::selectInterleaveCount(bool OptForSize,
return 1;
// Do not interleave loops with a relatively small trip count.
- unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
+ unsigned TC = SE->getSmallConstantTripCount(TheLoop);
if (TC > 1 && TC < TinyTripCountInterleaveThreshold)
return 1;
@@ -5164,15 +5177,15 @@ LoopVectorizationCostModel::calculateRegisterUsage(
// Ignore instructions that are never used within the loop.
if (!Ends.count(I)) continue;
+ // Skip ignored values.
+ if (ValuesToIgnore.count(I))
+ continue;
+
// Remove all of the instructions that end at this location.
InstrList &List = TransposeEnds[i];
for (unsigned int j = 0, e = List.size(); j < e; ++j)
OpenIntervals.erase(List[j]);
- // Skip ignored values.
- if (ValuesToIgnore.count(I))
- continue;
-
// For each VF find the maximum usage of registers.
for (unsigned j = 0, e = VFs.size(); j < e; ++j) {
if (VFs[j] == 1) {
@@ -5182,12 +5195,8 @@ LoopVectorizationCostModel::calculateRegisterUsage(
// Count the number of live intervals.
unsigned RegUsage = 0;
- for (auto Inst : OpenIntervals) {
- // Skip ignored values for VF > 1.
- if (VecValuesToIgnore.count(Inst))
- continue;
+ for (auto Inst : OpenIntervals)
RegUsage += GetRegUsage(Inst->getType(), VFs[j]);
- }
MaxUsages[j] = std::max(MaxUsages[j], RegUsage);
}
@@ -5331,7 +5340,6 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I, unsigned VF) {
if (VF > 1 && MinBWs.count(I))
RetTy = IntegerType::get(RetTy->getContext(), MinBWs[I]);
Type *VectorTy = ToVectorTy(RetTy, VF);
- auto SE = PSE.getSE();
// TODO: We need to estimate the cost of intrinsic calls.
switch (I->getOpcode()) {
@@ -5633,79 +5641,6 @@ bool LoopVectorizationCostModel::isConsecutiveLoadOrStore(Instruction *Inst) {
return false;
}
-void LoopVectorizationCostModel::collectValuesToIgnore() {
- // Ignore ephemeral values.
- CodeMetrics::collectEphemeralValues(TheLoop, AC, ValuesToIgnore);
-
- // Ignore type-promoting instructions we identified during reduction
- // detection.
- for (auto &Reduction : *Legal->getReductionVars()) {
- RecurrenceDescriptor &RedDes = Reduction.second;
- SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
- VecValuesToIgnore.insert(Casts.begin(), Casts.end());
- }
-
- // Ignore induction phis that are only used in either GetElementPtr or ICmp
- // instruction to exit loop. Induction variables usually have large types and
- // can have big impact when estimating register usage.
- // This is for when VF > 1.
- for (auto &Induction : *Legal->getInductionVars()) {
- auto *PN = Induction.first;
- auto *UpdateV = PN->getIncomingValueForBlock(TheLoop->getLoopLatch());
-
- // Check that the PHI is only used by the induction increment (UpdateV) or
- // by GEPs. Then check that UpdateV is only used by a compare instruction or
- // the loop header PHI.
- // FIXME: Need precise def-use analysis to determine if this instruction
- // variable will be vectorized.
- if (std::all_of(PN->user_begin(), PN->user_end(),
- [&](const User *U) -> bool {
- return U == UpdateV || isa<GetElementPtrInst>(U);
- }) &&
- std::all_of(UpdateV->user_begin(), UpdateV->user_end(),
- [&](const User *U) -> bool {
- return U == PN || isa<ICmpInst>(U);
- })) {
- VecValuesToIgnore.insert(PN);
- VecValuesToIgnore.insert(UpdateV);
- }
- }
-
- // Ignore instructions that will not be vectorized.
- // This is for when VF > 1.
- for (auto bb = TheLoop->block_begin(), be = TheLoop->block_end(); bb != be;
- ++bb) {
- for (auto &Inst : **bb) {
- switch (Inst.getOpcode()) {
- case Instruction::GetElementPtr: {
- // Ignore GEP if its last operand is an induction variable so that it is
- // a consecutive load/store and won't be vectorized as scatter/gather
- // pattern.
-
- GetElementPtrInst *Gep = cast<GetElementPtrInst>(&Inst);
- unsigned NumOperands = Gep->getNumOperands();
- unsigned InductionOperand = getGEPInductionOperand(Gep);
- bool GepToIgnore = true;
-
- // Check that all of the gep indices are uniform except for the
- // induction operand.
- for (unsigned i = 0; i != NumOperands; ++i) {
- if (i != InductionOperand &&
- !PSE.getSE()->isLoopInvariant(PSE.getSCEV(Gep->getOperand(i)),
- TheLoop)) {
- GepToIgnore = false;
- break;
- }
- }
-
- if (GepToIgnore)
- VecValuesToIgnore.insert(&Inst);
- break;
- }
- }
- }
- }
-}
void InnerLoopUnroller::scalarizeInstruction(Instruction *Instr,
bool IfPredicateStore) {
OpenPOWER on IntegriCloud