summaryrefslogtreecommitdiffstats
path: root/lib/CodeGen/SelectionDAG
diff options
context:
space:
mode:
Diffstat (limited to 'lib/CodeGen/SelectionDAG')
-rw-r--r--lib/CodeGen/SelectionDAG/DAGCombiner.cpp6
-rw-r--r--lib/CodeGen/SelectionDAG/SelectionDAG.cpp94
-rw-r--r--lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp57
-rw-r--r--lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h2
-rw-r--r--lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp3
-rw-r--r--lib/CodeGen/SelectionDAG/StatepointLowering.cpp41
6 files changed, 130 insertions, 73 deletions
diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 0872d7a..bc2405b9 100644
--- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -6843,9 +6843,13 @@ SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
uint64_t PtrOff = ShAmt / 8;
unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff);
SDLoc DL(LN0);
+ // The original load itself didn't wrap, so an offset within it doesn't.
+ SDNodeFlags Flags;
+ Flags.setNoUnsignedWrap(true);
SDValue NewPtr = DAG.getNode(ISD::ADD, DL,
PtrType, LN0->getBasePtr(),
- DAG.getConstant(PtrOff, DL, PtrType));
+ DAG.getConstant(PtrOff, DL, PtrType),
+ &Flags);
AddToWorklist(NewPtr.getNode());
SDValue Load;
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index abbc48e..96bf914 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2843,6 +2843,43 @@ bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
return (AZero | BZero).isAllOnesValue();
}
+static SDValue FoldCONCAT_VECTORS(SDLoc DL, EVT VT, ArrayRef<SDValue> Ops,
+ llvm::SelectionDAG &DAG) {
+ if (Ops.size() == 1)
+ return Ops[0];
+
+ // Concat of UNDEFs is UNDEF.
+ if (std::all_of(Ops.begin(), Ops.end(),
+ [](SDValue Op) { return Op.isUndef(); }))
+ return DAG.getUNDEF(VT);
+
+ // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified
+ // to one big BUILD_VECTOR.
+ // FIXME: Add support for UNDEF and SCALAR_TO_VECTOR as well.
+ if (!std::all_of(Ops.begin(), Ops.end(), [](SDValue Op) {
+ return Op.getOpcode() == ISD::BUILD_VECTOR;
+ }))
+ return SDValue();
+
+ EVT SVT = VT.getScalarType();
+ SmallVector<SDValue, 16> Elts;
+ for (SDValue Op : Ops)
+ Elts.append(Op->op_begin(), Op->op_end());
+
+ // BUILD_VECTOR requires all inputs to be of the same type, find the
+ // maximum type and extend them all.
+ for (SDValue Op : Elts)
+ SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
+
+ if (SVT.bitsGT(VT.getScalarType()))
+ for (SDValue &Op : Elts)
+ Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
+ ? DAG.getZExtOrTrunc(Op, DL, SVT)
+ : DAG.getSExtOrTrunc(Op, DL, SVT);
+
+ return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
+}
+
/// getNode - Gets or creates the specified node.
///
SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
@@ -3426,34 +3463,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
if (N2.getOpcode() == ISD::EntryToken) return N1;
if (N1 == N2) return N1;
break;
- case ISD::CONCAT_VECTORS:
- // Concat of UNDEFs is UNDEF.
- if (N1.getOpcode() == ISD::UNDEF &&
- N2.getOpcode() == ISD::UNDEF)
- return getUNDEF(VT);
-
- // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
- // one big BUILD_VECTOR.
- if (N1.getOpcode() == ISD::BUILD_VECTOR &&
- N2.getOpcode() == ISD::BUILD_VECTOR) {
- SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
- N1.getNode()->op_end());
- Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
-
- // BUILD_VECTOR requires all inputs to be of the same type, find the
- // maximum type and extend them all.
- EVT SVT = VT.getScalarType();
- for (SDValue Op : Elts)
- SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
- if (SVT.bitsGT(VT.getScalarType()))
- for (SDValue &Op : Elts)
- Op = TLI->isZExtFree(Op.getValueType(), SVT)
- ? getZExtOrTrunc(Op, DL, SVT)
- : getSExtOrTrunc(Op, DL, SVT);
-
- return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
- }
+ case ISD::CONCAT_VECTORS: {
+ // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
+ SDValue Ops[] = {N1, N2};
+ if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
+ return V;
break;
+ }
case ISD::AND:
assert(VT.isInteger() && "This operator does not apply to FP types!");
assert(N1.getValueType() == N2.getValueType() &&
@@ -3911,19 +3927,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
}
break;
}
- case ISD::CONCAT_VECTORS:
- // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
- // one big BUILD_VECTOR.
- if (N1.getOpcode() == ISD::BUILD_VECTOR &&
- N2.getOpcode() == ISD::BUILD_VECTOR &&
- N3.getOpcode() == ISD::BUILD_VECTOR) {
- SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
- N1.getNode()->op_end());
- Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
- Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
- return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
- }
+ case ISD::CONCAT_VECTORS: {
+ // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
+ SDValue Ops[] = {N1, N2, N3};
+ if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
+ return V;
break;
+ }
case ISD::SETCC: {
// Use FoldSetCC to simplify SETCC's.
if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
@@ -5462,6 +5472,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
switch (Opcode) {
default: break;
+ case ISD::CONCAT_VECTORS: {
+ // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
+ if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
+ return V;
+ break;
+ }
case ISD::SELECT_CC: {
assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
assert(Ops[0].getValueType() == Ops[1].getValueType() &&
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index d2ea85ab..e446a93 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1329,12 +1329,18 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
unsigned NumValues = ValueVTs.size();
+ // An aggregate return value cannot wrap around the address space, so
+ // offsets to its parts don't wrap either.
+ SDNodeFlags Flags;
+ Flags.setNoUnsignedWrap(true);
+
SmallVector<SDValue, 4> Chains(NumValues);
for (unsigned i = 0; i != NumValues; ++i) {
SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
RetPtr.getValueType(), RetPtr,
DAG.getIntPtrConstant(Offsets[i],
- getCurSDLoc()));
+ getCurSDLoc()),
+ &Flags);
Chains[i] =
DAG.getStore(Chain, getCurSDLoc(),
SDValue(RetOp.getNode(), RetOp.getResNo() + i),
@@ -2994,8 +3000,15 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
if (Field) {
// N = N + Offset
uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
+
+ // In an inbouds GEP with an offset that is nonnegative even when
+ // interpreted as signed, assume there is no unsigned overflow.
+ SDNodeFlags Flags;
+ if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
+ Flags.setNoUnsignedWrap(true);
+
N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
- DAG.getConstant(Offset, dl, N.getValueType()));
+ DAG.getConstant(Offset, dl, N.getValueType()), &Flags);
}
Ty = StTy->getElementType(Field);
@@ -3020,7 +3033,14 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
SDValue OffsVal = VectorWidth ?
DAG.getConstant(Offs, dl, MVT::getVectorVT(PtrTy, VectorWidth)) :
DAG.getConstant(Offs, dl, PtrTy);
- N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal);
+
+ // In an inbouds GEP with an offset that is nonnegative even when
+ // interpreted as signed, assume there is no unsigned overflow.
+ SDNodeFlags Flags;
+ if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
+ Flags.setNoUnsignedWrap(true);
+
+ N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, &Flags);
continue;
}
@@ -3092,10 +3112,13 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
Align = 0;
// Round the size of the allocation up to the stack alignment size
- // by add SA-1 to the size.
+ // by add SA-1 to the size. This doesn't overflow because we're computing
+ // an address inside an alloca.
+ SDNodeFlags Flags;
+ Flags.setNoUnsignedWrap(true);
AllocSize = DAG.getNode(ISD::ADD, dl,
AllocSize.getValueType(), AllocSize,
- DAG.getIntPtrConstant(StackAlign - 1, dl));
+ DAG.getIntPtrConstant(StackAlign - 1, dl), &Flags);
// Mask out the low bits for alignment purposes.
AllocSize = DAG.getNode(ISD::AND, dl,
@@ -3168,6 +3191,11 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
if (isVolatile)
Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
+ // An aggregate load cannot wrap around the address space, so offsets to its
+ // parts don't wrap either.
+ SDNodeFlags Flags;
+ Flags.setNoUnsignedWrap(true);
+
SmallVector<SDValue, 4> Values(NumValues);
SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
EVT PtrVT = Ptr.getValueType();
@@ -3188,7 +3216,8 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
}
SDValue A = DAG.getNode(ISD::ADD, dl,
PtrVT, Ptr,
- DAG.getConstant(Offsets[i], dl, PtrVT));
+ DAG.getConstant(Offsets[i], dl, PtrVT),
+ &Flags);
SDValue L = DAG.getLoad(ValueVTs[i], dl, Root,
A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
isNonTemporal, isInvariant, Alignment, AAInfo,
@@ -3243,6 +3272,11 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
AAMDNodes AAInfo;
I.getAAMetadata(AAInfo);
+ // An aggregate load cannot wrap around the address space, so offsets to its
+ // parts don't wrap either.
+ SDNodeFlags Flags;
+ Flags.setNoUnsignedWrap(true);
+
unsigned ChainI = 0;
for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
// See visitLoad comments.
@@ -3253,7 +3287,7 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
ChainI = 0;
}
SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
- DAG.getConstant(Offsets[i], dl, PtrVT));
+ DAG.getConstant(Offsets[i], dl, PtrVT), &Flags);
SDValue St = DAG.getStore(Root, dl,
SDValue(Src.getNode(), Src.getResNo() + i),
Add, MachinePointerInfo(PtrV, Offsets[i]),
@@ -5189,7 +5223,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
return nullptr;
}
case Intrinsic::experimental_gc_relocate: {
- visitGCRelocate(I);
+ visitGCRelocate(cast<GCRelocateInst>(I));
return nullptr;
}
case Intrinsic::instrprof_increment:
@@ -7202,10 +7236,15 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
ReturnValues.resize(NumValues);
SmallVector<SDValue, 4> Chains(NumValues);
+ // An aggregate return value cannot wrap around the address space, so
+ // offsets to its parts don't wrap either.
+ SDNodeFlags Flags;
+ Flags.setNoUnsignedWrap(true);
+
for (unsigned i = 0; i < NumValues; ++i) {
SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
CLI.DAG.getConstant(Offsets[i], CLI.DL,
- PtrVT));
+ PtrVT), &Flags);
SDValue L = CLI.DAG.getLoad(
RetTys[i], CLI.DL, CLI.Chain, Add,
MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 49a3872..8fb85ff 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -855,7 +855,7 @@ private:
// These three are implemented in StatepointLowering.cpp
void visitStatepoint(const CallInst &I);
- void visitGCRelocate(const CallInst &I);
+ void visitGCRelocate(const GCRelocateInst &I);
void visitGCResult(const CallInst &I);
void visitUserOp1(const Instruction &I) {
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 853a21a..9f8759d 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -633,6 +633,9 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
MRI.replaceRegWith(From, To);
}
+ if (TLI->hasCopyImplyingStackAdjustment(MF))
+ MFI->setHasOpaqueSPAdjustment(true);
+
// Freeze the set of reserved registers now that MachineFrameInfo has been
// set up. All the information required by getReservedRegs() should be
// available now.
diff --git a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 050ec21..6547a62 100644
--- a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -128,13 +128,11 @@ static Optional<int> findPreviousSpillSlot(const Value *Val,
return Optional<int>();
// Spill location is known for gc relocates
- if (isGCRelocate(Val)) {
- GCRelocateOperands RelocOps(cast<Instruction>(Val));
-
+ if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap =
- Builder.FuncInfo.StatepointRelocatedValues[RelocOps.getStatepoint()];
+ Builder.FuncInfo.StatepointRelocatedValues[Relocate->getStatepoint()];
- auto It = SpillMap.find(RelocOps.getDerivedPtr());
+ auto It = SpillMap.find(Relocate->getDerivedPtr());
if (It == SpillMap.end())
return Optional<int>();
@@ -401,10 +399,10 @@ static void getIncomingStatepointGCValues(
SmallVectorImpl<const Value *> &Bases, SmallVectorImpl<const Value *> &Ptrs,
SmallVectorImpl<const Value *> &Relocs, ImmutableStatepoint StatepointSite,
SelectionDAGBuilder &Builder) {
- for (GCRelocateOperands relocateOpers : StatepointSite.getRelocates()) {
- Relocs.push_back(relocateOpers.getUnderlyingCallSite().getInstruction());
- Bases.push_back(relocateOpers.getBasePtr());
- Ptrs.push_back(relocateOpers.getDerivedPtr());
+ for (const GCRelocateInst *Relocate : StatepointSite.getRelocates()) {
+ Relocs.push_back(Relocate);
+ Bases.push_back(Relocate->getBasePtr());
+ Ptrs.push_back(Relocate->getDerivedPtr());
}
// Remove any redundant llvm::Values which map to the same SDValue as another
@@ -602,8 +600,8 @@ static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap =
Builder.FuncInfo.StatepointRelocatedValues[StatepointInstr];
- for (GCRelocateOperands RelocateOpers : StatepointSite.getRelocates()) {
- const Value *V = RelocateOpers.getDerivedPtr();
+ for (const GCRelocateInst *Relocate : StatepointSite.getRelocates()) {
+ const Value *V = Relocate->getDerivedPtr();
SDValue SDV = Builder.getValue(V);
SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
@@ -624,8 +622,7 @@ static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
// uses of the corresponding values so that it would automatically
// export them. Relocates of the spilled values does not use original
// value.
- if (RelocateOpers.getUnderlyingCallSite().getParent() !=
- StatepointInstr->getParent())
+ if (Relocate->getParent() != StatepointInstr->getParent())
Builder.ExportFromCurrentBlock(V);
}
}
@@ -656,7 +653,7 @@ void SelectionDAGBuilder::LowerStatepoint(
// statepoint.
for (const User *U : CS->users()) {
const CallInst *Call = cast<CallInst>(U);
- if (isGCRelocate(Call) && Call->getParent() == CS.getParent())
+ if (isa<GCRelocateInst>(Call) && Call->getParent() == CS.getParent())
StatepointLowering.scheduleRelocCall(*Call);
}
#endif
@@ -859,24 +856,22 @@ void SelectionDAGBuilder::visitGCResult(const CallInst &CI) {
}
}
-void SelectionDAGBuilder::visitGCRelocate(const CallInst &CI) {
- GCRelocateOperands RelocateOpers(&CI);
-
+void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
#ifndef NDEBUG
// Consistency check
// We skip this check for relocates not in the same basic block as thier
// statepoint. It would be too expensive to preserve validation info through
// different basic blocks.
- if (RelocateOpers.getStatepoint()->getParent() == CI.getParent()) {
- StatepointLowering.relocCallVisited(CI);
+ if (Relocate.getStatepoint()->getParent() == Relocate.getParent()) {
+ StatepointLowering.relocCallVisited(Relocate);
}
#endif
- const Value *DerivedPtr = RelocateOpers.getDerivedPtr();
+ const Value *DerivedPtr = Relocate.getDerivedPtr();
SDValue SD = getValue(DerivedPtr);
FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap =
- FuncInfo.StatepointRelocatedValues[RelocateOpers.getStatepoint()];
+ FuncInfo.StatepointRelocatedValues[Relocate.getStatepoint()];
// We should have recorded location for this pointer
assert(SpillMap.count(DerivedPtr) && "Relocating not lowered gc value");
@@ -885,7 +880,7 @@ void SelectionDAGBuilder::visitGCRelocate(const CallInst &CI) {
// We didn't need to spill these special cases (constants and allocas).
// See the handling in spillIncomingValueForStatepoint for detail.
if (!DerivedPtrLocation) {
- setValue(&CI, SD);
+ setValue(&Relocate, SD);
return;
}
@@ -907,5 +902,5 @@ void SelectionDAGBuilder::visitGCRelocate(const CallInst &CI) {
DAG.setRoot(SpillLoad.getValue(1));
assert(SpillLoad.getNode());
- setValue(&CI, SpillLoad);
+ setValue(&Relocate, SpillLoad);
}
OpenPOWER on IntegriCloud