diff options
Diffstat (limited to 'lib/CodeGen/SelectionDAG')
25 files changed, 1961 insertions, 1568 deletions
diff --git a/lib/CodeGen/SelectionDAG/CMakeLists.txt b/lib/CodeGen/SelectionDAG/CMakeLists.txt index 80c7d7c..0cfd5e1 100644 --- a/lib/CodeGen/SelectionDAG/CMakeLists.txt +++ b/lib/CodeGen/SelectionDAG/CMakeLists.txt @@ -20,6 +20,7 @@ add_llvm_library(LLVMSelectionDAG SelectionDAGISel.cpp SelectionDAGPrinter.cpp TargetLowering.cpp + TargetSelectionDAGInfo.cpp ) target_link_libraries (LLVMSelectionDAG LLVMAnalysis LLVMAsmPrinter LLVMCodeGen) diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index a336e0a..3639f80 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -129,6 +129,14 @@ namespace { bool CombineToPreIndexedLoadStore(SDNode *N); bool CombineToPostIndexedLoadStore(SDNode *N); + void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad); + SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace); + SDValue SExtPromoteOperand(SDValue Op, EVT PVT); + SDValue ZExtPromoteOperand(SDValue Op, EVT PVT); + SDValue PromoteIntBinOp(SDValue Op); + SDValue PromoteIntShiftOp(SDValue Op); + SDValue PromoteExtend(SDValue Op); + bool PromoteLoad(SDValue Op); /// combine - call the node-specific routine that knows how to fold each /// particular type of node. If that doesn't do anything, try the @@ -254,24 +262,28 @@ namespace { /// looking for a better chain (aliasing node.) SDValue FindBetterChain(SDNode *N, SDValue Chain); - /// getShiftAmountTy - Returns a type large enough to hold any valid - /// shift amount - before type legalization these can be huge. - EVT getShiftAmountTy() { - return LegalTypes ? TLI.getShiftAmountTy() : TLI.getPointerTy(); - } - -public: + public: DAGCombiner(SelectionDAG &D, AliasAnalysis &A, CodeGenOpt::Level OL) - : DAG(D), - TLI(D.getTargetLoweringInfo()), - Level(Unrestricted), - OptLevel(OL), - LegalOperations(false), - LegalTypes(false), - AA(A) {} + : DAG(D), TLI(D.getTargetLoweringInfo()), Level(Unrestricted), + OptLevel(OL), LegalOperations(false), LegalTypes(false), AA(A) {} /// Run - runs the dag combiner on all nodes in the work list void Run(CombineLevel AtLevel); + + SelectionDAG &getDAG() const { return DAG; } + + /// getShiftAmountTy - Returns a type large enough to hold any valid + /// shift amount - before type legalization these can be huge. + EVT getShiftAmountTy() { + return LegalTypes ? TLI.getShiftAmountTy() : TLI.getPointerTy(); + } + + /// isTypeLegal - This method returns true if we are running before type + /// legalization or if the specified VT is legal. + bool isTypeLegal(const EVT &VT) { + if (!LegalTypes) return true; + return TLI.isTypeLegal(VT); + } }; } @@ -577,9 +589,8 @@ SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo, return SDValue(N, 0); } -void -DAGCombiner::CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt & - TLO) { +void DAGCombiner:: +CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) { // Replace all uses. If any nodes become isomorphic to other nodes and // are deleted, make sure to remove them from our worklist. WorkListRemover DeadNodes(*this); @@ -609,7 +620,7 @@ DAGCombiner::CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt & /// it can be simplified or if things it uses can be simplified by bit /// propagation. If so, return true. bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) { - TargetLowering::TargetLoweringOpt TLO(DAG); + TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations); APInt KnownZero, KnownOne; if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO)) return false; @@ -629,6 +640,274 @@ bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) { return true; } +void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) { + DebugLoc dl = Load->getDebugLoc(); + EVT VT = Load->getValueType(0); + SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0)); + + DEBUG(dbgs() << "\nReplacing.9 "; + Load->dump(&DAG); + dbgs() << "\nWith: "; + Trunc.getNode()->dump(&DAG); + dbgs() << '\n'); + WorkListRemover DeadNodes(*this); + DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc, &DeadNodes); + DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1), + &DeadNodes); + removeFromWorkList(Load); + DAG.DeleteNode(Load); + AddToWorkList(Trunc.getNode()); +} + +SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) { + Replace = false; + DebugLoc dl = Op.getDebugLoc(); + if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { + EVT MemVT = LD->getMemoryVT(); + ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) + ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD : ISD::EXTLOAD) + : LD->getExtensionType(); + Replace = true; + return DAG.getExtLoad(ExtType, dl, PVT, + LD->getChain(), LD->getBasePtr(), + LD->getSrcValue(), LD->getSrcValueOffset(), + MemVT, LD->isVolatile(), + LD->isNonTemporal(), LD->getAlignment()); + } + + unsigned Opc = Op.getOpcode(); + switch (Opc) { + default: break; + case ISD::AssertSext: + return DAG.getNode(ISD::AssertSext, dl, PVT, + SExtPromoteOperand(Op.getOperand(0), PVT), + Op.getOperand(1)); + case ISD::AssertZext: + return DAG.getNode(ISD::AssertZext, dl, PVT, + ZExtPromoteOperand(Op.getOperand(0), PVT), + Op.getOperand(1)); + case ISD::Constant: { + unsigned ExtOpc = + Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; + return DAG.getNode(ExtOpc, dl, PVT, Op); + } + } + + if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT)) + return SDValue(); + return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op); +} + +SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) { + if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT)) + return SDValue(); + EVT OldVT = Op.getValueType(); + DebugLoc dl = Op.getDebugLoc(); + bool Replace = false; + SDValue NewOp = PromoteOperand(Op, PVT, Replace); + if (NewOp.getNode() == 0) + return SDValue(); + AddToWorkList(NewOp.getNode()); + + if (Replace) + ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode()); + return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp, + DAG.getValueType(OldVT)); +} + +SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) { + EVT OldVT = Op.getValueType(); + DebugLoc dl = Op.getDebugLoc(); + bool Replace = false; + SDValue NewOp = PromoteOperand(Op, PVT, Replace); + if (NewOp.getNode() == 0) + return SDValue(); + AddToWorkList(NewOp.getNode()); + + if (Replace) + ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode()); + return DAG.getZeroExtendInReg(NewOp, dl, OldVT); +} + +/// PromoteIntBinOp - Promote the specified integer binary operation if the +/// target indicates it is beneficial. e.g. On x86, it's usually better to +/// promote i16 operations to i32 since i16 instructions are longer. +SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) { + if (!LegalOperations) + return SDValue(); + + EVT VT = Op.getValueType(); + if (VT.isVector() || !VT.isInteger()) + return SDValue(); + + // If operation type is 'undesirable', e.g. i16 on x86, consider + // promoting it. + unsigned Opc = Op.getOpcode(); + if (TLI.isTypeDesirableForOp(Opc, VT)) + return SDValue(); + + EVT PVT = VT; + // Consult target whether it is a good idea to promote this operation and + // what's the right type to promote it to. + if (TLI.IsDesirableToPromoteOp(Op, PVT)) { + assert(PVT != VT && "Don't know what type to promote to!"); + + bool Replace0 = false; + SDValue N0 = Op.getOperand(0); + SDValue NN0 = PromoteOperand(N0, PVT, Replace0); + if (NN0.getNode() == 0) + return SDValue(); + + bool Replace1 = false; + SDValue N1 = Op.getOperand(1); + SDValue NN1 = PromoteOperand(N1, PVT, Replace1); + if (NN1.getNode() == 0) + return SDValue(); + + AddToWorkList(NN0.getNode()); + AddToWorkList(NN1.getNode()); + + if (Replace0) + ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode()); + if (Replace1) + ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode()); + + DEBUG(dbgs() << "\nPromoting "; + Op.getNode()->dump(&DAG)); + DebugLoc dl = Op.getDebugLoc(); + return DAG.getNode(ISD::TRUNCATE, dl, VT, + DAG.getNode(Opc, dl, PVT, NN0, NN1)); + } + return SDValue(); +} + +/// PromoteIntShiftOp - Promote the specified integer shift operation if the +/// target indicates it is beneficial. e.g. On x86, it's usually better to +/// promote i16 operations to i32 since i16 instructions are longer. +SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) { + if (!LegalOperations) + return SDValue(); + + EVT VT = Op.getValueType(); + if (VT.isVector() || !VT.isInteger()) + return SDValue(); + + // If operation type is 'undesirable', e.g. i16 on x86, consider + // promoting it. + unsigned Opc = Op.getOpcode(); + if (TLI.isTypeDesirableForOp(Opc, VT)) + return SDValue(); + + EVT PVT = VT; + // Consult target whether it is a good idea to promote this operation and + // what's the right type to promote it to. + if (TLI.IsDesirableToPromoteOp(Op, PVT)) { + assert(PVT != VT && "Don't know what type to promote to!"); + + bool Replace = false; + SDValue N0 = Op.getOperand(0); + if (Opc == ISD::SRA) + N0 = SExtPromoteOperand(Op.getOperand(0), PVT); + else if (Opc == ISD::SRL) + N0 = ZExtPromoteOperand(Op.getOperand(0), PVT); + else + N0 = PromoteOperand(N0, PVT, Replace); + if (N0.getNode() == 0) + return SDValue(); + + AddToWorkList(N0.getNode()); + if (Replace) + ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode()); + + DEBUG(dbgs() << "\nPromoting "; + Op.getNode()->dump(&DAG)); + DebugLoc dl = Op.getDebugLoc(); + return DAG.getNode(ISD::TRUNCATE, dl, VT, + DAG.getNode(Opc, dl, PVT, N0, Op.getOperand(1))); + } + return SDValue(); +} + +SDValue DAGCombiner::PromoteExtend(SDValue Op) { + if (!LegalOperations) + return SDValue(); + + EVT VT = Op.getValueType(); + if (VT.isVector() || !VT.isInteger()) + return SDValue(); + + // If operation type is 'undesirable', e.g. i16 on x86, consider + // promoting it. + unsigned Opc = Op.getOpcode(); + if (TLI.isTypeDesirableForOp(Opc, VT)) + return SDValue(); + + EVT PVT = VT; + // Consult target whether it is a good idea to promote this operation and + // what's the right type to promote it to. + if (TLI.IsDesirableToPromoteOp(Op, PVT)) { + assert(PVT != VT && "Don't know what type to promote to!"); + // fold (aext (aext x)) -> (aext x) + // fold (aext (zext x)) -> (zext x) + // fold (aext (sext x)) -> (sext x) + DEBUG(dbgs() << "\nPromoting "; + Op.getNode()->dump(&DAG)); + return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), VT, Op.getOperand(0)); + } + return SDValue(); +} + +bool DAGCombiner::PromoteLoad(SDValue Op) { + if (!LegalOperations) + return false; + + EVT VT = Op.getValueType(); + if (VT.isVector() || !VT.isInteger()) + return false; + + // If operation type is 'undesirable', e.g. i16 on x86, consider + // promoting it. + unsigned Opc = Op.getOpcode(); + if (TLI.isTypeDesirableForOp(Opc, VT)) + return false; + + EVT PVT = VT; + // Consult target whether it is a good idea to promote this operation and + // what's the right type to promote it to. + if (TLI.IsDesirableToPromoteOp(Op, PVT)) { + assert(PVT != VT && "Don't know what type to promote to!"); + + DebugLoc dl = Op.getDebugLoc(); + SDNode *N = Op.getNode(); + LoadSDNode *LD = cast<LoadSDNode>(N); + EVT MemVT = LD->getMemoryVT(); + ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) + ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD : ISD::EXTLOAD) + : LD->getExtensionType(); + SDValue NewLD = DAG.getExtLoad(ExtType, dl, PVT, + LD->getChain(), LD->getBasePtr(), + LD->getSrcValue(), LD->getSrcValueOffset(), + MemVT, LD->isVolatile(), + LD->isNonTemporal(), LD->getAlignment()); + SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD); + + DEBUG(dbgs() << "\nPromoting "; + N->dump(&DAG); + dbgs() << "\nTo: "; + Result.getNode()->dump(&DAG); + dbgs() << '\n'); + WorkListRemover DeadNodes(*this); + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result, &DeadNodes); + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1), &DeadNodes); + removeFromWorkList(N); + DAG.DeleteNode(N); + AddToWorkList(Result.getNode()); + return true; + } + return false; +} + + //===----------------------------------------------------------------------===// // Main DAG Combiner implementation //===----------------------------------------------------------------------===// @@ -732,7 +1011,7 @@ void DAGCombiner::Run(CombineLevel AtLevel) { } SDValue DAGCombiner::visit(SDNode *N) { - switch(N->getOpcode()) { + switch (N->getOpcode()) { default: break; case ISD::TokenFactor: return visitTokenFactor(N); case ISD::MERGE_VALUES: return visitMERGE_VALUES(N); @@ -817,6 +1096,35 @@ SDValue DAGCombiner::combine(SDNode *N) { } } + // If nothing happened still, try promoting the operation. + if (RV.getNode() == 0) { + switch (N->getOpcode()) { + default: break; + case ISD::ADD: + case ISD::SUB: + case ISD::MUL: + case ISD::AND: + case ISD::OR: + case ISD::XOR: + RV = PromoteIntBinOp(SDValue(N, 0)); + break; + case ISD::SHL: + case ISD::SRA: + case ISD::SRL: + RV = PromoteIntShiftOp(SDValue(N, 0)); + break; + case ISD::SIGN_EXTEND: + case ISD::ZERO_EXTEND: + case ISD::ANY_EXTEND: + RV = PromoteExtend(SDValue(N, 0)); + break; + case ISD::LOAD: + if (PromoteLoad(SDValue(N, 0))) + RV = SDValue(N, 0); + break; + } + } + // If N is a commutative binary node, try commuting it to enable more // sdisel CSE. if (RV.getNode() == 0 && @@ -1720,8 +2028,10 @@ SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) { // into a vsetcc. EVT Op0VT = N0.getOperand(0).getValueType(); if ((N0.getOpcode() == ISD::ZERO_EXTEND || - N0.getOpcode() == ISD::ANY_EXTEND || N0.getOpcode() == ISD::SIGN_EXTEND || + // Avoid infinite looping with PromoteIntBinOp. + (N0.getOpcode() == ISD::ANY_EXTEND && + (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) || (N0.getOpcode() == ISD::TRUNCATE && TLI.isTypeLegal(Op0VT))) && !VT.isVector() && Op0VT == N1.getOperand(0).getValueType() && @@ -2579,7 +2889,13 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { HiBitsMask); } - return N1C ? visitShiftByConstant(N, N1C->getZExtValue()) : SDValue(); + if (N1C) { + SDValue NewSHL = visitShiftByConstant(N, N1C->getZExtValue()); + if (NewSHL.getNode()) + return NewSHL; + } + + return SDValue(); } SDValue DAGCombiner::visitSRA(SDNode *N) { @@ -2693,7 +3009,13 @@ SDValue DAGCombiner::visitSRA(SDNode *N) { if (DAG.SignBitIsZero(N0)) return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0, N1); - return N1C ? visitShiftByConstant(N, N1C->getZExtValue()) : SDValue(); + if (N1C) { + SDValue NewSRA = visitShiftByConstant(N, N1C->getZExtValue()); + if (NewSRA.getNode()) + return NewSRA; + } + + return SDValue(); } SDValue DAGCombiner::visitSRL(SDNode *N) { @@ -2731,6 +3053,15 @@ SDValue DAGCombiner::visitSRL(SDNode *N) { return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0.getOperand(0), DAG.getConstant(c1 + c2, N1.getValueType())); } + + // fold (srl (shl x, c), c) -> (and x, cst2) + if (N1C && N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1 && + N0.getValueSizeInBits() <= 64) { + uint64_t ShAmt = N1C->getZExtValue()+64-N0.getValueSizeInBits(); + return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0.getOperand(0), + DAG.getConstant(~0ULL >> ShAmt, VT)); + } + // fold (srl (anyextend x), c) -> (anyextend (srl x, c)) if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { @@ -2739,10 +3070,12 @@ SDValue DAGCombiner::visitSRL(SDNode *N) { if (N1C->getZExtValue() >= SmallVT.getSizeInBits()) return DAG.getUNDEF(VT); - SDValue SmallShift = DAG.getNode(ISD::SRL, N0.getDebugLoc(), SmallVT, - N0.getOperand(0), N1); - AddToWorkList(SmallShift.getNode()); - return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, SmallShift); + if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) { + SDValue SmallShift = DAG.getNode(ISD::SRL, N0.getDebugLoc(), SmallVT, + N0.getOperand(0), N1); + AddToWorkList(SmallShift.getNode()); + return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, SmallShift); + } } // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign @@ -3205,24 +3538,40 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) { if (N0.getOpcode() == ISD::SETCC) { // sext(setcc) -> sext_in_reg(vsetcc) for vectors. - if (VT.isVector() && + // Only do this before legalize for now. + if (VT.isVector() && !LegalOperations) { + EVT N0VT = N0.getOperand(0).getValueType(); // We know that the # elements of the results is the same as the // # elements of the compare (and the # elements of the compare result // for that matter). Check to see that they are the same size. If so, // we know that the element size of the sext'd result matches the // element size of the compare operands. - VT.getSizeInBits() == N0.getOperand(0).getValueType().getSizeInBits() && - - // Only do this before legalize for now. - !LegalOperations) { - return DAG.getVSetCC(N->getDebugLoc(), VT, N0.getOperand(0), - N0.getOperand(1), - cast<CondCodeSDNode>(N0.getOperand(2))->get()); + if (VT.getSizeInBits() == N0VT.getSizeInBits()) + return DAG.getVSetCC(N->getDebugLoc(), VT, N0.getOperand(0), + N0.getOperand(1), + cast<CondCodeSDNode>(N0.getOperand(2))->get()); + // If the desired elements are smaller or larger than the source + // elements we can use a matching integer vector type and then + // truncate/sign extend + else { + EVT MatchingElementType = + EVT::getIntegerVT(*DAG.getContext(), + N0VT.getScalarType().getSizeInBits()); + EVT MatchingVectorType = + EVT::getVectorVT(*DAG.getContext(), MatchingElementType, + N0VT.getVectorNumElements()); + SDValue VsetCC = + DAG.getVSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0), + N0.getOperand(1), + cast<CondCodeSDNode>(N0.getOperand(2))->get()); + return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT); + } } - + // sext(setcc x, y, cc) -> (select_cc x, y, -1, 0, cc) + unsigned ElementWidth = VT.getScalarType().getSizeInBits(); SDValue NegOne = - DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT); + DAG.getConstant(APInt::getAllOnesValue(ElementWidth), VT); SDValue SCC = SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1), NegOne, DAG.getConstant(0, VT), @@ -3624,7 +3973,7 @@ SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) { // Do not generate loads of non-round integer types since these can // be expensive (and would be wrong if the type is not byte sized). if (isa<LoadSDNode>(N0) && N0.hasOneUse() && ExtVT.isRound() && - cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits() > EVTBits && + cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits() >= EVTBits && // Do not change the width of a volatile load. !cast<LoadSDNode>(N0)->isVolatile()) { LoadSDNode *LN0 = cast<LoadSDNode>(N0); @@ -3694,7 +4043,8 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) { // if x is small enough. if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) { SDValue N00 = N0.getOperand(0); - if (N00.getValueType().getScalarType().getSizeInBits() < EVTBits) + if (N00.getValueType().getScalarType().getSizeInBits() <= EVTBits && + (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, N00, N1); } @@ -3779,7 +4129,8 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { if (N0.getOpcode() == ISD::TRUNCATE) return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, N0.getOperand(0)); // fold (truncate (ext x)) -> (ext x) or (truncate x) or x - if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::SIGN_EXTEND|| + if (N0.getOpcode() == ISD::ZERO_EXTEND || + N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) { if (N0.getOperand(0).getValueType().bitsLT(VT)) // if the source is smaller than the dest, we still need an extend @@ -3805,7 +4156,9 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { // fold (truncate (load x)) -> (smaller load x) // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits)) - return ReduceLoadWidth(N); + if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) + return ReduceLoadWidth(N); + return SDValue(); } static SDNode *getBuildPairElt(SDNode *N, unsigned i) { @@ -3949,7 +4302,7 @@ SDValue DAGCombiner::visitBIT_CONVERT(SDNode *N) { VT.isInteger() && !VT.isVector()) { unsigned OrigXWidth = N0.getOperand(1).getValueType().getSizeInBits(); EVT IntXVT = EVT::getIntegerVT(*DAG.getContext(), OrigXWidth); - if (TLI.isTypeLegal(IntXVT) || !LegalTypes) { + if (isTypeLegal(IntXVT)) { SDValue X = DAG.getNode(ISD::BIT_CONVERT, N0.getDebugLoc(), IntXVT, N0.getOperand(1)); AddToWorkList(X.getNode()); @@ -4075,8 +4428,8 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) { if (Op.getOpcode() == ISD::UNDEF) continue; EltIsUndef = false; - NewBits |= (APInt(cast<ConstantSDNode>(Op)->getAPIntValue()). - zextOrTrunc(SrcBitSize).zext(DstBitSize)); + NewBits |= APInt(cast<ConstantSDNode>(Op)->getAPIntValue()). + zextOrTrunc(SrcBitSize).zext(DstBitSize); } if (EltIsUndef) @@ -4464,7 +4817,7 @@ SDValue DAGCombiner::visitFP_ROUND_INREG(SDNode *N) { ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); // fold (fp_round_inreg c1fp) -> c1fp - if (N0CFP && (TLI.isTypeLegal(EVT) || !LegalTypes)) { + if (N0CFP && isTypeLegal(EVT)) { SDValue Round = DAG.getConstantFP(*N0CFP->getConstantFPValue(), EVT); return DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), VT, Round); } @@ -4676,7 +5029,7 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) { if (Op0.getOpcode() == Op1.getOpcode()) { // Avoid missing important xor optimizations. SDValue Tmp = visitXOR(TheXor); - if (Tmp.getNode()) { + if (Tmp.getNode() && Tmp.getNode() != TheXor) { DEBUG(dbgs() << "\nReplacing.8 "; TheXor->dump(&DAG); dbgs() << "\nWith: "; @@ -5145,6 +5498,136 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) { return SDValue(); } +/// CheckForMaskedLoad - Check to see if V is (and load (ptr), imm), where the +/// load is having specific bytes cleared out. If so, return the byte size +/// being masked out and the shift amount. +static std::pair<unsigned, unsigned> +CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) { + std::pair<unsigned, unsigned> Result(0, 0); + + // Check for the structure we're looking for. + if (V->getOpcode() != ISD::AND || + !isa<ConstantSDNode>(V->getOperand(1)) || + !ISD::isNormalLoad(V->getOperand(0).getNode())) + return Result; + + // Check the chain and pointer. + LoadSDNode *LD = cast<LoadSDNode>(V->getOperand(0)); + if (LD->getBasePtr() != Ptr) return Result; // Not from same pointer. + + // The store should be chained directly to the load or be an operand of a + // tokenfactor. + if (LD == Chain.getNode()) + ; // ok. + else if (Chain->getOpcode() != ISD::TokenFactor) + return Result; // Fail. + else { + bool isOk = false; + for (unsigned i = 0, e = Chain->getNumOperands(); i != e; ++i) + if (Chain->getOperand(i).getNode() == LD) { + isOk = true; + break; + } + if (!isOk) return Result; + } + + // This only handles simple types. + if (V.getValueType() != MVT::i16 && + V.getValueType() != MVT::i32 && + V.getValueType() != MVT::i64) + return Result; + + // Check the constant mask. Invert it so that the bits being masked out are + // 0 and the bits being kept are 1. Use getSExtValue so that leading bits + // follow the sign bit for uniformity. + uint64_t NotMask = ~cast<ConstantSDNode>(V->getOperand(1))->getSExtValue(); + unsigned NotMaskLZ = CountLeadingZeros_64(NotMask); + if (NotMaskLZ & 7) return Result; // Must be multiple of a byte. + unsigned NotMaskTZ = CountTrailingZeros_64(NotMask); + if (NotMaskTZ & 7) return Result; // Must be multiple of a byte. + if (NotMaskLZ == 64) return Result; // All zero mask. + + // See if we have a continuous run of bits. If so, we have 0*1+0* + if (CountTrailingOnes_64(NotMask >> NotMaskTZ)+NotMaskTZ+NotMaskLZ != 64) + return Result; + + // Adjust NotMaskLZ down to be from the actual size of the int instead of i64. + if (V.getValueType() != MVT::i64 && NotMaskLZ) + NotMaskLZ -= 64-V.getValueSizeInBits(); + + unsigned MaskedBytes = (V.getValueSizeInBits()-NotMaskLZ-NotMaskTZ)/8; + switch (MaskedBytes) { + case 1: + case 2: + case 4: break; + default: return Result; // All one mask, or 5-byte mask. + } + + // Verify that the first bit starts at a multiple of mask so that the access + // is aligned the same as the access width. + if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes) return Result; + + Result.first = MaskedBytes; + Result.second = NotMaskTZ/8; + return Result; +} + + +/// ShrinkLoadReplaceStoreWithStore - Check to see if IVal is something that +/// provides a value as specified by MaskInfo. If so, replace the specified +/// store with a narrower store of truncated IVal. +static SDNode * +ShrinkLoadReplaceStoreWithStore(const std::pair<unsigned, unsigned> &MaskInfo, + SDValue IVal, StoreSDNode *St, + DAGCombiner *DC) { + unsigned NumBytes = MaskInfo.first; + unsigned ByteShift = MaskInfo.second; + SelectionDAG &DAG = DC->getDAG(); + + // Check to see if IVal is all zeros in the part being masked in by the 'or' + // that uses this. If not, this is not a replacement. + APInt Mask = ~APInt::getBitsSet(IVal.getValueSizeInBits(), + ByteShift*8, (ByteShift+NumBytes)*8); + if (!DAG.MaskedValueIsZero(IVal, Mask)) return 0; + + // Check that it is legal on the target to do this. It is legal if the new + // VT we're shrinking to (i8/i16/i32) is legal or we're still before type + // legalization. + MVT VT = MVT::getIntegerVT(NumBytes*8); + if (!DC->isTypeLegal(VT)) + return 0; + + // Okay, we can do this! Replace the 'St' store with a store of IVal that is + // shifted by ByteShift and truncated down to NumBytes. + if (ByteShift) + IVal = DAG.getNode(ISD::SRL, IVal->getDebugLoc(), IVal.getValueType(), IVal, + DAG.getConstant(ByteShift*8, DC->getShiftAmountTy())); + + // Figure out the offset for the store and the alignment of the access. + unsigned StOffset; + unsigned NewAlign = St->getAlignment(); + + if (DAG.getTargetLoweringInfo().isLittleEndian()) + StOffset = ByteShift; + else + StOffset = IVal.getValueType().getStoreSize() - ByteShift - NumBytes; + + SDValue Ptr = St->getBasePtr(); + if (StOffset) { + Ptr = DAG.getNode(ISD::ADD, IVal->getDebugLoc(), Ptr.getValueType(), + Ptr, DAG.getConstant(StOffset, Ptr.getValueType())); + NewAlign = MinAlign(NewAlign, StOffset); + } + + // Truncate down to the new size. + IVal = DAG.getNode(ISD::TRUNCATE, IVal->getDebugLoc(), VT, IVal); + + ++OpsNarrowed; + return DAG.getStore(St->getChain(), St->getDebugLoc(), IVal, Ptr, + St->getSrcValue(), St->getSrcValueOffset()+StOffset, + false, false, NewAlign).getNode(); +} + /// ReduceLoadOpStoreWidth - Look for sequence of load / op / store where op is /// one of 'or', 'xor', and 'and' of immediates. If 'op' is only touching some @@ -5164,6 +5647,28 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) { return SDValue(); unsigned Opc = Value.getOpcode(); + + // If this is "store (or X, Y), P" and X is "(and (load P), cst)", where cst + // is a byte mask indicating a consecutive number of bytes, check to see if + // Y is known to provide just those bytes. If so, we try to replace the + // load + replace + store sequence with a single (narrower) store, which makes + // the load dead. + if (Opc == ISD::OR) { + std::pair<unsigned, unsigned> MaskedLoad; + MaskedLoad = CheckForMaskedLoad(Value.getOperand(0), Ptr, Chain); + if (MaskedLoad.first) + if (SDNode *NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad, + Value.getOperand(1), ST,this)) + return SDValue(NewST, 0); + + // Or is commutative, so try swapping X and Y. + MaskedLoad = CheckForMaskedLoad(Value.getOperand(1), Ptr, Chain); + if (MaskedLoad.first) + if (SDNode *NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad, + Value.getOperand(0), ST,this)) + return SDValue(NewST, 0); + } + if ((Opc != ISD::OR && Opc != ISD::XOR && Opc != ISD::AND) || Value.getOperand(1).getOpcode() != ISD::Constant) return SDValue(); @@ -5211,8 +5716,8 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) { PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff; unsigned NewAlign = MinAlign(LD->getAlignment(), PtrOff); - if (NewAlign < - TLI.getTargetData()->getABITypeAlignment(NewVT.getTypeForEVT(*DAG.getContext()))) + const Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext()); + if (NewAlign < TLI.getTargetData()->getABITypeAlignment(NewVTTy)) return SDValue(); SDValue NewPtr = DAG.getNode(ISD::ADD, LD->getDebugLoc(), @@ -5282,8 +5787,7 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { case MVT::ppcf128: break; case MVT::f32: - if (((TLI.isTypeLegal(MVT::i32) || !LegalTypes) && !LegalOperations && - !ST->isVolatile()) || + if ((isTypeLegal(MVT::i32) && !LegalOperations && !ST->isVolatile()) || TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { Tmp = DAG.getConstant((uint32_t)CFP->getValueAPF(). bitcastToAPInt().getZExtValue(), MVT::i32); @@ -5294,7 +5798,7 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { } break; case MVT::f64: - if (((TLI.isTypeLegal(MVT::i64) || !LegalTypes) && !LegalOperations && + if ((TLI.isTypeLegal(MVT::i64) && !LegalOperations && !ST->isVolatile()) || TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) { Tmp = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). @@ -5551,7 +6055,7 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { InVec = InVec.getOperand(0); if (ISD::isNormalLoad(InVec.getNode())) { LN0 = cast<LoadSDNode>(InVec); - Elt = (Idx < (int)NumElems) ? Idx : Idx - NumElems; + Elt = (Idx < (int)NumElems) ? Idx : Idx - (int)NumElems; } } @@ -5659,7 +6163,7 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { } // Add count and size info. - if (!TLI.isTypeLegal(VT) && LegalTypes) + if (!isTypeLegal(VT)) return SDValue(); // Return the new VECTOR_SHUFFLE node. @@ -6287,7 +6791,7 @@ SDValue DAGCombiner::BuildUDIV(SDNode *N) { /// FindBaseOffset - Return true if base is a frame index, which is known not // to alias with anything but itself. Provides base object and offset as results. static bool FindBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset, - GlobalValue *&GV, void *&CV) { + const GlobalValue *&GV, void *&CV) { // Assume it is a primitive operation. Base = Ptr; Offset = 0; GV = 0; CV = 0; @@ -6335,7 +6839,7 @@ bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1, // Gather base node and offset information. SDValue Base1, Base2; int64_t Offset1, Offset2; - GlobalValue *GV1, *GV2; + const GlobalValue *GV1, *GV2; void *CV1, *CV2; bool isFrameIndex1 = FindBaseOffset(Ptr1, Base1, Offset1, GV1, CV1); bool isFrameIndex2 = FindBaseOffset(Ptr2, Base2, Offset2, GV2, CV2); diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp index 4bf41f2..b4c3833 100644 --- a/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1,4 +1,4 @@ -///===-- FastISel.cpp - Implementation of the FastISel class --------------===// +//===-- FastISel.cpp - Implementation of the FastISel class ---------------===// // // The LLVM Compiler Infrastructure // @@ -52,10 +52,11 @@ #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Support/ErrorHandling.h" #include "FunctionLoweringInfo.h" using namespace llvm; -unsigned FastISel::getRegForValue(Value *V) { +unsigned FastISel::getRegForValue(const Value *V) { EVT RealVT = TLI.getValueType(V->getType(), /*AllowUnknown=*/true); // Don't handle non-simple values in FastISel. if (!RealVT.isSimple()) @@ -83,7 +84,16 @@ unsigned FastISel::getRegForValue(Value *V) { if (Reg != 0) return Reg; - if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) { + return materializeRegForValue(V, VT); +} + +/// materializeRegForValue - Helper for getRegForVale. This function is +/// called when the value isn't already available in a register and must +/// be materialized with new instructions. +unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) { + unsigned Reg = 0; + + if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) { if (CI->getValue().getActiveBits() <= 64) Reg = FastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue()); } else if (isa<AllocaInst>(V)) { @@ -93,10 +103,12 @@ unsigned FastISel::getRegForValue(Value *V) { // local-CSE'd with actual integer zeros. Reg = getRegForValue(Constant::getNullValue(TD.getIntPtrType(V->getContext()))); - } else if (ConstantFP *CF = dyn_cast<ConstantFP>(V)) { + } else if (const ConstantFP *CF = dyn_cast<ConstantFP>(V)) { + // Try to emit the constant directly. Reg = FastEmit_f(VT, VT, ISD::ConstantFP, CF); if (!Reg) { + // Try to emit the constant by using an integer constant with a cast. const APFloat &Flt = CF->getValueAPF(); EVT IntVT = TLI.getPointerTy(); @@ -114,9 +126,9 @@ unsigned FastISel::getRegForValue(Value *V) { Reg = FastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg); } } - } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) { - if (!SelectOperator(CE, CE->getOpcode())) return 0; - Reg = LocalValueMap[CE]; + } else if (const Operator *Op = dyn_cast<Operator>(V)) { + if (!SelectOperator(Op, Op->getOpcode())) return 0; + Reg = LocalValueMap[Op]; } else if (isa<UndefValue>(V)) { Reg = createResultReg(TLI.getRegClassFor(VT)); BuildMI(MBB, DL, TII.get(TargetOpcode::IMPLICIT_DEF), Reg); @@ -134,11 +146,11 @@ unsigned FastISel::getRegForValue(Value *V) { return Reg; } -unsigned FastISel::lookUpRegForValue(Value *V) { +unsigned FastISel::lookUpRegForValue(const Value *V) { // Look up the value to see if we already have a register for it. We // cache values defined by Instructions across blocks, and other values // only locally. This is because Instructions already have the SSA - // def-dominatess-use requirement enforced. + // def-dominates-use requirement enforced. if (ValueMap.count(V)) return ValueMap[V]; return LocalValueMap[V]; @@ -150,7 +162,7 @@ unsigned FastISel::lookUpRegForValue(Value *V) { /// NOTE: This is only necessary because we might select a block that uses /// a value before we select the block that defines the value. It might be /// possible to fix this by selecting blocks in reverse postorder. -unsigned FastISel::UpdateValueMap(Value* I, unsigned Reg) { +unsigned FastISel::UpdateValueMap(const Value *I, unsigned Reg) { if (!isa<Instruction>(I)) { LocalValueMap[I] = Reg; return Reg; @@ -167,7 +179,7 @@ unsigned FastISel::UpdateValueMap(Value* I, unsigned Reg) { return AssignedReg; } -unsigned FastISel::getRegForGEPIndex(Value *Idx) { +unsigned FastISel::getRegForGEPIndex(const Value *Idx) { unsigned IdxN = getRegForValue(Idx); if (IdxN == 0) // Unhandled operand. Halt "fast" selection and bail. @@ -186,7 +198,7 @@ unsigned FastISel::getRegForGEPIndex(Value *Idx) { /// SelectBinaryOp - Select and emit code for a binary operator instruction, /// which has an opcode which directly corresponds to the given ISD opcode. /// -bool FastISel::SelectBinaryOp(User *I, unsigned ISDOpcode) { +bool FastISel::SelectBinaryOp(const User *I, unsigned ISDOpcode) { EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true); if (VT == MVT::Other || !VT.isSimple()) // Unhandled type. Halt "fast" selection and bail. @@ -252,7 +264,7 @@ bool FastISel::SelectBinaryOp(User *I, unsigned ISDOpcode) { return true; } -bool FastISel::SelectGetElementPtr(User *I) { +bool FastISel::SelectGetElementPtr(const User *I) { unsigned N = getRegForValue(I->getOperand(0)); if (N == 0) // Unhandled operand. Halt "fast" selection and bail. @@ -260,9 +272,9 @@ bool FastISel::SelectGetElementPtr(User *I) { const Type *Ty = I->getOperand(0)->getType(); MVT VT = TLI.getPointerTy(); - for (GetElementPtrInst::op_iterator OI = I->op_begin()+1, E = I->op_end(); - OI != E; ++OI) { - Value *Idx = *OI; + for (GetElementPtrInst::const_op_iterator OI = I->op_begin()+1, + E = I->op_end(); OI != E; ++OI) { + const Value *Idx = *OI; if (const StructType *StTy = dyn_cast<StructType>(Ty)) { unsigned Field = cast<ConstantInt>(Idx)->getZExtValue(); if (Field) { @@ -280,7 +292,7 @@ bool FastISel::SelectGetElementPtr(User *I) { Ty = cast<SequentialType>(Ty)->getElementType(); // If this is a constant subscript, handle it quickly. - if (ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) { + if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) { if (CI->getZExtValue() == 0) continue; uint64_t Offs = TD.getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue(); @@ -316,51 +328,56 @@ bool FastISel::SelectGetElementPtr(User *I) { return true; } -bool FastISel::SelectCall(User *I) { - Function *F = cast<CallInst>(I)->getCalledFunction(); +bool FastISel::SelectCall(const User *I) { + const Function *F = cast<CallInst>(I)->getCalledFunction(); if (!F) return false; + // Handle selected intrinsic function calls. unsigned IID = F->getIntrinsicID(); switch (IID) { default: break; case Intrinsic::dbg_declare: { - DbgDeclareInst *DI = cast<DbgDeclareInst>(I); + const DbgDeclareInst *DI = cast<DbgDeclareInst>(I); if (!DIDescriptor::ValidDebugInfo(DI->getVariable(), CodeGenOpt::None) || !MF.getMMI().hasDebugInfo()) return true; - Value *Address = DI->getAddress(); + const Value *Address = DI->getAddress(); if (!Address) return true; - AllocaInst *AI = dyn_cast<AllocaInst>(Address); + if (isa<UndefValue>(Address)) + return true; + const AllocaInst *AI = dyn_cast<AllocaInst>(Address); // Don't handle byval struct arguments or VLAs, for example. - if (!AI) break; - DenseMap<const AllocaInst*, int>::iterator SI = - StaticAllocaMap.find(AI); - if (SI == StaticAllocaMap.end()) break; // VLAs. - int FI = SI->second; - if (!DI->getDebugLoc().isUnknown()) - MF.getMMI().setVariableDbgInfo(DI->getVariable(), FI, DI->getDebugLoc()); - - // Building the map above is target independent. Generating DBG_VALUE - // inline is target dependent; do this now. - (void)TargetSelectInstruction(cast<Instruction>(I)); + // Note that if we have a byval struct argument, fast ISel is turned off; + // those are handled in SelectionDAGBuilder. + if (AI) { + DenseMap<const AllocaInst*, int>::iterator SI = + StaticAllocaMap.find(AI); + if (SI == StaticAllocaMap.end()) break; // VLAs. + int FI = SI->second; + if (!DI->getDebugLoc().isUnknown()) + MF.getMMI().setVariableDbgInfo(DI->getVariable(), FI, DI->getDebugLoc()); + } else + // Building the map above is target independent. Generating DBG_VALUE + // inline is target dependent; do this now. + (void)TargetSelectInstruction(cast<Instruction>(I)); return true; } case Intrinsic::dbg_value: { - // This requires target support, but right now X86 is the only Fast target. - DbgValueInst *DI = cast<DbgValueInst>(I); + // This form of DBG_VALUE is target-independent. + const DbgValueInst *DI = cast<DbgValueInst>(I); const TargetInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE); - Value *V = DI->getValue(); + const Value *V = DI->getValue(); if (!V) { // Currently the optimizer can produce this; insert an undef to // help debugging. Probably the optimizer should not do this. BuildMI(MBB, DL, II).addReg(0U).addImm(DI->getOffset()). addMetadata(DI->getVariable()); - } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) { + } else if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) { BuildMI(MBB, DL, II).addImm(CI->getZExtValue()).addImm(DI->getOffset()). addMetadata(DI->getVariable()); - } else if (ConstantFP *CF = dyn_cast<ConstantFP>(V)) { + } else if (const ConstantFP *CF = dyn_cast<ConstantFP>(V)) { BuildMI(MBB, DL, II).addFPImm(CF).addImm(DI->getOffset()). addMetadata(DI->getVariable()); } else if (unsigned Reg = lookUpRegForValue(V)) { @@ -438,10 +455,12 @@ bool FastISel::SelectCall(User *I) { break; } } + + // An arbitrary call. Bail. return false; } -bool FastISel::SelectCast(User *I, unsigned Opcode) { +bool FastISel::SelectCast(const User *I, unsigned Opcode) { EVT SrcVT = TLI.getValueType(I->getOperand(0)->getType()); EVT DstVT = TLI.getValueType(I->getType()); @@ -493,7 +512,7 @@ bool FastISel::SelectCast(User *I, unsigned Opcode) { return true; } -bool FastISel::SelectBitCast(User *I) { +bool FastISel::SelectBitCast(const User *I) { // If the bitcast doesn't change the type, just use the operand value. if (I->getType() == I->getOperand(0)->getType()) { unsigned Reg = getRegForValue(I->getOperand(0)); @@ -544,15 +563,28 @@ bool FastISel::SelectBitCast(User *I) { } bool -FastISel::SelectInstruction(Instruction *I) { +FastISel::SelectInstruction(const Instruction *I) { + // Just before the terminator instruction, insert instructions to + // feed PHI nodes in successor blocks. + if (isa<TerminatorInst>(I)) + if (!HandlePHINodesInSuccessorBlocks(I->getParent())) + return false; + + DL = I->getDebugLoc(); + // First, try doing target-independent selection. - if (SelectOperator(I, I->getOpcode())) + if (SelectOperator(I, I->getOpcode())) { + DL = DebugLoc(); return true; + } // Next, try calling the target to attempt to handle the instruction. - if (TargetSelectInstruction(I)) + if (TargetSelectInstruction(I)) { + DL = DebugLoc(); return true; + } + DL = DebugLoc(); return false; } @@ -573,7 +605,7 @@ FastISel::FastEmitBranch(MachineBasicBlock *MSucc) { /// SelectFNeg - Emit an FNeg operation. /// bool -FastISel::SelectFNeg(User *I) { +FastISel::SelectFNeg(const User *I) { unsigned OpReg = getRegForValue(BinaryOperator::getFNegArgument(I)); if (OpReg == 0) return false; @@ -614,7 +646,7 @@ FastISel::SelectFNeg(User *I) { } bool -FastISel::SelectOperator(User *I, unsigned Opcode) { +FastISel::SelectOperator(const User *I, unsigned Opcode) { switch (Opcode) { case Instruction::Add: return SelectBinaryOp(I, ISD::ADD); @@ -660,10 +692,10 @@ FastISel::SelectOperator(User *I, unsigned Opcode) { return SelectGetElementPtr(I); case Instruction::Br: { - BranchInst *BI = cast<BranchInst>(I); + const BranchInst *BI = cast<BranchInst>(I); if (BI->isUnconditional()) { - BasicBlock *LLVMSucc = BI->getSuccessor(0); + const BasicBlock *LLVMSucc = BI->getSuccessor(0); MachineBasicBlock *MSucc = MBBMap[LLVMSucc]; FastEmitBranch(MSucc); return true; @@ -678,10 +710,6 @@ FastISel::SelectOperator(User *I, unsigned Opcode) { // Nothing to emit. return true; - case Instruction::PHI: - // PHI nodes are already emitted. - return true; - case Instruction::Alloca: // FunctionLowering has the static-sized case covered. if (StaticAllocaMap.count(cast<AllocaInst>(I))) @@ -721,6 +749,9 @@ FastISel::SelectOperator(User *I, unsigned Opcode) { return true; } + case Instruction::PHI: + llvm_unreachable("FastISel shouldn't visit PHI nodes!"); + default: // Unhandled instruction. Halt "fast" selection and bail. return false; @@ -730,15 +761,17 @@ FastISel::SelectOperator(User *I, unsigned Opcode) { FastISel::FastISel(MachineFunction &mf, DenseMap<const Value *, unsigned> &vm, DenseMap<const BasicBlock *, MachineBasicBlock *> &bm, - DenseMap<const AllocaInst *, int> &am + DenseMap<const AllocaInst *, int> &am, + std::vector<std::pair<MachineInstr*, unsigned> > &pn #ifndef NDEBUG - , SmallSet<Instruction*, 8> &cil + , SmallSet<const Instruction *, 8> &cil #endif ) : MBB(0), ValueMap(vm), MBBMap(bm), StaticAllocaMap(am), + PHINodesToUpdate(pn), #ifndef NDEBUG CatchInfoLost(cil), #endif @@ -775,7 +808,7 @@ unsigned FastISel::FastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) { } unsigned FastISel::FastEmit_f(MVT, MVT, - unsigned, ConstantFP * /*FPImm*/) { + unsigned, const ConstantFP * /*FPImm*/) { return 0; } @@ -787,7 +820,7 @@ unsigned FastISel::FastEmit_ri(MVT, MVT, unsigned FastISel::FastEmit_rf(MVT, MVT, unsigned, unsigned /*Op0*/, - ConstantFP * /*FPImm*/) { + const ConstantFP * /*FPImm*/) { return 0; } @@ -820,7 +853,7 @@ unsigned FastISel::FastEmit_ri_(MVT VT, unsigned Opcode, /// FastEmit_rf. If that fails, it materializes the immediate into a register /// and try FastEmit_rr instead. unsigned FastISel::FastEmit_rf_(MVT VT, unsigned Opcode, - unsigned Op0, ConstantFP *FPImm, + unsigned Op0, const ConstantFP *FPImm, MVT ImmType) { // First check if immediate type is legal. If not, we can't use the rf form. unsigned ResultReg = FastEmit_rf(VT, VT, Opcode, Op0, FPImm); @@ -930,7 +963,7 @@ unsigned FastISel::FastEmitInst_ri(unsigned MachineInstOpcode, unsigned FastISel::FastEmitInst_rf(unsigned MachineInstOpcode, const TargetRegisterClass *RC, - unsigned Op0, ConstantFP *FPImm) { + unsigned Op0, const ConstantFP *FPImm) { unsigned ResultReg = createResultReg(RC); const TargetInstrDesc &II = TII.get(MachineInstOpcode); @@ -1006,3 +1039,67 @@ unsigned FastISel::FastEmitInst_extractsubreg(MVT RetVT, unsigned FastISel::FastEmitZExtFromI1(MVT VT, unsigned Op) { return FastEmit_ri(VT, VT, ISD::AND, Op, 1); } + +/// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks. +/// Emit code to ensure constants are copied into registers when needed. +/// Remember the virtual registers that need to be added to the Machine PHI +/// nodes as input. We cannot just directly add them, because expansion +/// might result in multiple MBB's for one BB. As such, the start of the +/// BB might correspond to a different MBB than the end. +bool FastISel::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { + const TerminatorInst *TI = LLVMBB->getTerminator(); + + SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled; + unsigned OrigNumPHINodesToUpdate = PHINodesToUpdate.size(); + + // Check successor nodes' PHI nodes that expect a constant to be available + // from this block. + for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) { + const BasicBlock *SuccBB = TI->getSuccessor(succ); + if (!isa<PHINode>(SuccBB->begin())) continue; + MachineBasicBlock *SuccMBB = MBBMap[SuccBB]; + + // If this terminator has multiple identical successors (common for + // switches), only handle each succ once. + if (!SuccsHandled.insert(SuccMBB)) continue; + + MachineBasicBlock::iterator MBBI = SuccMBB->begin(); + + // At this point we know that there is a 1-1 correspondence between LLVM PHI + // nodes and Machine PHI nodes, but the incoming operands have not been + // emitted yet. + for (BasicBlock::const_iterator I = SuccBB->begin(); + const PHINode *PN = dyn_cast<PHINode>(I); ++I) { + // Ignore dead phi's. + if (PN->use_empty()) continue; + + // Only handle legal types. Two interesting things to note here. First, + // by bailing out early, we may leave behind some dead instructions, + // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its + // own moves. Second, this check is necessary becuase FastISel doesn't + // use CreateRegForValue to create registers, so it always creates + // exactly one register for each non-void instruction. + EVT VT = TLI.getValueType(PN->getType(), /*AllowUnknown=*/true); + if (VT == MVT::Other || !TLI.isTypeLegal(VT)) { + // Promote MVT::i1. + if (VT == MVT::i1) + VT = TLI.getTypeToTransformTo(LLVMBB->getContext(), VT); + else { + PHINodesToUpdate.resize(OrigNumPHINodesToUpdate); + return false; + } + } + + const Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB); + + unsigned Reg = getRegForValue(PHIOp); + if (Reg == 0) { + PHINodesToUpdate.resize(OrigNumPHINodesToUpdate); + return false; + } + PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg)); + } + } + + return true; +} diff --git a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp index 4fb2aa2..65c36c1 100644 --- a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -14,19 +14,18 @@ #define DEBUG_TYPE "function-lowering-info" #include "FunctionLoweringInfo.h" -#include "llvm/CallingConv.h" #include "llvm/DerivedTypes.h" #include "llvm/Function.h" #include "llvm/Instructions.h" #include "llvm/IntrinsicInst.h" #include "llvm/LLVMContext.h" #include "llvm/Module.h" +#include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/Analysis/DebugInfo.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetFrameInfo.h" @@ -34,99 +33,21 @@ #include "llvm/Target/TargetIntrinsicInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetOptions.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" -#include "llvm/Support/raw_ostream.h" #include <algorithm> using namespace llvm; -/// ComputeLinearIndex - Given an LLVM IR aggregate type and a sequence -/// of insertvalue or extractvalue indices that identify a member, return -/// the linearized index of the start of the member. -/// -unsigned llvm::ComputeLinearIndex(const TargetLowering &TLI, const Type *Ty, - const unsigned *Indices, - const unsigned *IndicesEnd, - unsigned CurIndex) { - // Base case: We're done. - if (Indices && Indices == IndicesEnd) - return CurIndex; - - // Given a struct type, recursively traverse the elements. - if (const StructType *STy = dyn_cast<StructType>(Ty)) { - for (StructType::element_iterator EB = STy->element_begin(), - EI = EB, - EE = STy->element_end(); - EI != EE; ++EI) { - if (Indices && *Indices == unsigned(EI - EB)) - return ComputeLinearIndex(TLI, *EI, Indices+1, IndicesEnd, CurIndex); - CurIndex = ComputeLinearIndex(TLI, *EI, 0, 0, CurIndex); - } - return CurIndex; - } - // Given an array type, recursively traverse the elements. - else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { - const Type *EltTy = ATy->getElementType(); - for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) { - if (Indices && *Indices == i) - return ComputeLinearIndex(TLI, EltTy, Indices+1, IndicesEnd, CurIndex); - CurIndex = ComputeLinearIndex(TLI, EltTy, 0, 0, CurIndex); - } - return CurIndex; - } - // We haven't found the type we're looking for, so keep searching. - return CurIndex + 1; -} - -/// ComputeValueVTs - Given an LLVM IR type, compute a sequence of -/// EVTs that represent all the individual underlying -/// non-aggregate types that comprise it. -/// -/// If Offsets is non-null, it points to a vector to be filled in -/// with the in-memory offsets of each of the individual values. -/// -void llvm::ComputeValueVTs(const TargetLowering &TLI, const Type *Ty, - SmallVectorImpl<EVT> &ValueVTs, - SmallVectorImpl<uint64_t> *Offsets, - uint64_t StartingOffset) { - // Given a struct type, recursively traverse the elements. - if (const StructType *STy = dyn_cast<StructType>(Ty)) { - const StructLayout *SL = TLI.getTargetData()->getStructLayout(STy); - for (StructType::element_iterator EB = STy->element_begin(), - EI = EB, - EE = STy->element_end(); - EI != EE; ++EI) - ComputeValueVTs(TLI, *EI, ValueVTs, Offsets, - StartingOffset + SL->getElementOffset(EI - EB)); - return; - } - // Given an array type, recursively traverse the elements. - if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { - const Type *EltTy = ATy->getElementType(); - uint64_t EltSize = TLI.getTargetData()->getTypeAllocSize(EltTy); - for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) - ComputeValueVTs(TLI, EltTy, ValueVTs, Offsets, - StartingOffset + i * EltSize); - return; - } - // Interpret void as zero return values. - if (Ty->isVoidTy()) - return; - // Base case: we can get an EVT for this LLVM IR type. - ValueVTs.push_back(TLI.getValueType(Ty)); - if (Offsets) - Offsets->push_back(StartingOffset); -} - /// isUsedOutsideOfDefiningBlock - Return true if this instruction is used by /// PHI nodes or outside of the basic block that defines it, or used by a /// switch or atomic instruction, which may expand to multiple basic blocks. -static bool isUsedOutsideOfDefiningBlock(Instruction *I) { +static bool isUsedOutsideOfDefiningBlock(const Instruction *I) { + if (I->use_empty()) return false; if (isa<PHINode>(I)) return true; - BasicBlock *BB = I->getParent(); - for (Value::use_iterator UI = I->use_begin(), E = I->use_end(); UI != E; ++UI) + const BasicBlock *BB = I->getParent(); + for (Value::const_use_iterator UI = I->use_begin(), E = I->use_end(); + UI != E; ++UI) if (cast<Instruction>(*UI)->getParent() != BB || isa<PHINode>(*UI)) return true; return false; @@ -135,26 +56,25 @@ static bool isUsedOutsideOfDefiningBlock(Instruction *I) { /// isOnlyUsedInEntryBlock - If the specified argument is only used in the /// entry block, return true. This includes arguments used by switches, since /// the switch may expand into multiple basic blocks. -static bool isOnlyUsedInEntryBlock(Argument *A, bool EnableFastISel) { +static bool isOnlyUsedInEntryBlock(const Argument *A, bool EnableFastISel) { // With FastISel active, we may be splitting blocks, so force creation // of virtual registers for all non-dead arguments. - // Don't force virtual registers for byval arguments though, because - // fast-isel can't handle those in all cases. - if (EnableFastISel && !A->hasByValAttr()) + if (EnableFastISel) return A->use_empty(); - BasicBlock *Entry = A->getParent()->begin(); - for (Value::use_iterator UI = A->use_begin(), E = A->use_end(); UI != E; ++UI) + const BasicBlock *Entry = A->getParent()->begin(); + for (Value::const_use_iterator UI = A->use_begin(), E = A->use_end(); + UI != E; ++UI) if (cast<Instruction>(*UI)->getParent() != Entry || isa<SwitchInst>(*UI)) return false; // Use not in entry block. return true; } -FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli) +FunctionLoweringInfo::FunctionLoweringInfo(const TargetLowering &tli) : TLI(tli) { } -void FunctionLoweringInfo::set(Function &fn, MachineFunction &mf, +void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf, bool EnableFastISel) { Fn = &fn; MF = &mf; @@ -162,7 +82,7 @@ void FunctionLoweringInfo::set(Function &fn, MachineFunction &mf, // Create a vreg for each argument register that is not dead and is used // outside of the entry block for the function. - for (Function::arg_iterator AI = Fn->arg_begin(), E = Fn->arg_end(); + for (Function::const_arg_iterator AI = Fn->arg_begin(), E = Fn->arg_end(); AI != E; ++AI) if (!isOnlyUsedInEntryBlock(AI, EnableFastISel)) InitializeRegForValue(AI); @@ -170,10 +90,10 @@ void FunctionLoweringInfo::set(Function &fn, MachineFunction &mf, // Initialize the mapping of values to registers. This is only set up for // instruction values that are used outside of the block that defines // them. - Function::iterator BB = Fn->begin(), EB = Fn->end(); - for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) - if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) - if (ConstantInt *CUI = dyn_cast<ConstantInt>(AI->getArraySize())) { + Function::const_iterator BB = Fn->begin(), EB = Fn->end(); + for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E; ++I) + if (const AllocaInst *AI = dyn_cast<AllocaInst>(I)) + if (const ConstantInt *CUI = dyn_cast<ConstantInt>(AI->getArraySize())) { const Type *Ty = AI->getAllocatedType(); uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(Ty); unsigned Align = @@ -187,8 +107,8 @@ void FunctionLoweringInfo::set(Function &fn, MachineFunction &mf, } for (; BB != EB; ++BB) - for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) - if (!I->use_empty() && isUsedOutsideOfDefiningBlock(I)) + for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E; ++I) + if (isUsedOutsideOfDefiningBlock(I)) if (!isa<AllocaInst>(I) || !StaticAllocaMap.count(cast<AllocaInst>(I))) InitializeRegForValue(I); @@ -196,7 +116,7 @@ void FunctionLoweringInfo::set(Function &fn, MachineFunction &mf, // Create an initial MachineBasicBlock for each LLVM BasicBlock in F. This // also creates the initial PHI MachineInstrs, though none of the input // operands are populated. - for (BB = Fn->begin(), EB = Fn->end(); BB != EB; ++BB) { + for (BB = Fn->begin(); BB != EB; ++BB) { MachineBasicBlock *MBB = mf.CreateMachineBasicBlock(BB); MBBMap[BB] = MBB; MF->push_back(MBB); @@ -209,14 +129,11 @@ void FunctionLoweringInfo::set(Function &fn, MachineFunction &mf, // Create Machine PHI nodes for LLVM PHI nodes, lowering them as // appropriate. - PHINode *PN; - DebugLoc DL; - for (BasicBlock::iterator - I = BB->begin(), E = BB->end(); I != E; ++I) { - - PN = dyn_cast<PHINode>(I); - if (!PN || PN->use_empty()) continue; + for (BasicBlock::const_iterator I = BB->begin(); + const PHINode *PN = dyn_cast<PHINode>(I); ++I) { + if (PN->use_empty()) continue; + DebugLoc DL = PN->getDebugLoc(); unsigned PHIReg = ValueMap[PN]; assert(PHIReg && "PHI node does not have an assigned virtual register!"); @@ -232,12 +149,20 @@ void FunctionLoweringInfo::set(Function &fn, MachineFunction &mf, } } } + + // Mark landing pad blocks. + for (BB = Fn->begin(); BB != EB; ++BB) + if (const InvokeInst *Invoke = dyn_cast<InvokeInst>(BB->getTerminator())) + MBBMap[Invoke->getSuccessor(1)]->setIsLandingPad(); } /// clear - Clear out all the function-specific state. This returns this /// FunctionLoweringInfo to an empty state, ready to be used for a /// different function. void FunctionLoweringInfo::clear() { + assert(CatchInfoFound.size() == CatchInfoLost.size() && + "Not all catch info was assigned to a landing pad!"); + MBBMap.clear(); ValueMap.clear(); StaticAllocaMap.clear(); @@ -246,6 +171,7 @@ void FunctionLoweringInfo::clear() { CatchInfoFound.clear(); #endif LiveOutRegInfo.clear(); + ArgDbgValues.clear(); } unsigned FunctionLoweringInfo::MakeReg(EVT VT) { @@ -277,30 +203,12 @@ unsigned FunctionLoweringInfo::CreateRegForValue(const Value *V) { return FirstReg; } -/// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V. -GlobalVariable *llvm::ExtractTypeInfo(Value *V) { - V = V->stripPointerCasts(); - GlobalVariable *GV = dyn_cast<GlobalVariable>(V); - - if (GV && GV->getName() == ".llvm.eh.catch.all.value") { - assert(GV->hasInitializer() && - "The EH catch-all value must have an initializer"); - Value *Init = GV->getInitializer(); - GV = dyn_cast<GlobalVariable>(Init); - if (!GV) V = cast<ConstantPointerNull>(Init); - } - - assert((GV || isa<ConstantPointerNull>(V)) && - "TypeInfo must be a global variable or NULL"); - return GV; -} - /// AddCatchInfo - Extract the personality and type infos from an eh.selector /// call, and add them to the specified machine basic block. -void llvm::AddCatchInfo(CallInst &I, MachineModuleInfo *MMI, +void llvm::AddCatchInfo(const CallInst &I, MachineModuleInfo *MMI, MachineBasicBlock *MBB) { // Inform the MachineModuleInfo of the personality for this landing pad. - ConstantExpr *CE = cast<ConstantExpr>(I.getOperand(2)); + const ConstantExpr *CE = cast<ConstantExpr>(I.getOperand(2)); assert(CE->getOpcode() == Instruction::BitCast && isa<Function>(CE->getOperand(0)) && "Personality should be a function"); @@ -308,11 +216,11 @@ void llvm::AddCatchInfo(CallInst &I, MachineModuleInfo *MMI, // Gather all the type infos for this landing pad and pass them along to // MachineModuleInfo. - std::vector<GlobalVariable *> TyInfo; + std::vector<const GlobalVariable *> TyInfo; unsigned N = I.getNumOperands(); for (unsigned i = N - 1; i > 2; --i) { - if (ConstantInt *CI = dyn_cast<ConstantInt>(I.getOperand(i))) { + if (const ConstantInt *CI = dyn_cast<ConstantInt>(I.getOperand(i))) { unsigned FilterLength = CI->getZExtValue(); unsigned FirstCatch = i + FilterLength + !FilterLength; assert (FirstCatch <= N && "Invalid filter length"); @@ -349,10 +257,11 @@ void llvm::AddCatchInfo(CallInst &I, MachineModuleInfo *MMI, } } -void llvm::CopyCatchInfo(BasicBlock *SrcBB, BasicBlock *DestBB, +void llvm::CopyCatchInfo(const BasicBlock *SrcBB, const BasicBlock *DestBB, MachineModuleInfo *MMI, FunctionLoweringInfo &FLI) { - for (BasicBlock::iterator I = SrcBB->begin(), E = --SrcBB->end(); I != E; ++I) - if (EHSelectorInst *EHSel = dyn_cast<EHSelectorInst>(I)) { + for (BasicBlock::const_iterator I = SrcBB->begin(), E = --SrcBB->end(); + I != E; ++I) + if (const EHSelectorInst *EHSel = dyn_cast<EHSelectorInst>(I)) { // Apply the catch info to DestBB. AddCatchInfo(*EHSel, MMI, FLI.MBBMap[DestBB]); #ifndef NDEBUG diff --git a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.h b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.h index d851e64..4067a5b 100644 --- a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.h +++ b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.h @@ -15,12 +15,17 @@ #ifndef FUNCTIONLOWERINGINFO_H #define FUNCTIONLOWERINGINFO_H +#include "llvm/InlineAsm.h" +#include "llvm/Instructions.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" #ifndef NDEBUG #include "llvm/ADT/SmallSet.h" #endif #include "llvm/CodeGen/ValueTypes.h" +#include "llvm/CodeGen/ISDOpcodes.h" +#include "llvm/Support/CallSite.h" #include <vector> namespace llvm { @@ -31,6 +36,7 @@ class CallInst; class Function; class GlobalVariable; class Instruction; +class MachineInstr; class MachineBasicBlock; class MachineFunction; class MachineModuleInfo; @@ -44,8 +50,8 @@ class Value; /// class FunctionLoweringInfo { public: - TargetLowering &TLI; - Function *Fn; + const TargetLowering &TLI; + const Function *Fn; MachineFunction *MF; MachineRegisterInfo *RegInfo; @@ -57,13 +63,6 @@ public: /// allocated to hold a pointer to the hidden sret parameter. unsigned DemoteRegister; - explicit FunctionLoweringInfo(TargetLowering &TLI); - - /// set - Initialize this FunctionLoweringInfo with the given Function - /// and its associated MachineFunction. - /// - void set(Function &Fn, MachineFunction &MF, bool EnableFastISel); - /// MBBMap - A mapping from LLVM basic blocks to their machine code entry. DenseMap<const BasicBlock*, MachineBasicBlock *> MBBMap; @@ -77,27 +76,15 @@ public: /// anywhere in the function. DenseMap<const AllocaInst*, int> StaticAllocaMap; + /// ArgDbgValues - A list of DBG_VALUE instructions created during isel for + /// function arguments that are inserted after scheduling is completed. + SmallVector<MachineInstr*, 8> ArgDbgValues; + #ifndef NDEBUG - SmallSet<Instruction*, 8> CatchInfoLost; - SmallSet<Instruction*, 8> CatchInfoFound; + SmallSet<const Instruction *, 8> CatchInfoLost; + SmallSet<const Instruction *, 8> CatchInfoFound; #endif - unsigned MakeReg(EVT VT); - - /// isExportedInst - Return true if the specified value is an instruction - /// exported from its block. - bool isExportedInst(const Value *V) { - return ValueMap.count(V); - } - - unsigned CreateRegForValue(const Value *V); - - unsigned InitializeRegForValue(const Value *V) { - unsigned &R = ValueMap[V]; - assert(R == 0 && "Already initialized this value register!"); - return R = CreateRegForValue(V); - } - struct LiveOutInfo { unsigned NumSignBits; APInt KnownOne, KnownZero; @@ -108,42 +95,48 @@ public: /// register number offset by 'FirstVirtualRegister'. std::vector<LiveOutInfo> LiveOutRegInfo; + /// PHINodesToUpdate - A list of phi instructions whose operand list will + /// be updated after processing the current basic block. + /// TODO: This isn't per-function state, it's per-basic-block state. But + /// there's no other convenient place for it to live right now. + std::vector<std::pair<MachineInstr*, unsigned> > PHINodesToUpdate; + + explicit FunctionLoweringInfo(const TargetLowering &TLI); + + /// set - Initialize this FunctionLoweringInfo with the given Function + /// and its associated MachineFunction. + /// + void set(const Function &Fn, MachineFunction &MF, bool EnableFastISel); + /// clear - Clear out all the function-specific state. This returns this /// FunctionLoweringInfo to an empty state, ready to be used for a /// different function. void clear(); -}; -/// ComputeLinearIndex - Given an LLVM IR aggregate type and a sequence -/// of insertvalue or extractvalue indices that identify a member, return -/// the linearized index of the start of the member. -/// -unsigned ComputeLinearIndex(const TargetLowering &TLI, const Type *Ty, - const unsigned *Indices, - const unsigned *IndicesEnd, - unsigned CurIndex = 0); - -/// ComputeValueVTs - Given an LLVM IR type, compute a sequence of -/// EVTs that represent all the individual underlying -/// non-aggregate types that comprise it. -/// -/// If Offsets is non-null, it points to a vector to be filled in -/// with the in-memory offsets of each of the individual values. -/// -void ComputeValueVTs(const TargetLowering &TLI, const Type *Ty, - SmallVectorImpl<EVT> &ValueVTs, - SmallVectorImpl<uint64_t> *Offsets = 0, - uint64_t StartingOffset = 0); + unsigned MakeReg(EVT VT); + + /// isExportedInst - Return true if the specified value is an instruction + /// exported from its block. + bool isExportedInst(const Value *V) { + return ValueMap.count(V); + } -/// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V. -GlobalVariable *ExtractTypeInfo(Value *V); + unsigned CreateRegForValue(const Value *V); + + unsigned InitializeRegForValue(const Value *V) { + unsigned &R = ValueMap[V]; + assert(R == 0 && "Already initialized this value register!"); + return R = CreateRegForValue(V); + } +}; /// AddCatchInfo - Extract the personality and type infos from an eh.selector /// call, and add them to the specified machine basic block. -void AddCatchInfo(CallInst &I, MachineModuleInfo *MMI, MachineBasicBlock *MBB); +void AddCatchInfo(const CallInst &I, + MachineModuleInfo *MMI, MachineBasicBlock *MBB); /// CopyCatchInfo - Copy catch information from DestBB to SrcBB. -void CopyCatchInfo(BasicBlock *SrcBB, BasicBlock *DestBB, +void CopyCatchInfo(const BasicBlock *SrcBB, const BasicBlock *DestBB, MachineModuleInfo *MMI, FunctionLoweringInfo &FLI); } // end namespace llvm diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 28ba343..c5dae82 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -296,8 +296,19 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, } } + // If this value has only one use, that use is a kill. This is a + // conservative approximation. Tied operands are never killed, so we need + // to check that. And that means we need to determine the index of the + // operand. + unsigned Idx = MI->getNumOperands(); + while (Idx > 0 && + MI->getOperand(Idx-1).isReg() && MI->getOperand(Idx-1).isImplicit()) + --Idx; + bool isTied = MI->getDesc().getOperandConstraint(Idx, TOI::TIED_TO) != -1; + bool isKill = Op.hasOneUse() && !isTied && !IsDebug; + MI->addOperand(MachineOperand::CreateReg(VReg, isOptDef, - false/*isImp*/, false/*isKill*/, + false/*isImp*/, isKill, false/*isDead*/, false/*isUndef*/, false/*isEarlyClobber*/, 0/*SubReg*/, IsDebug)); @@ -440,8 +451,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, unsigned SubIdx = cast<ConstantSDNode>(N2)->getZExtValue(); const TargetRegisterClass *TRC = MRI->getRegClass(SubReg); const TargetRegisterClass *SRC = - getSuperRegisterRegClass(TRC, SubIdx, - Node->getValueType(0)); + getSuperRegisterRegClass(TRC, SubIdx, Node->getValueType(0)); // Figure out the register class to create for the destreg. // Note that if we're going to directly use an existing register, @@ -504,41 +514,83 @@ InstrEmitter::EmitCopyToRegClassNode(SDNode *Node, assert(isNew && "Node emitted out of order - early"); } +/// EmitRegSequence - Generate machine code for REG_SEQUENCE nodes. +/// +void InstrEmitter::EmitRegSequence(SDNode *Node, + DenseMap<SDValue, unsigned> &VRBaseMap) { + const TargetRegisterClass *RC = TLI->getRegClassFor(Node->getValueType(0)); + unsigned NewVReg = MRI->createVirtualRegister(RC); + MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), + TII->get(TargetOpcode::REG_SEQUENCE), NewVReg); + unsigned NumOps = Node->getNumOperands(); + assert((NumOps & 1) == 0 && + "REG_SEQUENCE must have an even number of operands!"); + const TargetInstrDesc &II = TII->get(TargetOpcode::REG_SEQUENCE); + for (unsigned i = 0; i != NumOps; ++i) { + SDValue Op = Node->getOperand(i); +#ifndef NDEBUG + if (i & 1) { + unsigned SubIdx = cast<ConstantSDNode>(Op)->getZExtValue(); + unsigned SubReg = getVR(Node->getOperand(i-1), VRBaseMap); + const TargetRegisterClass *TRC = MRI->getRegClass(SubReg); + const TargetRegisterClass *SRC = + getSuperRegisterRegClass(TRC, SubIdx, Node->getValueType(0)); + assert(SRC == RC && "Invalid subregister index in REG_SEQUENCE"); + } +#endif + AddOperand(MI, Op, i+1, &II, VRBaseMap); + } + + MBB->insert(InsertPos, MI); + SDValue Op(Node, 0); + bool isNew = VRBaseMap.insert(std::make_pair(Op, NewVReg)).second; + isNew = isNew; // Silence compiler warning. + assert(isNew && "Node emitted out of order - early"); +} + /// EmitDbgValue - Generate machine instruction for a dbg_value node. /// -MachineInstr *InstrEmitter::EmitDbgValue(SDDbgValue *SD, - MachineBasicBlock *InsertBB, - DenseMap<SDValue, unsigned> &VRBaseMap, - DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) { +MachineInstr * +InstrEmitter::EmitDbgValue(SDDbgValue *SD, + DenseMap<SDValue, unsigned> &VRBaseMap) { uint64_t Offset = SD->getOffset(); MDNode* MDPtr = SD->getMDPtr(); DebugLoc DL = SD->getDebugLoc(); + if (SD->getKind() == SDDbgValue::FRAMEIX) { + // Stack address; this needs to be lowered in target-dependent fashion. + // EmitTargetCodeForFrameDebugValue is responsible for allocation. + unsigned FrameIx = SD->getFrameIx(); + return TII->emitFrameIndexDebugValue(*MF, FrameIx, Offset, MDPtr, DL); + } + // Otherwise, we're going to create an instruction here. const TargetInstrDesc &II = TII->get(TargetOpcode::DBG_VALUE); MachineInstrBuilder MIB = BuildMI(*MF, DL, II); if (SD->getKind() == SDDbgValue::SDNODE) { - AddOperand(&*MIB, SDValue(SD->getSDNode(), SD->getResNo()), - (*MIB).getNumOperands(), &II, VRBaseMap, true /*IsDebug*/); + SDNode *Node = SD->getSDNode(); + SDValue Op = SDValue(Node, SD->getResNo()); + // It's possible we replaced this SDNode with other(s) and therefore + // didn't generate code for it. It's better to catch these cases where + // they happen and transfer the debug info, but trying to guarantee that + // in all cases would be very fragile; this is a safeguard for any + // that were missed. + DenseMap<SDValue, unsigned>::iterator I = VRBaseMap.find(Op); + if (I==VRBaseMap.end()) + MIB.addReg(0U); // undef + else + AddOperand(&*MIB, Op, (*MIB).getNumOperands(), &II, VRBaseMap, + true /*IsDebug*/); } else if (SD->getKind() == SDDbgValue::CONST) { - Value *V = SD->getConst(); - if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) { + const Value *V = SD->getConst(); + if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) { MIB.addImm(CI->getSExtValue()); - } else if (ConstantFP *CF = dyn_cast<ConstantFP>(V)) { + } else if (const ConstantFP *CF = dyn_cast<ConstantFP>(V)) { MIB.addFPImm(CF); } else { // Could be an Undef. In any case insert an Undef so we can see what we // dropped. MIB.addReg(0U); } - } else if (SD->getKind() == SDDbgValue::FRAMEIX) { - unsigned FrameIx = SD->getFrameIx(); - // Stack address; this needs to be lowered in target-dependent fashion. - // FIXME test that the target supports this somehow; if not emit Undef. - // Create a pseudo for EmitInstrWithCustomInserter's consumption. - MIB.addImm(FrameIx).addImm(Offset).addMetadata(MDPtr); - abort(); - TLI->EmitInstrWithCustomInserter(&*MIB, InsertBB, EM); - return 0; } else { // Insert an Undef so we can see what we dropped. MIB.addReg(0U); @@ -553,8 +605,7 @@ MachineInstr *InstrEmitter::EmitDbgValue(SDDbgValue *SD, /// void InstrEmitter:: EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, - DenseMap<SDValue, unsigned> &VRBaseMap, - DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) { + DenseMap<SDValue, unsigned> &VRBaseMap) { unsigned Opc = Node->getMachineOpcode(); // Handle subreg insert/extract specially @@ -571,6 +622,12 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, return; } + // Handle REG_SEQUENCE specially. + if (Opc == TargetOpcode::REG_SEQUENCE) { + EmitRegSequence(Node, VRBaseMap); + return; + } + if (Opc == TargetOpcode::IMPLICIT_DEF) // We want a unique VR for each IMPLICIT_DEF use. return; @@ -615,7 +672,7 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, if (II.usesCustomInsertionHook()) { // Insert this instruction into the basic block using a target // specific inserter which may returns a new basic block. - MBB = TLI->EmitInstrWithCustomInserter(MI, MBB, EM); + MBB = TLI->EmitInstrWithCustomInserter(MI, MBB); InsertPos = MBB->end(); return; } @@ -646,7 +703,6 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, i != e; ++i) MI->addRegisterDead(IDList[i-II.getNumDefs()], TRI); } - return; } /// EmitSpecialNode - Generate machine code for a target-independent node and @@ -720,12 +776,12 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, TII->get(TargetOpcode::INLINEASM)); // Add the asm string as an external symbol operand. - const char *AsmStr = - cast<ExternalSymbolSDNode>(Node->getOperand(1))->getSymbol(); + SDValue AsmStrV = Node->getOperand(InlineAsm::Op_AsmString); + const char *AsmStr = cast<ExternalSymbolSDNode>(AsmStrV)->getSymbol(); MI->addOperand(MachineOperand::CreateES(AsmStr)); // Add all of the operand registers to the instruction. - for (unsigned i = 2; i != NumOps;) { + for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) { unsigned Flags = cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue(); unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); @@ -733,24 +789,24 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, MI->addOperand(MachineOperand::CreateImm(Flags)); ++i; // Skip the ID value. - switch (Flags & 7) { + switch (InlineAsm::getKind(Flags)) { default: llvm_unreachable("Bad flags!"); - case 2: // Def of register. + case InlineAsm::Kind_RegDef: for (; NumVals; --NumVals, ++i) { unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); MI->addOperand(MachineOperand::CreateReg(Reg, true)); } break; - case 6: // Def of earlyclobber register. + case InlineAsm::Kind_RegDefEarlyClobber: for (; NumVals; --NumVals, ++i) { unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); MI->addOperand(MachineOperand::CreateReg(Reg, true, false, false, false, false, true)); } break; - case 1: // Use of register. - case 3: // Immediate. - case 4: // Addressing mode. + case InlineAsm::Kind_RegUse: // Use of register. + case InlineAsm::Kind_Imm: // Immediate. + case InlineAsm::Kind_Mem: // Addressing mode. // The addressing mode has been selected, just add all of the // operands to the machine instruction. for (; NumVals; --NumVals, ++i) @@ -758,6 +814,13 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, break; } } + + // Get the mdnode from the asm if it exists and add it to the instruction. + SDValue MDV = Node->getOperand(InlineAsm::Op_MDNode); + const MDNode *MD = cast<MDNodeSDNode>(MDV)->getMD(); + if (MD) + MI->addOperand(MachineOperand::CreateMetadata(MD)); + MBB->insert(InsertPos, MI); break; } diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.h b/lib/CodeGen/SelectionDAG/InstrEmitter.h index baabb75..c7e7c71 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.h +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.h @@ -88,6 +88,9 @@ class InstrEmitter { void EmitCopyToRegClassNode(SDNode *Node, DenseMap<SDValue, unsigned> &VRBaseMap); + /// EmitRegSequence - Generate machine code for REG_SEQUENCE nodes. + /// + void EmitRegSequence(SDNode *Node, DenseMap<SDValue, unsigned> &VRBaseMap); public: /// CountResults - The results of target nodes have register or immediate /// operands first, then an optional chain, and optional flag operands @@ -103,17 +106,14 @@ public: /// EmitDbgValue - Generate machine instruction for a dbg_value node. /// MachineInstr *EmitDbgValue(SDDbgValue *SD, - MachineBasicBlock *InsertBB, - DenseMap<SDValue, unsigned> &VRBaseMap, - DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM); + DenseMap<SDValue, unsigned> &VRBaseMap); /// EmitNode - Generate machine code for a node and needed dependencies. /// void EmitNode(SDNode *Node, bool IsClone, bool IsCloned, - DenseMap<SDValue, unsigned> &VRBaseMap, - DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) { + DenseMap<SDValue, unsigned> &VRBaseMap) { if (Node->isMachineOpcode()) - EmitMachineNode(Node, IsClone, IsCloned, VRBaseMap, EM); + EmitMachineNode(Node, IsClone, IsCloned, VRBaseMap); else EmitSpecialNode(Node, IsClone, IsCloned, VRBaseMap); } @@ -130,8 +130,7 @@ public: private: void EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, - DenseMap<SDValue, unsigned> &VRBaseMap, - DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM); + DenseMap<SDValue, unsigned> &VRBaseMap); void EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, DenseMap<SDValue, unsigned> &VRBaseMap); }; diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index d35f0da..bedfa57 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -32,13 +32,11 @@ #include "llvm/LLVMContext.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SmallPtrSet.h" -#include <map> using namespace llvm; //===----------------------------------------------------------------------===// @@ -55,7 +53,8 @@ using namespace llvm; /// namespace { class SelectionDAGLegalize { - TargetLowering &TLI; + const TargetMachine &TM; + const TargetLowering &TLI; SelectionDAG &DAG; CodeGenOpt::Level OptLevel; @@ -213,7 +212,8 @@ SelectionDAGLegalize::ShuffleWithNarrowerEltType(EVT NVT, EVT VT, DebugLoc dl, SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag, CodeGenOpt::Level ol) - : TLI(dag.getTargetLoweringInfo()), DAG(dag), OptLevel(ol), + : TM(dag.getTarget()), TLI(dag.getTargetLoweringInfo()), + DAG(dag), OptLevel(ol), ValueTypeActions(TLI.getValueTypeActions()) { assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE && "Too many value types for ValueTypeActions to hold!"); @@ -409,7 +409,9 @@ SDValue ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, // to the final destination using (unaligned) integer loads and stores. EVT StoredVT = ST->getMemoryVT(); EVT RegVT = - TLI.getRegisterType(*DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), StoredVT.getSizeInBits())); + TLI.getRegisterType(*DAG.getContext(), + EVT::getIntegerVT(*DAG.getContext(), + StoredVT.getSizeInBits())); unsigned StoredBytes = StoredVT.getSizeInBits() / 8; unsigned RegBytes = RegVT.getSizeInBits() / 8; unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes; @@ -445,7 +447,8 @@ SDValue ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, // The last store may be partial. Do a truncating store. On big-endian // machines this requires an extending load from the stack slot to ensure // that the bits are in the right place. - EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 8 * (StoredBytes - Offset)); + EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), + 8 * (StoredBytes - Offset)); // Load from the stack slot. SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr, @@ -548,7 +551,8 @@ SDValue ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, } // The last copy may be partial. Do an extending load. - EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 8 * (LoadedBytes - Offset)); + EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), + 8 * (LoadedBytes - Offset)); SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, LD->getSrcValue(), SVOffset + Offset, MemVT, LD->isVolatile(), @@ -968,11 +972,11 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Node->dump( &DAG); dbgs() << "\n"; #endif - llvm_unreachable("Do not know how to legalize this operator!"); + assert(0 && "Do not know how to legalize this operator!"); case ISD::BUILD_VECTOR: switch (TLI.getOperationAction(ISD::BUILD_VECTOR, Node->getValueType(0))) { - default: llvm_unreachable("This action is not supported yet!"); + default: assert(0 && "This action is not supported yet!"); case TargetLowering::Custom: Tmp3 = TLI.LowerOperation(Result, DAG); if (Tmp3.getNode()) { @@ -1089,7 +1093,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp4 = Result.getValue(1); switch (TLI.getOperationAction(Node->getOpcode(), VT)) { - default: llvm_unreachable("This action is not supported yet!"); + default: assert(0 && "This action is not supported yet!"); case TargetLowering::Legal: // If this is an unaligned load and the target doesn't support it, // expand it. @@ -1259,7 +1263,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp2 = LegalizeOp(Ch); } else { switch (TLI.getLoadExtAction(ExtType, SrcVT)) { - default: llvm_unreachable("This action is not supported yet!"); + default: assert(0 && "This action is not supported yet!"); case TargetLowering::Custom: isCustom = true; // FALLTHROUGH @@ -1357,7 +1361,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { EVT VT = Tmp3.getValueType(); switch (TLI.getOperationAction(ISD::STORE, VT)) { - default: llvm_unreachable("This action is not supported yet!"); + default: assert(0 && "This action is not supported yet!"); case TargetLowering::Legal: // If this is an unaligned store and the target doesn't support it, // expand it. @@ -1394,7 +1398,8 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // Promote to a byte-sized store with upper bits zero if not // storing an integral number of bytes. For example, promote // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) - EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StVT.getStoreSizeInBits()); + EVT NVT = EVT::getIntegerVT(*DAG.getContext(), + StVT.getStoreSizeInBits()); Tmp3 = DAG.getZeroExtendInReg(Tmp3, dl, StVT); Result = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getSrcValue(), SVOffset, NVT, isVolatile, isNonTemporal, @@ -1459,7 +1464,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { ST->getOffset()); switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { - default: llvm_unreachable("This action is not supported yet!"); + default: assert(0 && "This action is not supported yet!"); case TargetLowering::Legal: // If this is an unaligned store and the target doesn't support it, // expand it. @@ -1658,8 +1663,7 @@ void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); Chain = SP.getValue(1); unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); - unsigned StackAlign = - TLI.getTargetMachine().getFrameInfo()->getStackAlignment(); + unsigned StackAlign = TM.getFrameInfo()->getStackAlignment(); if (Align > StackAlign) SP = DAG.getNode(ISD::AND, dl, VT, SP, DAG.getConstant(-(uint64_t)Align, VT)); @@ -1683,7 +1687,7 @@ void SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, EVT OpVT = LHS.getValueType(); ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); switch (TLI.getCondCodeAction(CCCode, OpVT)) { - default: llvm_unreachable("Unknown condition code action!"); + default: assert(0 && "Unknown condition code action!"); case TargetLowering::Legal: // Nothing to do. break; @@ -1691,7 +1695,7 @@ void SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; unsigned Opc = 0; switch (CCCode) { - default: llvm_unreachable("Don't know how to expand this condition!"); + default: assert(0 && "Don't know how to expand this condition!"); case ISD::SETOEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETO; Opc = ISD::AND; break; case ISD::SETOGT: CC1 = ISD::SETGT; CC2 = ISD::SETO; Opc = ISD::AND; break; case ISD::SETOGE: CC1 = ISD::SETGE; CC2 = ISD::SETO; Opc = ISD::AND; break; @@ -1738,8 +1742,8 @@ SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); unsigned SlotSize = SlotVT.getSizeInBits(); unsigned DestSize = DestVT.getSizeInBits(); - unsigned DestAlign = - TLI.getTargetData()->getPrefTypeAlignment(DestVT.getTypeForEVT(*DAG.getContext())); + const Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); + unsigned DestAlign = TLI.getTargetData()->getPrefTypeAlignment(DestType); // Emit a store to the stack slot. Use a truncstore if the input value is // later than DestVT. @@ -1930,7 +1934,7 @@ SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, RTLIB::Libcall Call_PPCF128) { RTLIB::Libcall LC; switch (Node->getValueType(0).getSimpleVT().SimpleTy) { - default: llvm_unreachable("Unexpected request for libcall!"); + default: assert(0 && "Unexpected request for libcall!"); case MVT::f32: LC = Call_F32; break; case MVT::f64: LC = Call_F64; break; case MVT::f80: LC = Call_F80; break; @@ -1947,7 +1951,7 @@ SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, RTLIB::Libcall Call_I128) { RTLIB::Libcall LC; switch (Node->getValueType(0).getSimpleVT().SimpleTy) { - default: llvm_unreachable("Unexpected request for libcall!"); + default: assert(0 && "Unexpected request for libcall!"); case MVT::i8: LC = Call_I8; break; case MVT::i16: LC = Call_I16; break; case MVT::i32: LC = Call_I32; break; @@ -2062,7 +2066,7 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, // offset depending on the data type. uint64_t FF; switch (Op0.getValueType().getSimpleVT().SimpleTy) { - default: llvm_unreachable("Unsupported integer type!"); + default: assert(0 && "Unsupported integer type!"); case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) @@ -2182,7 +2186,7 @@ SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, DebugLoc dl) { EVT SHVT = TLI.getShiftAmountTy(); SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; switch (VT.getSimpleVT().SimpleTy) { - default: llvm_unreachable("Unhandled Expand type in BSWAP!"); + default: assert(0 && "Unhandled Expand type in BSWAP!"); case MVT::i16: Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); @@ -2227,7 +2231,7 @@ SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, DebugLoc dl) { SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, DebugLoc dl) { switch (Opc) { - default: llvm_unreachable("Cannot expand this yet!"); + default: assert(0 && "Cannot expand this yet!"); case ISD::CTPOP: { static const uint64_t mask[6] = { 0x5555555555555555ULL, 0x3333333333333333ULL, @@ -2333,10 +2337,10 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, EVT VT = Node->getValueType(0); if (VT.isInteger()) Results.push_back(DAG.getConstant(0, VT)); - else if (VT.isFloatingPoint()) + else { + assert(VT.isFloatingPoint() && "Unknown value type!"); Results.push_back(DAG.getConstantFP(0, VT)); - else - llvm_unreachable("Unknown value type!"); + } break; } case ISD::TRAP: { @@ -2431,7 +2435,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, // Increment the pointer, VAList, to the next vaarg Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, DAG.getConstant(TLI.getTargetData()-> - getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())), + getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())), TLI.getPointerTy())); // Store the incremented VAList to the legalized pointer Tmp3 = DAG.getStore(VAList.getValue(1), dl, Tmp3, Tmp2, V, 0, @@ -2842,7 +2846,14 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, RHS); TopHalf = BottomHalf.getValue(1); - } else if (TLI.isTypeLegal(EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2))) { + } else { + // FIXME: We should be able to fall back to a libcall with an illegal + // type in some cases. + // Also, we can fall back to a division in some cases, but that's a big + // performance hit in the general case. + assert(TLI.isTypeLegal(EVT::getIntegerVT(*DAG.getContext(), + VT.getSizeInBits() * 2)) && + "Don't know how to expand this operation yet!"); EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2); LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS); RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS); @@ -2851,12 +2862,6 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, DAG.getIntPtrConstant(0)); TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, DAG.getIntPtrConstant(1)); - } else { - // FIXME: We should be able to fall back to a libcall with an illegal - // type in some cases. - // Also, we can fall back to a division in some cases, but that's a big - // performance hit in the general case. - llvm_unreachable("Don't know how to expand this operation yet!"); } if (isSigned) { Tmp1 = DAG.getConstant(VT.getSizeInBits() - 1, TLI.getShiftAmountTy()); @@ -2916,7 +2921,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, PseudoSourceValue::getJumpTable(), 0, MemVT, false, false, 0); Addr = LD; - if (TLI.getTargetMachine().getRelocationModel() == Reloc::PIC_) { + if (TM.getRelocationModel() == Reloc::PIC_) { // For PIC, the sequence is: // BRIND(load(Jumptable + index) + RelocBase) // RelocBase can be JumpTable, GOT or some sort of global base. @@ -3078,11 +3083,10 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node, if (OVT.isVector()) { ExtOp = ISD::BIT_CONVERT; TruncOp = ISD::BIT_CONVERT; - } else if (OVT.isInteger()) { + } else { + assert(OVT.isInteger() && "Cannot promote logic operation"); ExtOp = ISD::ANY_EXTEND; TruncOp = ISD::TRUNCATE; - } else { - llvm_report_error("Cannot promote logic operation"); } // Promote each of the values to the new type. Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); diff --git a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index 665b21f..e3eb949 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -109,14 +109,16 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_BIT_CONVERT(SDNode *N) { SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) { // Convert the inputs to integers, and build a new pair out of them. return DAG.getNode(ISD::BUILD_PAIR, N->getDebugLoc(), - TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), + TLI.getTypeToTransformTo(*DAG.getContext(), + N->getValueType(0)), BitConvertToInteger(N->getOperand(0)), BitConvertToInteger(N->getOperand(1))); } SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(ConstantFPSDNode *N) { return DAG.getConstant(N->getValueAPF().bitcastToAPInt(), - TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0))); + TLI.getTypeToTransformTo(*DAG.getContext(), + N->getValueType(0))); } SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) { @@ -338,7 +340,8 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP32(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue Op = N->getOperand(0); - return MakeLibCall(RTLIB::FPEXT_F16_F32, NVT, &Op, 1, false, N->getDebugLoc()); + return MakeLibCall(RTLIB::FPEXT_F16_F32, NVT, &Op, 1, false, + N->getDebugLoc()); } SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { @@ -489,7 +492,8 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) { } SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) { - return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0))); + return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), + N->getValueType(0))); } SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) { @@ -531,7 +535,8 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) { // Sign/zero extend the argument if the libcall takes a larger type. SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); - return MakeLibCall(LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT), &Op, 1, false, dl); + return MakeLibCall(LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT), + &Op, 1, false, dl); } @@ -1403,7 +1408,8 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { SDValue Chain = ST->getChain(); SDValue Ptr = ST->getBasePtr(); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ST->getValue().getValueType()); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), + ST->getValue().getValueType()); assert(NVT.isByteSized() && "Expanded type not byte sized!"); assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?"); diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 48f64c3..548454c 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -204,7 +204,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BIT_CONVERT(SDNode *N) { std::swap(Lo, Hi); InOp = DAG.getNode(ISD::ANY_EXTEND, dl, - EVT::getIntegerVT(*DAG.getContext(), NOutVT.getSizeInBits()), + EVT::getIntegerVT(*DAG.getContext(), + NOutVT.getSizeInBits()), JoinIntegers(Lo, Hi)); return DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, InOp); } @@ -464,7 +465,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { return DAG.getNode(ISD::SHL, N->getDebugLoc(), - TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), + TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)), GetPromotedInteger(N->getOperand(0)), N->getOperand(1)); } @@ -555,7 +556,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) { } SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { - return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0))); + return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), + N->getValueType(0))); } SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { @@ -1383,7 +1385,8 @@ void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, if (NVTBits < EVTBits) { Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, - DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), EVTBits - NVTBits))); + DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), + EVTBits - NVTBits))); } else { Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); // The high part replicates the sign bit of Lo, make it explicit. @@ -1403,7 +1406,8 @@ void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, if (NVTBits < EVTBits) { Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, - DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), EVTBits - NVTBits))); + DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), + EVTBits - NVTBits))); } else { Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); // The high part must be zero, make it explicit. @@ -1846,7 +1850,8 @@ void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, unsigned ExcessBits = Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, - DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), ExcessBits))); + DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), + ExcessBits))); } } @@ -1968,7 +1973,8 @@ void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, SplitInteger(Res, Lo, Hi); unsigned ExcessBits = Op.getValueType().getSizeInBits() - NVT.getSizeInBits(); - Hi = DAG.getZeroExtendInReg(Hi, dl, EVT::getIntegerVT(*DAG.getContext(), ExcessBits)); + Hi = DAG.getZeroExtendInReg(Hi, dl, + EVT::getIntegerVT(*DAG.getContext(), ExcessBits)); } } @@ -2269,7 +2275,8 @@ SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { unsigned EBytes = ExtVT.getStoreSize(); unsigned IncrementSize = NVT.getSizeInBits()/8; unsigned ExcessBits = (EBytes - IncrementSize)*8; - EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), ExtVT.getSizeInBits() - ExcessBits); + EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), + ExtVT.getSizeInBits() - ExcessBits); if (ExcessBits < NVT.getSizeInBits()) { // Transfer high bits from the top of Lo to the bottom of Hi. diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp index f3e7ca4f..17f131b 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp @@ -721,7 +721,8 @@ void DAGTypeLegalizer::ReplaceValueWith(SDValue From, SDValue To) { } void DAGTypeLegalizer::SetPromotedInteger(SDValue Op, SDValue Result) { - assert(Result.getValueType() == TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) && + assert(Result.getValueType() == + TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) && "Invalid type for promoted integer"); AnalyzeNewValue(Result); @@ -731,7 +732,8 @@ void DAGTypeLegalizer::SetPromotedInteger(SDValue Op, SDValue Result) { } void DAGTypeLegalizer::SetSoftenedFloat(SDValue Op, SDValue Result) { - assert(Result.getValueType() == TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) && + assert(Result.getValueType() == + TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) && "Invalid type for softened float"); AnalyzeNewValue(Result); @@ -762,7 +764,8 @@ void DAGTypeLegalizer::GetExpandedInteger(SDValue Op, SDValue &Lo, void DAGTypeLegalizer::SetExpandedInteger(SDValue Op, SDValue Lo, SDValue Hi) { - assert(Lo.getValueType() == TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) && + assert(Lo.getValueType() == + TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) && Hi.getValueType() == Lo.getValueType() && "Invalid type for expanded integer"); // Lo/Hi may have been newly allocated, if so, add nodeid's as relevant. @@ -788,7 +791,8 @@ void DAGTypeLegalizer::GetExpandedFloat(SDValue Op, SDValue &Lo, void DAGTypeLegalizer::SetExpandedFloat(SDValue Op, SDValue Lo, SDValue Hi) { - assert(Lo.getValueType() == TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) && + assert(Lo.getValueType() == + TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) && Hi.getValueType() == Lo.getValueType() && "Invalid type for expanded float"); // Lo/Hi may have been newly allocated, if so, add nodeid's as relevant. @@ -832,7 +836,8 @@ void DAGTypeLegalizer::SetSplitVector(SDValue Op, SDValue Lo, } void DAGTypeLegalizer::SetWidenedVector(SDValue Op, SDValue Result) { - assert(Result.getValueType() == TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) && + assert(Result.getValueType() == + TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) && "Invalid type for widened vector"); AnalyzeNewValue(Result); @@ -940,7 +945,8 @@ void DAGTypeLegalizer::GetSplitDestVTs(EVT InVT, EVT &LoVT, EVT &HiVT) { } else { unsigned NumElements = InVT.getVectorNumElements(); assert(!(NumElements & 1) && "Splitting vector, but not in half!"); - LoVT = HiVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), NumElements/2); + LoVT = HiVT = EVT::getVectorVT(*DAG.getContext(), + InVT.getVectorElementType(), NumElements/2); } } @@ -980,7 +986,8 @@ SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) { DebugLoc dlLo = Lo.getDebugLoc(); EVT LVT = Lo.getValueType(); EVT HVT = Hi.getValueType(); - EVT NVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits() + HVT.getSizeInBits()); + EVT NVT = EVT::getIntegerVT(*DAG.getContext(), + LVT.getSizeInBits() + HVT.getSizeInBits()); Lo = DAG.getNode(ISD::ZERO_EXTEND, dlLo, NVT, Lo); Hi = DAG.getNode(ISD::ANY_EXTEND, dlHi, NVT, Hi); @@ -1082,7 +1089,8 @@ void DAGTypeLegalizer::SplitInteger(SDValue Op, /// type half the size of Op's. void DAGTypeLegalizer::SplitInteger(SDValue Op, SDValue &Lo, SDValue &Hi) { - EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Op.getValueType().getSizeInBits()/2); + EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), + Op.getValueType().getSizeInBits()/2); SplitInteger(Op, HalfVT, HalfVT, Lo, Hi); } diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 9dd9796..d60ad60 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -33,7 +33,7 @@ namespace llvm { /// into small values. /// class VISIBILITY_HIDDEN DAGTypeLegalizer { - TargetLowering &TLI; + const TargetLowering &TLI; SelectionDAG &DAG; public: // NodeIdFlags - This pass uses the NodeId on the SDNodes to hold information diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp index 5e83b4b..88e1e62 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp @@ -173,8 +173,9 @@ void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT); SDValue NewVec = DAG.getNode(ISD::BIT_CONVERT, dl, - EVT::getVectorVT(*DAG.getContext(), NewVT, 2*OldElts), - OldVec); + EVT::getVectorVT(*DAG.getContext(), + NewVT, 2*OldElts), + OldVec); // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector. SDValue Idx = N->getOperand(1); @@ -268,7 +269,9 @@ SDValue DAGTypeLegalizer::ExpandOp_BIT_CONVERT(SDNode *N) { // is no point, and it might create expansion loops). For example, on // x86 this turns v1i64 = BIT_CONVERT i64 into v1i64 = BIT_CONVERT v2i32. EVT OVT = N->getOperand(0).getValueType(); - EVT NVT = EVT::getVectorVT(*DAG.getContext(), TLI.getTypeToTransformTo(*DAG.getContext(), OVT), 2); + EVT NVT = EVT::getVectorVT(*DAG.getContext(), + TLI.getTypeToTransformTo(*DAG.getContext(), OVT), + 2); if (isTypeLegal(NVT)) { SDValue Parts[2]; @@ -312,8 +315,9 @@ SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) { } SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, - EVT::getVectorVT(*DAG.getContext(), NewVT, NewElts.size()), - &NewElts[0], NewElts.size()); + EVT::getVectorVT(*DAG.getContext(), + NewVT, NewElts.size()), + &NewElts[0], NewElts.size()); // Convert the new vector to the old vector type. return DAG.getNode(ISD::BIT_CONVERT, dl, VecVT, NewVec); @@ -380,7 +384,8 @@ SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) { DebugLoc dl = N->getDebugLoc(); StoreSDNode *St = cast<StoreSDNode>(N); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), St->getValue().getValueType()); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), + St->getValue().getValueType()); SDValue Chain = St->getChain(); SDValue Ptr = St->getBasePtr(); int SVOffset = St->getSrcValueOffset(); diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index b5f84c0..0e2bd02 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -34,7 +34,7 @@ using namespace llvm; namespace { class VectorLegalizer { SelectionDAG& DAG; - TargetLowering& TLI; + const TargetLowering &TLI; bool Changed; // Keep track of whether anything changed /// LegalizedNodes - For nodes that are of legal width, and that have more diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index ed5f24c..7efeea1 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -705,8 +705,9 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, // Store the new element. This may be larger than the vector element type, // so use a truncating store. SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); + const Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); unsigned Alignment = - TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForEVT(*DAG.getContext())); + TLI.getTargetData()->getPrefTypeAlignment(VecType); Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT, false, false, 0); @@ -1419,7 +1420,8 @@ SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { ShOp = GetWidenedVector(ShOp); ShVT = ShOp.getValueType(); } - EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), ShVT.getVectorElementType(), + EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), + ShVT.getVectorElementType(), WidenVT.getVectorNumElements()); if (ShVT != ShWidenVT) ShOp = ModifyToType(ShOp, ShWidenVT); @@ -1493,7 +1495,8 @@ SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) { unsigned NewNumElts = WidenSize / InSize; if (InVT.isVector()) { EVT InEltVT = InVT.getVectorElementType(); - NewInVT= EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenSize / InEltVT.getSizeInBits()); + NewInVT= EVT::getVectorVT(*DAG.getContext(), InEltVT, + WidenSize / InEltVT.getSizeInBits()); } else { NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); } @@ -1617,7 +1620,8 @@ SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { SDValue RndOp = N->getOperand(3); SDValue SatOp = N->getOperand(4); - EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), + N->getValueType(0)); unsigned WidenNumElts = WidenVT.getVectorNumElements(); EVT InVT = InOp.getValueType(); @@ -1791,7 +1795,8 @@ SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { EVT CondVT = Cond1.getValueType(); if (CondVT.isVector()) { EVT CondEltVT = CondVT.getVectorElementType(); - EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), CondEltVT, WidenNumElts); + EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), + CondEltVT, WidenNumElts); if (getTypeAction(CondVT) == WidenVector) Cond1 = GetWidenedVector(Cond1); @@ -1859,7 +1864,8 @@ SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { SDValue InOp1 = N->getOperand(0); EVT InVT = InOp1.getValueType(); assert(InVT.isVector() && "can not widen non vector type"); - EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), WidenNumElts); + EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), + InVT.getVectorElementType(), WidenNumElts); InOp1 = GetWidenedVector(InOp1); SDValue InOp2 = GetWidenedVector(N->getOperand(1)); @@ -2124,7 +2130,7 @@ SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain, // The routines chops the vector into the largest vector loads with the same // element type or scalar loads and then recombines it to the widen vector // type. - EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); + EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); unsigned WidenWidth = WidenVT.getSizeInBits(); EVT LdVT = LD->getMemoryVT(); DebugLoc dl = LD->getDebugLoc(); diff --git a/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h b/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h index 9d1568f..ac2d338 100644 --- a/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h +++ b/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h @@ -41,7 +41,7 @@ private: SDNode *Node; // valid for expressions unsigned ResNo; // valid for expressions } s; - Value *Const; // valid for constants + const Value *Const; // valid for constants unsigned FrameIx; // valid for stack objects } u; MDNode *mdPtr; @@ -60,7 +60,8 @@ public: } // Constructor for constants. - SDDbgValue(MDNode *mdP, Value *C, uint64_t off, DebugLoc dl, unsigned O) : + SDDbgValue(MDNode *mdP, const Value *C, uint64_t off, DebugLoc dl, + unsigned O) : mdPtr(mdP), Offset(off), DL(dl), Order(O), Invalid(false) { kind = CONST; u.Const = C; @@ -86,7 +87,7 @@ public: unsigned getResNo() { assert (kind==SDNODE); return u.s.ResNo; } // Returns the Value* for a constant - Value *getConst() { assert (kind==CONST); return u.Const; } + const Value *getConst() { assert (kind==CONST); return u.Const; } // Returns the FrameIx for a stack object unsigned getFrameIx() { assert (kind==FRAMEIX); return u.FrameIx; } diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index 3f1766d..da02850 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -17,18 +17,19 @@ #define DEBUG_TYPE "pre-RA-sched" #include "ScheduleDAGSDNodes.h" +#include "llvm/InlineAsm.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAGISel.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" #include "llvm/ADT/PriorityQueue.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include <climits> using namespace llvm; @@ -647,13 +648,14 @@ bool ScheduleDAGRRList::DelayForLiveRegsBottomUp(SUnit *SU, if (Node->getOperand(NumOps-1).getValueType() == MVT::Flag) --NumOps; // Ignore the flag operand. - for (unsigned i = 2; i != NumOps;) { + for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) { unsigned Flags = cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue(); - unsigned NumVals = (Flags & 0xffff) >> 3; + unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags); ++i; // Skip the ID value. - if ((Flags & 7) == 2 || (Flags & 7) == 6) { + if (InlineAsm::isRegDefKind(Flags) || + InlineAsm::isRegDefEarlyClobberKind(Flags)) { // Check for def of register or earlyclobber register. for (; NumVals; --NumVals, ++i) { unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index e7ab2f0..76e4771 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -353,8 +353,8 @@ void ScheduleDAGSDNodes::AddSchedEdges() { const SDep& dep = SDep(OpSU, isChain ? SDep::Order : SDep::Data, OpSU->Latency, PhysReg); if (!isChain && !UnitLatencies) { - ComputeOperandLatency(OpSU, SU, (SDep &)dep); - ST.adjustSchedDependency(OpSU, SU, (SDep &)dep); + ComputeOperandLatency(OpSU, SU, const_cast<SDep &>(dep)); + ST.adjustSchedDependency(OpSU, SU, const_cast<SDep &>(dep)); } SU->addPred(dep); @@ -422,7 +422,6 @@ namespace { // instructions in the right order. static void ProcessSourceNode(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter, - DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM, DenseMap<SDValue, unsigned> &VRBaseMap, SmallVector<std::pair<unsigned, MachineInstr*>, 32> &Orders, SmallSet<unsigned, 8> &Seen) { @@ -449,9 +448,11 @@ static void ProcessSourceNode(SDNode *N, SelectionDAG *DAG, continue; unsigned DVOrder = DVs[i]->getOrder(); if (DVOrder == ++Order) { - MachineInstr *DbgMI = Emitter.EmitDbgValue(DVs[i], BB, VRBaseMap, EM); - Orders.push_back(std::make_pair(DVOrder, DbgMI)); - BB->insert(InsertPos, DbgMI); + MachineInstr *DbgMI = Emitter.EmitDbgValue(DVs[i], VRBaseMap); + if (DbgMI) { + Orders.push_back(std::make_pair(DVOrder, DbgMI)); + BB->insert(InsertPos, DbgMI); + } DVs[i]->setIsInvalidated(); } } @@ -459,8 +460,7 @@ static void ProcessSourceNode(SDNode *N, SelectionDAG *DAG, /// EmitSchedule - Emit the machine code in scheduled order. -MachineBasicBlock *ScheduleDAGSDNodes:: -EmitSchedule(DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) { +MachineBasicBlock *ScheduleDAGSDNodes::EmitSchedule() { InstrEmitter Emitter(BB, InsertPos); DenseMap<SDValue, unsigned> VRBaseMap; DenseMap<SUnit*, unsigned> CopyVRBaseMap; @@ -468,6 +468,17 @@ EmitSchedule(DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) { SmallSet<unsigned, 8> Seen; bool HasDbg = DAG->hasDebugValues(); + // If this is the first BB, emit byval parameter dbg_value's. + if (HasDbg && BB->getParent()->begin() == MachineFunction::iterator(BB)) { + SDDbgInfo::DbgIterator PDI = DAG->ByvalParmDbgBegin(); + SDDbgInfo::DbgIterator PDE = DAG->ByvalParmDbgEnd(); + for (; PDI != PDE; ++PDI) { + MachineInstr *DbgMI= Emitter.EmitDbgValue(*PDI, VRBaseMap); + if (DbgMI) + BB->insert(BB->end(), DbgMI); + } + } + for (unsigned i = 0, e = Sequence.size(); i != e; i++) { SUnit *SU = Sequence[i]; if (!SU) { @@ -491,21 +502,21 @@ EmitSchedule(DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) { while (!FlaggedNodes.empty()) { SDNode *N = FlaggedNodes.back(); Emitter.EmitNode(FlaggedNodes.back(), SU->OrigNode != SU, SU->isCloned, - VRBaseMap, EM); - // Remember the the source order of the inserted instruction. + VRBaseMap); + // Remember the source order of the inserted instruction. if (HasDbg) - ProcessSourceNode(N, DAG, Emitter, EM, VRBaseMap, Orders, Seen); + ProcessSourceNode(N, DAG, Emitter, VRBaseMap, Orders, Seen); FlaggedNodes.pop_back(); } Emitter.EmitNode(SU->getNode(), SU->OrigNode != SU, SU->isCloned, - VRBaseMap, EM); - // Remember the the source order of the inserted instruction. + VRBaseMap); + // Remember the source order of the inserted instruction. if (HasDbg) - ProcessSourceNode(SU->getNode(), DAG, Emitter, EM, VRBaseMap, Orders, + ProcessSourceNode(SU->getNode(), DAG, Emitter, VRBaseMap, Orders, Seen); } - // Insert all the dbg_value which have not already been inserted in source + // Insert all the dbg_values which have not already been inserted in source // order sequence. if (HasDbg) { MachineBasicBlock::iterator BBBegin = BB->empty() ? BB->end() : BB->begin(); @@ -540,13 +551,15 @@ EmitSchedule(DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) { #endif if ((*DI)->isInvalidated()) continue; - MachineInstr *DbgMI = Emitter.EmitDbgValue(*DI, MIBB, VRBaseMap, EM); - if (!LastOrder) - // Insert to start of the BB (after PHIs). - BB->insert(BBBegin, DbgMI); - else { - MachineBasicBlock::iterator Pos = MI; - MIBB->insert(llvm::next(Pos), DbgMI); + MachineInstr *DbgMI = Emitter.EmitDbgValue(*DI, VRBaseMap); + if (DbgMI) { + if (!LastOrder) + // Insert to start of the BB (after PHIs). + BB->insert(BBBegin, DbgMI); + else { + MachineBasicBlock::iterator Pos = MI; + MIBB->insert(llvm::next(Pos), DbgMI); + } } } LastOrder = Order; @@ -558,8 +571,9 @@ EmitSchedule(DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) { MachineBasicBlock *InsertBB = Emitter.getBlock(); MachineBasicBlock::iterator Pos= Emitter.getBlock()->getFirstTerminator(); if (!(*DI)->isInvalidated()) { - MachineInstr *DbgMI= Emitter.EmitDbgValue(*DI, InsertBB, VRBaseMap, EM); - InsertBB->insert(Pos, DbgMI); + MachineInstr *DbgMI= Emitter.EmitDbgValue(*DI, VRBaseMap); + if (DbgMI) + InsertBB->insert(Pos, DbgMI); } ++DI; } diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h index 6b829b6..7ae8ec2 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h @@ -59,7 +59,8 @@ namespace llvm { if (isa<JumpTableSDNode>(Node)) return true; if (isa<ExternalSymbolSDNode>(Node)) return true; if (isa<BlockAddressSDNode>(Node)) return true; - if (Node->getOpcode() == ISD::EntryToken) return true; + if (Node->getOpcode() == ISD::EntryToken || + isa<MDNodeSDNode>(Node)) return true; return false; } @@ -93,8 +94,7 @@ namespace llvm { /// virtual void ComputeLatency(SUnit *SU); - virtual MachineBasicBlock * - EmitSchedule(DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM); + virtual MachineBasicBlock *EmitSchedule(); /// Schedule - Order nodes according to selected style, filling /// in the Sequence member. diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 8c0554d..e6df742 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -304,10 +304,6 @@ ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, return Result; } -const TargetMachine &SelectionDAG::getTarget() const { - return MF->getTarget(); -} - //===----------------------------------------------------------------------===// // SDNode Profile Support //===----------------------------------------------------------------------===// @@ -792,8 +788,8 @@ unsigned SelectionDAG::getEVTAlignment(EVT VT) const { } // EntryNode could meaningfully have debug info if we can find it... -SelectionDAG::SelectionDAG(TargetLowering &tli, FunctionLoweringInfo &fli) - : TLI(tli), FLI(fli), +SelectionDAG::SelectionDAG(const TargetMachine &tm, FunctionLoweringInfo &fli) + : TM(tm), TLI(*tm.getTargetLowering()), FLI(fli), EntryNode(ISD::EntryToken, DebugLoc(), getVTList(MVT::Other)), Root(getEntryNode()), Ordering(0) { AllNodes.push_back(&EntryNode); @@ -1048,7 +1044,7 @@ SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, return SDValue(N, 0); } -SDValue SelectionDAG::getConstantPool(Constant *C, EVT VT, +SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, unsigned Alignment, int Offset, bool isTarget, unsigned char TargetFlags) { @@ -1319,7 +1315,7 @@ SDValue SelectionDAG::getEHLabel(DebugLoc dl, SDValue Root, MCSymbol *Label) { } -SDValue SelectionDAG::getBlockAddress(BlockAddress *BA, EVT VT, +SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, bool isTarget, unsigned char TargetFlags) { unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; @@ -1356,6 +1352,23 @@ SDValue SelectionDAG::getSrcValue(const Value *V) { return SDValue(N, 0); } +/// getMDNode - Return an MDNodeSDNode which holds an MDNode. +SDValue SelectionDAG::getMDNode(const MDNode *MD) { + FoldingSetNodeID ID; + AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); + ID.AddPointer(MD); + + void *IP = 0; + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) + return SDValue(E, 0); + + SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); + CSEMap.InsertNode(N, IP); + AllNodes.push_back(N); + return SDValue(N, 0); +} + + /// getShiftAmountOperand - Return the specified value casted to /// the target's desired shift amount type. SDValue SelectionDAG::getShiftAmountOperand(SDValue Op) { @@ -1904,7 +1917,8 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, // Output known-0 bits are known if clear or set in both the low clear bits // common to both LHS & RHS. For example, 8+(X<<3) is known to have the // low 3 bits clear. - APInt Mask2 = APInt::getLowBitsSet(BitWidth, Mask.countTrailingOnes()); + APInt Mask2 = APInt::getLowBitsSet(BitWidth, + BitWidth - Mask.countLeadingZeros()); ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero2, KnownOne2, Depth+1); assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); @@ -2253,7 +2267,7 @@ bool SelectionDAG::isVerifiedDebugInfoDesc(SDValue Op) const { GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op); if (!GA) return false; if (GA->getOffset() != 0) return false; - GlobalVariable *GV = dyn_cast<GlobalVariable>(GA->getGlobal()); + const GlobalVariable *GV = dyn_cast<GlobalVariable>(GA->getGlobal()); if (!GV) return false; return MF->getMMI().hasDebugInfo(); } @@ -2778,14 +2792,19 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, // If the indices are the same, return the inserted element else // if the indices are known different, extract the element from // the original vector. - if (N1.getOperand(2) == N2) { - if (VT == N1.getOperand(1).getValueType()) - return N1.getOperand(1); - else - return getSExtOrTrunc(N1.getOperand(1), DL, VT); - } else if (isa<ConstantSDNode>(N1.getOperand(2)) && - isa<ConstantSDNode>(N2)) + SDValue N1Op2 = N1.getOperand(2); + ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode()); + + if (N1Op2C && N2C) { + if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { + if (VT == N1.getOperand(1).getValueType()) + return N1.getOperand(1); + else + return getSExtOrTrunc(N1.getOperand(1), DL, VT); + } + return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); + } } break; case ISD::EXTRACT_ELEMENT: @@ -3178,7 +3197,7 @@ static bool isMemSrcFromString(SDValue Src, std::string &Str) { if (!G) return false; - GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal()); + const GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal()); if (GV && GetConstantStringInfo(GV, Str, SrcDelta, false)) return true; @@ -3193,6 +3212,7 @@ static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, unsigned Limit, uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool NonScalarIntSafe, + bool MemcpyStrSrc, SelectionDAG &DAG, const TargetLowering &TLI) { assert((SrcAlign == 0 || SrcAlign >= DstAlign) && @@ -3201,9 +3221,12 @@ static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, // the value, i.e. memset or memcpy from constant string. Otherwise, it's // the inferred alignment of the source. 'DstAlign', on the other hand, is the // specified alignment of the memory operation. If it is zero, that means - // it's possible to change the alignment of the destination. + // it's possible to change the alignment of the destination. 'MemcpyStrSrc' + // indicates whether the memcpy source is constant so it does not need to be + // loaded. EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, - NonScalarIntSafe, DAG); + NonScalarIntSafe, MemcpyStrSrc, + DAG.getMachineFunction()); if (VT == MVT::Other) { if (DstAlign >= TLI.getTargetData()->getPointerPrefAlignment() || @@ -3269,9 +3292,6 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, // below a certain threshold. const TargetLowering &TLI = DAG.getTargetLoweringInfo(); std::vector<EVT> MemOps; - uint64_t Limit = -1ULL; - if (!AlwaysInline) - Limit = TLI.getMaxStoresPerMemcpy(); bool DstAlignCanChange = false; MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); @@ -3283,9 +3303,13 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, std::string Str; bool CopyFromStr = isMemSrcFromString(Src, Str); bool isZeroStr = CopyFromStr && Str.empty(); + uint64_t Limit = -1ULL; + if (!AlwaysInline) + Limit = TLI.getMaxStoresPerMemcpy(); if (!FindOptimalMemOpLowering(MemOps, Limit, Size, (DstAlignCanChange ? 0 : Align), - (isZeroStr ? 0 : SrcAlign), true, DAG, TLI)) + (isZeroStr ? 0 : SrcAlign), + true, CopyFromStr, DAG, TLI)) return SDValue(); if (DstAlignCanChange) { @@ -3373,7 +3397,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, if (!FindOptimalMemOpLowering(MemOps, Limit, Size, (DstAlignCanChange ? 0 : Align), - SrcAlign, true, DAG, TLI)) + SrcAlign, true, false, DAG, TLI)) return SDValue(); if (DstAlignCanChange) { @@ -3445,7 +3469,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(), Size, (DstAlignCanChange ? 0 : Align), 0, - NonScalarIntSafe, DAG, TLI)) + NonScalarIntSafe, false, DAG, TLI)) return SDValue(); if (DstAlignCanChange) { @@ -3571,8 +3595,10 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, if (Result.getNode()) return Result; + // FIXME: If the memmove is volatile, lowering it to plain libc memmove may + // not be safe. See memcpy above for more details. + // Emit a library call. - assert(!isVol && "library memmove does not support volatile"); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext()); @@ -3620,8 +3646,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, if (Result.getNode()) return Result; - // Emit a library call. - assert(!isVol && "library memset does not support volatile"); + // Emit a library call. const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext()); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; @@ -4913,7 +4938,7 @@ SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, } SDDbgValue * -SelectionDAG::getDbgValue(MDNode *MDPtr, Value *C, uint64_t Off, +SelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, DebugLoc DL, unsigned O) { return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); } @@ -5321,8 +5346,8 @@ unsigned SelectionDAG::GetOrdering(const SDNode *SD) const { /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the /// value is produced by SD. -void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD) { - DbgInfo->add(DB, SD); +void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { + DbgInfo->add(DB, SD, isParameter); if (SD) SD->setHasDebugValue(true); } @@ -5338,7 +5363,7 @@ HandleSDNode::~HandleSDNode() { GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, const GlobalValue *GA, EVT VT, int64_t o, unsigned char TF) : SDNode(Opc, DebugLoc(), getSDVTList(VT)), Offset(o), TargetFlags(TF) { - TheGlobal = const_cast<GlobalValue*>(GA); + TheGlobal = GA; } MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, @@ -5558,6 +5583,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::PCMARKER: return "PCMarker"; case ISD::READCYCLECOUNTER: return "ReadCycleCounter"; case ISD::SRCVALUE: return "SrcValue"; + case ISD::MDNODE_SDNODE: return "MDNode"; case ISD::EntryToken: return "EntryToken"; case ISD::TokenFactor: return "TokenFactor"; case ISD::AssertSext: return "AssertSext"; @@ -5926,6 +5952,11 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { OS << "<" << M->getValue() << ">"; else OS << "<null>"; + } else if (const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(this)) { + if (MD->getMD()) + OS << "<" << MD->getMD() << ">"; + else + OS << "<null>"; } else if (const VTSDNode *N = dyn_cast<VTSDNode>(this)) { OS << ":" << N->getVT().getEVTString(); } @@ -6063,7 +6094,7 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { unsigned i; for (i= 0; i != NE; ++i) { - for (unsigned j = 0; j != N->getNumOperands(); ++j) { + for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { SDValue Operand = N->getOperand(j); EVT OperandVT = Operand.getValueType(); if (OperandVT.isVector()) { @@ -6141,8 +6172,8 @@ bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, return true; } - GlobalValue *GV1 = NULL; - GlobalValue *GV2 = NULL; + const GlobalValue *GV1 = NULL; + const GlobalValue *GV2 = NULL; int64_t Offset1 = 0; int64_t Offset2 = 0; bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); @@ -6157,14 +6188,14 @@ bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, /// it cannot be inferred. unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { // If this is a GlobalAddress + cst, return the alignment. - GlobalValue *GV; + const GlobalValue *GV; int64_t GVOffset = 0; if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { // If GV has specified alignment, then use it. Otherwise, use the preferred // alignment. unsigned Align = GV->getAlignment(); if (!Align) { - if (GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV)) { + if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV)) { if (GVar->hasInitializer()) { const TargetData *TD = TLI.getTargetData(); Align = TD->getPreferredAlignment(GVar); @@ -6326,8 +6357,8 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, if (OpVal.getOpcode() == ISD::UNDEF) SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) - SplatValue |= (APInt(CN->getAPIntValue()).zextOrTrunc(EltBitSize). - zextOrTrunc(sz) << BitPos); + SplatValue |= APInt(CN->getAPIntValue()).zextOrTrunc(EltBitSize). + zextOrTrunc(sz) << BitPos; else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; else diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 4bbb3de..a38b204 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -28,7 +28,9 @@ #include "llvm/Instructions.h" #include "llvm/Intrinsics.h" #include "llvm/IntrinsicInst.h" +#include "llvm/LLVMContext.h" #include "llvm/Module.h" +#include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/FastISel.h" #include "llvm/CodeGen/GCStrategy.h" #include "llvm/CodeGen/GCMetadata.h" @@ -168,7 +170,7 @@ namespace { /// AddInlineAsmOperands - Add this value to the specified inlineasm node /// operand list. This adds the code marker, matching input operand index /// (if applicable), and includes the number of values added into it. - void AddInlineAsmOperands(unsigned Code, + void AddInlineAsmOperands(unsigned Kind, bool HasMatching, unsigned MatchingIdx, SelectionDAG &DAG, std::vector<SDValue> &Ops) const; @@ -533,7 +535,7 @@ void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa) { TD = DAG.getTarget().getTargetData(); } -/// clear - Clear out the curret SelectionDAG and the associated +/// clear - Clear out the current SelectionDAG and the associated /// state and prepare this SelectionDAGBuilder object to be used /// for a new block. This doesn't clear out information about /// additional blocks that are needed to complete switch lowering @@ -543,8 +545,6 @@ void SelectionDAGBuilder::clear() { NodeMap.clear(); PendingLoads.clear(); PendingExports.clear(); - EdgeMapping.clear(); - DAG.clear(); CurDebugLoc = DebugLoc(); HasTailCall = false; } @@ -612,11 +612,26 @@ void SelectionDAGBuilder::AssignOrderingToNode(const SDNode *Node) { AssignOrderingToNode(Node->getOperand(I).getNode()); } -void SelectionDAGBuilder::visit(Instruction &I) { +void SelectionDAGBuilder::visit(const Instruction &I) { + // Set up outgoing PHI node register values before emitting the terminator. + if (isa<TerminatorInst>(&I)) + HandlePHINodesInSuccessorBlocks(I.getParent()); + + CurDebugLoc = I.getDebugLoc(); + visit(I.getOpcode(), I); + + if (!isa<TerminatorInst>(&I) && !HasTailCall) + CopyToExportRegsIfNeeded(&I); + + CurDebugLoc = DebugLoc(); } -void SelectionDAGBuilder::visit(unsigned Opcode, User &I) { +void SelectionDAGBuilder::visitPHI(const PHINode &) { + llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!"); +} + +void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) { // Note: this doesn't use InstVisitor, because it has to work with // ConstantExpr's in addition to instructions. switch (Opcode) { @@ -638,28 +653,28 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { SDValue &N = NodeMap[V]; if (N.getNode()) return N; - if (Constant *C = const_cast<Constant*>(dyn_cast<Constant>(V))) { + if (const Constant *C = dyn_cast<Constant>(V)) { EVT VT = TLI.getValueType(V->getType(), true); - if (ConstantInt *CI = dyn_cast<ConstantInt>(C)) + if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) return N = DAG.getConstant(*CI, VT); - if (GlobalValue *GV = dyn_cast<GlobalValue>(C)) + if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) return N = DAG.getGlobalAddress(GV, VT); if (isa<ConstantPointerNull>(C)) return N = DAG.getConstant(0, TLI.getPointerTy()); - if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) + if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) return N = DAG.getConstantFP(*CFP, VT); if (isa<UndefValue>(C) && !V->getType()->isAggregateType()) return N = DAG.getUNDEF(VT); - if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { + if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { visit(CE->getOpcode(), *CE); SDValue N1 = NodeMap[V]; - assert(N1.getNode() && "visit didn't populate the ValueMap!"); + assert(N1.getNode() && "visit didn't populate the NodeMap!"); return N1; } @@ -704,7 +719,7 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { getCurDebugLoc()); } - if (BlockAddress *BA = dyn_cast<BlockAddress>(C)) + if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) return DAG.getBlockAddress(BA, VT); const VectorType *VecTy = cast<VectorType>(V->getType()); @@ -713,7 +728,7 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { // Now that we know the number and type of the elements, get that number of // elements into the Ops array based on what kind of constant it is. SmallVector<SDValue, 16> Ops; - if (ConstantVector *CP = dyn_cast<ConstantVector>(C)) { + if (const ConstantVector *CP = dyn_cast<ConstantVector>(C)) { for (unsigned i = 0; i != NumElements; ++i) Ops.push_back(getValue(CP->getOperand(i))); } else { @@ -756,7 +771,7 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { static void getReturnInfo(const Type* ReturnType, Attributes attr, SmallVectorImpl<EVT> &OutVTs, SmallVectorImpl<ISD::ArgFlagsTy> &OutFlags, - TargetLowering &TLI, + const TargetLowering &TLI, SmallVectorImpl<uint64_t> *Offsets = 0) { SmallVector<EVT, 4> ValueVTs; ComputeValueVTs(TLI, ReturnType, ValueVTs); @@ -811,7 +826,7 @@ static void getReturnInfo(const Type* ReturnType, } } -void SelectionDAGBuilder::visitRet(ReturnInst &I) { +void SelectionDAGBuilder::visitRet(const ReturnInst &I) { SDValue Chain = getControlRoot(); SmallVector<ISD::OutputArg, 8> Outs; FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); @@ -916,18 +931,18 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) { /// CopyToExportRegsIfNeeded - If the given value has virtual registers /// created for it, emit nodes to copy the value into the virtual /// registers. -void SelectionDAGBuilder::CopyToExportRegsIfNeeded(Value *V) { - if (!V->use_empty()) { - DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V); - if (VMI != FuncInfo.ValueMap.end()) - CopyValueToVirtualRegister(V, VMI->second); +void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) { + DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V); + if (VMI != FuncInfo.ValueMap.end()) { + assert(!V->use_empty() && "Unused value assigned virtual registers!"); + CopyValueToVirtualRegister(V, VMI->second); } } /// ExportFromCurrentBlock - If this condition isn't known to be exported from /// the current basic block, add it to ValueMap now so that we'll get a /// CopyTo/FromReg. -void SelectionDAGBuilder::ExportFromCurrentBlock(Value *V) { +void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) { // No need to export constants. if (!isa<Instruction>(V) && !isa<Argument>(V)) return; @@ -938,11 +953,11 @@ void SelectionDAGBuilder::ExportFromCurrentBlock(Value *V) { CopyValueToVirtualRegister(V, Reg); } -bool SelectionDAGBuilder::isExportableFromCurrentBlock(Value *V, +bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB) { // The operands of the setcc have to be in this block. We don't know // how to export them from some other block. - if (Instruction *VI = dyn_cast<Instruction>(V)) { + if (const Instruction *VI = dyn_cast<Instruction>(V)) { // Can export from current BB. if (VI->getParent() == FromBB) return true; @@ -971,85 +986,31 @@ static bool InBlock(const Value *V, const BasicBlock *BB) { return true; } -/// getFCmpCondCode - Return the ISD condition code corresponding to -/// the given LLVM IR floating-point condition code. This includes -/// consideration of global floating-point math flags. -/// -static ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred) { - ISD::CondCode FPC, FOC; - switch (Pred) { - case FCmpInst::FCMP_FALSE: FOC = FPC = ISD::SETFALSE; break; - case FCmpInst::FCMP_OEQ: FOC = ISD::SETEQ; FPC = ISD::SETOEQ; break; - case FCmpInst::FCMP_OGT: FOC = ISD::SETGT; FPC = ISD::SETOGT; break; - case FCmpInst::FCMP_OGE: FOC = ISD::SETGE; FPC = ISD::SETOGE; break; - case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break; - case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break; - case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break; - case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break; - case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break; - case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break; - case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break; - case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break; - case FCmpInst::FCMP_ULT: FOC = ISD::SETLT; FPC = ISD::SETULT; break; - case FCmpInst::FCMP_ULE: FOC = ISD::SETLE; FPC = ISD::SETULE; break; - case FCmpInst::FCMP_UNE: FOC = ISD::SETNE; FPC = ISD::SETUNE; break; - case FCmpInst::FCMP_TRUE: FOC = FPC = ISD::SETTRUE; break; - default: - llvm_unreachable("Invalid FCmp predicate opcode!"); - FOC = FPC = ISD::SETFALSE; - break; - } - if (FiniteOnlyFPMath()) - return FOC; - else - return FPC; -} - -/// getICmpCondCode - Return the ISD condition code corresponding to -/// the given LLVM IR integer condition code. -/// -static ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred) { - switch (Pred) { - case ICmpInst::ICMP_EQ: return ISD::SETEQ; - case ICmpInst::ICMP_NE: return ISD::SETNE; - case ICmpInst::ICMP_SLE: return ISD::SETLE; - case ICmpInst::ICMP_ULE: return ISD::SETULE; - case ICmpInst::ICMP_SGE: return ISD::SETGE; - case ICmpInst::ICMP_UGE: return ISD::SETUGE; - case ICmpInst::ICMP_SLT: return ISD::SETLT; - case ICmpInst::ICMP_ULT: return ISD::SETULT; - case ICmpInst::ICMP_SGT: return ISD::SETGT; - case ICmpInst::ICMP_UGT: return ISD::SETUGT; - default: - llvm_unreachable("Invalid ICmp predicate opcode!"); - return ISD::SETNE; - } -} - /// EmitBranchForMergedCondition - Helper method for FindMergedConditions. /// This function emits a branch and is used at the leaves of an OR or an /// AND operator tree. /// void -SelectionDAGBuilder::EmitBranchForMergedCondition(Value *Cond, +SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, - MachineBasicBlock *CurBB) { + MachineBasicBlock *CurBB, + MachineBasicBlock *SwitchBB) { const BasicBlock *BB = CurBB->getBasicBlock(); // If the leaf of the tree is a comparison, merge the condition into // the caseblock. - if (CmpInst *BOp = dyn_cast<CmpInst>(Cond)) { + if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) { // The operands of the cmp have to be in this block. We don't know // how to export them from some other block. If this is the first block // of the sequence, no exporting is needed. - if (CurBB == CurMBB || + if (CurBB == SwitchBB || (isExportableFromCurrentBlock(BOp->getOperand(0), BB) && isExportableFromCurrentBlock(BOp->getOperand(1), BB))) { ISD::CondCode Condition; - if (ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) { + if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) { Condition = getICmpCondCode(IC->getPredicate()); - } else if (FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) { + } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) { Condition = getFCmpCondCode(FC->getPredicate()); } else { Condition = ISD::SETEQ; // silence warning. @@ -1070,19 +1031,20 @@ SelectionDAGBuilder::EmitBranchForMergedCondition(Value *Cond, } /// FindMergedConditions - If Cond is an expression like -void SelectionDAGBuilder::FindMergedConditions(Value *Cond, +void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, + MachineBasicBlock *SwitchBB, unsigned Opc) { // If this node is not part of the or/and tree, emit it as a branch. - Instruction *BOp = dyn_cast<Instruction>(Cond); + const Instruction *BOp = dyn_cast<Instruction>(Cond); if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) || (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() || BOp->getParent() != CurBB->getBasicBlock() || !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) || !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) { - EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB); + EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB); return; } @@ -1102,10 +1064,10 @@ void SelectionDAGBuilder::FindMergedConditions(Value *Cond, // // Emit the LHS condition. - FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, Opc); + FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc); // Emit the RHS condition into TmpBB. - FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, Opc); + FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc); } else { assert(Opc == Instruction::And && "Unknown merge op!"); // Codegen X & Y as: @@ -1118,10 +1080,10 @@ void SelectionDAGBuilder::FindMergedConditions(Value *Cond, // This requires creation of TmpBB after CurBB. // Emit the LHS condition. - FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, Opc); + FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc); // Emit the RHS condition into TmpBB. - FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, Opc); + FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc); } } @@ -1156,19 +1118,21 @@ SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases){ return true; } -void SelectionDAGBuilder::visitBr(BranchInst &I) { +void SelectionDAGBuilder::visitBr(const BranchInst &I) { + MachineBasicBlock *BrMBB = FuncInfo.MBBMap[I.getParent()]; + // Update machine-CFG edges. MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)]; // Figure out which block is immediately after the current one. MachineBasicBlock *NextBlock = 0; - MachineFunction::iterator BBI = CurMBB; + MachineFunction::iterator BBI = BrMBB; if (++BBI != FuncInfo.MF->end()) NextBlock = BBI; if (I.isUnconditional()) { // Update machine-CFG edges. - CurMBB->addSuccessor(Succ0MBB); + BrMBB->addSuccessor(Succ0MBB); // If this is not a fall-through branch, emit the branch. if (Succ0MBB != NextBlock) @@ -1181,7 +1145,7 @@ void SelectionDAGBuilder::visitBr(BranchInst &I) { // If this condition is one of the special cases we handle, do special stuff // now. - Value *CondVal = I.getCondition(); + const Value *CondVal = I.getCondition(); MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)]; // If this is a series of conditions that are or'd or and'd together, emit @@ -1199,15 +1163,16 @@ void SelectionDAGBuilder::visitBr(BranchInst &I) { // cmp D, E // jle foo // - if (BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) { + if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) { if (BOp->hasOneUse() && (BOp->getOpcode() == Instruction::And || BOp->getOpcode() == Instruction::Or)) { - FindMergedConditions(BOp, Succ0MBB, Succ1MBB, CurMBB, BOp->getOpcode()); + FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, + BOp->getOpcode()); // If the compares in later blocks need to use values not currently // exported from this block, export them now. This block should always // be the first entry. - assert(SwitchCases[0].ThisBB == CurMBB && "Unexpected lowering!"); + assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"); // Allow some cases to be rejected. if (ShouldEmitAsBranches(SwitchCases)) { @@ -1217,7 +1182,7 @@ void SelectionDAGBuilder::visitBr(BranchInst &I) { } // Emit the branch for this block. - visitSwitchCase(SwitchCases[0]); + visitSwitchCase(SwitchCases[0], BrMBB); SwitchCases.erase(SwitchCases.begin()); return; } @@ -1233,16 +1198,17 @@ void SelectionDAGBuilder::visitBr(BranchInst &I) { // Create a CaseBlock record representing this branch. CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()), - NULL, Succ0MBB, Succ1MBB, CurMBB); + NULL, Succ0MBB, Succ1MBB, BrMBB); // Use visitSwitchCase to actually insert the fast branch sequence for this // cond branch. - visitSwitchCase(CB); + visitSwitchCase(CB, BrMBB); } /// visitSwitchCase - Emits the necessary code to represent a single node in /// the binary search tree resulting from lowering a switch instruction. -void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB) { +void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, + MachineBasicBlock *SwitchBB) { SDValue Cond; SDValue CondLHS = getValue(CB.CmpLHS); DebugLoc dl = getCurDebugLoc(); @@ -1281,13 +1247,13 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB) { } // Update successor info - CurMBB->addSuccessor(CB.TrueBB); - CurMBB->addSuccessor(CB.FalseBB); + SwitchBB->addSuccessor(CB.TrueBB); + SwitchBB->addSuccessor(CB.FalseBB); // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; - MachineFunction::iterator BBI = CurMBB; + MachineFunction::iterator BBI = SwitchBB; if (++BBI != FuncInfo.MF->end()) NextBlock = BBI; @@ -1305,11 +1271,11 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB) { // If the branch was constant folded, fix up the CFG. if (BrCond.getOpcode() == ISD::BR) { - CurMBB->removeSuccessor(CB.FalseBB); + SwitchBB->removeSuccessor(CB.FalseBB); } else { // Otherwise, go ahead and insert the false branch. if (BrCond == getControlRoot()) - CurMBB->removeSuccessor(CB.TrueBB); + SwitchBB->removeSuccessor(CB.TrueBB); if (CB.FalseBB != NextBlock) BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, @@ -1336,7 +1302,8 @@ void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) { /// visitJumpTableHeader - This function emits necessary code to produce index /// in the JumpTable from switch case. void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, - JumpTableHeader &JTH) { + JumpTableHeader &JTH, + MachineBasicBlock *SwitchBB) { // Subtract the lowest switch case value from the value being switched on and // conditional branch to default mbb if the result is greater than the // difference between smallest and largest cases. @@ -1368,7 +1335,7 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; - MachineFunction::iterator BBI = CurMBB; + MachineFunction::iterator BBI = SwitchBB; if (++BBI != FuncInfo.MF->end()) NextBlock = BBI; @@ -1386,7 +1353,8 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, /// visitBitTestHeader - This function emits necessary code to produce value /// suitable for "bit tests" -void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B) { +void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, + MachineBasicBlock *SwitchBB) { // Subtract the minimum value SDValue SwitchOp = getValue(B.SValue); EVT VT = SwitchOp.getValueType(); @@ -1409,14 +1377,14 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B) { // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; - MachineFunction::iterator BBI = CurMBB; + MachineFunction::iterator BBI = SwitchBB; if (++BBI != FuncInfo.MF->end()) NextBlock = BBI; MachineBasicBlock* MBB = B.Cases[0].ThisBB; - CurMBB->addSuccessor(B.Default); - CurMBB->addSuccessor(MBB); + SwitchBB->addSuccessor(B.Default); + SwitchBB->addSuccessor(MBB); SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurDebugLoc(), MVT::Other, CopyTo, RangeCmp, @@ -1432,7 +1400,8 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B) { /// visitBitTestCase - this function produces one "bit test" void SelectionDAGBuilder::visitBitTestCase(MachineBasicBlock* NextMBB, unsigned Reg, - BitTestCase &B) { + BitTestCase &B, + MachineBasicBlock *SwitchBB) { // Make desired shift SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurDebugLoc(), Reg, TLI.getPointerTy()); @@ -1450,8 +1419,8 @@ void SelectionDAGBuilder::visitBitTestCase(MachineBasicBlock* NextMBB, AndOp, DAG.getConstant(0, TLI.getPointerTy()), ISD::SETNE); - CurMBB->addSuccessor(B.TargetBB); - CurMBB->addSuccessor(NextMBB); + SwitchBB->addSuccessor(B.TargetBB); + SwitchBB->addSuccessor(NextMBB); SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurDebugLoc(), MVT::Other, getControlRoot(), @@ -1460,7 +1429,7 @@ void SelectionDAGBuilder::visitBitTestCase(MachineBasicBlock* NextMBB, // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; - MachineFunction::iterator BBI = CurMBB; + MachineFunction::iterator BBI = SwitchBB; if (++BBI != FuncInfo.MF->end()) NextBlock = BBI; @@ -1471,7 +1440,9 @@ void SelectionDAGBuilder::visitBitTestCase(MachineBasicBlock* NextMBB, DAG.setRoot(BrAnd); } -void SelectionDAGBuilder::visitInvoke(InvokeInst &I) { +void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { + MachineBasicBlock *InvokeMBB = FuncInfo.MBBMap[I.getParent()]; + // Retrieve successors. MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)]; MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)]; @@ -1487,8 +1458,8 @@ void SelectionDAGBuilder::visitInvoke(InvokeInst &I) { CopyToExportRegsIfNeeded(&I); // Update successor info - CurMBB->addSuccessor(Return); - CurMBB->addSuccessor(LandingPad); + InvokeMBB->addSuccessor(Return); + InvokeMBB->addSuccessor(LandingPad); // Drop into normal successor. DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(), @@ -1496,15 +1467,16 @@ void SelectionDAGBuilder::visitInvoke(InvokeInst &I) { DAG.getBasicBlock(Return))); } -void SelectionDAGBuilder::visitUnwind(UnwindInst &I) { +void SelectionDAGBuilder::visitUnwind(const UnwindInst &I) { } /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for /// small case ranges). bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, CaseRecVector& WorkList, - Value* SV, - MachineBasicBlock* Default) { + const Value* SV, + MachineBasicBlock *Default, + MachineBasicBlock *SwitchBB) { Case& BackCase = *(CR.Range.second-1); // Size is the number of Cases represented by this range. @@ -1557,7 +1529,7 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, FallThrough = Default; } - Value *RHS, *LHS, *MHS; + const Value *RHS, *LHS, *MHS; ISD::CondCode CC; if (I->High == I->Low) { // This is just small small case range :) containing exactly 1 case @@ -1573,8 +1545,8 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, // code into the current block. Otherwise, push the CaseBlock onto the // vector to be later processed by SDISel, and insert the node's MBB // before the next MBB. - if (CurBlock == CurMBB) - visitSwitchCase(CB); + if (CurBlock == SwitchBB) + visitSwitchCase(CB, SwitchBB); else SwitchCases.push_back(CB); @@ -1600,8 +1572,9 @@ static APInt ComputeRange(const APInt &First, const APInt &Last) { /// handleJTSwitchCase - Emit jumptable for current switch case range bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec& CR, CaseRecVector& WorkList, - Value* SV, - MachineBasicBlock* Default) { + const Value* SV, + MachineBasicBlock* Default, + MachineBasicBlock *SwitchBB) { Case& FrontCase = *CR.Range.first; Case& BackCase = *(CR.Range.second-1); @@ -1613,7 +1586,7 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec& CR, I!=E; ++I) TSize += I->size(); - if (!areJTsAllowed(TLI) || TSize.ult(APInt(First.getBitWidth(), 4))) + if (!areJTsAllowed(TLI) || TSize.ult(4)) return false; APInt Range = ComputeRange(First, Last); @@ -1682,9 +1655,9 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec& CR, // Set the jump table information so that we can codegen it as a second // MachineBasicBlock JumpTable JT(-1U, JTI, JumpTableBB, Default); - JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == CurMBB)); - if (CR.CaseBB == CurMBB) - visitJumpTableHeader(JT, JTH); + JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB)); + if (CR.CaseBB == SwitchBB) + visitJumpTableHeader(JT, JTH, SwitchBB); JTCases.push_back(JumpTableBlock(JTH, JT)); @@ -1695,8 +1668,9 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec& CR, /// 2 subtrees. bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR, CaseRecVector& WorkList, - Value* SV, - MachineBasicBlock* Default) { + const Value* SV, + MachineBasicBlock *Default, + MachineBasicBlock *SwitchBB) { // Get the MachineFunction which holds the current MBB. This is used when // inserting any additional MBBs necessary to represent the switch. MachineFunction *CurMF = FuncInfo.MF; @@ -1810,8 +1784,8 @@ bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR, // Otherwise, branch to LHS. CaseBlock CB(ISD::SETLT, SV, C, NULL, TrueBB, FalseBB, CR.CaseBB); - if (CR.CaseBB == CurMBB) - visitSwitchCase(CB); + if (CR.CaseBB == SwitchBB) + visitSwitchCase(CB, SwitchBB); else SwitchCases.push_back(CB); @@ -1823,8 +1797,9 @@ bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR, /// of masks and emit bit tests with these masks. bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, CaseRecVector& WorkList, - Value* SV, - MachineBasicBlock* Default){ + const Value* SV, + MachineBasicBlock* Default, + MachineBasicBlock *SwitchBB){ EVT PTy = TLI.getPointerTy(); unsigned IntPtrBits = PTy.getSizeInBits(); @@ -1867,7 +1842,7 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, << "Low bound: " << minValue << '\n' << "High bound: " << maxValue << '\n'); - if (cmpRange.uge(APInt(cmpRange.getBitWidth(), IntPtrBits)) || + if (cmpRange.uge(IntPtrBits) || (!(Dests.size() == 1 && numCmps >= 3) && !(Dests.size() == 2 && numCmps >= 5) && !(Dests.size() >= 3 && numCmps >= 6))) @@ -1879,8 +1854,7 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, // Optimize the case where all the case values fit in a // word without having to subtract minValue. In this case, // we can optimize away the subtraction. - if (minValue.isNonNegative() && - maxValue.slt(APInt(maxValue.getBitWidth(), IntPtrBits))) { + if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) { cmpRange = maxValue; } else { lowBound = minValue; @@ -1940,11 +1914,11 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR, } BitTestBlock BTB(lowBound, cmpRange, SV, - -1U, (CR.CaseBB == CurMBB), + -1U, (CR.CaseBB == SwitchBB), CR.CaseBB, Default, BTC); - if (CR.CaseBB == CurMBB) - visitBitTestHeader(BTB); + if (CR.CaseBB == SwitchBB) + visitBitTestHeader(BTB, SwitchBB); BitTestCases.push_back(BTB); @@ -1994,7 +1968,9 @@ size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases, return numCmps; } -void SelectionDAGBuilder::visitSwitch(SwitchInst &SI) { +void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { + MachineBasicBlock *SwitchMBB = FuncInfo.MBBMap[SI.getParent()]; + // Figure out which block is immediately after the current one. MachineBasicBlock *NextBlock = 0; MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()]; @@ -2005,7 +1981,7 @@ void SelectionDAGBuilder::visitSwitch(SwitchInst &SI) { // Update machine-CFG edges. // If this is not a fall-through branch, emit the branch. - CurMBB->addSuccessor(Default); + SwitchMBB->addSuccessor(Default); if (Default != NextBlock) DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, getControlRoot(), @@ -2026,38 +2002,41 @@ void SelectionDAGBuilder::visitSwitch(SwitchInst &SI) { // Get the Value to be switched on and default basic blocks, which will be // inserted into CaseBlock records, representing basic blocks in the binary // search tree. - Value *SV = SI.getOperand(0); + const Value *SV = SI.getOperand(0); // Push the initial CaseRec onto the worklist CaseRecVector WorkList; - WorkList.push_back(CaseRec(CurMBB,0,0,CaseRange(Cases.begin(),Cases.end()))); + WorkList.push_back(CaseRec(SwitchMBB,0,0, + CaseRange(Cases.begin(),Cases.end()))); while (!WorkList.empty()) { // Grab a record representing a case range to process off the worklist CaseRec CR = WorkList.back(); WorkList.pop_back(); - if (handleBitTestsSwitchCase(CR, WorkList, SV, Default)) + if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB)) continue; // If the range has few cases (two or less) emit a series of specific // tests. - if (handleSmallSwitchRange(CR, WorkList, SV, Default)) + if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB)) continue; // If the switch has more than 5 blocks, and at least 40% dense, and the // target supports indirect branches, then emit a jump table rather than // lowering the switch to a binary tree of conditional branches. - if (handleJTSwitchCase(CR, WorkList, SV, Default)) + if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB)) continue; // Emit binary tree. We need to pick a pivot, and push left and right ranges // onto the worklist. Leafs are handled via handleSmallSwitchRange() call. - handleBTSplitSwitchCase(CR, WorkList, SV, Default); + handleBTSplitSwitchCase(CR, WorkList, SV, Default, SwitchMBB); } } -void SelectionDAGBuilder::visitIndirectBr(IndirectBrInst &I) { +void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) { + MachineBasicBlock *IndirectBrMBB = FuncInfo.MBBMap[I.getParent()]; + // Update machine-CFG edges with unique successors. SmallVector<BasicBlock*, 32> succs; succs.reserve(I.getNumSuccessors()); @@ -2066,14 +2045,14 @@ void SelectionDAGBuilder::visitIndirectBr(IndirectBrInst &I) { array_pod_sort(succs.begin(), succs.end()); succs.erase(std::unique(succs.begin(), succs.end()), succs.end()); for (unsigned i = 0, e = succs.size(); i != e; ++i) - CurMBB->addSuccessor(FuncInfo.MBBMap[succs[i]]); + IndirectBrMBB->addSuccessor(FuncInfo.MBBMap[succs[i]]); DAG.setRoot(DAG.getNode(ISD::BRIND, getCurDebugLoc(), MVT::Other, getControlRoot(), getValue(I.getAddress()))); } -void SelectionDAGBuilder::visitFSub(User &I) { +void SelectionDAGBuilder::visitFSub(const User &I) { // -0.0 - X --> fneg const Type *Ty = I.getType(); if (Ty->isVectorTy()) { @@ -2103,14 +2082,14 @@ void SelectionDAGBuilder::visitFSub(User &I) { visitBinary(I, ISD::FSUB); } -void SelectionDAGBuilder::visitBinary(User &I, unsigned OpCode) { +void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) { SDValue Op1 = getValue(I.getOperand(0)); SDValue Op2 = getValue(I.getOperand(1)); setValue(&I, DAG.getNode(OpCode, getCurDebugLoc(), Op1.getValueType(), Op1, Op2)); } -void SelectionDAGBuilder::visitShift(User &I, unsigned Opcode) { +void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { SDValue Op1 = getValue(I.getOperand(0)); SDValue Op2 = getValue(I.getOperand(1)); if (!I.getType()->isVectorTy() && @@ -2144,11 +2123,11 @@ void SelectionDAGBuilder::visitShift(User &I, unsigned Opcode) { Op1.getValueType(), Op1, Op2)); } -void SelectionDAGBuilder::visitICmp(User &I) { +void SelectionDAGBuilder::visitICmp(const User &I) { ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE; - if (ICmpInst *IC = dyn_cast<ICmpInst>(&I)) + if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I)) predicate = IC->getPredicate(); - else if (ConstantExpr *IC = dyn_cast<ConstantExpr>(&I)) + else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I)) predicate = ICmpInst::Predicate(IC->getPredicate()); SDValue Op1 = getValue(I.getOperand(0)); SDValue Op2 = getValue(I.getOperand(1)); @@ -2158,11 +2137,11 @@ void SelectionDAGBuilder::visitICmp(User &I) { setValue(&I, DAG.getSetCC(getCurDebugLoc(), DestVT, Op1, Op2, Opcode)); } -void SelectionDAGBuilder::visitFCmp(User &I) { +void SelectionDAGBuilder::visitFCmp(const User &I) { FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE; - if (FCmpInst *FC = dyn_cast<FCmpInst>(&I)) + if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I)) predicate = FC->getPredicate(); - else if (ConstantExpr *FC = dyn_cast<ConstantExpr>(&I)) + else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I)) predicate = FCmpInst::Predicate(FC->getPredicate()); SDValue Op1 = getValue(I.getOperand(0)); SDValue Op2 = getValue(I.getOperand(1)); @@ -2171,7 +2150,7 @@ void SelectionDAGBuilder::visitFCmp(User &I) { setValue(&I, DAG.getSetCC(getCurDebugLoc(), DestVT, Op1, Op2, Condition)); } -void SelectionDAGBuilder::visitSelect(User &I) { +void SelectionDAGBuilder::visitSelect(const User &I) { SmallVector<EVT, 4> ValueVTs; ComputeValueVTs(TLI, I.getType(), ValueVTs); unsigned NumValues = ValueVTs.size(); @@ -2196,14 +2175,14 @@ void SelectionDAGBuilder::visitSelect(User &I) { &Values[0], NumValues)); } -void SelectionDAGBuilder::visitTrunc(User &I) { +void SelectionDAGBuilder::visitTrunc(const User &I) { // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurDebugLoc(), DestVT, N)); } -void SelectionDAGBuilder::visitZExt(User &I) { +void SelectionDAGBuilder::visitZExt(const User &I) { // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). // ZExt also can't be a cast to bool for same reason. So, nothing much to do SDValue N = getValue(I.getOperand(0)); @@ -2211,7 +2190,7 @@ void SelectionDAGBuilder::visitZExt(User &I) { setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurDebugLoc(), DestVT, N)); } -void SelectionDAGBuilder::visitSExt(User &I) { +void SelectionDAGBuilder::visitSExt(const User &I) { // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). // SExt also can't be a cast to bool for same reason. So, nothing much to do SDValue N = getValue(I.getOperand(0)); @@ -2219,7 +2198,7 @@ void SelectionDAGBuilder::visitSExt(User &I) { setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurDebugLoc(), DestVT, N)); } -void SelectionDAGBuilder::visitFPTrunc(User &I) { +void SelectionDAGBuilder::visitFPTrunc(const User &I) { // FPTrunc is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); @@ -2227,42 +2206,42 @@ void SelectionDAGBuilder::visitFPTrunc(User &I) { DestVT, N, DAG.getIntPtrConstant(0))); } -void SelectionDAGBuilder::visitFPExt(User &I){ +void SelectionDAGBuilder::visitFPExt(const User &I){ // FPTrunc is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurDebugLoc(), DestVT, N)); } -void SelectionDAGBuilder::visitFPToUI(User &I) { +void SelectionDAGBuilder::visitFPToUI(const User &I) { // FPToUI is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurDebugLoc(), DestVT, N)); } -void SelectionDAGBuilder::visitFPToSI(User &I) { +void SelectionDAGBuilder::visitFPToSI(const User &I) { // FPToSI is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurDebugLoc(), DestVT, N)); } -void SelectionDAGBuilder::visitUIToFP(User &I) { +void SelectionDAGBuilder::visitUIToFP(const User &I) { // UIToFP is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurDebugLoc(), DestVT, N)); } -void SelectionDAGBuilder::visitSIToFP(User &I){ +void SelectionDAGBuilder::visitSIToFP(const User &I){ // SIToFP is never a no-op cast, no need to check SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurDebugLoc(), DestVT, N)); } -void SelectionDAGBuilder::visitPtrToInt(User &I) { +void SelectionDAGBuilder::visitPtrToInt(const User &I) { // What to do depends on the size of the integer and the size of the pointer. // We can either truncate, zero extend, or no-op, accordingly. SDValue N = getValue(I.getOperand(0)); @@ -2271,7 +2250,7 @@ void SelectionDAGBuilder::visitPtrToInt(User &I) { setValue(&I, DAG.getZExtOrTrunc(N, getCurDebugLoc(), DestVT)); } -void SelectionDAGBuilder::visitIntToPtr(User &I) { +void SelectionDAGBuilder::visitIntToPtr(const User &I) { // What to do depends on the size of the integer and the size of the pointer. // We can either truncate, zero extend, or no-op, accordingly. SDValue N = getValue(I.getOperand(0)); @@ -2280,7 +2259,7 @@ void SelectionDAGBuilder::visitIntToPtr(User &I) { setValue(&I, DAG.getZExtOrTrunc(N, getCurDebugLoc(), DestVT)); } -void SelectionDAGBuilder::visitBitCast(User &I) { +void SelectionDAGBuilder::visitBitCast(const User &I) { SDValue N = getValue(I.getOperand(0)); EVT DestVT = TLI.getValueType(I.getType()); @@ -2293,7 +2272,7 @@ void SelectionDAGBuilder::visitBitCast(User &I) { setValue(&I, N); // noop cast. } -void SelectionDAGBuilder::visitInsertElement(User &I) { +void SelectionDAGBuilder::visitInsertElement(const User &I) { SDValue InVec = getValue(I.getOperand(0)); SDValue InVal = getValue(I.getOperand(1)); SDValue InIdx = DAG.getNode(ISD::ZERO_EXTEND, getCurDebugLoc(), @@ -2304,7 +2283,7 @@ void SelectionDAGBuilder::visitInsertElement(User &I) { InVec, InVal, InIdx)); } -void SelectionDAGBuilder::visitExtractElement(User &I) { +void SelectionDAGBuilder::visitExtractElement(const User &I) { SDValue InVec = getValue(I.getOperand(0)); SDValue InIdx = DAG.getNode(ISD::ZERO_EXTEND, getCurDebugLoc(), TLI.getPointerTy(), @@ -2323,7 +2302,7 @@ static bool SequentialMask(SmallVectorImpl<int> &Mask, unsigned SIndx) { return true; } -void SelectionDAGBuilder::visitShuffleVector(User &I) { +void SelectionDAGBuilder::visitShuffleVector(const User &I) { SmallVector<int, 8> Mask; SDValue Src1 = getValue(I.getOperand(0)); SDValue Src2 = getValue(I.getOperand(1)); @@ -2504,7 +2483,7 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) { VT, &Ops[0], Ops.size())); } -void SelectionDAGBuilder::visitInsertValue(InsertValueInst &I) { +void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { const Value *Op0 = I.getOperand(0); const Value *Op1 = I.getOperand(1); const Type *AggTy = I.getType(); @@ -2545,7 +2524,7 @@ void SelectionDAGBuilder::visitInsertValue(InsertValueInst &I) { &Values[0], NumAggValues)); } -void SelectionDAGBuilder::visitExtractValue(ExtractValueInst &I) { +void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) { const Value *Op0 = I.getOperand(0); const Type *AggTy = Op0->getType(); const Type *ValTy = I.getType(); @@ -2573,13 +2552,13 @@ void SelectionDAGBuilder::visitExtractValue(ExtractValueInst &I) { &Values[0], NumValValues)); } -void SelectionDAGBuilder::visitGetElementPtr(User &I) { +void SelectionDAGBuilder::visitGetElementPtr(const User &I) { SDValue N = getValue(I.getOperand(0)); const Type *Ty = I.getOperand(0)->getType(); - for (GetElementPtrInst::op_iterator OI = I.op_begin()+1, E = I.op_end(); + for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end(); OI != E; ++OI) { - Value *Idx = *OI; + const Value *Idx = *OI; if (const StructType *StTy = dyn_cast<StructType>(Ty)) { unsigned Field = cast<ConstantInt>(Idx)->getZExtValue(); if (Field) { @@ -2599,7 +2578,7 @@ void SelectionDAGBuilder::visitGetElementPtr(User &I) { Ty = cast<SequentialType>(Ty)->getElementType(); // If this is a constant subscript, handle it quickly. - if (ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) { + if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) { if (CI->getZExtValue() == 0) continue; uint64_t Offs = TD->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue(); @@ -2650,7 +2629,7 @@ void SelectionDAGBuilder::visitGetElementPtr(User &I) { setValue(&I, N); } -void SelectionDAGBuilder::visitAlloca(AllocaInst &I) { +void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { // If this is a fixed sized alloca in the entry block of the function, // allocate it statically on the stack. if (FuncInfo.StaticAllocaMap.count(&I)) @@ -2674,8 +2653,7 @@ void SelectionDAGBuilder::visitAlloca(AllocaInst &I) { // Handle alignment. If the requested alignment is less than or equal to // the stack alignment, ignore it. If the size is greater than or equal to // the stack alignment, we note this in the DYNAMIC_STACKALLOC node. - unsigned StackAlign = - TLI.getTargetMachine().getFrameInfo()->getStackAlignment(); + unsigned StackAlign = TM.getFrameInfo()->getStackAlignment(); if (Align <= StackAlign) Align = 0; @@ -2702,7 +2680,7 @@ void SelectionDAGBuilder::visitAlloca(AllocaInst &I) { FuncInfo.MF->getFrameInfo()->CreateVariableSizedObject(); } -void SelectionDAGBuilder::visitLoad(LoadInst &I) { +void SelectionDAGBuilder::visitLoad(const LoadInst &I) { const Value *SV = I.getOperand(0); SDValue Ptr = getValue(SV); @@ -2762,9 +2740,9 @@ void SelectionDAGBuilder::visitLoad(LoadInst &I) { &Values[0], NumValues)); } -void SelectionDAGBuilder::visitStore(StoreInst &I) { - Value *SrcV = I.getOperand(0); - Value *PtrV = I.getOperand(1); +void SelectionDAGBuilder::visitStore(const StoreInst &I) { + const Value *SrcV = I.getOperand(0); + const Value *PtrV = I.getOperand(1); SmallVector<EVT, 4> ValueVTs; SmallVector<uint64_t, 4> Offsets; @@ -2801,7 +2779,7 @@ void SelectionDAGBuilder::visitStore(StoreInst &I) { /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC /// node. -void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I, +void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic) { bool HasChain = !I.doesNotAccessMemory(); bool OnlyLoad = HasChain && I.onlyReadsMemory(); @@ -2927,7 +2905,8 @@ getF32Constant(SelectionDAG &DAG, unsigned Flt) { /// visitIntrinsicCall: I is a call instruction /// Op is the associated NodeType for I const char * -SelectionDAGBuilder::implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op) { +SelectionDAGBuilder::implVisitBinaryAtomic(const CallInst& I, + ISD::NodeType Op) { SDValue Root = getRoot(); SDValue L = DAG.getAtomic(Op, getCurDebugLoc(), @@ -2943,7 +2922,7 @@ SelectionDAGBuilder::implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op) { // implVisitAluOverflow - Lower arithmetic overflow instrinsics. const char * -SelectionDAGBuilder::implVisitAluOverflow(CallInst &I, ISD::NodeType Op) { +SelectionDAGBuilder::implVisitAluOverflow(const CallInst &I, ISD::NodeType Op) { SDValue Op1 = getValue(I.getOperand(1)); SDValue Op2 = getValue(I.getOperand(2)); @@ -2955,7 +2934,7 @@ SelectionDAGBuilder::implVisitAluOverflow(CallInst &I, ISD::NodeType Op) { /// visitExp - Lower an exp intrinsic. Handles the special sequences for /// limited-precision mode. void -SelectionDAGBuilder::visitExp(CallInst &I) { +SelectionDAGBuilder::visitExp(const CallInst &I) { SDValue result; DebugLoc dl = getCurDebugLoc(); @@ -3081,7 +3060,7 @@ SelectionDAGBuilder::visitExp(CallInst &I) { /// visitLog - Lower a log intrinsic. Handles the special sequences for /// limited-precision mode. void -SelectionDAGBuilder::visitLog(CallInst &I) { +SelectionDAGBuilder::visitLog(const CallInst &I) { SDValue result; DebugLoc dl = getCurDebugLoc(); @@ -3191,7 +3170,7 @@ SelectionDAGBuilder::visitLog(CallInst &I) { /// visitLog2 - Lower a log2 intrinsic. Handles the special sequences for /// limited-precision mode. void -SelectionDAGBuilder::visitLog2(CallInst &I) { +SelectionDAGBuilder::visitLog2(const CallInst &I) { SDValue result; DebugLoc dl = getCurDebugLoc(); @@ -3300,7 +3279,7 @@ SelectionDAGBuilder::visitLog2(CallInst &I) { /// visitLog10 - Lower a log10 intrinsic. Handles the special sequences for /// limited-precision mode. void -SelectionDAGBuilder::visitLog10(CallInst &I) { +SelectionDAGBuilder::visitLog10(const CallInst &I) { SDValue result; DebugLoc dl = getCurDebugLoc(); @@ -3402,7 +3381,7 @@ SelectionDAGBuilder::visitLog10(CallInst &I) { /// visitExp2 - Lower an exp2 intrinsic. Handles the special sequences for /// limited-precision mode. void -SelectionDAGBuilder::visitExp2(CallInst &I) { +SelectionDAGBuilder::visitExp2(const CallInst &I) { SDValue result; DebugLoc dl = getCurDebugLoc(); @@ -3516,9 +3495,9 @@ SelectionDAGBuilder::visitExp2(CallInst &I) { /// visitPow - Lower a pow intrinsic. Handles the special sequences for /// limited-precision mode with x == 10.0f. void -SelectionDAGBuilder::visitPow(CallInst &I) { +SelectionDAGBuilder::visitPow(const CallInst &I) { SDValue result; - Value *Val = I.getOperand(1); + const Value *Val = I.getOperand(1); DebugLoc dl = getCurDebugLoc(); bool IsExp10 = false; @@ -3664,7 +3643,7 @@ static SDValue ExpandPowI(DebugLoc DL, SDValue LHS, SDValue RHS, if (Val == 0) return DAG.getConstantFP(1.0, LHS.getValueType()); - Function *F = DAG.getMachineFunction().getFunction(); + const Function *F = DAG.getMachineFunction().getFunction(); if (!F->hasFnAttr(Attribute::OptimizeForSize) || // If optimizing for size, don't insert too many multiplies. This // inserts up to 5 multiplies. @@ -3700,12 +3679,58 @@ static SDValue ExpandPowI(DebugLoc DL, SDValue LHS, SDValue RHS, return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); } +/// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function +/// argument, create the corresponding DBG_VALUE machine instruction for it now. +/// At the end of instruction selection, they will be inserted to the entry BB. +bool +SelectionDAGBuilder::EmitFuncArgumentDbgValue(const DbgValueInst &DI, + const Value *V, MDNode *Variable, + uint64_t Offset, + const SDValue &N) { + if (!isa<Argument>(V)) + return false; + + MachineFunction &MF = DAG.getMachineFunction(); + // Ignore inlined function arguments here. + DIVariable DV(Variable); + if (DV.isInlinedFnArgument(MF.getFunction())) + return false; + + MachineBasicBlock *MBB = FuncInfo.MBBMap[DI.getParent()]; + if (MBB != &MF.front()) + return false; + + unsigned Reg = 0; + if (N.getOpcode() == ISD::CopyFromReg) { + Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg(); + if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) { + MachineRegisterInfo &RegInfo = MF.getRegInfo(); + unsigned PR = RegInfo.getLiveInPhysReg(Reg); + if (PR) + Reg = PR; + } + } + + if (!Reg) { + DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V); + if (VMI == FuncInfo.ValueMap.end()) + return false; + Reg = VMI->second; + } + + const TargetInstrInfo *TII = DAG.getTarget().getInstrInfo(); + MachineInstrBuilder MIB = BuildMI(MF, getCurDebugLoc(), + TII->get(TargetOpcode::DBG_VALUE)) + .addReg(Reg, RegState::Debug).addImm(Offset).addMetadata(Variable); + FuncInfo.ArgDbgValues.push_back(&*MIB); + return true; +} /// visitIntrinsicCall - Lower the call to the specified intrinsic function. If /// we want to emit this as a call to a named external function, return the name /// otherwise lower it and return null. const char * -SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { +SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { DebugLoc dl = getCurDebugLoc(); SDValue Res; @@ -3792,44 +3817,76 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return 0; } case Intrinsic::dbg_declare: { - // FIXME: currently, we get here only if OptLevel != CodeGenOpt::None. - // The real handling of this intrinsic is in FastISel. - if (OptLevel != CodeGenOpt::None) - // FIXME: Variable debug info is not supported here. - return 0; - DbgDeclareInst &DI = cast<DbgDeclareInst>(I); + const DbgDeclareInst &DI = cast<DbgDeclareInst>(I); if (!DIDescriptor::ValidDebugInfo(DI.getVariable(), CodeGenOpt::None)) return 0; MDNode *Variable = DI.getVariable(); - Value *Address = DI.getAddress(); + // Parameters are handled specially. + bool isParameter = + DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable; + const Value *Address = DI.getAddress(); if (!Address) return 0; - if (BitCastInst *BCI = dyn_cast<BitCastInst>(Address)) + if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address)) Address = BCI->getOperand(0); - AllocaInst *AI = dyn_cast<AllocaInst>(Address); - // Don't handle byval struct arguments or VLAs, for example. - if (!AI) - return 0; - DenseMap<const AllocaInst*, int>::iterator SI = - FuncInfo.StaticAllocaMap.find(AI); - if (SI == FuncInfo.StaticAllocaMap.end()) - return 0; // VLAs. - int FI = SI->second; + const AllocaInst *AI = dyn_cast<AllocaInst>(Address); + if (AI) { + // Don't handle byval arguments or VLAs, for example. + // Non-byval arguments are handled here (they refer to the stack temporary + // alloca at this point). + DenseMap<const AllocaInst*, int>::iterator SI = + FuncInfo.StaticAllocaMap.find(AI); + if (SI == FuncInfo.StaticAllocaMap.end()) + return 0; // VLAs. + int FI = SI->second; + + MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); + if (!DI.getDebugLoc().isUnknown() && MMI.hasDebugInfo()) + MMI.setVariableDbgInfo(Variable, FI, DI.getDebugLoc()); + } - MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); - if (!DI.getDebugLoc().isUnknown() && MMI.hasDebugInfo()) - MMI.setVariableDbgInfo(Variable, FI, DI.getDebugLoc()); + // Build an entry in DbgOrdering. Debug info input nodes get an SDNodeOrder + // but do not always have a corresponding SDNode built. The SDNodeOrder + // absolute, but not relative, values are different depending on whether + // debug info exists. + ++SDNodeOrder; + SDValue &N = NodeMap[Address]; + SDDbgValue *SDV; + if (N.getNode()) { + if (isParameter && !AI) { + FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode()); + if (FINode) + // Byval parameter. We have a frame index at this point. + SDV = DAG.getDbgValue(Variable, FINode->getIndex(), + 0, dl, SDNodeOrder); + else + // Can't do anything with other non-AI cases yet. This might be a + // parameter of a callee function that got inlined, for example. + return 0; + } else if (AI) + SDV = DAG.getDbgValue(Variable, N.getNode(), N.getResNo(), + 0, dl, SDNodeOrder); + else + // Can't do anything with other non-AI cases yet. + return 0; + DAG.AddDbgValue(SDV, N.getNode(), isParameter); + } else { + // This isn't useful, but it shows what we're missing. + SDV = DAG.getDbgValue(Variable, UndefValue::get(Address->getType()), + 0, dl, SDNodeOrder); + DAG.AddDbgValue(SDV, 0, isParameter); + } return 0; } case Intrinsic::dbg_value: { - DbgValueInst &DI = cast<DbgValueInst>(I); + const DbgValueInst &DI = cast<DbgValueInst>(I); if (!DIDescriptor::ValidDebugInfo(DI.getVariable(), CodeGenOpt::None)) return 0; MDNode *Variable = DI.getVariable(); uint64_t Offset = DI.getOffset(); - Value *V = DI.getValue(); + const Value *V = DI.getValue(); if (!V) return 0; @@ -3838,26 +3895,31 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { // absolute, but not relative, values are different depending on whether // debug info exists. ++SDNodeOrder; + SDDbgValue *SDV; if (isa<ConstantInt>(V) || isa<ConstantFP>(V)) { - DAG.AddDbgValue(DAG.getDbgValue(Variable, V, Offset, dl, SDNodeOrder)); + SDV = DAG.getDbgValue(Variable, V, Offset, dl, SDNodeOrder); + DAG.AddDbgValue(SDV, 0, false); } else { SDValue &N = NodeMap[V]; - if (N.getNode()) - DAG.AddDbgValue(DAG.getDbgValue(Variable, N.getNode(), - N.getResNo(), Offset, dl, SDNodeOrder), - N.getNode()); - else + if (N.getNode()) { + if (!EmitFuncArgumentDbgValue(DI, V, Variable, Offset, N)) { + SDV = DAG.getDbgValue(Variable, N.getNode(), + N.getResNo(), Offset, dl, SDNodeOrder); + DAG.AddDbgValue(SDV, N.getNode(), false); + } + } else { // We may expand this to cover more cases. One case where we have no // data available is an unreferenced parameter; we need this fallback. - DAG.AddDbgValue(DAG.getDbgValue(Variable, - UndefValue::get(V->getType()), - Offset, dl, SDNodeOrder)); + SDV = DAG.getDbgValue(Variable, UndefValue::get(V->getType()), + Offset, dl, SDNodeOrder); + DAG.AddDbgValue(SDV, 0, false); + } } // Build a debug info table entry. - if (BitCastInst *BCI = dyn_cast<BitCastInst>(V)) + if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V)) V = BCI->getOperand(0); - AllocaInst *AI = dyn_cast<AllocaInst>(V); + const AllocaInst *AI = dyn_cast<AllocaInst>(V); // Don't handle byval struct arguments or VLAs, for example. if (!AI) return 0; @@ -3874,7 +3936,8 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::eh_exception: { // Insert the EXCEPTIONADDR instruction. - assert(CurMBB->isLandingPad() &&"Call to eh.exception not in landing pad!"); + assert(FuncInfo.MBBMap[I.getParent()]->isLandingPad() && + "Call to eh.exception not in landing pad!"); SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other); SDValue Ops[1]; Ops[0] = DAG.getRoot(); @@ -3885,16 +3948,17 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::eh_selector: { + MachineBasicBlock *CallMBB = FuncInfo.MBBMap[I.getParent()]; MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); - if (CurMBB->isLandingPad()) - AddCatchInfo(I, &MMI, CurMBB); + if (CallMBB->isLandingPad()) + AddCatchInfo(I, &MMI, CallMBB); else { #ifndef NDEBUG FuncInfo.CatchInfoLost.insert(&I); #endif // FIXME: Mark exception selector register as live in. Hack for PR1508. unsigned Reg = TLI.getExceptionSelectorRegister(); - if (Reg) CurMBB->addLiveIn(Reg); + if (Reg) FuncInfo.MBBMap[I.getParent()]->addLiveIn(Reg); } // Insert the EHSELECTION instruction. @@ -3977,7 +4041,7 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::convertuu: Code = ISD::CVT_UU; break; } EVT DestVT = TLI.getValueType(I.getType()); - Value *Op1 = I.getOperand(1); + const Value *Op1 = I.getOperand(1); Res = DAG.getConvertRndSat(DestVT, getCurDebugLoc(), getValue(Op1), DAG.getValueType(DestVT), DAG.getValueType(getValue(Op1).getValueType()), @@ -4146,8 +4210,8 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::gcroot: if (GFI) { - Value *Alloca = I.getOperand(1); - Constant *TypeMap = cast<Constant>(I.getOperand(2)); + const Value *Alloca = I.getOperand(1); + const Constant *TypeMap = cast<Constant>(I.getOperand(2)); FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode()); GFI->addStackRoot(FI->getIndex(), TypeMap); @@ -4244,93 +4308,7 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } } -/// Test if the given instruction is in a position to be optimized -/// with a tail-call. This roughly means that it's in a block with -/// a return and there's nothing that needs to be scheduled -/// between it and the return. -/// -/// This function only tests target-independent requirements. -static bool -isInTailCallPosition(CallSite CS, Attributes CalleeRetAttr, - const TargetLowering &TLI) { - const Instruction *I = CS.getInstruction(); - const BasicBlock *ExitBB = I->getParent(); - const TerminatorInst *Term = ExitBB->getTerminator(); - const ReturnInst *Ret = dyn_cast<ReturnInst>(Term); - const Function *F = ExitBB->getParent(); - - // The block must end in a return statement or unreachable. - // - // FIXME: Decline tailcall if it's not guaranteed and if the block ends in - // an unreachable, for now. The way tailcall optimization is currently - // implemented means it will add an epilogue followed by a jump. That is - // not profitable. Also, if the callee is a special function (e.g. - // longjmp on x86), it can end up causing miscompilation that has not - // been fully understood. - if (!Ret && - (!GuaranteedTailCallOpt || !isa<UnreachableInst>(Term))) return false; - - // If I will have a chain, make sure no other instruction that will have a - // chain interposes between I and the return. - if (I->mayHaveSideEffects() || I->mayReadFromMemory() || - !I->isSafeToSpeculativelyExecute()) - for (BasicBlock::const_iterator BBI = prior(prior(ExitBB->end())); ; - --BBI) { - if (&*BBI == I) - break; - // Debug info intrinsics do not get in the way of tail call optimization. - if (isa<DbgInfoIntrinsic>(BBI)) - continue; - if (BBI->mayHaveSideEffects() || BBI->mayReadFromMemory() || - !BBI->isSafeToSpeculativelyExecute()) - return false; - } - - // If the block ends with a void return or unreachable, it doesn't matter - // what the call's return type is. - if (!Ret || Ret->getNumOperands() == 0) return true; - - // If the return value is undef, it doesn't matter what the call's - // return type is. - if (isa<UndefValue>(Ret->getOperand(0))) return true; - - // Conservatively require the attributes of the call to match those of - // the return. Ignore noalias because it doesn't affect the call sequence. - unsigned CallerRetAttr = F->getAttributes().getRetAttributes(); - if ((CalleeRetAttr ^ CallerRetAttr) & ~Attribute::NoAlias) - return false; - - // It's not safe to eliminate the sign / zero extension of the return value. - if ((CallerRetAttr & Attribute::ZExt) || (CallerRetAttr & Attribute::SExt)) - return false; - - // Otherwise, make sure the unmodified return value of I is the return value. - for (const Instruction *U = dyn_cast<Instruction>(Ret->getOperand(0)); ; - U = dyn_cast<Instruction>(U->getOperand(0))) { - if (!U) - return false; - if (!U->hasOneUse()) - return false; - if (U == I) - break; - // Check for a truly no-op truncate. - if (isa<TruncInst>(U) && - TLI.isTruncateFree(U->getOperand(0)->getType(), U->getType())) - continue; - // Check for a truly no-op bitcast. - if (isa<BitCastInst>(U) && - (U->getOperand(0)->getType() == U->getType() || - (U->getOperand(0)->getType()->isPointerTy() && - U->getType()->isPointerTy()))) - continue; - // Otherwise it's not a true no-op. - return false; - } - - return true; -} - -void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee, +void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, bool isTailCall, MachineBasicBlock *LandingPad) { const PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType()); @@ -4378,7 +4356,7 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee, RetTy = Type::getVoidTy(FTy->getContext()); } - for (CallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); + for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); i != e; ++i) { SDValue ArgNode = getValue(*i); Entry.Node = ArgNode; Entry.Ty = (*i)->getType(); @@ -4509,12 +4487,12 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee, /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the /// value is equal or not-equal to zero. -static bool IsOnlyUsedInZeroEqualityComparison(Value *V) { - for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); +static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) { + for (Value::const_use_iterator UI = V->use_begin(), E = V->use_end(); UI != E; ++UI) { - if (ICmpInst *IC = dyn_cast<ICmpInst>(*UI)) + if (const ICmpInst *IC = dyn_cast<ICmpInst>(*UI)) if (IC->isEquality()) - if (Constant *C = dyn_cast<Constant>(IC->getOperand(1))) + if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1))) if (C->isNullValue()) continue; // Unknown instruction. @@ -4523,17 +4501,20 @@ static bool IsOnlyUsedInZeroEqualityComparison(Value *V) { return true; } -static SDValue getMemCmpLoad(Value *PtrVal, MVT LoadVT, const Type *LoadTy, +static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, + const Type *LoadTy, SelectionDAGBuilder &Builder) { // Check to see if this load can be trivially constant folded, e.g. if the // input is from a string literal. - if (Constant *LoadInput = dyn_cast<Constant>(PtrVal)) { + if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) { // Cast pointer to the type we really want to load. - LoadInput = ConstantExpr::getBitCast(LoadInput, + LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput), PointerType::getUnqual(LoadTy)); - if (Constant *LoadCst = ConstantFoldLoadFromConstPtr(LoadInput, Builder.TD)) + if (const Constant *LoadCst = + ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput), + Builder.TD)) return Builder.getValue(LoadCst); } @@ -4566,18 +4547,18 @@ static SDValue getMemCmpLoad(Value *PtrVal, MVT LoadVT, const Type *LoadTy, /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form. /// If so, return true and lower it, otherwise return false and it will be /// lowered like a normal call. -bool SelectionDAGBuilder::visitMemCmpCall(CallInst &I) { +bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) { // Verify that the prototype makes sense. int memcmp(void*,void*,size_t) if (I.getNumOperands() != 4) return false; - Value *LHS = I.getOperand(1), *RHS = I.getOperand(2); + const Value *LHS = I.getOperand(1), *RHS = I.getOperand(2); if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() || !I.getOperand(3)->getType()->isIntegerTy() || !I.getType()->isIntegerTy()) return false; - ConstantInt *Size = dyn_cast<ConstantInt>(I.getOperand(3)); + const ConstantInt *Size = dyn_cast<ConstantInt>(I.getOperand(3)); // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0 @@ -4643,11 +4624,11 @@ bool SelectionDAGBuilder::visitMemCmpCall(CallInst &I) { } -void SelectionDAGBuilder::visitCall(CallInst &I) { +void SelectionDAGBuilder::visitCall(const CallInst &I) { const char *RenameFn = 0; if (Function *F = I.getCalledFunction()) { if (F->isDeclaration()) { - const TargetIntrinsicInfo *II = TLI.getTargetMachine().getIntrinsicInfo(); + const TargetIntrinsicInfo *II = TM.getIntrinsicInfo(); if (II) { if (unsigned IID = II->getIntrinsicID(F)) { RenameFn = visitIntrinsicCall(I, IID); @@ -4871,14 +4852,13 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl, /// AddInlineAsmOperands - Add this value to the specified inlineasm node /// operand list. This adds the code marker and includes the number of /// values added into it. -void RegsForValue::AddInlineAsmOperands(unsigned Code, - bool HasMatching,unsigned MatchingIdx, +void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, + unsigned MatchingIdx, SelectionDAG &DAG, std::vector<SDValue> &Ops) const { - assert(Regs.size() < (1 << 13) && "Too many inline asm outputs!"); - unsigned Flag = Code | (Regs.size() << 3); + unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size()); if (HasMatching) - Flag |= 0x80000000 | (MatchingIdx << 16); + Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx); SDValue Res = DAG.getTargetConstant(Flag, MVT::i32); Ops.push_back(Res); @@ -4994,7 +4974,7 @@ public: if (isIndirect) { const llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy); if (!PtrTy) - llvm_report_error("Indirect operand for inline asm not a pointer!"); + report_fatal_error("Indirect operand for inline asm not a pointer!"); OpTy = PtrTy->getElementType(); } @@ -5214,31 +5194,10 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, // Otherwise, we couldn't allocate enough registers for this. } -/// hasInlineAsmMemConstraint - Return true if the inline asm instruction being -/// processed uses a memory 'm' constraint. -static bool -hasInlineAsmMemConstraint(std::vector<InlineAsm::ConstraintInfo> &CInfos, - const TargetLowering &TLI) { - for (unsigned i = 0, e = CInfos.size(); i != e; ++i) { - InlineAsm::ConstraintInfo &CI = CInfos[i]; - for (unsigned j = 0, ee = CI.Codes.size(); j != ee; ++j) { - TargetLowering::ConstraintType CType = TLI.getConstraintType(CI.Codes[j]); - if (CType == TargetLowering::C_Memory) - return true; - } - - // Indirect operand accesses access memory. - if (CI.isIndirect) - return true; - } - - return false; -} - /// visitInlineAsm - Handle a call to an InlineAsm object. /// -void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { - InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue()); +void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { + const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue()); /// ConstraintOperands - Information about all of the constraints. std::vector<SDISelAsmOperandInfo> ConstraintOperands; @@ -5274,7 +5233,7 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { case InlineAsm::isOutput: // Indirect outputs just consume an argument. if (OpInfo.isIndirect) { - OpInfo.CallOperandVal = CS.getArgument(ArgNo++); + OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++)); break; } @@ -5291,7 +5250,7 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { ++ResNo; break; case InlineAsm::isInput: - OpInfo.CallOperandVal = CS.getArgument(ArgNo++); + OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++)); break; case InlineAsm::isClobber: // Nothing to do. @@ -5304,7 +5263,7 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // Strip bitcasts, if any. This mostly comes up for functions. OpInfo.CallOperandVal = OpInfo.CallOperandVal->stripPointerCasts(); - if (BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) { + if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) { OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]); } else { OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); @@ -5327,14 +5286,15 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // error. if (OpInfo.hasMatchingInput()) { SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput]; + if (OpInfo.ConstraintVT != Input.ConstraintVT) { if ((OpInfo.ConstraintVT.isInteger() != Input.ConstraintVT.isInteger()) || (OpInfo.ConstraintVT.getSizeInBits() != Input.ConstraintVT.getSizeInBits())) { - llvm_report_error("Unsupported asm: input constraint" - " with a matching output constraint of incompatible" - " type!"); + report_fatal_error("Unsupported asm: input constraint" + " with a matching output constraint of" + " incompatible type!"); } Input.ConstraintVT = OpInfo.ConstraintVT; } @@ -5356,7 +5316,7 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // If the operand is a float, integer, or vector constant, spill to a // constant pool entry to get its address. - Value *OpVal = OpInfo.CallOperandVal; + const Value *OpVal = OpInfo.CallOperandVal; if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) || isa<ConstantVector>(OpVal)) { OpInfo.CallOperand = DAG.getConstantPool(cast<Constant>(OpVal), @@ -5409,6 +5369,11 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { DAG.getTargetExternalSymbol(IA->getAsmString().c_str(), TLI.getPointerTy())); + // If we have a !srcloc metadata node associated with it, we want to attach + // this to the ultimately generated inline asm machineinstr. To do this, we + // pass in the third operand as this (potentially null) inline asm MDNode. + const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc"); + AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc)); // Loop over all of the inputs, copying the operand values into the // appropriate registers and processing the output regs. @@ -5428,8 +5393,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { assert(OpInfo.isIndirect && "Memory output must be indirect operand"); // Add information to the INLINEASM node to know about this output. - unsigned ResOpType = 4/*MEM*/ | (1<<3); - AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, + unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); + AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, TLI.getPointerTy())); AsmNodeOperands.push_back(OpInfo.CallOperand); break; @@ -5439,10 +5404,9 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // Copy the output from the appropriate register. Find a register that // we can use. - if (OpInfo.AssignedRegs.Regs.empty()) { - llvm_report_error("Couldn't allocate output reg for" - " constraint '" + OpInfo.ConstraintCode + "'!"); - } + if (OpInfo.AssignedRegs.Regs.empty()) + report_fatal_error("Couldn't allocate output reg for constraint '" + + Twine(OpInfo.ConstraintCode) + "'!"); // If this is an indirect operand, store through the pointer after the // asm. @@ -5459,8 +5423,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // Add information to the INLINEASM node to know that this register is // set. OpInfo.AssignedRegs.AddInlineAsmOperands(OpInfo.isEarlyClobber ? - 6 /* EARLYCLOBBER REGDEF */ : - 2 /* REGDEF */ , + InlineAsm::Kind_RegDefEarlyClobber : + InlineAsm::Kind_RegDef, false, 0, DAG, @@ -5477,27 +5441,30 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // Scan until we find the definition we already emitted of this operand. // When we find it, create a RegsForValue operand. - unsigned CurOp = 2; // The first operand. + unsigned CurOp = InlineAsm::Op_FirstOperand; for (; OperandNo; --OperandNo) { // Advance to the next operand. unsigned OpFlag = cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); - assert(((OpFlag & 7) == 2 /*REGDEF*/ || - (OpFlag & 7) == 6 /*EARLYCLOBBER REGDEF*/ || - (OpFlag & 7) == 4 /*MEM*/) && - "Skipped past definitions?"); + assert((InlineAsm::isRegDefKind(OpFlag) || + InlineAsm::isRegDefEarlyClobberKind(OpFlag) || + InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"); CurOp += InlineAsm::getNumOperandRegisters(OpFlag)+1; } unsigned OpFlag = cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); - if ((OpFlag & 7) == 2 /*REGDEF*/ - || (OpFlag & 7) == 6 /* EARLYCLOBBER REGDEF */) { + if (InlineAsm::isRegDefKind(OpFlag) || + InlineAsm::isRegDefEarlyClobberKind(OpFlag)) { // Add (OpFlag&0xffff)>>3 registers to MatchedRegs. if (OpInfo.isIndirect) { - llvm_report_error("Don't know how to handle tied indirect " - "register inputs yet!"); + // This happens on gcc/testsuite/gcc.dg/pr8788-1.c + LLVMContext &Ctx = *DAG.getContext(); + Ctx.emitError(CS.getInstruction(), "inline asm not supported yet:" + " don't know how to handle tied " + "indirect register inputs"); } + RegsForValue MatchedRegs; MatchedRegs.TLI = &TLI; MatchedRegs.ValueVTs.push_back(InOperandVal.getValueType()); @@ -5512,22 +5479,23 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // Use the produced MatchedRegs object to MatchedRegs.getCopyToRegs(InOperandVal, DAG, getCurDebugLoc(), Chain, &Flag); - MatchedRegs.AddInlineAsmOperands(1 /*REGUSE*/, + MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, true, OpInfo.getMatchedOperand(), DAG, AsmNodeOperands); break; - } else { - assert(((OpFlag & 7) == 4) && "Unknown matching constraint!"); - assert((InlineAsm::getNumOperandRegisters(OpFlag)) == 1 && - "Unexpected number of operands"); - // Add information to the INLINEASM node to know about this input. - // See InlineAsm.h isUseOperandTiedToDef. - OpFlag |= 0x80000000 | (OpInfo.getMatchedOperand() << 16); - AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlag, - TLI.getPointerTy())); - AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]); - break; } + + assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!"); + assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 && + "Unexpected number of operands"); + // Add information to the INLINEASM node to know about this input. + // See InlineAsm.h isUseOperandTiedToDef. + OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag, + OpInfo.getMatchedOperand()); + AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlag, + TLI.getPointerTy())); + AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]); + break; } if (OpInfo.ConstraintType == TargetLowering::C_Other) { @@ -5537,24 +5505,26 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { std::vector<SDValue> Ops; TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode[0], hasMemory, Ops, DAG); - if (Ops.empty()) { - llvm_report_error("Invalid operand for inline asm" - " constraint '" + OpInfo.ConstraintCode + "'!"); - } + if (Ops.empty()) + report_fatal_error("Invalid operand for inline asm constraint '" + + Twine(OpInfo.ConstraintCode) + "'!"); // Add information to the INLINEASM node to know about this input. - unsigned ResOpType = 3 /*IMM*/ | (Ops.size() << 3); + unsigned ResOpType = + InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size()); AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, TLI.getPointerTy())); AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end()); break; - } else if (OpInfo.ConstraintType == TargetLowering::C_Memory) { + } + + if (OpInfo.ConstraintType == TargetLowering::C_Memory) { assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!"); assert(InOperandVal.getValueType() == TLI.getPointerTy() && "Memory operands expect pointer values"); // Add information to the INLINEASM node to know about this input. - unsigned ResOpType = 4/*MEM*/ | (1<<3); + unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, TLI.getPointerTy())); AsmNodeOperands.push_back(InOperandVal); @@ -5569,15 +5539,14 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // Copy the input into the appropriate registers. if (OpInfo.AssignedRegs.Regs.empty() || - !OpInfo.AssignedRegs.areValueTypesLegal()) { - llvm_report_error("Couldn't allocate input reg for" - " constraint '"+ OpInfo.ConstraintCode +"'!"); - } + !OpInfo.AssignedRegs.areValueTypesLegal()) + report_fatal_error("Couldn't allocate input reg for constraint '" + + Twine(OpInfo.ConstraintCode) + "'!"); OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, getCurDebugLoc(), Chain, &Flag); - OpInfo.AssignedRegs.AddInlineAsmOperands(1/*REGUSE*/, false, 0, + OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0, DAG, AsmNodeOperands); break; } @@ -5585,7 +5554,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { // Add the clobbered value to the operand list, so that the register // allocator is aware that the physreg got clobbered. if (!OpInfo.AssignedRegs.Regs.empty()) - OpInfo.AssignedRegs.AddInlineAsmOperands(6 /* EARLYCLOBBER REGDEF */, + OpInfo.AssignedRegs.AddInlineAsmOperands( + InlineAsm::Kind_RegDefEarlyClobber, false, 0, DAG, AsmNodeOperands); break; @@ -5593,7 +5563,7 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { } } - // Finish up input operands. + // Finish up input operands. Set the input chain and add the flag last. AsmNodeOperands[0] = Chain; if (Flag.getNode()) AsmNodeOperands.push_back(Flag); @@ -5638,17 +5608,16 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { return; } - std::vector<std::pair<SDValue, Value*> > StoresToEmit; + std::vector<std::pair<SDValue, const Value *> > StoresToEmit; // Process indirect outputs, first output all of the flagged copies out of // physregs. for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) { RegsForValue &OutRegs = IndirectStoresToEmit[i].first; - Value *Ptr = IndirectStoresToEmit[i].second; + const Value *Ptr = IndirectStoresToEmit[i].second; SDValue OutVal = OutRegs.getCopyFromRegs(DAG, getCurDebugLoc(), Chain, &Flag); StoresToEmit.push_back(std::make_pair(OutVal, Ptr)); - } // Emit the non-flagged stores from the physregs. @@ -5669,14 +5638,14 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) { DAG.setRoot(Chain); } -void SelectionDAGBuilder::visitVAStart(CallInst &I) { +void SelectionDAGBuilder::visitVAStart(const CallInst &I) { DAG.setRoot(DAG.getNode(ISD::VASTART, getCurDebugLoc(), MVT::Other, getRoot(), getValue(I.getOperand(1)), DAG.getSrcValue(I.getOperand(1)))); } -void SelectionDAGBuilder::visitVAArg(VAArgInst &I) { +void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) { SDValue V = DAG.getVAArg(TLI.getValueType(I.getType()), getCurDebugLoc(), getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0))); @@ -5684,14 +5653,14 @@ void SelectionDAGBuilder::visitVAArg(VAArgInst &I) { DAG.setRoot(V.getValue(1)); } -void SelectionDAGBuilder::visitVAEnd(CallInst &I) { +void SelectionDAGBuilder::visitVAEnd(const CallInst &I) { DAG.setRoot(DAG.getNode(ISD::VAEND, getCurDebugLoc(), MVT::Other, getRoot(), getValue(I.getOperand(1)), DAG.getSrcValue(I.getOperand(1)))); } -void SelectionDAGBuilder::visitVACopy(CallInst &I) { +void SelectionDAGBuilder::visitVACopy(const CallInst &I) { DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurDebugLoc(), MVT::Other, getRoot(), getValue(I.getOperand(1)), @@ -5711,7 +5680,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, CallingConv::ID CallConv, bool isTailCall, bool isReturnValueUsed, SDValue Callee, - ArgListTy &Args, SelectionDAG &DAG, DebugLoc dl) { + ArgListTy &Args, SelectionDAG &DAG, + DebugLoc dl) const { // Handle all of the outgoing arguments. SmallVector<ISD::OutputArg, 32> Outs; for (unsigned i = 0, e = Args.size(); i != e; ++i) { @@ -5862,18 +5832,19 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy, void TargetLowering::LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results, - SelectionDAG &DAG) { + SelectionDAG &DAG) const { SDValue Res = LowerOperation(SDValue(N, 0), DAG); if (Res.getNode()) Results.push_back(Res); } -SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) { +SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { llvm_unreachable("LowerOperation not implemented for this target!"); return SDValue(); } -void SelectionDAGBuilder::CopyValueToVirtualRegister(Value *V, unsigned Reg) { +void +SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) { SDValue Op = getValue(V); assert((Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && @@ -5888,9 +5859,9 @@ void SelectionDAGBuilder::CopyValueToVirtualRegister(Value *V, unsigned Reg) { #include "llvm/CodeGen/SelectionDAGISel.h" -void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { +void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { // If this is the entry block, emit arguments. - Function &F = *LLVMBB->getParent(); + const Function &F = *LLVMBB->getParent(); SelectionDAG &DAG = SDB->DAG; SDValue OldRoot = DAG.getRoot(); DebugLoc dl = SDB->getCurDebugLoc(); @@ -5915,14 +5886,14 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { // or one register. ISD::ArgFlagsTy Flags; Flags.setSRet(); - EVT RegisterVT = TLI.getRegisterType(*CurDAG->getContext(), ValueVTs[0]); + EVT RegisterVT = TLI.getRegisterType(*DAG.getContext(), ValueVTs[0]); ISD::InputArg RetArg(Flags, RegisterVT, true); Ins.push_back(RetArg); } // Set up the incoming argument description vector. unsigned Idx = 1; - for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); + for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++Idx) { SmallVector<EVT, 4> ValueVTs; ComputeValueVTs(TLI, I->getType(), ValueVTs); @@ -6024,7 +5995,7 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { ++i; } - for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; + for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++Idx) { SmallVector<SDValue, 4> ArgValues; SmallVector<EVT, 4> ValueVTs; @@ -6067,7 +6038,7 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { // Finally, if the target has anything special to do, allow it to do so. // FIXME: this should insert code into the DAG! - EmitFunctionEntryCode(F, SDB->DAG.getMachineFunction()); + EmitFunctionEntryCode(); } /// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to @@ -6078,51 +6049,50 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { /// the end. /// void -SelectionDAGISel::HandlePHINodesInSuccessorBlocks(BasicBlock *LLVMBB) { - TerminatorInst *TI = LLVMBB->getTerminator(); +SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { + const TerminatorInst *TI = LLVMBB->getTerminator(); SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled; // Check successor nodes' PHI nodes that expect a constant to be available // from this block. for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) { - BasicBlock *SuccBB = TI->getSuccessor(succ); + const BasicBlock *SuccBB = TI->getSuccessor(succ); if (!isa<PHINode>(SuccBB->begin())) continue; - MachineBasicBlock *SuccMBB = FuncInfo->MBBMap[SuccBB]; + MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB]; // If this terminator has multiple identical successors (common for // switches), only handle each succ once. if (!SuccsHandled.insert(SuccMBB)) continue; MachineBasicBlock::iterator MBBI = SuccMBB->begin(); - PHINode *PN; // At this point we know that there is a 1-1 correspondence between LLVM PHI // nodes and Machine PHI nodes, but the incoming operands have not been // emitted yet. - for (BasicBlock::iterator I = SuccBB->begin(); - (PN = dyn_cast<PHINode>(I)); ++I) { + for (BasicBlock::const_iterator I = SuccBB->begin(); + const PHINode *PN = dyn_cast<PHINode>(I); ++I) { // Ignore dead phi's. if (PN->use_empty()) continue; unsigned Reg; - Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB); + const Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB); - if (Constant *C = dyn_cast<Constant>(PHIOp)) { - unsigned &RegOut = SDB->ConstantsOut[C]; + if (const Constant *C = dyn_cast<Constant>(PHIOp)) { + unsigned &RegOut = ConstantsOut[C]; if (RegOut == 0) { - RegOut = FuncInfo->CreateRegForValue(C); - SDB->CopyValueToVirtualRegister(C, RegOut); + RegOut = FuncInfo.CreateRegForValue(C); + CopyValueToVirtualRegister(C, RegOut); } Reg = RegOut; } else { - Reg = FuncInfo->ValueMap[PHIOp]; + Reg = FuncInfo.ValueMap[PHIOp]; if (Reg == 0) { assert(isa<AllocaInst>(PHIOp) && - FuncInfo->StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && + FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && "Didn't codegen value into a register!??"); - Reg = FuncInfo->CreateRegForValue(PHIOp); - SDB->CopyValueToVirtualRegister(PHIOp, Reg); + Reg = FuncInfo.CreateRegForValue(PHIOp); + CopyValueToVirtualRegister(PHIOp, Reg); } } @@ -6132,77 +6102,12 @@ SelectionDAGISel::HandlePHINodesInSuccessorBlocks(BasicBlock *LLVMBB) { ComputeValueVTs(TLI, PN->getType(), ValueVTs); for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) { EVT VT = ValueVTs[vti]; - unsigned NumRegisters = TLI.getNumRegisters(*CurDAG->getContext(), VT); + unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT); for (unsigned i = 0, e = NumRegisters; i != e; ++i) - SDB->PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); + FuncInfo.PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); Reg += NumRegisters; } } } - SDB->ConstantsOut.clear(); -} - -/// This is the Fast-ISel version of HandlePHINodesInSuccessorBlocks. It only -/// supports legal types, and it emits MachineInstrs directly instead of -/// creating SelectionDAG nodes. -/// -bool -SelectionDAGISel::HandlePHINodesInSuccessorBlocksFast(BasicBlock *LLVMBB, - FastISel *F) { - TerminatorInst *TI = LLVMBB->getTerminator(); - - SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled; - unsigned OrigNumPHINodesToUpdate = SDB->PHINodesToUpdate.size(); - - // Check successor nodes' PHI nodes that expect a constant to be available - // from this block. - for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) { - BasicBlock *SuccBB = TI->getSuccessor(succ); - if (!isa<PHINode>(SuccBB->begin())) continue; - MachineBasicBlock *SuccMBB = FuncInfo->MBBMap[SuccBB]; - - // If this terminator has multiple identical successors (common for - // switches), only handle each succ once. - if (!SuccsHandled.insert(SuccMBB)) continue; - - MachineBasicBlock::iterator MBBI = SuccMBB->begin(); - PHINode *PN; - - // At this point we know that there is a 1-1 correspondence between LLVM PHI - // nodes and Machine PHI nodes, but the incoming operands have not been - // emitted yet. - for (BasicBlock::iterator I = SuccBB->begin(); - (PN = dyn_cast<PHINode>(I)); ++I) { - // Ignore dead phi's. - if (PN->use_empty()) continue; - - // Only handle legal types. Two interesting things to note here. First, - // by bailing out early, we may leave behind some dead instructions, - // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its - // own moves. Second, this check is necessary becuase FastISel doesn't - // use CreateRegForValue to create registers, so it always creates - // exactly one register for each non-void instruction. - EVT VT = TLI.getValueType(PN->getType(), /*AllowUnknown=*/true); - if (VT == MVT::Other || !TLI.isTypeLegal(VT)) { - // Promote MVT::i1. - if (VT == MVT::i1) - VT = TLI.getTypeToTransformTo(*CurDAG->getContext(), VT); - else { - SDB->PHINodesToUpdate.resize(OrigNumPHINodesToUpdate); - return false; - } - } - - Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB); - - unsigned Reg = F->getRegForValue(PHIOp); - if (Reg == 0) { - SDB->PHINodesToUpdate.resize(OrigNumPHINodesToUpdate); - return false; - } - SDB->PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg)); - } - } - - return true; + ConstantsOut.clear(); } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index fdcba0f..3fcd4b9 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -36,6 +36,7 @@ class BasicBlock; class BitCastInst; class BranchInst; class CallInst; +class DbgValueInst; class ExtractElementInst; class ExtractValueInst; class FCmpInst; @@ -58,6 +59,7 @@ class LoadInst; class MachineBasicBlock; class MachineInstr; class MachineRegisterInfo; +class MDNode; class PHINode; class PtrToIntInst; class ReturnInst; @@ -80,11 +82,8 @@ class ZExtInst; //===----------------------------------------------------------------------===// /// SelectionDAGBuilder - This is the common target-independent lowering /// implementation that is parameterized by a TargetLowering object. -/// Also, targets can overload any lowering method. /// class SelectionDAGBuilder { - MachineBasicBlock *CurMBB; - /// CurDebugLoc - current file + line number. Changes as we build the DAG. DebugLoc CurDebugLoc; @@ -143,15 +142,16 @@ private: /// CaseRec - A struct with ctor used in lowering switches to a binary tree /// of conditional branches. struct CaseRec { - CaseRec(MachineBasicBlock *bb, Constant *lt, Constant *ge, CaseRange r) : + CaseRec(MachineBasicBlock *bb, const Constant *lt, const Constant *ge, + CaseRange r) : CaseBB(bb), LT(lt), GE(ge), Range(r) {} /// CaseBB - The MBB in which to emit the compare and branch MachineBasicBlock *CaseBB; /// LT, GE - If nonzero, we know the current case value must be less-than or /// greater-than-or-equal-to these Constants. - Constant *LT; - Constant *GE; + const Constant *LT; + const Constant *GE; /// Range - A pair of iterators representing the range of case values to be /// processed at this point in the binary search tree. CaseRange Range; @@ -182,7 +182,8 @@ private: /// SelectionDAGBuilder and SDISel for the code generation of additional basic /// blocks needed by multi-case switch statements. struct CaseBlock { - CaseBlock(ISD::CondCode cc, Value *cmplhs, Value *cmprhs, Value *cmpmiddle, + CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs, + const Value *cmpmiddle, MachineBasicBlock *truebb, MachineBasicBlock *falsebb, MachineBasicBlock *me) : CC(cc), CmpLHS(cmplhs), CmpMHS(cmpmiddle), CmpRHS(cmprhs), @@ -192,7 +193,7 @@ private: // CmpLHS/CmpRHS/CmpMHS - The LHS/MHS/RHS of the comparison to emit. // Emit by default LHS op RHS. MHS is used for range comparisons: // If MHS is not null: (LHS <= MHS) and (MHS <= RHS). - Value *CmpLHS, *CmpMHS, *CmpRHS; + const Value *CmpLHS, *CmpMHS, *CmpRHS; // TrueBB/FalseBB - the block to branch to if the setcc is true/false. MachineBasicBlock *TrueBB, *FalseBB; // ThisBB - the block into which to emit the code for the setcc and branches @@ -214,12 +215,12 @@ private: MachineBasicBlock *Default; }; struct JumpTableHeader { - JumpTableHeader(APInt F, APInt L, Value *SV, MachineBasicBlock *H, + JumpTableHeader(APInt F, APInt L, const Value *SV, MachineBasicBlock *H, bool E = false): First(F), Last(L), SValue(SV), HeaderBB(H), Emitted(E) {} APInt First; APInt Last; - Value *SValue; + const Value *SValue; MachineBasicBlock *HeaderBB; bool Emitted; }; @@ -236,7 +237,7 @@ private: typedef SmallVector<BitTestCase, 3> BitTestInfo; struct BitTestBlock { - BitTestBlock(APInt F, APInt R, Value* SV, + BitTestBlock(APInt F, APInt R, const Value* SV, unsigned Rg, bool E, MachineBasicBlock* P, MachineBasicBlock* D, const BitTestInfo& C): @@ -244,7 +245,7 @@ private: Parent(P), Default(D), Cases(C) { } APInt First; APInt Range; - Value *SValue; + const Value *SValue; unsigned Reg; bool Emitted; MachineBasicBlock *Parent; @@ -256,7 +257,8 @@ public: // TLI - This is information that describes the available target features we // need for lowering. This indicates when operations are unavailable, // implemented with a libcall, etc. - TargetLowering &TLI; + const TargetMachine &TM; + const TargetLowering &TLI; SelectionDAG &DAG; const TargetData *TD; AliasAnalysis *AA; @@ -271,17 +273,9 @@ public: /// SwitchInst code generation information. std::vector<BitTestBlock> BitTestCases; - /// PHINodesToUpdate - A list of phi instructions whose operand list will - /// be updated after processing the current basic block. - std::vector<std::pair<MachineInstr*, unsigned> > PHINodesToUpdate; - - /// EdgeMapping - If an edge from CurMBB to any MBB is changed (e.g. due to - /// scheduler custom lowering), track the change here. - DenseMap<MachineBasicBlock*, MachineBasicBlock*> EdgeMapping; - // Emit PHI-node-operand constants only once even if used by multiple // PHI nodes. - DenseMap<Constant*, unsigned> ConstantsOut; + DenseMap<const Constant *, unsigned> ConstantsOut; /// FuncInfo - Information about the function as a whole. /// @@ -302,16 +296,16 @@ public: LLVMContext *Context; - SelectionDAGBuilder(SelectionDAG &dag, TargetLowering &tli, - FunctionLoweringInfo &funcinfo, + SelectionDAGBuilder(SelectionDAG &dag, FunctionLoweringInfo &funcinfo, CodeGenOpt::Level ol) - : SDNodeOrder(0), TLI(tli), DAG(dag), FuncInfo(funcinfo), OptLevel(ol), + : SDNodeOrder(0), TM(dag.getTarget()), TLI(dag.getTargetLoweringInfo()), + DAG(dag), FuncInfo(funcinfo), OptLevel(ol), HasTailCall(false), Context(dag.getContext()) { } void init(GCFunctionInfo *gfi, AliasAnalysis &aa); - /// clear - Clear out the curret SelectionDAG and the associated + /// clear - Clear out the current SelectionDAG and the associated /// state and prepare this SelectionDAGBuilder object to be used /// for a new block. This doesn't clear out information about /// additional blocks that are needed to complete switch lowering @@ -333,22 +327,19 @@ public: SDValue getControlRoot(); DebugLoc getCurDebugLoc() const { return CurDebugLoc; } - void setCurDebugLoc(DebugLoc dl) { CurDebugLoc = dl; } unsigned getSDNodeOrder() const { return SDNodeOrder; } - void CopyValueToVirtualRegister(Value *V, unsigned Reg); + void CopyValueToVirtualRegister(const Value *V, unsigned Reg); /// AssignOrderingToNode - Assign an ordering to the node. The order is gotten /// from how the code appeared in the source. The ordering is used by the /// scheduler to effectively turn off scheduling. void AssignOrderingToNode(const SDNode *Node); - void visit(Instruction &I); - - void visit(unsigned Opcode, User &I); + void visit(const Instruction &I); - void setCurrentBasicBlock(MachineBasicBlock *MBB) { CurMBB = MBB; } + void visit(unsigned Opcode, const User &I); SDValue getValue(const Value *V); @@ -362,136 +353,154 @@ public: std::set<unsigned> &OutputRegs, std::set<unsigned> &InputRegs); - void FindMergedConditions(Value *Cond, MachineBasicBlock *TBB, + void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, - unsigned Opc); - void EmitBranchForMergedCondition(Value *Cond, MachineBasicBlock *TBB, + MachineBasicBlock *SwitchBB, unsigned Opc); + void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, - MachineBasicBlock *CurBB); + MachineBasicBlock *CurBB, + MachineBasicBlock *SwitchBB); bool ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases); - bool isExportableFromCurrentBlock(Value *V, const BasicBlock *FromBB); - void CopyToExportRegsIfNeeded(Value *V); - void ExportFromCurrentBlock(Value *V); - void LowerCallTo(CallSite CS, SDValue Callee, bool IsTailCall, + bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB); + void CopyToExportRegsIfNeeded(const Value *V); + void ExportFromCurrentBlock(const Value *V); + void LowerCallTo(ImmutableCallSite CS, SDValue Callee, bool IsTailCall, MachineBasicBlock *LandingPad = NULL); private: // Terminator instructions. - void visitRet(ReturnInst &I); - void visitBr(BranchInst &I); - void visitSwitch(SwitchInst &I); - void visitIndirectBr(IndirectBrInst &I); - void visitUnreachable(UnreachableInst &I) { /* noop */ } + void visitRet(const ReturnInst &I); + void visitBr(const BranchInst &I); + void visitSwitch(const SwitchInst &I); + void visitIndirectBr(const IndirectBrInst &I); + void visitUnreachable(const UnreachableInst &I) { /* noop */ } // Helpers for visitSwitch bool handleSmallSwitchRange(CaseRec& CR, CaseRecVector& WorkList, - Value* SV, - MachineBasicBlock* Default); + const Value* SV, + MachineBasicBlock* Default, + MachineBasicBlock *SwitchBB); bool handleJTSwitchCase(CaseRec& CR, CaseRecVector& WorkList, - Value* SV, - MachineBasicBlock* Default); + const Value* SV, + MachineBasicBlock* Default, + MachineBasicBlock *SwitchBB); bool handleBTSplitSwitchCase(CaseRec& CR, CaseRecVector& WorkList, - Value* SV, - MachineBasicBlock* Default); + const Value* SV, + MachineBasicBlock* Default, + MachineBasicBlock *SwitchBB); bool handleBitTestsSwitchCase(CaseRec& CR, CaseRecVector& WorkList, - Value* SV, - MachineBasicBlock* Default); + const Value* SV, + MachineBasicBlock* Default, + MachineBasicBlock *SwitchBB); public: - void visitSwitchCase(CaseBlock &CB); - void visitBitTestHeader(BitTestBlock &B); + void visitSwitchCase(CaseBlock &CB, + MachineBasicBlock *SwitchBB); + void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB); void visitBitTestCase(MachineBasicBlock* NextMBB, unsigned Reg, - BitTestCase &B); + BitTestCase &B, + MachineBasicBlock *SwitchBB); void visitJumpTable(JumpTable &JT); - void visitJumpTableHeader(JumpTable &JT, JumpTableHeader &JTH); + void visitJumpTableHeader(JumpTable &JT, JumpTableHeader &JTH, + MachineBasicBlock *SwitchBB); private: // These all get lowered before this pass. - void visitInvoke(InvokeInst &I); - void visitUnwind(UnwindInst &I); - - void visitBinary(User &I, unsigned OpCode); - void visitShift(User &I, unsigned Opcode); - void visitAdd(User &I) { visitBinary(I, ISD::ADD); } - void visitFAdd(User &I) { visitBinary(I, ISD::FADD); } - void visitSub(User &I) { visitBinary(I, ISD::SUB); } - void visitFSub(User &I); - void visitMul(User &I) { visitBinary(I, ISD::MUL); } - void visitFMul(User &I) { visitBinary(I, ISD::FMUL); } - void visitURem(User &I) { visitBinary(I, ISD::UREM); } - void visitSRem(User &I) { visitBinary(I, ISD::SREM); } - void visitFRem(User &I) { visitBinary(I, ISD::FREM); } - void visitUDiv(User &I) { visitBinary(I, ISD::UDIV); } - void visitSDiv(User &I) { visitBinary(I, ISD::SDIV); } - void visitFDiv(User &I) { visitBinary(I, ISD::FDIV); } - void visitAnd (User &I) { visitBinary(I, ISD::AND); } - void visitOr (User &I) { visitBinary(I, ISD::OR); } - void visitXor (User &I) { visitBinary(I, ISD::XOR); } - void visitShl (User &I) { visitShift(I, ISD::SHL); } - void visitLShr(User &I) { visitShift(I, ISD::SRL); } - void visitAShr(User &I) { visitShift(I, ISD::SRA); } - void visitICmp(User &I); - void visitFCmp(User &I); + void visitInvoke(const InvokeInst &I); + void visitUnwind(const UnwindInst &I); + + void visitBinary(const User &I, unsigned OpCode); + void visitShift(const User &I, unsigned Opcode); + void visitAdd(const User &I) { visitBinary(I, ISD::ADD); } + void visitFAdd(const User &I) { visitBinary(I, ISD::FADD); } + void visitSub(const User &I) { visitBinary(I, ISD::SUB); } + void visitFSub(const User &I); + void visitMul(const User &I) { visitBinary(I, ISD::MUL); } + void visitFMul(const User &I) { visitBinary(I, ISD::FMUL); } + void visitURem(const User &I) { visitBinary(I, ISD::UREM); } + void visitSRem(const User &I) { visitBinary(I, ISD::SREM); } + void visitFRem(const User &I) { visitBinary(I, ISD::FREM); } + void visitUDiv(const User &I) { visitBinary(I, ISD::UDIV); } + void visitSDiv(const User &I) { visitBinary(I, ISD::SDIV); } + void visitFDiv(const User &I) { visitBinary(I, ISD::FDIV); } + void visitAnd (const User &I) { visitBinary(I, ISD::AND); } + void visitOr (const User &I) { visitBinary(I, ISD::OR); } + void visitXor (const User &I) { visitBinary(I, ISD::XOR); } + void visitShl (const User &I) { visitShift(I, ISD::SHL); } + void visitLShr(const User &I) { visitShift(I, ISD::SRL); } + void visitAShr(const User &I) { visitShift(I, ISD::SRA); } + void visitICmp(const User &I); + void visitFCmp(const User &I); // Visit the conversion instructions - void visitTrunc(User &I); - void visitZExt(User &I); - void visitSExt(User &I); - void visitFPTrunc(User &I); - void visitFPExt(User &I); - void visitFPToUI(User &I); - void visitFPToSI(User &I); - void visitUIToFP(User &I); - void visitSIToFP(User &I); - void visitPtrToInt(User &I); - void visitIntToPtr(User &I); - void visitBitCast(User &I); - - void visitExtractElement(User &I); - void visitInsertElement(User &I); - void visitShuffleVector(User &I); - - void visitExtractValue(ExtractValueInst &I); - void visitInsertValue(InsertValueInst &I); - - void visitGetElementPtr(User &I); - void visitSelect(User &I); - - void visitAlloca(AllocaInst &I); - void visitLoad(LoadInst &I); - void visitStore(StoreInst &I); - void visitPHI(PHINode &I) { } // PHI nodes are handled specially. - void visitCall(CallInst &I); - bool visitMemCmpCall(CallInst &I); + void visitTrunc(const User &I); + void visitZExt(const User &I); + void visitSExt(const User &I); + void visitFPTrunc(const User &I); + void visitFPExt(const User &I); + void visitFPToUI(const User &I); + void visitFPToSI(const User &I); + void visitUIToFP(const User &I); + void visitSIToFP(const User &I); + void visitPtrToInt(const User &I); + void visitIntToPtr(const User &I); + void visitBitCast(const User &I); + + void visitExtractElement(const User &I); + void visitInsertElement(const User &I); + void visitShuffleVector(const User &I); + + void visitExtractValue(const ExtractValueInst &I); + void visitInsertValue(const InsertValueInst &I); + + void visitGetElementPtr(const User &I); + void visitSelect(const User &I); + + void visitAlloca(const AllocaInst &I); + void visitLoad(const LoadInst &I); + void visitStore(const StoreInst &I); + void visitPHI(const PHINode &I); + void visitCall(const CallInst &I); + bool visitMemCmpCall(const CallInst &I); - void visitInlineAsm(CallSite CS); - const char *visitIntrinsicCall(CallInst &I, unsigned Intrinsic); - void visitTargetIntrinsic(CallInst &I, unsigned Intrinsic); - - void visitPow(CallInst &I); - void visitExp2(CallInst &I); - void visitExp(CallInst &I); - void visitLog(CallInst &I); - void visitLog2(CallInst &I); - void visitLog10(CallInst &I); - - void visitVAStart(CallInst &I); - void visitVAArg(VAArgInst &I); - void visitVAEnd(CallInst &I); - void visitVACopy(CallInst &I); - - void visitUserOp1(Instruction &I) { + void visitInlineAsm(ImmutableCallSite CS); + const char *visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); + void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic); + + void visitPow(const CallInst &I); + void visitExp2(const CallInst &I); + void visitExp(const CallInst &I); + void visitLog(const CallInst &I); + void visitLog2(const CallInst &I); + void visitLog10(const CallInst &I); + + void visitVAStart(const CallInst &I); + void visitVAArg(const VAArgInst &I); + void visitVAEnd(const CallInst &I); + void visitVACopy(const CallInst &I); + + void visitUserOp1(const Instruction &I) { llvm_unreachable("UserOp1 should not exist at instruction selection time!"); } - void visitUserOp2(Instruction &I) { + void visitUserOp2(const Instruction &I) { llvm_unreachable("UserOp2 should not exist at instruction selection time!"); } - const char *implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op); - const char *implVisitAluOverflow(CallInst &I, ISD::NodeType Op); + const char *implVisitBinaryAtomic(const CallInst& I, ISD::NodeType Op); + const char *implVisitAluOverflow(const CallInst &I, ISD::NodeType Op); + + void HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB); + + /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a + /// function argument, create the corresponding DBG_VALUE machine instruction + /// for it now. At the end of instruction selection, they will be inserted to + /// the entry BB. + bool EmitFuncArgumentDbgValue(const DbgValueInst &DI, + const Value *V, MDNode *Variable, + uint64_t Offset, const SDValue &N); }; } // end namespace llvm diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 9b137a5..422cb7a 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -19,10 +19,7 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/DebugInfo.h" #include "llvm/Constants.h" -#include "llvm/CallingConv.h" -#include "llvm/DerivedTypes.h" #include "llvm/Function.h" -#include "llvm/GlobalVariable.h" #include "llvm/InlineAsm.h" #include "llvm/Instructions.h" #include "llvm/Intrinsics.h" @@ -32,18 +29,13 @@ #include "llvm/CodeGen/GCStrategy.h" #include "llvm/CodeGen/GCMetadata.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineFunctionAnalysis.h" -#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/ScheduleHazardRecognizer.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Target/TargetData.h" -#include "llvm/Target/TargetFrameInfo.h" #include "llvm/Target/TargetIntrinsicInfo.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" @@ -52,7 +44,6 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MathExtras.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" #include "llvm/ADT/Statistic.h" @@ -69,10 +60,6 @@ EnableFastISelVerbose("fast-isel-verbose", cl::Hidden, static cl::opt<bool> EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction fails")); -static cl::opt<bool> -SchedLiveInCopies("schedule-livein-copies", cl::Hidden, - cl::desc("Schedule copies of livein registers"), - cl::init(false)); #ifndef NDEBUG static cl::opt<bool> @@ -161,9 +148,9 @@ namespace llvm { // When new basic blocks are inserted and the edges from MBB to its successors // are modified, the method should insert pairs of <OldSucc, NewSucc> into the // DenseMap. -MachineBasicBlock *TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, - MachineBasicBlock *MBB, - DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) const { +MachineBasicBlock * +TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, + MachineBasicBlock *MBB) const { #ifndef NDEBUG dbgs() << "If a target marks an instruction with " "'usesCustomInserter', it must implement " @@ -173,115 +160,15 @@ MachineBasicBlock *TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, return 0; } -/// EmitLiveInCopy - Emit a copy for a live in physical register. If the -/// physical register has only a single copy use, then coalesced the copy -/// if possible. -static void EmitLiveInCopy(MachineBasicBlock *MBB, - MachineBasicBlock::iterator &InsertPos, - unsigned VirtReg, unsigned PhysReg, - const TargetRegisterClass *RC, - DenseMap<MachineInstr*, unsigned> &CopyRegMap, - const MachineRegisterInfo &MRI, - const TargetRegisterInfo &TRI, - const TargetInstrInfo &TII) { - unsigned NumUses = 0; - MachineInstr *UseMI = NULL; - for (MachineRegisterInfo::use_iterator UI = MRI.use_begin(VirtReg), - UE = MRI.use_end(); UI != UE; ++UI) { - UseMI = &*UI; - if (++NumUses > 1) - break; - } - - // If the number of uses is not one, or the use is not a move instruction, - // don't coalesce. Also, only coalesce away a virtual register to virtual - // register copy. - bool Coalesced = false; - unsigned SrcReg, DstReg, SrcSubReg, DstSubReg; - if (NumUses == 1 && - TII.isMoveInstr(*UseMI, SrcReg, DstReg, SrcSubReg, DstSubReg) && - TargetRegisterInfo::isVirtualRegister(DstReg)) { - VirtReg = DstReg; - Coalesced = true; - } - - // Now find an ideal location to insert the copy. - MachineBasicBlock::iterator Pos = InsertPos; - while (Pos != MBB->begin()) { - MachineInstr *PrevMI = prior(Pos); - DenseMap<MachineInstr*, unsigned>::iterator RI = CopyRegMap.find(PrevMI); - // copyRegToReg might emit multiple instructions to do a copy. - unsigned CopyDstReg = (RI == CopyRegMap.end()) ? 0 : RI->second; - if (CopyDstReg && !TRI.regsOverlap(CopyDstReg, PhysReg)) - // This is what the BB looks like right now: - // r1024 = mov r0 - // ... - // r1 = mov r1024 - // - // We want to insert "r1025 = mov r1". Inserting this copy below the - // move to r1024 makes it impossible for that move to be coalesced. - // - // r1025 = mov r1 - // r1024 = mov r0 - // ... - // r1 = mov 1024 - // r2 = mov 1025 - break; // Woot! Found a good location. - --Pos; - } - - bool Emitted = TII.copyRegToReg(*MBB, Pos, VirtReg, PhysReg, RC, RC); - assert(Emitted && "Unable to issue a live-in copy instruction!\n"); - (void) Emitted; - - CopyRegMap.insert(std::make_pair(prior(Pos), VirtReg)); - if (Coalesced) { - if (&*InsertPos == UseMI) ++InsertPos; - MBB->erase(UseMI); - } -} - -/// EmitLiveInCopies - If this is the first basic block in the function, -/// and if it has live ins that need to be copied into vregs, emit the -/// copies into the block. -static void EmitLiveInCopies(MachineBasicBlock *EntryMBB, - const MachineRegisterInfo &MRI, - const TargetRegisterInfo &TRI, - const TargetInstrInfo &TII) { - if (SchedLiveInCopies) { - // Emit the copies at a heuristically-determined location in the block. - DenseMap<MachineInstr*, unsigned> CopyRegMap; - MachineBasicBlock::iterator InsertPos = EntryMBB->begin(); - for (MachineRegisterInfo::livein_iterator LI = MRI.livein_begin(), - E = MRI.livein_end(); LI != E; ++LI) - if (LI->second) { - const TargetRegisterClass *RC = MRI.getRegClass(LI->second); - EmitLiveInCopy(EntryMBB, InsertPos, LI->second, LI->first, - RC, CopyRegMap, MRI, TRI, TII); - } - } else { - // Emit the copies into the top of the block. - for (MachineRegisterInfo::livein_iterator LI = MRI.livein_begin(), - E = MRI.livein_end(); LI != E; ++LI) - if (LI->second) { - const TargetRegisterClass *RC = MRI.getRegClass(LI->second); - bool Emitted = TII.copyRegToReg(*EntryMBB, EntryMBB->begin(), - LI->second, LI->first, RC, RC); - assert(Emitted && "Unable to issue a live-in copy instruction!\n"); - (void) Emitted; - } - } -} - //===----------------------------------------------------------------------===// // SelectionDAGISel code //===----------------------------------------------------------------------===// -SelectionDAGISel::SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL) : +SelectionDAGISel::SelectionDAGISel(const TargetMachine &tm, CodeGenOpt::Level OL) : MachineFunctionPass(&ID), TM(tm), TLI(*tm.getTargetLowering()), FuncInfo(new FunctionLoweringInfo(TLI)), - CurDAG(new SelectionDAG(TLI, *FuncInfo)), - SDB(new SelectionDAGBuilder(*CurDAG, TLI, *FuncInfo, OL)), + CurDAG(new SelectionDAG(tm, *FuncInfo)), + SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, OL)), GFI(), OptLevel(OL), DAGSize(0) @@ -293,10 +180,6 @@ SelectionDAGISel::~SelectionDAGISel() { delete FuncInfo; } -unsigned SelectionDAGISel::MakeReg(EVT VT) { - return RegInfo->createVirtualRegister(TLI.getRegClassFor(VT)); -} - void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired<AliasAnalysis>(); AU.addPreserved<AliasAnalysis>(); @@ -306,129 +189,75 @@ void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { } bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { - Function &Fn = *mf.getFunction(); - // Do some sanity-checking on the command-line options. assert((!EnableFastISelVerbose || EnableFastISel) && "-fast-isel-verbose requires -fast-isel"); assert((!EnableFastISelAbort || EnableFastISel) && "-fast-isel-abort requires -fast-isel"); - // Get alias analysis for load/store combining. - AA = &getAnalysis<AliasAnalysis>(); - - MF = &mf; + const Function &Fn = *mf.getFunction(); const TargetInstrInfo &TII = *TM.getInstrInfo(); const TargetRegisterInfo &TRI = *TM.getRegisterInfo(); - if (Fn.hasGC()) - GFI = &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn); - else - GFI = 0; + MF = &mf; RegInfo = &MF->getRegInfo(); + AA = &getAnalysis<AliasAnalysis>(); + GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : 0; + DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n"); CurDAG->init(*MF); FuncInfo->set(Fn, *MF, EnableFastISel); SDB->init(GFI, *AA); - for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) - if (InvokeInst *Invoke = dyn_cast<InvokeInst>(I->getTerminator())) - // Mark landing pad. - FuncInfo->MBBMap[Invoke->getSuccessor(1)]->setIsLandingPad(); - - SelectAllBasicBlocks(Fn, *MF, TII); + SelectAllBasicBlocks(Fn); // If the first basic block in the function has live ins that need to be // copied into vregs, emit the copies into the top of the block before // emitting the code for the block. - EmitLiveInCopies(MF->begin(), *RegInfo, TRI, TII); - - // Add function live-ins to entry block live-in set. - for (MachineRegisterInfo::livein_iterator I = RegInfo->livein_begin(), - E = RegInfo->livein_end(); I != E; ++I) - MF->begin()->addLiveIn(I->first); - -#ifndef NDEBUG - assert(FuncInfo->CatchInfoFound.size() == FuncInfo->CatchInfoLost.size() && - "Not all catch info was assigned to a landing pad!"); -#endif + MachineBasicBlock *EntryMBB = MF->begin(); + RegInfo->EmitLiveInCopies(EntryMBB, TRI, TII); + + // Insert DBG_VALUE instructions for function arguments to the entry block. + for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) { + MachineInstr *MI = FuncInfo->ArgDbgValues[e-i-1]; + unsigned Reg = MI->getOperand(0).getReg(); + if (TargetRegisterInfo::isPhysicalRegister(Reg)) + EntryMBB->insert(EntryMBB->begin(), MI); + else { + MachineInstr *Def = RegInfo->getVRegDef(Reg); + MachineBasicBlock::iterator InsertPos = Def; + // FIXME: VR def may not be in entry block. + Def->getParent()->insert(llvm::next(InsertPos), MI); + } + } + // Release function-specific state. SDB and CurDAG are already cleared + // at this point. FuncInfo->clear(); return true; } -/// SetDebugLoc - Update MF's and SDB's DebugLocs if debug information is -/// attached with this instruction. -static void SetDebugLoc(Instruction *I, SelectionDAGBuilder *SDB, - FastISel *FastIS, MachineFunction *MF) { - DebugLoc DL = I->getDebugLoc(); - if (DL.isUnknown()) return; - - SDB->setCurDebugLoc(DL); - - if (FastIS) - FastIS->setCurDebugLoc(DL); - - // If the function doesn't have a default debug location yet, set - // it. This is kind of a hack. - if (MF->getDefaultDebugLoc().isUnknown()) - MF->setDefaultDebugLoc(DL); -} - -/// ResetDebugLoc - Set MF's and SDB's DebugLocs to Unknown. -static void ResetDebugLoc(SelectionDAGBuilder *SDB, FastISel *FastIS) { - SDB->setCurDebugLoc(DebugLoc()); - if (FastIS) - FastIS->setCurDebugLoc(DebugLoc()); -} - -void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, - BasicBlock::iterator Begin, - BasicBlock::iterator End, - bool &HadTailCall) { - SDB->setCurrentBasicBlock(BB); - +MachineBasicBlock * +SelectionDAGISel::SelectBasicBlock(MachineBasicBlock *BB, + const BasicBlock *LLVMBB, + BasicBlock::const_iterator Begin, + BasicBlock::const_iterator End, + bool &HadTailCall) { // Lower all of the non-terminator instructions. If a call is emitted - // as a tail call, cease emitting nodes for this block. - for (BasicBlock::iterator I = Begin; I != End && !SDB->HasTailCall; ++I) { - SetDebugLoc(I, SDB, 0, MF); - - if (!isa<TerminatorInst>(I)) { - SDB->visit(*I); - - // Set the current debug location back to "unknown" so that it doesn't - // spuriously apply to subsequent instructions. - ResetDebugLoc(SDB, 0); - } - } - - if (!SDB->HasTailCall) { - // Ensure that all instructions which are used outside of their defining - // blocks are available as virtual registers. Invoke is handled elsewhere. - for (BasicBlock::iterator I = Begin; I != End; ++I) - if (!isa<PHINode>(I) && !isa<InvokeInst>(I)) - SDB->CopyToExportRegsIfNeeded(I); - - // Handle PHI nodes in successor blocks. - if (End == LLVMBB->end()) { - HandlePHINodesInSuccessorBlocks(LLVMBB); - - // Lower the terminator after the copies are emitted. - SetDebugLoc(LLVMBB->getTerminator(), SDB, 0, MF); - SDB->visit(*LLVMBB->getTerminator()); - ResetDebugLoc(SDB, 0); - } - } + // as a tail call, cease emitting nodes for this block. Terminators + // are handled below. + for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) + SDB->visit(*I); // Make sure the root of the DAG is up-to-date. CurDAG->setRoot(SDB->getControlRoot()); - - // Final step, emit the lowered DAG as machine code. - CodeGenAndEmitDAG(); HadTailCall = SDB->HasTailCall; SDB->clear(); + + // Final step, emit the lowered DAG as machine code. + return CodeGenAndEmitDAG(BB); } namespace { @@ -493,7 +322,7 @@ void SelectionDAGISel::ShrinkDemandedOps() { InWorklist.insert(I); } - TargetLowering::TargetLoweringOpt TLO(*CurDAG, true); + TargetLowering::TargetLoweringOpt TLO(*CurDAG, true, true, true); while (!Worklist.empty()) { SDNode *N = Worklist.pop_back_val(); InWorklist.erase(N); @@ -613,7 +442,7 @@ void SelectionDAGISel::ComputeLiveOutVRegInfo() { } while (!Worklist.empty()); } -void SelectionDAGISel::CodeGenAndEmitDAG() { +MachineBasicBlock *SelectionDAGISel::CodeGenAndEmitDAG(MachineBasicBlock *BB) { std::string GroupName; if (TimePassesIsEnabled) GroupName = "Instruction Selection and Scheduling"; @@ -763,9 +592,9 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { // inserted into. if (TimePassesIsEnabled) { NamedRegionTimer T("Instruction Creation", GroupName); - BB = Scheduler->EmitSchedule(&SDB->EdgeMapping); + BB = Scheduler->EmitSchedule(); } else { - BB = Scheduler->EmitSchedule(&SDB->EdgeMapping); + BB = Scheduler->EmitSchedule(); } // Free the scheduler state. @@ -776,8 +605,10 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { delete Scheduler; } - DEBUG(dbgs() << "Selected machine code:\n"); - DEBUG(BB->dump()); + // Free the SelectionDAG state, now that we're finished with it. + CurDAG->clear(); + + return BB; } void SelectionDAGISel::DoInstructionSelection() { @@ -836,128 +667,94 @@ void SelectionDAGISel::DoInstructionSelection() { PostprocessISelDAG(); } +/// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and +/// do other setup for EH landing-pad blocks. +void SelectionDAGISel::PrepareEHLandingPad(MachineBasicBlock *BB) { + // Add a label to mark the beginning of the landing pad. Deletion of the + // landing pad can thus be detected via the MachineModuleInfo. + MCSymbol *Label = MF->getMMI().addLandingPad(BB); + + const TargetInstrDesc &II = TM.getInstrInfo()->get(TargetOpcode::EH_LABEL); + BuildMI(BB, SDB->getCurDebugLoc(), II).addSym(Label); + + // Mark exception register as live in. + unsigned Reg = TLI.getExceptionAddressRegister(); + if (Reg) BB->addLiveIn(Reg); + + // Mark exception selector register as live in. + Reg = TLI.getExceptionSelectorRegister(); + if (Reg) BB->addLiveIn(Reg); + + // FIXME: Hack around an exception handling flaw (PR1508): the personality + // function and list of typeids logically belong to the invoke (or, if you + // like, the basic block containing the invoke), and need to be associated + // with it in the dwarf exception handling tables. Currently however the + // information is provided by an intrinsic (eh.selector) that can be moved + // to unexpected places by the optimizers: if the unwind edge is critical, + // then breaking it can result in the intrinsics being in the successor of + // the landing pad, not the landing pad itself. This results + // in exceptions not being caught because no typeids are associated with + // the invoke. This may not be the only way things can go wrong, but it + // is the only way we try to work around for the moment. + const BasicBlock *LLVMBB = BB->getBasicBlock(); + const BranchInst *Br = dyn_cast<BranchInst>(LLVMBB->getTerminator()); + + if (Br && Br->isUnconditional()) { // Critical edge? + BasicBlock::const_iterator I, E; + for (I = LLVMBB->begin(), E = --LLVMBB->end(); I != E; ++I) + if (isa<EHSelectorInst>(I)) + break; -void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, - MachineFunction &MF, - const TargetInstrInfo &TII) { + if (I == E) + // No catch info found - try to extract some from the successor. + CopyCatchInfo(Br->getSuccessor(0), LLVMBB, &MF->getMMI(), *FuncInfo); + } +} + +void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { // Initialize the Fast-ISel state, if needed. FastISel *FastIS = 0; if (EnableFastISel) - FastIS = TLI.createFastISel(MF, FuncInfo->ValueMap, FuncInfo->MBBMap, - FuncInfo->StaticAllocaMap + FastIS = TLI.createFastISel(*MF, FuncInfo->ValueMap, FuncInfo->MBBMap, + FuncInfo->StaticAllocaMap, + FuncInfo->PHINodesToUpdate #ifndef NDEBUG , FuncInfo->CatchInfoLost #endif ); // Iterate over all basic blocks in the function. - for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) { - BasicBlock *LLVMBB = &*I; - BB = FuncInfo->MBBMap[LLVMBB]; + for (Function::const_iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) { + const BasicBlock *LLVMBB = &*I; + MachineBasicBlock *BB = FuncInfo->MBBMap[LLVMBB]; - BasicBlock::iterator const Begin = LLVMBB->begin(); - BasicBlock::iterator const End = LLVMBB->end(); - BasicBlock::iterator BI = Begin; + BasicBlock::const_iterator const Begin = LLVMBB->getFirstNonPHI(); + BasicBlock::const_iterator const End = LLVMBB->end(); + BasicBlock::const_iterator BI = Begin; // Lower any arguments needed in this block if this is the entry block. - bool SuppressFastISel = false; - if (LLVMBB == &Fn.getEntryBlock()) { + if (LLVMBB == &Fn.getEntryBlock()) LowerArguments(LLVMBB); - // If any of the arguments has the byval attribute, forgo - // fast-isel in the entry block. - if (FastIS) { - unsigned j = 1; - for (Function::arg_iterator I = Fn.arg_begin(), E = Fn.arg_end(); - I != E; ++I, ++j) - if (Fn.paramHasAttr(j, Attribute::ByVal)) { - if (EnableFastISelVerbose || EnableFastISelAbort) - dbgs() << "FastISel skips entry block due to byval argument\n"; - SuppressFastISel = true; - break; - } - } - } - - if (BB->isLandingPad()) { - // Add a label to mark the beginning of the landing pad. Deletion of the - // landing pad can thus be detected via the MachineModuleInfo. - MCSymbol *Label = MF.getMMI().addLandingPad(BB); - - const TargetInstrDesc &II = TII.get(TargetOpcode::EH_LABEL); - BuildMI(BB, SDB->getCurDebugLoc(), II).addSym(Label); - - // Mark exception register as live in. - unsigned Reg = TLI.getExceptionAddressRegister(); - if (Reg) BB->addLiveIn(Reg); - - // Mark exception selector register as live in. - Reg = TLI.getExceptionSelectorRegister(); - if (Reg) BB->addLiveIn(Reg); - - // FIXME: Hack around an exception handling flaw (PR1508): the personality - // function and list of typeids logically belong to the invoke (or, if you - // like, the basic block containing the invoke), and need to be associated - // with it in the dwarf exception handling tables. Currently however the - // information is provided by an intrinsic (eh.selector) that can be moved - // to unexpected places by the optimizers: if the unwind edge is critical, - // then breaking it can result in the intrinsics being in the successor of - // the landing pad, not the landing pad itself. This results - // in exceptions not being caught because no typeids are associated with - // the invoke. This may not be the only way things can go wrong, but it - // is the only way we try to work around for the moment. - BranchInst *Br = dyn_cast<BranchInst>(LLVMBB->getTerminator()); - - if (Br && Br->isUnconditional()) { // Critical edge? - BasicBlock::iterator I, E; - for (I = LLVMBB->begin(), E = --LLVMBB->end(); I != E; ++I) - if (isa<EHSelectorInst>(I)) - break; - - if (I == E) - // No catch info found - try to extract some from the successor. - CopyCatchInfo(Br->getSuccessor(0), LLVMBB, &MF.getMMI(), *FuncInfo); - } - } - + // Setup an EH landing-pad block. + if (BB->isLandingPad()) + PrepareEHLandingPad(BB); + // Before doing SelectionDAG ISel, see if FastISel has been requested. - if (FastIS && !SuppressFastISel) { + if (FastIS) { // Emit code for any incoming arguments. This must happen before // beginning FastISel on the entry block. if (LLVMBB == &Fn.getEntryBlock()) { CurDAG->setRoot(SDB->getControlRoot()); - CodeGenAndEmitDAG(); SDB->clear(); + BB = CodeGenAndEmitDAG(BB); } FastIS->startNewBlock(BB); // Do FastISel on as many instructions as possible. for (; BI != End; ++BI) { - // Just before the terminator instruction, insert instructions to - // feed PHI nodes in successor blocks. - if (isa<TerminatorInst>(BI)) - if (!HandlePHINodesInSuccessorBlocksFast(LLVMBB, FastIS)) { - ++NumFastIselFailures; - ResetDebugLoc(SDB, FastIS); - if (EnableFastISelVerbose || EnableFastISelAbort) { - dbgs() << "FastISel miss: "; - BI->dump(); - } - assert(!EnableFastISelAbort && - "FastISel didn't handle a PHI in a successor"); - break; - } - - SetDebugLoc(BI, SDB, FastIS, &MF); - // Try to select the instruction with FastISel. - if (FastIS->SelectInstruction(BI)) { - ResetDebugLoc(SDB, FastIS); + if (FastIS->SelectInstruction(BI)) continue; - } - - // Clear out the debug location so that it doesn't carry over to - // unrelated instructions. - ResetDebugLoc(SDB, FastIS); // Then handle certain instructions as single-LLVM-Instruction blocks. if (isa<CallInst>(BI)) { @@ -967,14 +764,14 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, BI->dump(); } - if (!BI->getType()->isVoidTy()) { + if (!BI->getType()->isVoidTy() && !BI->use_empty()) { unsigned &R = FuncInfo->ValueMap[BI]; if (!R) R = FuncInfo->CreateRegForValue(BI); } bool HadTailCall = false; - SelectBasicBlock(LLVMBB, BI, llvm::next(BI), HadTailCall); + BB = SelectBasicBlock(BB, LLVMBB, BI, llvm::next(BI), HadTailCall); // If the call was emitted as a tail call, we're done with the block. if (HadTailCall) { @@ -1010,44 +807,41 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, // block. if (BI != End) { bool HadTailCall; - SelectBasicBlock(LLVMBB, BI, End, HadTailCall); + BB = SelectBasicBlock(BB, LLVMBB, BI, End, HadTailCall); } - FinishBasicBlock(); + FinishBasicBlock(BB); + FuncInfo->PHINodesToUpdate.clear(); } delete FastIS; } void -SelectionDAGISel::FinishBasicBlock() { - - DEBUG(dbgs() << "Target-post-processed machine code:\n"); - DEBUG(BB->dump()); +SelectionDAGISel::FinishBasicBlock(MachineBasicBlock *BB) { DEBUG(dbgs() << "Total amount of phi nodes to update: " - << SDB->PHINodesToUpdate.size() << "\n"); - DEBUG(for (unsigned i = 0, e = SDB->PHINodesToUpdate.size(); i != e; ++i) + << FuncInfo->PHINodesToUpdate.size() << "\n"); + DEBUG(for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) dbgs() << "Node " << i << " : (" - << SDB->PHINodesToUpdate[i].first - << ", " << SDB->PHINodesToUpdate[i].second << ")\n"); + << FuncInfo->PHINodesToUpdate[i].first + << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n"); // Next, now that we know what the last MBB the LLVM BB expanded is, update // PHI nodes in successors. if (SDB->SwitchCases.empty() && SDB->JTCases.empty() && SDB->BitTestCases.empty()) { - for (unsigned i = 0, e = SDB->PHINodesToUpdate.size(); i != e; ++i) { - MachineInstr *PHI = SDB->PHINodesToUpdate[i].first; + for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) { + MachineInstr *PHI = FuncInfo->PHINodesToUpdate[i].first; assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); if (!BB->isSuccessor(PHI->getParent())) continue; - PHI->addOperand(MachineOperand::CreateReg(SDB->PHINodesToUpdate[i].second, - false)); + PHI->addOperand( + MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[i].second, false)); PHI->addOperand(MachineOperand::CreateMBB(BB)); } - SDB->PHINodesToUpdate.clear(); return; } @@ -1056,37 +850,38 @@ SelectionDAGISel::FinishBasicBlock() { if (!SDB->BitTestCases[i].Emitted) { // Set the current basic block to the mbb we wish to insert the code into BB = SDB->BitTestCases[i].Parent; - SDB->setCurrentBasicBlock(BB); // Emit the code - SDB->visitBitTestHeader(SDB->BitTestCases[i]); + SDB->visitBitTestHeader(SDB->BitTestCases[i], BB); CurDAG->setRoot(SDB->getRoot()); - CodeGenAndEmitDAG(); SDB->clear(); + BB = CodeGenAndEmitDAG(BB); } for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) { // Set the current basic block to the mbb we wish to insert the code into BB = SDB->BitTestCases[i].Cases[j].ThisBB; - SDB->setCurrentBasicBlock(BB); // Emit the code if (j+1 != ej) SDB->visitBitTestCase(SDB->BitTestCases[i].Cases[j+1].ThisBB, SDB->BitTestCases[i].Reg, - SDB->BitTestCases[i].Cases[j]); + SDB->BitTestCases[i].Cases[j], + BB); else SDB->visitBitTestCase(SDB->BitTestCases[i].Default, SDB->BitTestCases[i].Reg, - SDB->BitTestCases[i].Cases[j]); + SDB->BitTestCases[i].Cases[j], + BB); CurDAG->setRoot(SDB->getRoot()); - CodeGenAndEmitDAG(); SDB->clear(); + BB = CodeGenAndEmitDAG(BB); } // Update PHI Nodes - for (unsigned pi = 0, pe = SDB->PHINodesToUpdate.size(); pi != pe; ++pi) { - MachineInstr *PHI = SDB->PHINodesToUpdate[pi].first; + for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size(); + pi != pe; ++pi) { + MachineInstr *PHI = FuncInfo->PHINodesToUpdate[pi].first; MachineBasicBlock *PHIBB = PHI->getParent(); assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); @@ -1094,10 +889,12 @@ SelectionDAGISel::FinishBasicBlock() { // from last "case" BB. if (PHIBB == SDB->BitTestCases[i].Default) { PHI->addOperand(MachineOperand:: - CreateReg(SDB->PHINodesToUpdate[pi].second, false)); + CreateReg(FuncInfo->PHINodesToUpdate[pi].second, + false)); PHI->addOperand(MachineOperand::CreateMBB(SDB->BitTestCases[i].Parent)); PHI->addOperand(MachineOperand:: - CreateReg(SDB->PHINodesToUpdate[pi].second, false)); + CreateReg(FuncInfo->PHINodesToUpdate[pi].second, + false)); PHI->addOperand(MachineOperand::CreateMBB(SDB->BitTestCases[i].Cases. back().ThisBB)); } @@ -1107,7 +904,8 @@ SelectionDAGISel::FinishBasicBlock() { MachineBasicBlock* cBB = SDB->BitTestCases[i].Cases[j].ThisBB; if (cBB->isSuccessor(PHIBB)) { PHI->addOperand(MachineOperand:: - CreateReg(SDB->PHINodesToUpdate[pi].second, false)); + CreateReg(FuncInfo->PHINodesToUpdate[pi].second, + false)); PHI->addOperand(MachineOperand::CreateMBB(cBB)); } } @@ -1123,40 +921,42 @@ SelectionDAGISel::FinishBasicBlock() { if (!SDB->JTCases[i].first.Emitted) { // Set the current basic block to the mbb we wish to insert the code into BB = SDB->JTCases[i].first.HeaderBB; - SDB->setCurrentBasicBlock(BB); // Emit the code - SDB->visitJumpTableHeader(SDB->JTCases[i].second, SDB->JTCases[i].first); + SDB->visitJumpTableHeader(SDB->JTCases[i].second, SDB->JTCases[i].first, + BB); CurDAG->setRoot(SDB->getRoot()); - CodeGenAndEmitDAG(); SDB->clear(); + BB = CodeGenAndEmitDAG(BB); } // Set the current basic block to the mbb we wish to insert the code into BB = SDB->JTCases[i].second.MBB; - SDB->setCurrentBasicBlock(BB); // Emit the code SDB->visitJumpTable(SDB->JTCases[i].second); CurDAG->setRoot(SDB->getRoot()); - CodeGenAndEmitDAG(); SDB->clear(); + BB = CodeGenAndEmitDAG(BB); // Update PHI Nodes - for (unsigned pi = 0, pe = SDB->PHINodesToUpdate.size(); pi != pe; ++pi) { - MachineInstr *PHI = SDB->PHINodesToUpdate[pi].first; + for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size(); + pi != pe; ++pi) { + MachineInstr *PHI = FuncInfo->PHINodesToUpdate[pi].first; MachineBasicBlock *PHIBB = PHI->getParent(); assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); // "default" BB. We can go there only from header BB. if (PHIBB == SDB->JTCases[i].second.Default) { PHI->addOperand - (MachineOperand::CreateReg(SDB->PHINodesToUpdate[pi].second, false)); + (MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[pi].second, + false)); PHI->addOperand (MachineOperand::CreateMBB(SDB->JTCases[i].first.HeaderBB)); } // JT BB. Just iterate over successors here if (BB->isSuccessor(PHIBB)) { PHI->addOperand - (MachineOperand::CreateReg(SDB->PHINodesToUpdate[pi].second, false)); + (MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[pi].second, + false)); PHI->addOperand(MachineOperand::CreateMBB(BB)); } } @@ -1165,13 +965,13 @@ SelectionDAGISel::FinishBasicBlock() { // If the switch block involved a branch to one of the actual successors, we // need to update PHI nodes in that block. - for (unsigned i = 0, e = SDB->PHINodesToUpdate.size(); i != e; ++i) { - MachineInstr *PHI = SDB->PHINodesToUpdate[i].first; + for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) { + MachineInstr *PHI = FuncInfo->PHINodesToUpdate[i].first; assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); if (BB->isSuccessor(PHI->getParent())) { - PHI->addOperand(MachineOperand::CreateReg(SDB->PHINodesToUpdate[i].second, - false)); + PHI->addOperand( + MachineOperand::CreateReg(FuncInfo->PHINodesToUpdate[i].second, false)); PHI->addOperand(MachineOperand::CreateMBB(BB)); } } @@ -1181,26 +981,26 @@ SelectionDAGISel::FinishBasicBlock() { for (unsigned i = 0, e = SDB->SwitchCases.size(); i != e; ++i) { // Set the current basic block to the mbb we wish to insert the code into MachineBasicBlock *ThisBB = BB = SDB->SwitchCases[i].ThisBB; - SDB->setCurrentBasicBlock(BB); - // Emit the code - SDB->visitSwitchCase(SDB->SwitchCases[i]); + // Determine the unique successors. + SmallVector<MachineBasicBlock *, 2> Succs; + Succs.push_back(SDB->SwitchCases[i].TrueBB); + if (SDB->SwitchCases[i].TrueBB != SDB->SwitchCases[i].FalseBB) + Succs.push_back(SDB->SwitchCases[i].FalseBB); + + // Emit the code. Note that this could result in ThisBB being split, so + // we need to check for updates. + SDB->visitSwitchCase(SDB->SwitchCases[i], BB); CurDAG->setRoot(SDB->getRoot()); - CodeGenAndEmitDAG(); + SDB->clear(); + ThisBB = CodeGenAndEmitDAG(BB); // Handle any PHI nodes in successors of this chunk, as if we were coming // from the original BB before switch expansion. Note that PHI nodes can // occur multiple times in PHINodesToUpdate. We have to be very careful to // handle them the right number of times. - while ((BB = SDB->SwitchCases[i].TrueBB)) { // Handle LHS and RHS. - // If new BB's are created during scheduling, the edges may have been - // updated. That is, the edge from ThisBB to BB may have been split and - // BB's predecessor is now another block. - DenseMap<MachineBasicBlock*, MachineBasicBlock*>::iterator EI = - SDB->EdgeMapping.find(BB); - if (EI != SDB->EdgeMapping.end()) - ThisBB = EI->second; - + for (unsigned i = 0, e = Succs.size(); i != e; ++i) { + BB = Succs[i]; // BB may have been removed from the CFG if a branch was constant folded. if (ThisBB->isSuccessor(BB)) { for (MachineBasicBlock::iterator Phi = BB->begin(); @@ -1208,11 +1008,11 @@ SelectionDAGISel::FinishBasicBlock() { ++Phi) { // This value for this PHI node is recorded in PHINodesToUpdate. for (unsigned pn = 0; ; ++pn) { - assert(pn != SDB->PHINodesToUpdate.size() && + assert(pn != FuncInfo->PHINodesToUpdate.size() && "Didn't find PHI entry!"); - if (SDB->PHINodesToUpdate[pn].first == Phi) { + if (FuncInfo->PHINodesToUpdate[pn].first == Phi) { Phi->addOperand(MachineOperand:: - CreateReg(SDB->PHINodesToUpdate[pn].second, + CreateReg(FuncInfo->PHINodesToUpdate[pn].second, false)); Phi->addOperand(MachineOperand::CreateMBB(ThisBB)); break; @@ -1220,21 +1020,9 @@ SelectionDAGISel::FinishBasicBlock() { } } } - - // Don't process RHS if same block as LHS. - if (BB == SDB->SwitchCases[i].FalseBB) - SDB->SwitchCases[i].FalseBB = 0; - - // If we haven't handled the RHS, do so now. Otherwise, we're done. - SDB->SwitchCases[i].TrueBB = SDB->SwitchCases[i].FalseBB; - SDB->SwitchCases[i].FalseBB = 0; } - assert(SDB->SwitchCases[i].TrueBB == 0 && SDB->SwitchCases[i].FalseBB == 0); - SDB->clear(); } SDB->SwitchCases.clear(); - - SDB->PHINodesToUpdate.clear(); } @@ -1333,16 +1121,17 @@ SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops) { std::vector<SDValue> InOps; std::swap(InOps, Ops); - Ops.push_back(InOps[0]); // input chain. - Ops.push_back(InOps[1]); // input asm string. + Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0 + Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1 + Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc - unsigned i = 2, e = InOps.size(); + unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size(); if (InOps[e-1].getValueType() == MVT::Flag) --e; // Don't process a flag operand if it is here. while (i != e) { unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue(); - if ((Flags & 7) != 4 /*MEM*/) { + if (!InlineAsm::isMemKind(Flags)) { // Just skip over this operand, copying the operands verbatim. Ops.insert(Ops.end(), InOps.begin()+i, InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1); @@ -1352,14 +1141,14 @@ SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops) { "Memory operand with multiple values?"); // Otherwise, this is a memory operand. Ask the target to select it. std::vector<SDValue> SelOps; - if (SelectInlineAsmMemoryOperand(InOps[i+1], 'm', SelOps)) { - llvm_report_error("Could not match memory address. Inline asm" - " failure!"); - } + if (SelectInlineAsmMemoryOperand(InOps[i+1], 'm', SelOps)) + report_fatal_error("Could not match memory address. Inline asm" + " failure!"); // Add this to the output node. - Ops.push_back(CurDAG->getTargetConstant(4/*MEM*/ | (SelOps.size()<< 3), - MVT::i32)); + unsigned NewFlags = + InlineAsm::getFlagWord(InlineAsm::Kind_Mem, SelOps.size()); + Ops.push_back(CurDAG->getTargetConstant(NewFlags, MVT::i32)); Ops.insert(Ops.end(), SelOps.begin(), SelOps.end()); i += 2; } @@ -1436,7 +1225,8 @@ bool SelectionDAGISel::IsProfitableToFold(SDValue N, SDNode *U, /// IsLegalToFold - Returns true if the specific operand node N of /// U can be folded during instruction selection that starts at Root. bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, - bool IgnoreChains) const { + CodeGenOpt::Level OptLevel, + bool IgnoreChains) { if (OptLevel == CodeGenOpt::None) return false; // If Root use can somehow reach N through a path that that doesn't contain @@ -2011,6 +1801,7 @@ static unsigned IsPredicateKnownToFail(const unsigned char *Table, } } +namespace { struct MatchScope { /// FailIndex - If this match fails, this is the index to continue with. @@ -2032,6 +1823,8 @@ struct MatchScope { bool HasChainNodesMatched, HasFlagResultNodesMatched; }; +} + SDNode *SelectionDAGISel:: SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, unsigned TableSize) { @@ -2045,6 +1838,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, //case ISD::VALUETYPE: //case ISD::CONDCODE: case ISD::HANDLENODE: + case ISD::MDNODE_SDNODE: case ISD::TargetConstant: case ISD::TargetConstantFP: case ISD::TargetConstantPool: @@ -2383,7 +2177,8 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(), NodeToMatch) || !IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(), - NodeToMatch, true/*We validate our own chains*/)) + NodeToMatch, OptLevel, + true/*We validate our own chains*/)) break; continue; @@ -2776,7 +2571,7 @@ void SelectionDAGISel::CannotYetSelect(SDNode *N) { else Msg << "unknown intrinsic #" << iid; } - llvm_report_error(Msg.str()); + report_fatal_error(Msg.str()); } char SelectionDAGISel::ID = 0; diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index ea2ff2f..8a4a1b1 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -480,7 +480,8 @@ static void InitCmpLibcallCCs(ISD::CondCode *CCs) { } /// NOTE: The constructor takes ownership of TLOF. -TargetLowering::TargetLowering(TargetMachine &tm,TargetLoweringObjectFile *tlof) +TargetLowering::TargetLowering(const TargetMachine &tm, + const TargetLoweringObjectFile *tlof) : TM(tm), TD(TM.getTargetData()), TLOF(*tlof) { // All operations default to being supported. memset(OpActions, 0, sizeof(OpActions)); @@ -720,7 +721,7 @@ void TargetLowering::computeRegisterProperties() { unsigned NElts = VT.getVectorNumElements(); for (unsigned nVT = i+1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) { EVT SVT = (MVT::SimpleValueType)nVT; - if (isTypeLegal(SVT) && SVT.getVectorElementType() == EltVT && + if (isTypeSynthesizable(SVT) && SVT.getVectorElementType() == EltVT && SVT.getVectorNumElements() > NElts && NElts != 1) { TransformToType[i] = SVT; ValueTypeActions.setTypeAction(VT, Promote); @@ -1279,8 +1280,9 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op, // variable. The low bit of the shift cannot be an input sign bit unless // the shift amount is >= the size of the datatype, which is undefined. if (DemandedMask == 1) - return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, Op.getValueType(), - Op.getOperand(0), Op.getOperand(1))); + return TLO.CombineTo(Op, + TLO.DAG.getNode(ISD::SRL, dl, Op.getValueType(), + Op.getOperand(0), Op.getOperand(1))); if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { EVT VT = Op.getValueType(); @@ -1465,23 +1467,29 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op, case ISD::SRL: // Shrink SRL by a constant if none of the high bits shifted in are // demanded. - if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(In.getOperand(1))){ - APInt HighBits = APInt::getHighBitsSet(OperandBitWidth, - OperandBitWidth - BitWidth); - HighBits = HighBits.lshr(ShAmt->getZExtValue()); - HighBits.trunc(BitWidth); - - if (ShAmt->getZExtValue() < BitWidth && !(HighBits & NewMask)) { - // None of the shifted in bits are needed. Add a truncate of the - // shift input, then shift it. - SDValue NewTrunc = TLO.DAG.getNode(ISD::TRUNCATE, dl, - Op.getValueType(), - In.getOperand(0)); - return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, - Op.getValueType(), - NewTrunc, - In.getOperand(1))); - } + if (TLO.LegalTypes() && + !isTypeDesirableForOp(ISD::SRL, Op.getValueType())) + // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is + // undesirable. + break; + ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(In.getOperand(1)); + if (!ShAmt) + break; + APInt HighBits = APInt::getHighBitsSet(OperandBitWidth, + OperandBitWidth - BitWidth); + HighBits = HighBits.lshr(ShAmt->getZExtValue()); + HighBits.trunc(BitWidth); + + if (ShAmt->getZExtValue() < BitWidth && !(HighBits & NewMask)) { + // None of the shifted in bits are needed. Add a truncate of the + // shift input, then shift it. + SDValue NewTrunc = TLO.DAG.getNode(ISD::TRUNCATE, dl, + Op.getValueType(), + In.getOperand(0)); + return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, + Op.getValueType(), + NewTrunc, + In.getOperand(1))); } break; } @@ -1874,10 +1882,15 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, isa<ConstantSDNode>(Op0.getOperand(1)) && cast<ConstantSDNode>(Op0.getOperand(1))->getAPIntValue() == 1) { // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0. - if (Op0.getValueType() != VT) + if (Op0.getValueType().bitsGT(VT)) Op0 = DAG.getNode(ISD::AND, dl, VT, DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), DAG.getConstant(1, VT)); + else if (Op0.getValueType().bitsLT(VT)) + Op0 = DAG.getNode(ISD::AND, dl, VT, + DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), + DAG.getConstant(1, VT)); + return DAG.getSetCC(dl, VT, Op0, DAG.getConstant(0, Op0.getValueType()), Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); @@ -2245,7 +2258,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, /// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the /// node is a GlobalAddress + offset. -bool TargetLowering::isGAPlusOffset(SDNode *N, GlobalValue* &GA, +bool TargetLowering::isGAPlusOffset(SDNode *N, const GlobalValue* &GA, int64_t &Offset) const { if (isa<GlobalAddressSDNode>(N)) { GlobalAddressSDNode *GASD = cast<GlobalAddressSDNode>(N); diff --git a/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp b/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp new file mode 100644 index 0000000..d20477f --- /dev/null +++ b/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp @@ -0,0 +1,21 @@ +//===-- TargetSelectionDAGInfo.cpp - SelectionDAG Info --------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This implements the TargetSelectionDAGInfo class. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Target/TargetSelectionDAGInfo.h" +using namespace llvm; + +TargetSelectionDAGInfo::TargetSelectionDAGInfo() { +} + +TargetSelectionDAGInfo::~TargetSelectionDAGInfo() { +} |
