diff options
Diffstat (limited to 'lib/CodeGen/SelectionDAG')
| -rw-r--r-- | lib/CodeGen/SelectionDAG/CallingConvLower.cpp | 15 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 30 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/FastISel.cpp | 93 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 2 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/InstrEmitter.h | 2 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 48 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 21 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp | 491 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/SelectionDAGBuild.h | 14 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 34 | ||||
| -rw-r--r-- | lib/CodeGen/SelectionDAG/TargetLowering.cpp | 10 |
11 files changed, 420 insertions, 340 deletions
diff --git a/lib/CodeGen/SelectionDAG/CallingConvLower.cpp b/lib/CodeGen/SelectionDAG/CallingConvLower.cpp index fbe40b6..38839c4 100644 --- a/lib/CodeGen/SelectionDAG/CallingConvLower.cpp +++ b/lib/CodeGen/SelectionDAG/CallingConvLower.cpp @@ -77,6 +77,21 @@ CCState::AnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins, } } +/// CheckReturn - Analyze the return values of a function, returning true if +/// the return can be performed without sret-demotion, and false otherwise. +bool CCState::CheckReturn(const SmallVectorImpl<EVT> &OutTys, + const SmallVectorImpl<ISD::ArgFlagsTy> &ArgsFlags, + CCAssignFn Fn) { + // Determine which register each value should be copied into. + for (unsigned i = 0, e = OutTys.size(); i != e; ++i) { + EVT VT = OutTys[i]; + ISD::ArgFlagsTy ArgFlags = ArgsFlags[i]; + if (Fn(i, VT, VT, CCValAssign::Full, ArgFlags, *this)) + return false; + } + return true; +} + /// AnalyzeReturn - Analyze the returned values of a return, /// incorporating info about the result values into this state. void CCState::AnalyzeReturn(const SmallVectorImpl<ISD::OutputArg> &Outs, diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 5f70cb8..06ffdd6 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -37,7 +37,6 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> -#include <set> using namespace llvm; STATISTIC(NodesCombined , "Number of dag nodes combined"); @@ -4443,14 +4442,13 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) { SDValue Chain = N->getOperand(0); SDValue N1 = N->getOperand(1); SDValue N2 = N->getOperand(2); - ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); - // never taken branch, fold to chain - if (N1C && N1C->isNullValue()) - return Chain; - // unconditional branch - if (N1C && N1C->getAPIntValue() == 1) - return DAG.getNode(ISD::BR, N->getDebugLoc(), MVT::Other, Chain, N2); + // If N is a constant we could fold this into a fallthrough or unconditional + // branch. However that doesn't happen very often in normal code, because + // Instcombine/SimplifyCFG should have handled the available opportunities. + // If we did this folding here, it would be necessary to update the + // MachineBasicBlock CFG, which is awkward. + // fold a brcond with a setcc condition into a BR_CC node if BR_CC is legal // on the target. if (N1.getOpcode() == ISD::SETCC && @@ -4517,22 +4515,18 @@ SDValue DAGCombiner::visitBR_CC(SDNode *N) { CondCodeSDNode *CC = cast<CondCodeSDNode>(N->getOperand(1)); SDValue CondLHS = N->getOperand(2), CondRHS = N->getOperand(3); + // If N is a constant we could fold this into a fallthrough or unconditional + // branch. However that doesn't happen very often in normal code, because + // Instcombine/SimplifyCFG should have handled the available opportunities. + // If we did this folding here, it would be necessary to update the + // MachineBasicBlock CFG, which is awkward. + // Use SimplifySetCC to simplify SETCC's. SDValue Simp = SimplifySetCC(TLI.getSetCCResultType(CondLHS.getValueType()), CondLHS, CondRHS, CC->get(), N->getDebugLoc(), false); if (Simp.getNode()) AddToWorkList(Simp.getNode()); - ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(Simp.getNode()); - - // fold br_cc true, dest -> br dest (unconditional branch) - if (SCCC && !SCCC->isNullValue()) - return DAG.getNode(ISD::BR, N->getDebugLoc(), MVT::Other, - N->getOperand(0), N->getOperand(4)); - // fold br_cc false, dest -> unconditional fall through - if (SCCC && SCCC->isNullValue()) - return N->getOperand(0); - // fold to a simpler setcc if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC) return DAG.getNode(ISD::BR_CC, N->getDebugLoc(), MVT::Other, diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp index 8e955af..7dbc136 100644 --- a/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -43,6 +43,7 @@ #include "llvm/GlobalVariable.h" #include "llvm/Instructions.h" #include "llvm/IntrinsicInst.h" +#include "llvm/LLVMContext.h" #include "llvm/CodeGen/FastISel.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfo.h" @@ -324,82 +325,12 @@ bool FastISel::SelectCall(User *I) { unsigned IID = F->getIntrinsicID(); switch (IID) { default: break; - case Intrinsic::dbg_stoppoint: { - DbgStopPointInst *SPI = cast<DbgStopPointInst>(I); - if (isValidDebugInfoIntrinsic(*SPI, CodeGenOpt::None)) - setCurDebugLoc(ExtractDebugLocation(*SPI, MF.getDebugLocInfo())); + case Intrinsic::dbg_stoppoint: + case Intrinsic::dbg_region_start: + case Intrinsic::dbg_region_end: + case Intrinsic::dbg_func_start: + // FIXME - Remove this instructions once the dust settles. return true; - } - case Intrinsic::dbg_region_start: { - DbgRegionStartInst *RSI = cast<DbgRegionStartInst>(I); - if (isValidDebugInfoIntrinsic(*RSI, CodeGenOpt::None) && DW - && DW->ShouldEmitDwarfDebug()) { - unsigned ID = - DW->RecordRegionStart(RSI->getContext()); - const TargetInstrDesc &II = TII.get(TargetInstrInfo::DBG_LABEL); - BuildMI(MBB, DL, II).addImm(ID); - } - return true; - } - case Intrinsic::dbg_region_end: { - DbgRegionEndInst *REI = cast<DbgRegionEndInst>(I); - if (isValidDebugInfoIntrinsic(*REI, CodeGenOpt::None) && DW - && DW->ShouldEmitDwarfDebug()) { - unsigned ID = 0; - DISubprogram Subprogram(REI->getContext()); - if (isInlinedFnEnd(*REI, MF.getFunction())) { - // This is end of an inlined function. - const TargetInstrDesc &II = TII.get(TargetInstrInfo::DBG_LABEL); - ID = DW->RecordInlinedFnEnd(Subprogram); - if (ID) - // Returned ID is 0 if this is unbalanced "end of inlined - // scope". This could happen if optimizer eats dbg intrinsics - // or "beginning of inlined scope" is not recoginized due to - // missing location info. In such cases, ignore this region.end. - BuildMI(MBB, DL, II).addImm(ID); - } else { - const TargetInstrDesc &II = TII.get(TargetInstrInfo::DBG_LABEL); - ID = DW->RecordRegionEnd(REI->getContext()); - BuildMI(MBB, DL, II).addImm(ID); - } - } - return true; - } - case Intrinsic::dbg_func_start: { - DbgFuncStartInst *FSI = cast<DbgFuncStartInst>(I); - if (!isValidDebugInfoIntrinsic(*FSI, CodeGenOpt::None) || !DW - || !DW->ShouldEmitDwarfDebug()) - return true; - - if (isInlinedFnStart(*FSI, MF.getFunction())) { - // This is a beginning of an inlined function. - - // If llvm.dbg.func.start is seen in a new block before any - // llvm.dbg.stoppoint intrinsic then the location info is unknown. - // FIXME : Why DebugLoc is reset at the beginning of each block ? - DebugLoc PrevLoc = DL; - if (PrevLoc.isUnknown()) - return true; - // Record the source line. - setCurDebugLoc(ExtractDebugLocation(*FSI, MF.getDebugLocInfo())); - - DebugLocTuple PrevLocTpl = MF.getDebugLocTuple(PrevLoc); - DISubprogram SP(FSI->getSubprogram()); - unsigned LabelID = - DW->RecordInlinedFnStart(SP,DICompileUnit(PrevLocTpl.Scope), - PrevLocTpl.Line, PrevLocTpl.Col); - const TargetInstrDesc &II = TII.get(TargetInstrInfo::DBG_LABEL); - BuildMI(MBB, DL, II).addImm(LabelID); - return true; - } - - // This is a beginning of a new function. - MF.setDefaultDebugLoc(ExtractDebugLocation(*FSI, MF.getDebugLocInfo())); - - // llvm.dbg.func_start also defines beginning of function scope. - DW->RecordRegionStart(FSI->getSubprogram()); - return true; - } case Intrinsic::dbg_declare: { DbgDeclareInst *DI = cast<DbgDeclareInst>(I); if (!isValidDebugInfoIntrinsic(*DI, CodeGenOpt::None) || !DW @@ -416,11 +347,13 @@ bool FastISel::SelectCall(User *I) { StaticAllocaMap.find(AI); if (SI == StaticAllocaMap.end()) break; // VLAs. int FI = SI->second; - if (MMI) - MMI->setVariableDbgInfo(DI->getVariable(), FI); -#ifndef ATTACH_DEBUG_INFO_TO_AN_INSN - DW->RecordVariable(DI->getVariable(), FI); -#endif + if (MMI) { + MetadataContext &TheMetadata = + DI->getParent()->getContext().getMetadata(); + unsigned MDDbgKind = TheMetadata.getMDKind("dbg"); + MDNode *Dbg = TheMetadata.getMD(MDDbgKind, DI); + MMI->setVariableDbgInfo(DI->getVariable(), FI, Dbg); + } return true; } case Intrinsic::eh_exception: { diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index da311ed..52b0832 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -497,7 +497,7 @@ InstrEmitter::EmitCopyToRegClassNode(SDNode *Node, assert(isNew && "Node emitted out of order - early"); } -/// EmitNode - Generate machine code for an node and needed dependencies. +/// EmitNode - Generate machine code for a node and needed dependencies. /// void InstrEmitter::EmitNode(SDNode *Node, bool IsClone, bool IsCloned, DenseMap<SDValue, unsigned> &VRBaseMap, diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.h b/lib/CodeGen/SelectionDAG/InstrEmitter.h index bb4634d..91817e4 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.h +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.h @@ -97,7 +97,7 @@ public: /// MachineInstr. static unsigned CountOperands(SDNode *Node); - /// EmitNode - Generate machine code for an node and needed dependencies. + /// EmitNode - Generate machine code for a node and needed dependencies. /// void EmitNode(SDNode *Node, bool IsClone, bool IsCloned, DenseMap<SDValue, unsigned> &VRBaseMap, diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index f389f7f..4f0a229 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -148,8 +148,11 @@ private: SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_PPCF128); - SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I16, - RTLIB::Libcall Call_I32, RTLIB::Libcall Call_I64, + SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, + RTLIB::Libcall Call_I8, + RTLIB::Libcall Call_I16, + RTLIB::Libcall Call_I32, + RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128); SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, DebugLoc dl); @@ -1810,10 +1813,19 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); } else if (ConstantSDNode *V = dyn_cast<ConstantSDNode>(Node->getOperand(i))) { - CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); + if (OpVT==EltVT) + CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); + else { + // If OpVT and EltVT don't match, EltVT is not legal and the + // element values have been promoted/truncated earlier. Undo this; + // we don't want a v16i8 to become a v16i32 for example. + const ConstantInt *CI = V->getConstantIntValue(); + CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), + CI->getZExtValue())); + } } else { assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); - const Type *OpNTy = OpVT.getTypeForEVT(*DAG.getContext()); + const Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); CV.push_back(UndefValue::get(OpNTy)); } } @@ -1909,6 +1921,7 @@ SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, } SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, + RTLIB::Libcall Call_I8, RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32, RTLIB::Libcall Call_I64, @@ -1916,9 +1929,10 @@ SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, RTLIB::Libcall LC; switch (Node->getValueType(0).getSimpleVT().SimpleTy) { default: llvm_unreachable("Unexpected request for libcall!"); - case MVT::i16: LC = Call_I16; break; - case MVT::i32: LC = Call_I32; break; - case MVT::i64: LC = Call_I64; break; + case MVT::i8: LC = Call_I8; break; + case MVT::i16: LC = Call_I16; break; + case MVT::i32: LC = Call_I32; break; + case MVT::i64: LC = Call_I64; break; case MVT::i128: LC = Call_I128; break; } return ExpandLibCall(LC, Node, isSigned); @@ -2624,10 +2638,14 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); } else if (isSigned) { - Tmp1 = ExpandIntLibCall(Node, true, RTLIB::SREM_I16, RTLIB::SREM_I32, + Tmp1 = ExpandIntLibCall(Node, true, + RTLIB::SREM_I8, + RTLIB::SREM_I16, RTLIB::SREM_I32, RTLIB::SREM_I64, RTLIB::SREM_I128); } else { - Tmp1 = ExpandIntLibCall(Node, false, RTLIB::UREM_I16, RTLIB::UREM_I32, + Tmp1 = ExpandIntLibCall(Node, false, + RTLIB::UREM_I8, + RTLIB::UREM_I16, RTLIB::UREM_I32, RTLIB::UREM_I64, RTLIB::UREM_I128); } Results.push_back(Tmp1); @@ -2643,10 +2661,14 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), Node->getOperand(1)); else if (isSigned) - Tmp1 = ExpandIntLibCall(Node, true, RTLIB::SDIV_I16, RTLIB::SDIV_I32, + Tmp1 = ExpandIntLibCall(Node, true, + RTLIB::SDIV_I8, + RTLIB::SDIV_I16, RTLIB::SDIV_I32, RTLIB::SDIV_I64, RTLIB::SDIV_I128); else - Tmp1 = ExpandIntLibCall(Node, false, RTLIB::UDIV_I16, RTLIB::UDIV_I32, + Tmp1 = ExpandIntLibCall(Node, false, + RTLIB::UDIV_I8, + RTLIB::UDIV_I16, RTLIB::UDIV_I32, RTLIB::UDIV_I64, RTLIB::UDIV_I128); Results.push_back(Tmp1); break; @@ -2691,7 +2713,9 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node, Node->getOperand(1))); break; } - Tmp1 = ExpandIntLibCall(Node, false, RTLIB::MUL_I16, RTLIB::MUL_I32, + Tmp1 = ExpandIntLibCall(Node, false, + RTLIB::MUL_I8, + RTLIB::MUL_I16, RTLIB::MUL_I32, RTLIB::MUL_I64, RTLIB::MUL_I128); Results.push_back(Tmp1); break; diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 98e7317..4530ffc 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1270,11 +1270,12 @@ SDValue SelectionDAG::getConvertRndSat(EVT VT, DebugLoc dl, return Val; FoldingSetNodeID ID; + SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; + AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); void* IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); CvtRndSatSDNode *N = NodeAllocator.Allocate<CvtRndSatSDNode>(); - SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; new (N) CvtRndSatSDNode(VT, dl, Ops, 5, Code); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); @@ -1378,7 +1379,7 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { unsigned StackAlign = std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), minAlign); - int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign); + int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); return getFrameIndex(FrameIdx, TLI.getPointerTy()); } @@ -1394,7 +1395,7 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { TD->getPrefTypeAlignment(Ty2)); MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); - int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align); + int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); return getFrameIndex(FrameIdx, TLI.getPointerTy()); } @@ -5814,9 +5815,8 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const { print_types(OS, G); - OS << " "; for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { - if (i) OS << ", "; + if (i) OS << ", "; else OS << " "; OS << (void*)getOperand(i).getNode(); if (unsigned RN = getOperand(i).getResNo()) OS << ":" << RN; @@ -5916,7 +5916,8 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, - unsigned MinSplatBits) { + unsigned MinSplatBits, + bool isBigEndian) { EVT VT = getValueType(0); assert(VT.isVector() && "Expected a vector type"); unsigned sz = VT.getSizeInBits(); @@ -5933,12 +5934,14 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, unsigned int nOps = getNumOperands(); assert(nOps > 0 && "isConstantSplat has 0-size build vector"); unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); - for (unsigned i = 0; i < nOps; ++i) { + + for (unsigned j = 0; j < nOps; ++j) { + unsigned i = isBigEndian ? nOps-1-j : j; SDValue OpVal = getOperand(i); - unsigned BitPos = i * EltBitSize; + unsigned BitPos = j * EltBitSize; if (OpVal.getOpcode() == ISD::UNDEF) - SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos +EltBitSize); + SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) SplatValue |= (APInt(CN->getAPIntValue()).zextOrTrunc(EltBitSize). zextOrTrunc(sz) << BitPos); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp index c0d2a4d..90fd95e 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp @@ -26,6 +26,7 @@ #include "llvm/Instructions.h" #include "llvm/Intrinsics.h" #include "llvm/IntrinsicInst.h" +#include "llvm/LLVMContext.h" #include "llvm/Module.h" #include "llvm/CodeGen/FastISel.h" #include "llvm/CodeGen/GCStrategy.h" @@ -304,7 +305,7 @@ void FunctionLoweringInfo::set(Function &fn, MachineFunction &mf, TySize *= CUI->getZExtValue(); // Get total allocated size. if (TySize == 0) TySize = 1; // Don't create zero-sized stack objects. StaticAllocaMap[AI] = - MF->getFrameInfo()->CreateStackObject(TySize, Align); + MF->getFrameInfo()->CreateStackObject(TySize, Align, false); } for (; BB != EB; ++BB) @@ -334,25 +335,6 @@ void FunctionLoweringInfo::set(Function &fn, MachineFunction &mf, DebugLoc DL; for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) { - if (CallInst *CI = dyn_cast<CallInst>(I)) { - if (Function *F = CI->getCalledFunction()) { - switch (F->getIntrinsicID()) { - default: break; - case Intrinsic::dbg_stoppoint: { - DbgStopPointInst *SPI = cast<DbgStopPointInst>(I); - if (isValidDebugInfoIntrinsic(*SPI, CodeGenOpt::Default)) - DL = ExtractDebugLocation(*SPI, MF->getDebugLocInfo()); - break; - } - case Intrinsic::dbg_func_start: { - DbgFuncStartInst *FSI = cast<DbgFuncStartInst>(I); - if (isValidDebugInfoIntrinsic(*FSI, CodeGenOpt::Default)) - DL = ExtractDebugLocation(*FSI, MF->getDebugLocInfo()); - break; - } - } - } - } PN = dyn_cast<PHINode>(I); if (!PN || PN->use_empty()) continue; @@ -947,58 +929,143 @@ SDValue SelectionDAGLowering::getValue(const Value *V) { return RFV.getCopyFromRegs(DAG, getCurDebugLoc(), Chain, NULL); } +/// Get the EVTs and ArgFlags collections that represent the return type +/// of the given function. This does not require a DAG or a return value, and +/// is suitable for use before any DAGs for the function are constructed. +static void getReturnInfo(const Type* ReturnType, + Attributes attr, SmallVectorImpl<EVT> &OutVTs, + SmallVectorImpl<ISD::ArgFlagsTy> &OutFlags, + TargetLowering &TLI, + SmallVectorImpl<uint64_t> *Offsets = 0) { + SmallVector<EVT, 4> ValueVTs; + ComputeValueVTs(TLI, ReturnType, ValueVTs, Offsets); + unsigned NumValues = ValueVTs.size(); + if ( NumValues == 0 ) return; + + for (unsigned j = 0, f = NumValues; j != f; ++j) { + EVT VT = ValueVTs[j]; + ISD::NodeType ExtendKind = ISD::ANY_EXTEND; + + if (attr & Attribute::SExt) + ExtendKind = ISD::SIGN_EXTEND; + else if (attr & Attribute::ZExt) + ExtendKind = ISD::ZERO_EXTEND; + + // FIXME: C calling convention requires the return type to be promoted to + // at least 32-bit. But this is not necessary for non-C calling + // conventions. The frontend should mark functions whose return values + // require promoting with signext or zeroext attributes. + if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) { + EVT MinVT = TLI.getRegisterType(ReturnType->getContext(), MVT::i32); + if (VT.bitsLT(MinVT)) + VT = MinVT; + } + + unsigned NumParts = TLI.getNumRegisters(ReturnType->getContext(), VT); + EVT PartVT = TLI.getRegisterType(ReturnType->getContext(), VT); + // 'inreg' on function refers to return value + ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); + if (attr & Attribute::InReg) + Flags.setInReg(); + + // Propagate extension type if any + if (attr & Attribute::SExt) + Flags.setSExt(); + else if (attr & Attribute::ZExt) + Flags.setZExt(); + + for (unsigned i = 0; i < NumParts; ++i) { + OutVTs.push_back(PartVT); + OutFlags.push_back(Flags); + } + } +} void SelectionDAGLowering::visitRet(ReturnInst &I) { SDValue Chain = getControlRoot(); SmallVector<ISD::OutputArg, 8> Outs; - for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { + FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); + + if (!FLI.CanLowerReturn) { + unsigned DemoteReg = FLI.DemoteRegister; + const Function *F = I.getParent()->getParent(); + + // Emit a store of the return value through the virtual register. + // Leave Outs empty so that LowerReturn won't try to load return + // registers the usual way. + SmallVector<EVT, 1> PtrValueVTs; + ComputeValueVTs(TLI, PointerType::getUnqual(F->getReturnType()), + PtrValueVTs); + + SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]); + SDValue RetOp = getValue(I.getOperand(0)); + SmallVector<EVT, 4> ValueVTs; - ComputeValueVTs(TLI, I.getOperand(i)->getType(), ValueVTs); + SmallVector<uint64_t, 4> Offsets; + ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets); unsigned NumValues = ValueVTs.size(); - if (NumValues == 0) continue; - - SDValue RetOp = getValue(I.getOperand(i)); - for (unsigned j = 0, f = NumValues; j != f; ++j) { - EVT VT = ValueVTs[j]; - ISD::NodeType ExtendKind = ISD::ANY_EXTEND; - - const Function *F = I.getParent()->getParent(); - if (F->paramHasAttr(0, Attribute::SExt)) - ExtendKind = ISD::SIGN_EXTEND; - else if (F->paramHasAttr(0, Attribute::ZExt)) - ExtendKind = ISD::ZERO_EXTEND; + SmallVector<SDValue, 4> Chains(NumValues); + EVT PtrVT = PtrValueVTs[0]; + for (unsigned i = 0; i != NumValues; ++i) + Chains[i] = DAG.getStore(Chain, getCurDebugLoc(), + SDValue(RetOp.getNode(), RetOp.getResNo() + i), + DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, RetPtr, + DAG.getConstant(Offsets[i], PtrVT)), + NULL, Offsets[i], false, 0); + Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), + MVT::Other, &Chains[0], NumValues); + } + else { + for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { + SmallVector<EVT, 4> ValueVTs; + ComputeValueVTs(TLI, I.getOperand(i)->getType(), ValueVTs); + unsigned NumValues = ValueVTs.size(); + if (NumValues == 0) continue; + + SDValue RetOp = getValue(I.getOperand(i)); + for (unsigned j = 0, f = NumValues; j != f; ++j) { + EVT VT = ValueVTs[j]; + + ISD::NodeType ExtendKind = ISD::ANY_EXTEND; + + const Function *F = I.getParent()->getParent(); + if (F->paramHasAttr(0, Attribute::SExt)) + ExtendKind = ISD::SIGN_EXTEND; + else if (F->paramHasAttr(0, Attribute::ZExt)) + ExtendKind = ISD::ZERO_EXTEND; + + // FIXME: C calling convention requires the return type to be promoted to + // at least 32-bit. But this is not necessary for non-C calling + // conventions. The frontend should mark functions whose return values + // require promoting with signext or zeroext attributes. + if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) { + EVT MinVT = TLI.getRegisterType(*DAG.getContext(), MVT::i32); + if (VT.bitsLT(MinVT)) + VT = MinVT; + } - // FIXME: C calling convention requires the return type to be promoted to - // at least 32-bit. But this is not necessary for non-C calling - // conventions. The frontend should mark functions whose return values - // require promoting with signext or zeroext attributes. - if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) { - EVT MinVT = TLI.getRegisterType(*DAG.getContext(), MVT::i32); - if (VT.bitsLT(MinVT)) - VT = MinVT; + unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), VT); + EVT PartVT = TLI.getRegisterType(*DAG.getContext(), VT); + SmallVector<SDValue, 4> Parts(NumParts); + getCopyToParts(DAG, getCurDebugLoc(), + SDValue(RetOp.getNode(), RetOp.getResNo() + j), + &Parts[0], NumParts, PartVT, ExtendKind); + + // 'inreg' on function refers to return value + ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); + if (F->paramHasAttr(0, Attribute::InReg)) + Flags.setInReg(); + + // Propagate extension type if any + if (F->paramHasAttr(0, Attribute::SExt)) + Flags.setSExt(); + else if (F->paramHasAttr(0, Attribute::ZExt)) + Flags.setZExt(); + + for (unsigned i = 0; i < NumParts; ++i) + Outs.push_back(ISD::OutputArg(Flags, Parts[i], /*isfixed=*/true)); } - - unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), VT); - EVT PartVT = TLI.getRegisterType(*DAG.getContext(), VT); - SmallVector<SDValue, 4> Parts(NumParts); - getCopyToParts(DAG, getCurDebugLoc(), - SDValue(RetOp.getNode(), RetOp.getResNo() + j), - &Parts[0], NumParts, PartVT, ExtendKind); - - // 'inreg' on function refers to return value - ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); - if (F->paramHasAttr(0, Attribute::InReg)) - Flags.setInReg(); - - // Propagate extension type if any - if (F->paramHasAttr(0, Attribute::SExt)) - Flags.setSExt(); - else if (F->paramHasAttr(0, Attribute::ZExt)) - Flags.setZExt(); - - for (unsigned i = 0; i < NumParts; ++i) - Outs.push_back(ISD::OutputArg(Flags, Parts[i], /*isfixed=*/true)); } } @@ -1691,19 +1758,19 @@ bool SelectionDAGLowering::handleJTSwitchCase(CaseRec& CR, Case& FrontCase = *CR.Range.first; Case& BackCase = *(CR.Range.second-1); - const APInt& First = cast<ConstantInt>(FrontCase.Low)->getValue(); - const APInt& Last = cast<ConstantInt>(BackCase.High)->getValue(); + const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue(); + const APInt &Last = cast<ConstantInt>(BackCase.High)->getValue(); - size_t TSize = 0; + APInt TSize(First.getBitWidth(), 0); for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) TSize += I->size(); - if (!areJTsAllowed(TLI) || TSize <= 3) + if (!areJTsAllowed(TLI) || TSize.ult(APInt(First.getBitWidth(), 4))) return false; APInt Range = ComputeRange(First, Last); - double Density = (double)TSize / Range.roundToDouble(); + double Density = TSize.roundToDouble() / Range.roundToDouble(); if (Density < 0.4) return false; @@ -1797,32 +1864,34 @@ bool SelectionDAGLowering::handleBTSplitSwitchCase(CaseRec& CR, // Size is the number of Cases represented by this range. unsigned Size = CR.Range.second - CR.Range.first; - const APInt& First = cast<ConstantInt>(FrontCase.Low)->getValue(); - const APInt& Last = cast<ConstantInt>(BackCase.High)->getValue(); + const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue(); + const APInt &Last = cast<ConstantInt>(BackCase.High)->getValue(); double FMetric = 0; CaseItr Pivot = CR.Range.first + Size/2; // Select optimal pivot, maximizing sum density of LHS and RHS. This will // (heuristically) allow us to emit JumpTable's later. - size_t TSize = 0; + APInt TSize(First.getBitWidth(), 0); for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) TSize += I->size(); - size_t LSize = FrontCase.size(); - size_t RSize = TSize-LSize; + APInt LSize = FrontCase.size(); + APInt RSize = TSize-LSize; DEBUG(errs() << "Selecting best pivot: \n" << "First: " << First << ", Last: " << Last <<'\n' << "LSize: " << LSize << ", RSize: " << RSize << '\n'); for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second; J!=E; ++I, ++J) { - const APInt& LEnd = cast<ConstantInt>(I->High)->getValue(); - const APInt& RBegin = cast<ConstantInt>(J->Low)->getValue(); + const APInt &LEnd = cast<ConstantInt>(I->High)->getValue(); + const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue(); APInt Range = ComputeRange(LEnd, RBegin); assert((Range - 2ULL).isNonNegative() && "Invalid case distance"); - double LDensity = (double)LSize / (LEnd - First + 1ULL).roundToDouble(); - double RDensity = (double)RSize / (Last - RBegin + 1ULL).roundToDouble(); + double LDensity = (double)LSize.roundToDouble() / + (LEnd - First + 1ULL).roundToDouble(); + double RDensity = (double)RSize.roundToDouble() / + (Last - RBegin + 1ULL).roundToDouble(); double Metric = Range.logBase2()*(LDensity+RDensity); // Should always split in some non-trivial place DEBUG(errs() <<"=>Step\n" @@ -3842,112 +3911,12 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { I.getOperand(1), 0, I.getOperand(2), 0)); return 0; } - case Intrinsic::dbg_stoppoint: { - DbgStopPointInst &SPI = cast<DbgStopPointInst>(I); - if (isValidDebugInfoIntrinsic(SPI, CodeGenOpt::Default)) { - MachineFunction &MF = DAG.getMachineFunction(); - DebugLoc Loc = ExtractDebugLocation(SPI, MF.getDebugLocInfo()); - setCurDebugLoc(Loc); - - if (OptLevel == CodeGenOpt::None) - DAG.setRoot(DAG.getDbgStopPoint(Loc, getRoot(), - SPI.getLine(), - SPI.getColumn(), - SPI.getContext())); - } + case Intrinsic::dbg_stoppoint: + case Intrinsic::dbg_region_start: + case Intrinsic::dbg_region_end: + case Intrinsic::dbg_func_start: + // FIXME - Remove this instructions once the dust settles. return 0; - } - case Intrinsic::dbg_region_start: { - DwarfWriter *DW = DAG.getDwarfWriter(); - DbgRegionStartInst &RSI = cast<DbgRegionStartInst>(I); - if (isValidDebugInfoIntrinsic(RSI, OptLevel) && DW - && DW->ShouldEmitDwarfDebug()) { - unsigned LabelID = - DW->RecordRegionStart(RSI.getContext()); - DAG.setRoot(DAG.getLabel(ISD::DBG_LABEL, getCurDebugLoc(), - getRoot(), LabelID)); - } - return 0; - } - case Intrinsic::dbg_region_end: { - DwarfWriter *DW = DAG.getDwarfWriter(); - DbgRegionEndInst &REI = cast<DbgRegionEndInst>(I); - - if (!isValidDebugInfoIntrinsic(REI, OptLevel) || !DW - || !DW->ShouldEmitDwarfDebug()) - return 0; - - MachineFunction &MF = DAG.getMachineFunction(); - DISubprogram Subprogram(REI.getContext()); - - if (isInlinedFnEnd(REI, MF.getFunction())) { - // This is end of inlined function. Debugging information for inlined - // function is not handled yet (only supported by FastISel). - if (OptLevel == CodeGenOpt::None) { - unsigned ID = DW->RecordInlinedFnEnd(Subprogram); - if (ID != 0) - // Returned ID is 0 if this is unbalanced "end of inlined - // scope". This could happen if optimizer eats dbg intrinsics or - // "beginning of inlined scope" is not recoginized due to missing - // location info. In such cases, do ignore this region.end. - DAG.setRoot(DAG.getLabel(ISD::DBG_LABEL, getCurDebugLoc(), - getRoot(), ID)); - } - return 0; - } - - unsigned LabelID = - DW->RecordRegionEnd(REI.getContext()); - DAG.setRoot(DAG.getLabel(ISD::DBG_LABEL, getCurDebugLoc(), - getRoot(), LabelID)); - return 0; - } - case Intrinsic::dbg_func_start: { - DwarfWriter *DW = DAG.getDwarfWriter(); - DbgFuncStartInst &FSI = cast<DbgFuncStartInst>(I); - if (!isValidDebugInfoIntrinsic(FSI, CodeGenOpt::None)) - return 0; - - MachineFunction &MF = DAG.getMachineFunction(); - // This is a beginning of an inlined function. - if (isInlinedFnStart(FSI, MF.getFunction())) { - if (OptLevel != CodeGenOpt::None) - // FIXME: Debugging informaation for inlined function is only - // supported at CodeGenOpt::Node. - return 0; - - DebugLoc PrevLoc = CurDebugLoc; - // If llvm.dbg.func.start is seen in a new block before any - // llvm.dbg.stoppoint intrinsic then the location info is unknown. - // FIXME : Why DebugLoc is reset at the beginning of each block ? - if (PrevLoc.isUnknown()) - return 0; - - // Record the source line. - setCurDebugLoc(ExtractDebugLocation(FSI, MF.getDebugLocInfo())); - - if (!DW || !DW->ShouldEmitDwarfDebug()) - return 0; - DebugLocTuple PrevLocTpl = MF.getDebugLocTuple(PrevLoc); - DISubprogram SP(FSI.getSubprogram()); - DICompileUnit CU(PrevLocTpl.Scope); - unsigned LabelID = DW->RecordInlinedFnStart(SP, CU, - PrevLocTpl.Line, - PrevLocTpl.Col); - DAG.setRoot(DAG.getLabel(ISD::DBG_LABEL, getCurDebugLoc(), - getRoot(), LabelID)); - return 0; - } - - // This is a beginning of a new function. - MF.setDefaultDebugLoc(ExtractDebugLocation(FSI, MF.getDebugLocInfo())); - - if (!DW || !DW->ShouldEmitDwarfDebug()) - return 0; - // llvm.dbg.func_start also defines beginning of function scope. - DW->RecordRegionStart(FSI.getSubprogram()); - return 0; - } case Intrinsic::dbg_declare: { if (OptLevel != CodeGenOpt::None) // FIXME: Variable debug info is not supported here. @@ -3972,13 +3941,15 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { if (SI == FuncInfo.StaticAllocaMap.end()) return 0; // VLAs. int FI = SI->second; -#ifdef ATTACH_DEBUG_INFO_TO_AN_INSN + MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - if (MMI) - MMI->setVariableDbgInfo(Variable, FI); -#else - DW->RecordVariable(Variable, FI); -#endif + if (MMI) { + MetadataContext &TheMetadata = + DI.getParent()->getContext().getMetadata(); + unsigned MDDbgKind = TheMetadata.getMDKind("dbg"); + MDNode *Dbg = TheMetadata.getMD(MDDbgKind, &DI); + MMI->setVariableDbgInfo(Variable, FI, Dbg); + } return 0; } case Intrinsic::eh_exception: { @@ -4233,7 +4204,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { EVT Ty = Arg.getValueType(); if (CI->getZExtValue() < 2) - setValue(&I, DAG.getConstant(-1U, Ty)); + setValue(&I, DAG.getConstant(-1ULL, Ty)); else setValue(&I, DAG.getConstant(0, Ty)); return 0; @@ -4355,6 +4326,16 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMAX); case Intrinsic::atomic_swap: return implVisitBinaryAtomic(I, ISD::ATOMIC_SWAP); + + case Intrinsic::invariant_start: + case Intrinsic::lifetime_start: + // Discard region information. + setValue(&I, DAG.getUNDEF(TLI.getPointerTy())); + return 0; + case Intrinsic::invariant_end: + case Intrinsic::lifetime_end: + // Discard region information. + return 0; } } @@ -4368,7 +4349,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { /// TargetLowering::IsEligibleForTailCallOptimization. /// static bool -isInTailCallPosition(const Instruction *I, Attributes RetAttr, +isInTailCallPosition(const Instruction *I, Attributes CalleeRetAttr, const TargetLowering &TLI) { const BasicBlock *ExitBB = I->getParent(); const TerminatorInst *Term = ExitBB->getTerminator(); @@ -4395,9 +4376,14 @@ isInTailCallPosition(const Instruction *I, Attributes RetAttr, // what the call's return type is. if (!Ret || Ret->getNumOperands() == 0) return true; + // If the return value is undef, it doesn't matter what the call's + // return type is. + if (isa<UndefValue>(Ret->getOperand(0))) return true; + // Conservatively require the attributes of the call to match those of - // the return. - if (F->getAttributes().getRetAttributes() != RetAttr) + // the return. Ignore noalias because it doesn't affect the call sequence. + unsigned CallerRetAttr = F->getAttributes().getRetAttributes(); + if ((CalleeRetAttr ^ CallerRetAttr) & ~Attribute::NoAlias) return false; // Otherwise, make sure the unmodified return value of I is the return value. @@ -4431,15 +4417,52 @@ void SelectionDAGLowering::LowerCallTo(CallSite CS, SDValue Callee, MachineBasicBlock *LandingPad) { const PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType()); const FunctionType *FTy = cast<FunctionType>(PT->getElementType()); + const Type *RetTy = FTy->getReturnType(); MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); unsigned BeginLabel = 0, EndLabel = 0; TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; Args.reserve(CS.arg_size()); - unsigned j = 1; + + // Check whether the function can return without sret-demotion. + SmallVector<EVT, 4> OutVTs; + SmallVector<ISD::ArgFlagsTy, 4> OutsFlags; + SmallVector<uint64_t, 4> Offsets; + getReturnInfo(RetTy, CS.getAttributes().getRetAttributes(), + OutVTs, OutsFlags, TLI, &Offsets); + + + bool CanLowerReturn = TLI.CanLowerReturn(CS.getCallingConv(), + FTy->isVarArg(), OutVTs, OutsFlags, DAG); + + SDValue DemoteStackSlot; + + if (!CanLowerReturn) { + uint64_t TySize = TLI.getTargetData()->getTypeAllocSize( + FTy->getReturnType()); + unsigned Align = TLI.getTargetData()->getPrefTypeAlignment( + FTy->getReturnType()); + MachineFunction &MF = DAG.getMachineFunction(); + int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align, false); + const Type *StackSlotPtrType = PointerType::getUnqual(FTy->getReturnType()); + + DemoteStackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy()); + Entry.Node = DemoteStackSlot; + Entry.Ty = StackSlotPtrType; + Entry.isSExt = false; + Entry.isZExt = false; + Entry.isInReg = false; + Entry.isSRet = true; + Entry.isNest = false; + Entry.isByVal = false; + Entry.Alignment = Align; + Args.push_back(Entry); + RetTy = Type::getVoidTy(FTy->getContext()); + } + for (CallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); - i != e; ++i, ++j) { + i != e; ++i) { SDValue ArgNode = getValue(*i); Entry.Node = ArgNode; Entry.Ty = (*i)->getType(); @@ -4475,7 +4498,7 @@ void SelectionDAGLowering::LowerCallTo(CallSite CS, SDValue Callee, isTailCall = false; std::pair<SDValue,SDValue> Result = - TLI.LowerCallTo(getRoot(), CS.getType(), + TLI.LowerCallTo(getRoot(), RetTy, CS.paramHasAttr(0, Attribute::SExt), CS.paramHasAttr(0, Attribute::ZExt), FTy->isVarArg(), CS.paramHasAttr(0, Attribute::InReg), FTy->getNumParams(), @@ -4489,6 +4512,35 @@ void SelectionDAGLowering::LowerCallTo(CallSite CS, SDValue Callee, "Null value expected with tail call!"); if (Result.first.getNode()) setValue(CS.getInstruction(), Result.first); + else if (!CanLowerReturn && Result.second.getNode()) { + // The instruction result is the result of loading from the + // hidden sret parameter. + SmallVector<EVT, 1> PVTs; + const Type *PtrRetTy = PointerType::getUnqual(FTy->getReturnType()); + + ComputeValueVTs(TLI, PtrRetTy, PVTs); + assert(PVTs.size() == 1 && "Pointers should fit in one register"); + EVT PtrVT = PVTs[0]; + unsigned NumValues = OutVTs.size(); + SmallVector<SDValue, 4> Values(NumValues); + SmallVector<SDValue, 4> Chains(NumValues); + + for (unsigned i = 0; i < NumValues; ++i) { + SDValue L = DAG.getLoad(OutVTs[i], getCurDebugLoc(), Result.second, + DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, DemoteStackSlot, + DAG.getConstant(Offsets[i], PtrVT)), + NULL, Offsets[i], false, 1); + Values[i] = L; + Chains[i] = L.getValue(1); + } + SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), + MVT::Other, &Chains[0], NumValues); + PendingLoads.push_back(Chain); + + setValue(CS.getInstruction(), DAG.getNode(ISD::MERGE_VALUES, + getCurDebugLoc(), DAG.getVTList(&OutVTs[0], NumValues), + &Values[0], NumValues)); + } // As a special case, a null chain means that a tail call has // been emitted and the DAG root is already updated. if (Result.second.getNode()) @@ -5229,7 +5281,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(Ty); unsigned Align = TLI.getTargetData()->getPrefTypeAlignment(Ty); MachineFunction &MF = DAG.getMachineFunction(); - int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align); + int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align, false); SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy()); Chain = DAG.getStore(Chain, getCurDebugLoc(), OpInfo.CallOperand, StackSlot, NULL, 0); @@ -5757,9 +5809,32 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { SDValue OldRoot = DAG.getRoot(); DebugLoc dl = SDL->getCurDebugLoc(); const TargetData *TD = TLI.getTargetData(); + SmallVector<ISD::InputArg, 16> Ins; + + // Check whether the function can return without sret-demotion. + SmallVector<EVT, 4> OutVTs; + SmallVector<ISD::ArgFlagsTy, 4> OutsFlags; + getReturnInfo(F.getReturnType(), F.getAttributes().getRetAttributes(), + OutVTs, OutsFlags, TLI); + FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); + + FLI.CanLowerReturn = TLI.CanLowerReturn(F.getCallingConv(), F.isVarArg(), + OutVTs, OutsFlags, DAG); + if (!FLI.CanLowerReturn) { + // Put in an sret pointer parameter before all the other parameters. + SmallVector<EVT, 1> ValueVTs; + ComputeValueVTs(TLI, PointerType::getUnqual(F.getReturnType()), ValueVTs); + + // NOTE: Assuming that a pointer will never break down to more than one VT + // or one register. + ISD::ArgFlagsTy Flags; + Flags.setSRet(); + EVT RegisterVT = TLI.getRegisterType(*CurDAG->getContext(), ValueVTs[0]); + ISD::InputArg RetArg(Flags, RegisterVT, true); + Ins.push_back(RetArg); + } // Set up the incoming argument description vector. - SmallVector<ISD::InputArg, 16> Ins; unsigned Idx = 1; for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++Idx) { @@ -5837,6 +5912,28 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) { // Set up the argument values. unsigned i = 0; Idx = 1; + if (!FLI.CanLowerReturn) { + // Create a virtual register for the sret pointer, and put in a copy + // from the sret argument into it. + SmallVector<EVT, 1> ValueVTs; + ComputeValueVTs(TLI, PointerType::getUnqual(F.getReturnType()), ValueVTs); + EVT VT = ValueVTs[0]; + EVT RegVT = TLI.getRegisterType(*CurDAG->getContext(), VT); + ISD::NodeType AssertOp = ISD::DELETED_NODE; + SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, + VT, AssertOp); + + MachineFunction& MF = SDL->DAG.getMachineFunction(); + MachineRegisterInfo& RegInfo = MF.getRegInfo(); + unsigned SRetReg = RegInfo.createVirtualRegister(TLI.getRegClassFor(RegVT)); + FLI.DemoteRegister = SRetReg; + NewRoot = SDL->DAG.getCopyToReg(NewRoot, SDL->getCurDebugLoc(), SRetReg, ArgValue); + DAG.setRoot(NewRoot); + + // i indexes lowered arguments. Bump it past the hidden sret argument. + // Idx indexes LLVM arguments. Don't touch it. + ++i; + } for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++Idx) { SmallVector<SDValue, 4> ArgValues; diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.h index a0ec7aa..10f256c 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.h +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.h @@ -90,6 +90,14 @@ public: MachineFunction *MF; MachineRegisterInfo *RegInfo; + /// CanLowerReturn - true iff the function's return value can be lowered to + /// registers. + bool CanLowerReturn; + + /// DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg + /// allocated to hold a pointer to the hidden sret parameter. + unsigned DemoteRegister; + explicit FunctionLoweringInfo(TargetLowering &TLI); /// set - Initialize this FunctionLoweringInfo with the given Function @@ -193,9 +201,9 @@ class SelectionDAGLowering { Case() : Low(0), High(0), BB(0) { } Case(Constant* low, Constant* high, MachineBasicBlock* bb) : Low(low), High(high), BB(bb) { } - uint64_t size() const { - uint64_t rHigh = cast<ConstantInt>(High)->getSExtValue(); - uint64_t rLow = cast<ConstantInt>(Low)->getSExtValue(); + APInt size() const { + const APInt &rHigh = cast<ConstantInt>(High)->getValue(); + const APInt &rLow = cast<ConstantInt>(Low)->getValue(); return (rHigh - rLow + 1ULL); } }; diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index b63d5bb..ab5f21e 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -68,7 +68,7 @@ static cl::opt<bool> EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction fails")); static cl::opt<bool> -SchedLiveInCopies("schedule-livein-copies", +SchedLiveInCopies("schedule-livein-copies", cl::Hidden, cl::desc("Schedule copies of livein registers"), cl::init(false)); @@ -387,13 +387,14 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, if (MDDbgKind) { // Update DebugLoc if debug information is attached with this // instruction. - if (MDNode *Dbg = TheMetadata.getMD(MDDbgKind, I)) { - DILocation DILoc(Dbg); - DebugLoc Loc = ExtractDebugLocation(DILoc, MF->getDebugLocInfo()); - SDL->setCurDebugLoc(Loc); - if (MF->getDefaultDebugLoc().isUnknown()) - MF->setDefaultDebugLoc(Loc); - } + if (!isa<DbgInfoIntrinsic>(I)) + if (MDNode *Dbg = TheMetadata.getMD(MDDbgKind, I)) { + DILocation DILoc(Dbg); + DebugLoc Loc = ExtractDebugLocation(DILoc, MF->getDebugLocInfo()); + SDL->setCurDebugLoc(Loc); + if (MF->getDefaultDebugLoc().isUnknown()) + MF->setDefaultDebugLoc(Loc); + } } if (!isa<TerminatorInst>(I)) SDL->visit(*I); @@ -750,14 +751,15 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, if (MDDbgKind) { // Update DebugLoc if debug information is attached with this // instruction. - if (MDNode *Dbg = TheMetadata.getMD(MDDbgKind, BI)) { - DILocation DILoc(Dbg); - DebugLoc Loc = ExtractDebugLocation(DILoc, - MF.getDebugLocInfo()); - FastIS->setCurDebugLoc(Loc); - if (MF.getDefaultDebugLoc().isUnknown()) - MF.setDefaultDebugLoc(Loc); - } + if (!isa<DbgInfoIntrinsic>(BI)) + if (MDNode *Dbg = TheMetadata.getMD(MDDbgKind, BI)) { + DILocation DILoc(Dbg); + DebugLoc Loc = ExtractDebugLocation(DILoc, + MF.getDebugLocInfo()); + FastIS->setCurDebugLoc(Loc); + if (MF.getDefaultDebugLoc().isUnknown()) + MF.setDefaultDebugLoc(Loc); + } } // Just before the terminator instruction, insert instructions to diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 9f36b67..2ca52a4 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -22,7 +22,6 @@ #include "llvm/DerivedTypes.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/SelectionDAG.h" -#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" @@ -65,22 +64,27 @@ static void InitLibcallNames(const char **Names) { Names[RTLIB::SRA_I32] = "__ashrsi3"; Names[RTLIB::SRA_I64] = "__ashrdi3"; Names[RTLIB::SRA_I128] = "__ashrti3"; + Names[RTLIB::MUL_I8] = "__mulqi3"; Names[RTLIB::MUL_I16] = "__mulhi3"; Names[RTLIB::MUL_I32] = "__mulsi3"; Names[RTLIB::MUL_I64] = "__muldi3"; Names[RTLIB::MUL_I128] = "__multi3"; + Names[RTLIB::SDIV_I8] = "__divqi3"; Names[RTLIB::SDIV_I16] = "__divhi3"; Names[RTLIB::SDIV_I32] = "__divsi3"; Names[RTLIB::SDIV_I64] = "__divdi3"; Names[RTLIB::SDIV_I128] = "__divti3"; + Names[RTLIB::UDIV_I8] = "__udivqi3"; Names[RTLIB::UDIV_I16] = "__udivhi3"; Names[RTLIB::UDIV_I32] = "__udivsi3"; Names[RTLIB::UDIV_I64] = "__udivdi3"; Names[RTLIB::UDIV_I128] = "__udivti3"; + Names[RTLIB::SREM_I8] = "__modqi3"; Names[RTLIB::SREM_I16] = "__modhi3"; Names[RTLIB::SREM_I32] = "__modsi3"; Names[RTLIB::SREM_I64] = "__moddi3"; Names[RTLIB::SREM_I128] = "__modti3"; + Names[RTLIB::UREM_I8] = "__umodqi3"; Names[RTLIB::UREM_I16] = "__umodhi3"; Names[RTLIB::UREM_I32] = "__umodsi3"; Names[RTLIB::UREM_I64] = "__umoddi3"; @@ -2360,7 +2364,7 @@ getRegForInlineAsmConstraint(const std::string &Constraint, assert(*(Constraint.end()-1) == '}' && "Not a brace enclosed constraint?"); // Remove the braces from around the name. - std::string RegName(Constraint.begin()+1, Constraint.end()-1); + StringRef RegName(Constraint.data()+1, Constraint.size()-2); // Figure out which register class contains this reg. const TargetRegisterInfo *RI = TM.getRegisterInfo(); @@ -2383,7 +2387,7 @@ getRegForInlineAsmConstraint(const std::string &Constraint, for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end(); I != E; ++I) { - if (StringsEqualNoCase(RegName, RI->getName(*I))) + if (RegName.equals_lower(RI->getName(*I))) return std::make_pair(*I, RC); } } |
