diff options
Diffstat (limited to 'contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp')
| -rw-r--r-- | contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 330 |
1 files changed, 231 insertions, 99 deletions
diff --git a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 18ad910..b002825 100644 --- a/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/contrib/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -259,19 +259,25 @@ SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { (VT == MVT::f64) ? MVT::i64 : MVT::i32); } + APFloat APF = CFP->getValueAPF(); EVT OrigVT = VT; EVT SVT = VT; - while (SVT != MVT::f32 && SVT != MVT::f16) { - SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); - if (ConstantFPSDNode::isValueValidForType(SVT, CFP->getValueAPF()) && - // Only do this if the target has a native EXTLOAD instruction from - // smaller type. - TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && - TLI.ShouldShrinkFPConstant(OrigVT)) { - Type *SType = SVT.getTypeForEVT(*DAG.getContext()); - LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); - VT = SVT; - Extend = true; + + // We don't want to shrink SNaNs. Converting the SNaN back to its real type + // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ). + if (!APF.isSignaling()) { + while (SVT != MVT::f32 && SVT != MVT::f16) { + SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); + if (ConstantFPSDNode::isValueValidForType(SVT, APF) && + // Only do this if the target has a native EXTLOAD instruction from + // smaller type. + TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && + TLI.ShouldShrinkFPConstant(OrigVT)) { + Type *SType = SVT.getTypeForEVT(*DAG.getContext()); + LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); + VT = SVT; + Extend = true; + } } } @@ -324,8 +330,6 @@ SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec, // supported by the target. EVT VT = Tmp1.getValueType(); EVT EltVT = VT.getVectorElementType(); - EVT IdxVT = Tmp3.getValueType(); - EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); SDValue StackPtr = DAG.CreateStackTemporary(VT); int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); @@ -335,13 +339,8 @@ SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec, DAG.getEntryNode(), dl, Tmp1, StackPtr, MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); - // Truncate or zero extend offset to target pointer type. - Tmp3 = DAG.getZExtOrTrunc(Tmp3, dl, PtrVT); - // Add the offset to the index. - unsigned EltSize = EltVT.getSizeInBits()/8; - Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3, - DAG.getConstant(EltSize, dl, IdxVT)); - SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr); + SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3); + // Store the scalar value. Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT); // Load the updated vector. @@ -795,7 +794,7 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { default: llvm_unreachable("This action is not supported yet!"); case TargetLowering::Custom: isCustom = true; - // FALLTHROUGH + LLVM_FALLTHROUGH; case TargetLowering::Legal: { Value = SDValue(Node, 0); Chain = SDValue(Node, 1); @@ -1013,6 +1012,7 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { case ISD::ADJUST_TRAMPOLINE: case ISD::FRAMEADDR: case ISD::RETURNADDR: + case ISD::ADDROFRETURNADDR: // These operations lie about being legal: when they claim to be legal, // they should actually be custom-lowered. Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); @@ -1061,35 +1061,41 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { case ISD::SRL: case ISD::SRA: case ISD::ROTL: - case ISD::ROTR: + case ISD::ROTR: { // Legalizing shifts/rotates requires adjusting the shift amount // to the appropriate width. - if (!Node->getOperand(1).getValueType().isVector()) { - SDValue SAO = - DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), - Node->getOperand(1)); - HandleSDNode Handle(SAO); - LegalizeOp(SAO.getNode()); - NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), - Handle.getValue()); + SDValue Op0 = Node->getOperand(0); + SDValue Op1 = Node->getOperand(1); + if (!Op1.getValueType().isVector()) { + SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1); + // The getShiftAmountOperand() may create a new operand node or + // return the existing one. If new operand is created we need + // to update the parent node. + // Do not try to legalize SAO here! It will be automatically legalized + // in the next round. + if (SAO != Op1) + NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); } - break; + } + break; case ISD::SRL_PARTS: case ISD::SRA_PARTS: - case ISD::SHL_PARTS: + case ISD::SHL_PARTS: { // Legalizing shifts/rotates requires adjusting the shift amount // to the appropriate width. - if (!Node->getOperand(2).getValueType().isVector()) { - SDValue SAO = - DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), - Node->getOperand(2)); - HandleSDNode Handle(SAO); - LegalizeOp(SAO.getNode()); - NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), - Node->getOperand(1), - Handle.getValue()); + SDValue Op0 = Node->getOperand(0); + SDValue Op1 = Node->getOperand(1); + SDValue Op2 = Node->getOperand(2); + if (!Op2.getValueType().isVector()) { + SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2); + // The getShiftAmountOperand() may create a new operand node or + // return the existing one. If new operand is created we need + // to update the parent node. + if (SAO != Op2) + NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); } - break; + } + break; } if (NewNode != Node) { @@ -1118,12 +1124,12 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { ReplaceNode(Node, ResultVals.data()); return; } + LLVM_FALLTHROUGH; } - // FALL THROUGH case TargetLowering::Expand: if (ExpandNode(Node)) return; - // FALL THROUGH + LLVM_FALLTHROUGH; case TargetLowering::LibCall: ConvertNodeToLibcall(Node); return; @@ -1196,21 +1202,16 @@ SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { } } + EVT VecVT = Vec.getValueType(); + if (!Ch.getNode()) { // Store the value to a temporary stack slot, then LOAD the returned part. - StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); + StackPtr = DAG.CreateStackTemporary(VecVT); Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo()); } - // Add the offset to the index. - unsigned EltSize = - Vec.getValueType().getVectorElementType().getSizeInBits()/8; - Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, - DAG.getConstant(EltSize, SDLoc(Vec), Idx.getValueType())); - - Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy(DAG.getDataLayout())); - StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr); + StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); SDValue NewLoad; @@ -1220,7 +1221,7 @@ SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { else NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, MachinePointerInfo(), - Vec.getValueType().getVectorElementType()); + VecVT.getVectorElementType()); // Replace the chain going out of the store, by the one out of the load. DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1)); @@ -1244,8 +1245,8 @@ SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { SDLoc dl(Op); // Store the value to a temporary stack slot, then LOAD the returned part. - - SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); + EVT VecVT = Vec.getValueType(); + SDValue StackPtr = DAG.CreateStackTemporary(VecVT); int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); @@ -1254,17 +1255,7 @@ SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); // Then store the inserted part. - - // Add the offset to the index. - unsigned EltSize = - Vec.getValueType().getVectorElementType().getSizeInBits()/8; - - Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, - DAG.getConstant(EltSize, SDLoc(Vec), Idx.getValueType())); - Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy(DAG.getDataLayout())); - - SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, - StackPtr); + SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); // Store the subvector. Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo()); @@ -1593,6 +1584,7 @@ bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS, break; } // Fallthrough if we are unsigned integer. + LLVM_FALLTHROUGH; case ISD::SETLE: case ISD::SETGT: case ISD::SETGE: @@ -1650,7 +1642,7 @@ SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); - unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); + unsigned SrcSize = SrcOp.getValueSizeInBits(); unsigned SlotSize = SlotVT.getSizeInBits(); unsigned DestSize = DestVT.getSizeInBits(); Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); @@ -2521,13 +2513,56 @@ SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); } -/// Open code the operations for BITREVERSE. +/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts. SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) { EVT VT = Op.getValueType(); EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); unsigned Sz = VT.getScalarSizeInBits(); - SDValue Tmp, Tmp2; + SDValue Tmp, Tmp2, Tmp3; + + // If we can, perform BSWAP first and then the mask+swap the i4, then i2 + // and finally the i1 pairs. + // TODO: We can easily support i4/i2 legal types if any target ever does. + if (Sz >= 8 && isPowerOf2_32(Sz)) { + // Create the masks - repeating the pattern every byte. + APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0); + APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0); + for (unsigned J = 0; J != Sz; J += 8) { + MaskHi4 = MaskHi4.Or(APInt(Sz, 0xF0ull << J)); + MaskLo4 = MaskLo4.Or(APInt(Sz, 0x0Full << J)); + MaskHi2 = MaskHi2.Or(APInt(Sz, 0xCCull << J)); + MaskLo2 = MaskLo2.Or(APInt(Sz, 0x33ull << J)); + MaskHi1 = MaskHi1.Or(APInt(Sz, 0xAAull << J)); + MaskLo1 = MaskLo1.Or(APInt(Sz, 0x55ull << J)); + } + + // BSWAP if the type is wider than a single byte. + Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); + + // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4) + Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT)); + Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT)); + Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT)); + Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT)); + Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); + + // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2) + Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT)); + Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT)); + Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT)); + Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT)); + Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); + + // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1) + Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT)); + Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT)); + Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT)); + Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT)); + Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); + return Tmp; + } + Tmp = DAG.getConstant(0, dl, VT); for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) { if (I < J) @@ -2551,7 +2586,7 @@ SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) { EVT VT = Op.getValueType(); EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; - switch (VT.getSimpleVT().SimpleTy) { + switch (VT.getSimpleVT().getScalarType().SimpleTy) { default: llvm_unreachable("Unhandled Expand type in BSWAP!"); case MVT::i16: Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); @@ -2780,10 +2815,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { SDValue Swap = DAG.getAtomicCmpSwap( ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, Node->getOperand(0), Node->getOperand(1), Zero, Zero, - cast<AtomicSDNode>(Node)->getMemOperand(), - cast<AtomicSDNode>(Node)->getOrdering(), - cast<AtomicSDNode>(Node)->getOrdering(), - cast<AtomicSDNode>(Node)->getSynchScope()); + cast<AtomicSDNode>(Node)->getMemOperand()); Results.push_back(Swap.getValue(0)); Results.push_back(Swap.getValue(1)); break; @@ -2794,9 +2826,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { cast<AtomicSDNode>(Node)->getMemoryVT(), Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), - cast<AtomicSDNode>(Node)->getMemOperand(), - cast<AtomicSDNode>(Node)->getOrdering(), - cast<AtomicSDNode>(Node)->getSynchScope()); + cast<AtomicSDNode>(Node)->getMemOperand()); Results.push_back(Swap.getValue(1)); break; } @@ -2808,10 +2838,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { SDValue Res = DAG.getAtomicCmpSwap( ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), - Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand(), - cast<AtomicSDNode>(Node)->getSuccessOrdering(), - cast<AtomicSDNode>(Node)->getFailureOrdering(), - cast<AtomicSDNode>(Node)->getSynchScope()); + Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); SDValue ExtRes = Res; SDValue LHS = Res; @@ -2879,15 +2906,32 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { Results.push_back(Tmp1); break; case ISD::SIGN_EXTEND_INREG: { - // NOTE: we could fall back on load/store here too for targets without - // SAR. However, it is doubtful that any exist. EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); EVT VT = Node->getValueType(0); + + // An in-register sign-extend of a boolean is a negation: + // 'true' (1) sign-extended is -1. + // 'false' (0) sign-extended is 0. + // However, we must mask the high bits of the source operand because the + // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero. + + // TODO: Do this for vectors too? + if (ExtraVT.getSizeInBits() == 1) { + SDValue One = DAG.getConstant(1, dl, VT); + SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); + SDValue Zero = DAG.getConstant(0, dl, VT); + SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); + Results.push_back(Neg); + break; + } + + // NOTE: we could fall back on load/store here too for targets without + // SRA. However, it is doubtful that any exist. EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); if (VT.isVector()) ShiftAmountTy = VT; - unsigned BitsDiff = VT.getScalarType().getSizeInBits() - - ExtraVT.getScalarType().getSizeInBits(); + unsigned BitsDiff = VT.getScalarSizeInBits() - + ExtraVT.getScalarSizeInBits(); SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy); Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), Node->getOperand(0), ShiftCst); @@ -3248,17 +3292,49 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { } case ISD::MULHU: case ISD::MULHS: { - unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : - ISD::SMUL_LOHI; + unsigned ExpandOpcode = + Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; EVT VT = Node->getValueType(0); SDVTList VTs = DAG.getVTList(VT, VT); - assert(TLI.isOperationLegalOrCustom(ExpandOpcode, VT) && - "If this wasn't legal, it shouldn't have been created!"); + Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), Node->getOperand(1)); Results.push_back(Tmp1.getValue(1)); break; } + case ISD::UMUL_LOHI: + case ISD::SMUL_LOHI: { + SDValue LHS = Node->getOperand(0); + SDValue RHS = Node->getOperand(1); + MVT VT = LHS.getSimpleValueType(); + unsigned MULHOpcode = + Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; + + if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) { + Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); + Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS)); + break; + } + + SmallVector<SDValue, 4> Halves; + EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext()); + assert(TLI.isTypeLegal(HalfType)); + if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves, + HalfType, DAG, + TargetLowering::MulExpansionKind::Always)) { + for (unsigned i = 0; i < 2; ++i) { + SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); + SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); + SDValue Shift = DAG.getConstant( + HalfType.getScalarSizeInBits(), dl, + TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); + Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); + Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); + } + break; + } + break; + } case ISD::MUL: { EVT VT = Node->getValueType(0); SDVTList VTs = DAG.getVTList(VT, VT); @@ -3293,7 +3369,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && TLI.isOperationLegalOrCustom(ISD::SHL, VT) && TLI.isOperationLegalOrCustom(ISD::OR, VT) && - TLI.expandMUL(Node, Lo, Hi, HalfType, DAG)) { + TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, + TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) { Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); SDValue Shift = @@ -3416,8 +3493,18 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { // pre-lowered to the correct types. This all depends upon WideVT not // being a legal type for the architecture and thus has to be split to // two arguments. - SDValue Args[] = { LHS, HiLHS, RHS, HiRHS }; - SDValue Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); + SDValue Ret; + if(DAG.getDataLayout().isLittleEndian()) { + // Halves of WideVT are packed into registers in different order + // depending on platform endianness. This is usually handled by + // the C calling convention, but we can't defer to it in + // the legalizer. + SDValue Args[] = { LHS, HiLHS, RHS, HiRHS }; + Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); + } else { + SDValue Args[] = { HiLHS, LHS, HiRHS, RHS }; + Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); + } BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, DAG.getIntPtrConstant(0, dl)); TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, @@ -3441,6 +3528,15 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, DAG.getConstant(0, dl, VT), ISD::SETNE); } + + // Truncate the result if SetCC returns a larger type than needed. + EVT RType = Node->getValueType(1); + if (RType.getSizeInBits() < TopHalf.getValueSizeInBits()) + TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf); + + assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() && + "Unexpected result type for S/UMULO legalization"); + Results.push_back(BottomHalf); Results.push_back(TopHalf); break; @@ -3476,9 +3572,9 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { SDValue Table = Node->getOperand(1); SDValue Index = Node->getOperand(2); - EVT PTy = TLI.getPointerTy(DAG.getDataLayout()); - const DataLayout &TD = DAG.getDataLayout(); + EVT PTy = TLI.getPointerTy(TD); + unsigned EntrySize = DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); @@ -3492,7 +3588,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { ISD::SEXTLOAD, dl, PTy, Chain, Addr, MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT); Addr = LD; - if (TM.isPositionIndependent()) { + if (TLI.isJumpTableRelative()) { // For PIC, the sequence is: // BRIND(load(Jumptable + index) + RelocBase) // RelocBase can be JumpTable, GOT or some sort of global base. @@ -4019,10 +4115,11 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { } Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); break; + case ISD::BITREVERSE: case ISD::BSWAP: { unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); - Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); + Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); Tmp1 = DAG.getNode( ISD::SRL, dl, NVT, Tmp1, DAG.getConstant(DiffBits, dl, @@ -4073,6 +4170,10 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { ReplacedNode(Node); break; } + case ISD::SDIV: + case ISD::SREM: + case ISD::UDIV: + case ISD::UREM: case ISD::AND: case ISD::OR: case ISD::XOR: { @@ -4082,7 +4183,20 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { TruncOp = ISD::BITCAST; } else { assert(OVT.isInteger() && "Cannot promote logic operation"); - ExtOp = ISD::ANY_EXTEND; + + switch (Node->getOpcode()) { + default: + ExtOp = ISD::ANY_EXTEND; + break; + case ISD::SDIV: + case ISD::SREM: + ExtOp = ISD::SIGN_EXTEND; + break; + case ISD::UDIV: + case ISD::UREM: + ExtOp = ISD::ZERO_EXTEND; + break; + } TruncOp = ISD::TRUNCATE; } // Promote each of the values to the new type. @@ -4093,6 +4207,24 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); break; } + case ISD::UMUL_LOHI: + case ISD::SMUL_LOHI: { + // Promote to a multiply in a wider integer type. + unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND + : ISD::SIGN_EXTEND; + Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); + Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); + Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); + + auto &DL = DAG.getDataLayout(); + unsigned OriginalSize = OVT.getScalarSizeInBits(); + Tmp2 = DAG.getNode( + ISD::SRL, dl, NVT, Tmp1, + DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT))); + Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); + Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); + break; + } case ISD::SELECT: { unsigned ExtOp, TruncOp; if (Node->getValueType(0).isVector() || @@ -4351,7 +4483,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { MVT EltVT = OVT.getVectorElementType(); MVT NewEltVT = NVT.getVectorElementType(); - // Handle bitcasts to different vector type with the smae total bit size. + // Handle bitcasts to different vector type with the same total bit size. // // e.g. v2i64 = scalar_to_vector x:i64 // => |
