diff options
Diffstat (limited to 'contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp')
| -rw-r--r-- | contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp | 2233 |
1 files changed, 2233 insertions, 0 deletions
diff --git a/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp new file mode 100644 index 0000000..eb21b31 --- /dev/null +++ b/contrib/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -0,0 +1,2233 @@ +//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the SystemZTargetLowering class. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "systemz-lower" + +#include "SystemZISelLowering.h" +#include "SystemZCallingConv.h" +#include "SystemZConstantPoolValue.h" +#include "SystemZMachineFunctionInfo.h" +#include "SystemZTargetMachine.h" +#include "llvm/CodeGen/CallingConvLower.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" + +using namespace llvm; + +// Classify VT as either 32 or 64 bit. +static bool is32Bit(EVT VT) { + switch (VT.getSimpleVT().SimpleTy) { + case MVT::i32: + return true; + case MVT::i64: + return false; + default: + llvm_unreachable("Unsupported type"); + } +} + +// Return a version of MachineOperand that can be safely used before the +// final use. +static MachineOperand earlyUseOperand(MachineOperand Op) { + if (Op.isReg()) + Op.setIsKill(false); + return Op; +} + +SystemZTargetLowering::SystemZTargetLowering(SystemZTargetMachine &tm) + : TargetLowering(tm, new TargetLoweringObjectFileELF()), + Subtarget(*tm.getSubtargetImpl()), TM(tm) { + MVT PtrVT = getPointerTy(); + + // Set up the register classes. + addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass); + addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass); + addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass); + addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass); + addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass); + + // Compute derived properties from the register classes + computeRegisterProperties(); + + // Set up special registers. + setExceptionPointerRegister(SystemZ::R6D); + setExceptionSelectorRegister(SystemZ::R7D); + setStackPointerRegisterToSaveRestore(SystemZ::R15D); + + // TODO: It may be better to default to latency-oriented scheduling, however + // LLVM's current latency-oriented scheduler can't handle physreg definitions + // such as SystemZ has with PSW, so set this to the register-pressure + // scheduler, because it can. + setSchedulingPreference(Sched::RegPressure); + + setBooleanContents(ZeroOrOneBooleanContent); + setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct? + + // Instructions are strings of 2-byte aligned 2-byte values. + setMinFunctionAlignment(2); + + // Handle operations that are handled in a similar way for all types. + for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; + I <= MVT::LAST_FP_VALUETYPE; + ++I) { + MVT VT = MVT::SimpleValueType(I); + if (isTypeLegal(VT)) { + // Expand SETCC(X, Y, COND) into SELECT_CC(X, Y, 1, 0, COND). + setOperationAction(ISD::SETCC, VT, Expand); + + // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE). + setOperationAction(ISD::SELECT, VT, Expand); + + // Lower SELECT_CC and BR_CC into separate comparisons and branches. + setOperationAction(ISD::SELECT_CC, VT, Custom); + setOperationAction(ISD::BR_CC, VT, Custom); + } + } + + // Expand jump table branches as address arithmetic followed by an + // indirect jump. + setOperationAction(ISD::BR_JT, MVT::Other, Expand); + + // Expand BRCOND into a BR_CC (see above). + setOperationAction(ISD::BRCOND, MVT::Other, Expand); + + // Handle integer types. + for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; + I <= MVT::LAST_INTEGER_VALUETYPE; + ++I) { + MVT VT = MVT::SimpleValueType(I); + if (isTypeLegal(VT)) { + // Expand individual DIV and REMs into DIVREMs. + setOperationAction(ISD::SDIV, VT, Expand); + setOperationAction(ISD::UDIV, VT, Expand); + setOperationAction(ISD::SREM, VT, Expand); + setOperationAction(ISD::UREM, VT, Expand); + setOperationAction(ISD::SDIVREM, VT, Custom); + setOperationAction(ISD::UDIVREM, VT, Custom); + + // Expand ATOMIC_LOAD and ATOMIC_STORE using ATOMIC_CMP_SWAP. + // FIXME: probably much too conservative. + setOperationAction(ISD::ATOMIC_LOAD, VT, Expand); + setOperationAction(ISD::ATOMIC_STORE, VT, Expand); + + // No special instructions for these. + setOperationAction(ISD::CTPOP, VT, Expand); + setOperationAction(ISD::CTTZ, VT, Expand); + setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); + setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); + setOperationAction(ISD::ROTR, VT, Expand); + + // Use *MUL_LOHI where possible and a wider multiplication otherwise. + setOperationAction(ISD::MULHS, VT, Expand); + setOperationAction(ISD::MULHU, VT, Expand); + + // We have instructions for signed but not unsigned FP conversion. + setOperationAction(ISD::FP_TO_UINT, VT, Expand); + } + } + + // Type legalization will convert 8- and 16-bit atomic operations into + // forms that operate on i32s (but still keeping the original memory VT). + // Lower them into full i32 operations. + setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom); + setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom); + setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); + setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); + setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom); + setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom); + setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom); + setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom); + setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom); + setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom); + setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom); + setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); + + // We have instructions for signed but not unsigned FP conversion. + // Handle unsigned 32-bit types as signed 64-bit types. + setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote); + setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); + + // We have native support for a 64-bit CTLZ, via FLOGR. + setOperationAction(ISD::CTLZ, MVT::i32, Promote); + setOperationAction(ISD::CTLZ, MVT::i64, Legal); + + // Give LowerOperation the chance to replace 64-bit ORs with subregs. + setOperationAction(ISD::OR, MVT::i64, Custom); + + // The architecture has 32-bit SMUL_LOHI and UMUL_LOHI (MR and MLR), + // but they aren't really worth using. There is no 64-bit SMUL_LOHI, + // but there is a 64-bit UMUL_LOHI: MLGR. + setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); + setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); + setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); + setOperationAction(ISD::UMUL_LOHI, MVT::i64, Custom); + + // FIXME: Can we support these natively? + setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); + setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); + setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); + + // We have native instructions for i8, i16 and i32 extensions, but not i1. + setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); + setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote); + setLoadExtAction(ISD::EXTLOAD, MVT::i1, Promote); + setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); + + // Handle the various types of symbolic address. + setOperationAction(ISD::ConstantPool, PtrVT, Custom); + setOperationAction(ISD::GlobalAddress, PtrVT, Custom); + setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom); + setOperationAction(ISD::BlockAddress, PtrVT, Custom); + setOperationAction(ISD::JumpTable, PtrVT, Custom); + + // We need to handle dynamic allocations specially because of the + // 160-byte area at the bottom of the stack. + setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); + + // Use custom expanders so that we can force the function to use + // a frame pointer. + setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); + setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom); + + // Expand these using getExceptionSelectorRegister() and + // getExceptionPointerRegister(). + setOperationAction(ISD::EXCEPTIONADDR, PtrVT, Expand); + setOperationAction(ISD::EHSELECTION, PtrVT, Expand); + + // Handle floating-point types. + for (unsigned I = MVT::FIRST_FP_VALUETYPE; + I <= MVT::LAST_FP_VALUETYPE; + ++I) { + MVT VT = MVT::SimpleValueType(I); + if (isTypeLegal(VT)) { + // We can use FI for FRINT. + setOperationAction(ISD::FRINT, VT, Legal); + + // No special instructions for these. + setOperationAction(ISD::FSIN, VT, Expand); + setOperationAction(ISD::FCOS, VT, Expand); + setOperationAction(ISD::FREM, VT, Expand); + } + } + + // We have fused multiply-addition for f32 and f64 but not f128. + setOperationAction(ISD::FMA, MVT::f32, Legal); + setOperationAction(ISD::FMA, MVT::f64, Legal); + setOperationAction(ISD::FMA, MVT::f128, Expand); + + // Needed so that we don't try to implement f128 constant loads using + // a load-and-extend of a f80 constant (in cases where the constant + // would fit in an f80). + setLoadExtAction(ISD::EXTLOAD, MVT::f80, Expand); + + // Floating-point truncation and stores need to be done separately. + setTruncStoreAction(MVT::f64, MVT::f32, Expand); + setTruncStoreAction(MVT::f128, MVT::f32, Expand); + setTruncStoreAction(MVT::f128, MVT::f64, Expand); + + // We have 64-bit FPR<->GPR moves, but need special handling for + // 32-bit forms. + setOperationAction(ISD::BITCAST, MVT::i32, Custom); + setOperationAction(ISD::BITCAST, MVT::f32, Custom); + + // VASTART and VACOPY need to deal with the SystemZ-specific varargs + // structure, but VAEND is a no-op. + setOperationAction(ISD::VASTART, MVT::Other, Custom); + setOperationAction(ISD::VACOPY, MVT::Other, Custom); + setOperationAction(ISD::VAEND, MVT::Other, Expand); +} + +bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { + // We can load zero using LZ?R and negative zero using LZ?R;LC?BR. + return Imm.isZero() || Imm.isNegZero(); +} + +//===----------------------------------------------------------------------===// +// Inline asm support +//===----------------------------------------------------------------------===// + +TargetLowering::ConstraintType +SystemZTargetLowering::getConstraintType(const std::string &Constraint) const { + if (Constraint.size() == 1) { + switch (Constraint[0]) { + case 'a': // Address register + case 'd': // Data register (equivalent to 'r') + case 'f': // Floating-point register + case 'r': // General-purpose register + return C_RegisterClass; + + case 'Q': // Memory with base and unsigned 12-bit displacement + case 'R': // Likewise, plus an index + case 'S': // Memory with base and signed 20-bit displacement + case 'T': // Likewise, plus an index + case 'm': // Equivalent to 'T'. + return C_Memory; + + case 'I': // Unsigned 8-bit constant + case 'J': // Unsigned 12-bit constant + case 'K': // Signed 16-bit constant + case 'L': // Signed 20-bit displacement (on all targets we support) + case 'M': // 0x7fffffff + return C_Other; + + default: + break; + } + } + return TargetLowering::getConstraintType(Constraint); +} + +TargetLowering::ConstraintWeight SystemZTargetLowering:: +getSingleConstraintMatchWeight(AsmOperandInfo &info, + const char *constraint) const { + ConstraintWeight weight = CW_Invalid; + Value *CallOperandVal = info.CallOperandVal; + // If we don't have a value, we can't do a match, + // but allow it at the lowest weight. + if (CallOperandVal == NULL) + return CW_Default; + Type *type = CallOperandVal->getType(); + // Look at the constraint type. + switch (*constraint) { + default: + weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); + break; + + case 'a': // Address register + case 'd': // Data register (equivalent to 'r') + case 'r': // General-purpose register + if (CallOperandVal->getType()->isIntegerTy()) + weight = CW_Register; + break; + + case 'f': // Floating-point register + if (type->isFloatingPointTy()) + weight = CW_Register; + break; + + case 'I': // Unsigned 8-bit constant + if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) + if (isUInt<8>(C->getZExtValue())) + weight = CW_Constant; + break; + + case 'J': // Unsigned 12-bit constant + if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) + if (isUInt<12>(C->getZExtValue())) + weight = CW_Constant; + break; + + case 'K': // Signed 16-bit constant + if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) + if (isInt<16>(C->getSExtValue())) + weight = CW_Constant; + break; + + case 'L': // Signed 20-bit displacement (on all targets we support) + if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) + if (isInt<20>(C->getSExtValue())) + weight = CW_Constant; + break; + + case 'M': // 0x7fffffff + if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) + if (C->getZExtValue() == 0x7fffffff) + weight = CW_Constant; + break; + } + return weight; +} + +std::pair<unsigned, const TargetRegisterClass *> SystemZTargetLowering:: +getRegForInlineAsmConstraint(const std::string &Constraint, EVT VT) const { + if (Constraint.size() == 1) { + // GCC Constraint Letters + switch (Constraint[0]) { + default: break; + case 'd': // Data register (equivalent to 'r') + case 'r': // General-purpose register + if (VT == MVT::i64) + return std::make_pair(0U, &SystemZ::GR64BitRegClass); + else if (VT == MVT::i128) + return std::make_pair(0U, &SystemZ::GR128BitRegClass); + return std::make_pair(0U, &SystemZ::GR32BitRegClass); + + case 'a': // Address register + if (VT == MVT::i64) + return std::make_pair(0U, &SystemZ::ADDR64BitRegClass); + else if (VT == MVT::i128) + return std::make_pair(0U, &SystemZ::ADDR128BitRegClass); + return std::make_pair(0U, &SystemZ::ADDR32BitRegClass); + + case 'f': // Floating-point register + if (VT == MVT::f64) + return std::make_pair(0U, &SystemZ::FP64BitRegClass); + else if (VT == MVT::f128) + return std::make_pair(0U, &SystemZ::FP128BitRegClass); + return std::make_pair(0U, &SystemZ::FP32BitRegClass); + } + } + return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); +} + +void SystemZTargetLowering:: +LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, + std::vector<SDValue> &Ops, + SelectionDAG &DAG) const { + // Only support length 1 constraints for now. + if (Constraint.length() == 1) { + switch (Constraint[0]) { + case 'I': // Unsigned 8-bit constant + if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) + if (isUInt<8>(C->getZExtValue())) + Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), + Op.getValueType())); + return; + + case 'J': // Unsigned 12-bit constant + if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) + if (isUInt<12>(C->getZExtValue())) + Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), + Op.getValueType())); + return; + + case 'K': // Signed 16-bit constant + if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) + if (isInt<16>(C->getSExtValue())) + Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), + Op.getValueType())); + return; + + case 'L': // Signed 20-bit displacement (on all targets we support) + if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) + if (isInt<20>(C->getSExtValue())) + Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), + Op.getValueType())); + return; + + case 'M': // 0x7fffffff + if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) + if (C->getZExtValue() == 0x7fffffff) + Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), + Op.getValueType())); + return; + } + } + TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); +} + +//===----------------------------------------------------------------------===// +// Calling conventions +//===----------------------------------------------------------------------===// + +#include "SystemZGenCallingConv.inc" + +// Value is a value that has been passed to us in the location described by VA +// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining +// any loads onto Chain. +static SDValue convertLocVTToValVT(SelectionDAG &DAG, DebugLoc DL, + CCValAssign &VA, SDValue Chain, + SDValue Value) { + // If the argument has been promoted from a smaller type, insert an + // assertion to capture this. + if (VA.getLocInfo() == CCValAssign::SExt) + Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, + DAG.getValueType(VA.getValVT())); + else if (VA.getLocInfo() == CCValAssign::ZExt) + Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, + DAG.getValueType(VA.getValVT())); + + if (VA.isExtInLoc()) + Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); + else if (VA.getLocInfo() == CCValAssign::Indirect) + Value = DAG.getLoad(VA.getValVT(), DL, Chain, Value, + MachinePointerInfo(), false, false, false, 0); + else + assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo"); + return Value; +} + +// Value is a value of type VA.getValVT() that we need to copy into +// the location described by VA. Return a copy of Value converted to +// VA.getValVT(). The caller is responsible for handling indirect values. +static SDValue convertValVTToLocVT(SelectionDAG &DAG, DebugLoc DL, + CCValAssign &VA, SDValue Value) { + switch (VA.getLocInfo()) { + case CCValAssign::SExt: + return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); + case CCValAssign::ZExt: + return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); + case CCValAssign::AExt: + return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); + case CCValAssign::Full: + return Value; + default: + llvm_unreachable("Unhandled getLocInfo()"); + } +} + +SDValue SystemZTargetLowering:: +LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, + const SmallVectorImpl<ISD::InputArg> &Ins, + DebugLoc DL, SelectionDAG &DAG, + SmallVectorImpl<SDValue> &InVals) const { + MachineFunction &MF = DAG.getMachineFunction(); + MachineFrameInfo *MFI = MF.getFrameInfo(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + SystemZMachineFunctionInfo *FuncInfo = + MF.getInfo<SystemZMachineFunctionInfo>(); + const SystemZFrameLowering *TFL = + static_cast<const SystemZFrameLowering *>(TM.getFrameLowering()); + + // Assign locations to all of the incoming arguments. + SmallVector<CCValAssign, 16> ArgLocs; + CCState CCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); + CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ); + + unsigned NumFixedGPRs = 0; + unsigned NumFixedFPRs = 0; + for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { + SDValue ArgValue; + CCValAssign &VA = ArgLocs[I]; + EVT LocVT = VA.getLocVT(); + if (VA.isRegLoc()) { + // Arguments passed in registers + const TargetRegisterClass *RC; + switch (LocVT.getSimpleVT().SimpleTy) { + default: + // Integers smaller than i64 should be promoted to i64. + llvm_unreachable("Unexpected argument type"); + case MVT::i32: + NumFixedGPRs += 1; + RC = &SystemZ::GR32BitRegClass; + break; + case MVT::i64: + NumFixedGPRs += 1; + RC = &SystemZ::GR64BitRegClass; + break; + case MVT::f32: + NumFixedFPRs += 1; + RC = &SystemZ::FP32BitRegClass; + break; + case MVT::f64: + NumFixedFPRs += 1; + RC = &SystemZ::FP64BitRegClass; + break; + } + + unsigned VReg = MRI.createVirtualRegister(RC); + MRI.addLiveIn(VA.getLocReg(), VReg); + ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); + } else { + assert(VA.isMemLoc() && "Argument not register or memory"); + + // Create the frame index object for this incoming parameter. + int FI = MFI->CreateFixedObject(LocVT.getSizeInBits() / 8, + VA.getLocMemOffset(), true); + + // Create the SelectionDAG nodes corresponding to a load + // from this parameter. Unpromoted ints and floats are + // passed as right-justified 8-byte values. + EVT PtrVT = getPointerTy(); + SDValue FIN = DAG.getFrameIndex(FI, PtrVT); + if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) + FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4)); + ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, + MachinePointerInfo::getFixedStack(FI), + false, false, false, 0); + } + + // Convert the value of the argument register into the value that's + // being passed. + InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); + } + + if (IsVarArg) { + // Save the number of non-varargs registers for later use by va_start, etc. + FuncInfo->setVarArgsFirstGPR(NumFixedGPRs); + FuncInfo->setVarArgsFirstFPR(NumFixedFPRs); + + // Likewise the address (in the form of a frame index) of where the + // first stack vararg would be. The 1-byte size here is arbitrary. + int64_t StackSize = CCInfo.getNextStackOffset(); + FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize, true)); + + // ...and a similar frame index for the caller-allocated save area + // that will be used to store the incoming registers. + int64_t RegSaveOffset = TFL->getOffsetOfLocalArea(); + unsigned RegSaveIndex = MFI->CreateFixedObject(1, RegSaveOffset, true); + FuncInfo->setRegSaveFrameIndex(RegSaveIndex); + + // Store the FPR varargs in the reserved frame slots. (We store the + // GPRs as part of the prologue.) + if (NumFixedFPRs < SystemZ::NumArgFPRs) { + SDValue MemOps[SystemZ::NumArgFPRs]; + for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) { + unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]); + int FI = MFI->CreateFixedObject(8, RegSaveOffset + Offset, true); + SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); + unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I], + &SystemZ::FP64BitRegClass); + SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); + MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, + MachinePointerInfo::getFixedStack(FI), + false, false, 0); + + } + // Join the stores, which are independent of one another. + Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, + &MemOps[NumFixedFPRs], + SystemZ::NumArgFPRs - NumFixedFPRs); + } + } + + return Chain; +} + +SDValue +SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, + SmallVectorImpl<SDValue> &InVals) const { + SelectionDAG &DAG = CLI.DAG; + DebugLoc &DL = CLI.DL; + SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; + SmallVector<SDValue, 32> &OutVals = CLI.OutVals; + SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins; + SDValue Chain = CLI.Chain; + SDValue Callee = CLI.Callee; + bool &isTailCall = CLI.IsTailCall; + CallingConv::ID CallConv = CLI.CallConv; + bool IsVarArg = CLI.IsVarArg; + MachineFunction &MF = DAG.getMachineFunction(); + EVT PtrVT = getPointerTy(); + + // SystemZ target does not yet support tail call optimization. + isTailCall = false; + + // Analyze the operands of the call, assigning locations to each operand. + SmallVector<CCValAssign, 16> ArgLocs; + CCState ArgCCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); + ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); + + // Get a count of how many bytes are to be pushed on the stack. + unsigned NumBytes = ArgCCInfo.getNextStackOffset(); + + // Mark the start of the call. + Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumBytes, PtrVT, true)); + + // Copy argument values to their designated locations. + SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass; + SmallVector<SDValue, 8> MemOpChains; + SDValue StackPtr; + for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { + CCValAssign &VA = ArgLocs[I]; + SDValue ArgValue = OutVals[I]; + + if (VA.getLocInfo() == CCValAssign::Indirect) { + // Store the argument in a stack slot and pass its address. + SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); + int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); + MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, SpillSlot, + MachinePointerInfo::getFixedStack(FI), + false, false, 0)); + ArgValue = SpillSlot; + } else + ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); + + if (VA.isRegLoc()) + // Queue up the argument copies and emit them at the end. + RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); + else { + assert(VA.isMemLoc() && "Argument not register or memory"); + + // Work out the address of the stack slot. Unpromoted ints and + // floats are passed as right-justified 8-byte values. + if (!StackPtr.getNode()) + StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT); + unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset(); + if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) + Offset += 4; + SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, + DAG.getIntPtrConstant(Offset)); + + // Emit the store. + MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, Address, + MachinePointerInfo(), + false, false, 0)); + } + } + + // Join the stores, which are independent of one another. + if (!MemOpChains.empty()) + Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, + &MemOpChains[0], MemOpChains.size()); + + // Build a sequence of copy-to-reg nodes, chained and glued together. + SDValue Glue; + for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) { + Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, + RegsToPass[I].second, Glue); + Glue = Chain.getValue(1); + } + + // Accept direct calls by converting symbolic call addresses to the + // associated Target* opcodes. + if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { + Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); + Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); + } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { + Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT); + Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); + } + + // The first call operand is the chain and the second is the target address. + SmallVector<SDValue, 8> Ops; + Ops.push_back(Chain); + Ops.push_back(Callee); + + // Add argument registers to the end of the list so that they are + // known live into the call. + for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) + Ops.push_back(DAG.getRegister(RegsToPass[I].first, + RegsToPass[I].second.getValueType())); + + // Glue the call to the argument copies, if any. + if (Glue.getNode()) + Ops.push_back(Glue); + + // Emit the call. + SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); + Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, &Ops[0], Ops.size()); + Glue = Chain.getValue(1); + + // Mark the end of the call, which is glued to the call itself. + Chain = DAG.getCALLSEQ_END(Chain, + DAG.getConstant(NumBytes, PtrVT, true), + DAG.getConstant(0, PtrVT, true), + Glue); + Glue = Chain.getValue(1); + + // Assign locations to each value returned by this call. + SmallVector<CCValAssign, 16> RetLocs; + CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); + RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); + + // Copy all of the result registers out of their specified physreg. + for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { + CCValAssign &VA = RetLocs[I]; + + // Copy the value out, gluing the copy to the end of the call sequence. + SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), + VA.getLocVT(), Glue); + Chain = RetValue.getValue(1); + Glue = RetValue.getValue(2); + + // Convert the value of the return register into the value that's + // being returned. + InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); + } + + return Chain; +} + +SDValue +SystemZTargetLowering::LowerReturn(SDValue Chain, + CallingConv::ID CallConv, bool IsVarArg, + const SmallVectorImpl<ISD::OutputArg> &Outs, + const SmallVectorImpl<SDValue> &OutVals, + DebugLoc DL, SelectionDAG &DAG) const { + MachineFunction &MF = DAG.getMachineFunction(); + + // Assign locations to each returned value. + SmallVector<CCValAssign, 16> RetLocs; + CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); + RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ); + + // Quick exit for void returns + if (RetLocs.empty()) + return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain); + + // Copy the result values into the output registers. + SDValue Glue; + SmallVector<SDValue, 4> RetOps; + RetOps.push_back(Chain); + for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { + CCValAssign &VA = RetLocs[I]; + SDValue RetValue = OutVals[I]; + + // Make the return register live on exit. + assert(VA.isRegLoc() && "Can only return in registers!"); + + // Promote the value as required. + RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); + + // Chain and glue the copies together. + unsigned Reg = VA.getLocReg(); + Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); + Glue = Chain.getValue(1); + RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT())); + } + + // Update chain and glue. + RetOps[0] = Chain; + if (Glue.getNode()) + RetOps.push_back(Glue); + + return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, + RetOps.data(), RetOps.size()); +} + +// CC is a comparison that will be implemented using an integer or +// floating-point comparison. Return the condition code mask for +// a branch on true. In the integer case, CCMASK_CMP_UO is set for +// unsigned comparisons and clear for signed ones. In the floating-point +// case, CCMASK_CMP_UO has its normal mask meaning (unordered). +static unsigned CCMaskForCondCode(ISD::CondCode CC) { +#define CONV(X) \ + case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \ + case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \ + case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X + + switch (CC) { + default: + llvm_unreachable("Invalid integer condition!"); + + CONV(EQ); + CONV(NE); + CONV(GT); + CONV(GE); + CONV(LT); + CONV(LE); + + case ISD::SETO: return SystemZ::CCMASK_CMP_O; + case ISD::SETUO: return SystemZ::CCMASK_CMP_UO; + } +#undef CONV +} + +// If a comparison described by IsUnsigned, CCMask, CmpOp0 and CmpOp1 +// is suitable for CLI(Y), CHHSI or CLHHSI, adjust the operands as necessary. +static void adjustSubwordCmp(SelectionDAG &DAG, bool &IsUnsigned, + SDValue &CmpOp0, SDValue &CmpOp1, + unsigned &CCMask) { + // For us to make any changes, it must a comparison between a single-use + // load and a constant. + if (!CmpOp0.hasOneUse() || + CmpOp0.getOpcode() != ISD::LOAD || + CmpOp1.getOpcode() != ISD::Constant) + return; + + // We must have an 8- or 16-bit load. + LoadSDNode *Load = cast<LoadSDNode>(CmpOp0); + unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits(); + if (NumBits != 8 && NumBits != 16) + return; + + // The load must be an extending one and the constant must be within the + // range of the unextended value. + ConstantSDNode *Constant = cast<ConstantSDNode>(CmpOp1); + uint64_t Value = Constant->getZExtValue(); + uint64_t Mask = (1 << NumBits) - 1; + if (Load->getExtensionType() == ISD::SEXTLOAD) { + int64_t SignedValue = Constant->getSExtValue(); + if (uint64_t(SignedValue) + (1 << (NumBits - 1)) > Mask) + return; + // Unsigned comparison between two sign-extended values is equivalent + // to unsigned comparison between two zero-extended values. + if (IsUnsigned) + Value &= Mask; + else if (CCMask == SystemZ::CCMASK_CMP_EQ || + CCMask == SystemZ::CCMASK_CMP_NE) + // Any choice of IsUnsigned is OK for equality comparisons. + // We could use either CHHSI or CLHHSI for 16-bit comparisons, + // but since we use CLHHSI for zero extensions, it seems better + // to be consistent and do the same here. + Value &= Mask, IsUnsigned = true; + else if (NumBits == 8) { + // Try to treat the comparison as unsigned, so that we can use CLI. + // Adjust CCMask and Value as necessary. + if (Value == 0 && CCMask == SystemZ::CCMASK_CMP_LT) + // Test whether the high bit of the byte is set. + Value = 127, CCMask = SystemZ::CCMASK_CMP_GT, IsUnsigned = true; + else if (SignedValue == -1 && CCMask == SystemZ::CCMASK_CMP_GT) + // Test whether the high bit of the byte is clear. + Value = 128, CCMask = SystemZ::CCMASK_CMP_LT, IsUnsigned = true; + else + // No instruction exists for this combination. + return; + } + } else if (Load->getExtensionType() == ISD::ZEXTLOAD) { + if (Value > Mask) + return; + // Signed comparison between two zero-extended values is equivalent + // to unsigned comparison. + IsUnsigned = true; + } else + return; + + // Make sure that the first operand is an i32 of the right extension type. + ISD::LoadExtType ExtType = IsUnsigned ? ISD::ZEXTLOAD : ISD::SEXTLOAD; + if (CmpOp0.getValueType() != MVT::i32 || + Load->getExtensionType() != ExtType) + CmpOp0 = DAG.getExtLoad(ExtType, Load->getDebugLoc(), MVT::i32, + Load->getChain(), Load->getBasePtr(), + Load->getPointerInfo(), Load->getMemoryVT(), + Load->isVolatile(), Load->isNonTemporal(), + Load->getAlignment()); + + // Make sure that the second operand is an i32 with the right value. + if (CmpOp1.getValueType() != MVT::i32 || + Value != Constant->getZExtValue()) + CmpOp1 = DAG.getConstant(Value, MVT::i32); +} + +// Return true if a comparison described by CCMask, CmpOp0 and CmpOp1 +// is an equality comparison that is better implemented using unsigned +// rather than signed comparison instructions. +static bool preferUnsignedComparison(SelectionDAG &DAG, SDValue CmpOp0, + SDValue CmpOp1, unsigned CCMask) { + // The test must be for equality or inequality. + if (CCMask != SystemZ::CCMASK_CMP_EQ && CCMask != SystemZ::CCMASK_CMP_NE) + return false; + + if (CmpOp1.getOpcode() == ISD::Constant) { + uint64_t Value = cast<ConstantSDNode>(CmpOp1)->getSExtValue(); + + // If we're comparing with memory, prefer unsigned comparisons for + // values that are in the unsigned 16-bit range but not the signed + // 16-bit range. We want to use CLFHSI and CLGHSI. + if (CmpOp0.hasOneUse() && + ISD::isNormalLoad(CmpOp0.getNode()) && + (Value >= 32768 && Value < 65536)) + return true; + + // Use unsigned comparisons for values that are in the CLGFI range + // but not in the CGFI range. + if (CmpOp0.getValueType() == MVT::i64 && (Value >> 31) == 1) + return true; + + return false; + } + + // Prefer CL for zero-extended loads. + if (CmpOp1.getOpcode() == ISD::ZERO_EXTEND || + ISD::isZEXTLoad(CmpOp1.getNode())) + return true; + + // ...and for "in-register" zero extensions. + if (CmpOp1.getOpcode() == ISD::AND && CmpOp1.getValueType() == MVT::i64) { + SDValue Mask = CmpOp1.getOperand(1); + if (Mask.getOpcode() == ISD::Constant && + cast<ConstantSDNode>(Mask)->getZExtValue() == 0xffffffff) + return true; + } + + return false; +} + +// Return a target node that compares CmpOp0 and CmpOp1. Set CCMask to the +// 4-bit condition-code mask for CC. +static SDValue emitCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, + ISD::CondCode CC, unsigned &CCMask) { + bool IsUnsigned = false; + CCMask = CCMaskForCondCode(CC); + if (!CmpOp0.getValueType().isFloatingPoint()) { + IsUnsigned = CCMask & SystemZ::CCMASK_CMP_UO; + CCMask &= ~SystemZ::CCMASK_CMP_UO; + adjustSubwordCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask); + if (preferUnsignedComparison(DAG, CmpOp0, CmpOp1, CCMask)) + IsUnsigned = true; + } + + DebugLoc DL = CmpOp0.getDebugLoc(); + return DAG.getNode((IsUnsigned ? SystemZISD::UCMP : SystemZISD::CMP), + DL, MVT::Glue, CmpOp0, CmpOp1); +} + +// Lower a binary operation that produces two VT results, one in each +// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, +// Extend extends Op0 to a GR128, and Opcode performs the GR128 operation +// on the extended Op0 and (unextended) Op1. Store the even register result +// in Even and the odd register result in Odd. +static void lowerGR128Binary(SelectionDAG &DAG, DebugLoc DL, EVT VT, + unsigned Extend, unsigned Opcode, + SDValue Op0, SDValue Op1, + SDValue &Even, SDValue &Odd) { + SDNode *In128 = DAG.getMachineNode(Extend, DL, MVT::Untyped, Op0); + SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, + SDValue(In128, 0), Op1); + bool Is32Bit = is32Bit(VT); + SDValue SubReg0 = DAG.getTargetConstant(SystemZ::even128(Is32Bit), VT); + SDValue SubReg1 = DAG.getTargetConstant(SystemZ::odd128(Is32Bit), VT); + SDNode *Reg0 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, + VT, Result, SubReg0); + SDNode *Reg1 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, + VT, Result, SubReg1); + Even = SDValue(Reg0, 0); + Odd = SDValue(Reg1, 0); +} + +SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const { + SDValue Chain = Op.getOperand(0); + ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); + SDValue CmpOp0 = Op.getOperand(2); + SDValue CmpOp1 = Op.getOperand(3); + SDValue Dest = Op.getOperand(4); + DebugLoc DL = Op.getDebugLoc(); + + unsigned CCMask; + SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCMask); + return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(), + Chain, DAG.getConstant(CCMask, MVT::i32), Dest, Flags); +} + +SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op, + SelectionDAG &DAG) const { + SDValue CmpOp0 = Op.getOperand(0); + SDValue CmpOp1 = Op.getOperand(1); + SDValue TrueOp = Op.getOperand(2); + SDValue FalseOp = Op.getOperand(3); + ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); + DebugLoc DL = Op.getDebugLoc(); + + unsigned CCMask; + SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCMask); + + SmallVector<SDValue, 4> Ops; + Ops.push_back(TrueOp); + Ops.push_back(FalseOp); + Ops.push_back(DAG.getConstant(CCMask, MVT::i32)); + Ops.push_back(Flags); + + SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); + return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, &Ops[0], Ops.size()); +} + +SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node, + SelectionDAG &DAG) const { + DebugLoc DL = Node->getDebugLoc(); + const GlobalValue *GV = Node->getGlobal(); + int64_t Offset = Node->getOffset(); + EVT PtrVT = getPointerTy(); + Reloc::Model RM = TM.getRelocationModel(); + CodeModel::Model CM = TM.getCodeModel(); + + SDValue Result; + if (Subtarget.isPC32DBLSymbol(GV, RM, CM)) { + // Make sure that the offset is aligned to a halfword. If it isn't, + // create an "anchor" at the previous 12-bit boundary. + // FIXME check whether there is a better way of handling this. + if (Offset & 1) { + Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, + Offset & ~uint64_t(0xfff)); + Offset &= 0xfff; + } else { + Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Offset); + Offset = 0; + } + Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); + } else { + Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); + Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); + Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, + MachinePointerInfo::getGOT(), false, false, false, 0); + } + + // If there was a non-zero offset that we didn't fold, create an explicit + // addition for it. + if (Offset != 0) + Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, + DAG.getConstant(Offset, PtrVT)); + + return Result; +} + +SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node, + SelectionDAG &DAG) const { + DebugLoc DL = Node->getDebugLoc(); + const GlobalValue *GV = Node->getGlobal(); + EVT PtrVT = getPointerTy(); + TLSModel::Model model = TM.getTLSModel(GV); + + if (model != TLSModel::LocalExec) + llvm_unreachable("only local-exec TLS mode supported"); + + // The high part of the thread pointer is in access register 0. + SDValue TPHi = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, + DAG.getConstant(0, MVT::i32)); + TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); + + // The low part of the thread pointer is in access register 1. + SDValue TPLo = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, + DAG.getConstant(1, MVT::i32)); + TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); + + // Merge them into a single 64-bit address. + SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, + DAG.getConstant(32, PtrVT)); + SDValue TP = DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); + + // Get the offset of GA from the thread pointer. + SystemZConstantPoolValue *CPV = + SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF); + + // Force the offset into the constant pool and load it from there. + SDValue CPAddr = DAG.getConstantPool(CPV, PtrVT, 8); + SDValue Offset = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), + CPAddr, MachinePointerInfo::getConstantPool(), + false, false, false, 0); + + // Add the base and offset together. + return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); +} + +SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node, + SelectionDAG &DAG) const { + DebugLoc DL = Node->getDebugLoc(); + const BlockAddress *BA = Node->getBlockAddress(); + int64_t Offset = Node->getOffset(); + EVT PtrVT = getPointerTy(); + + SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset); + Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); + return Result; +} + +SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT, + SelectionDAG &DAG) const { + DebugLoc DL = JT->getDebugLoc(); + EVT PtrVT = getPointerTy(); + SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); + + // Use LARL to load the address of the table. + return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); +} + +SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP, + SelectionDAG &DAG) const { + DebugLoc DL = CP->getDebugLoc(); + EVT PtrVT = getPointerTy(); + + SDValue Result; + if (CP->isMachineConstantPoolEntry()) + Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, + CP->getAlignment()); + else + Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, + CP->getAlignment(), CP->getOffset()); + + // Use LARL to load the address of the constant pool entry. + return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); +} + +SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op, + SelectionDAG &DAG) const { + DebugLoc DL = Op.getDebugLoc(); + SDValue In = Op.getOperand(0); + EVT InVT = In.getValueType(); + EVT ResVT = Op.getValueType(); + + SDValue SubReg32 = DAG.getTargetConstant(SystemZ::subreg_32bit, MVT::i64); + SDValue Shift32 = DAG.getConstant(32, MVT::i64); + if (InVT == MVT::i32 && ResVT == MVT::f32) { + SDValue In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); + SDValue Shift = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, Shift32); + SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, Shift); + SDNode *Out = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, + MVT::f32, Out64, SubReg32); + return SDValue(Out, 0); + } + if (InVT == MVT::f32 && ResVT == MVT::i32) { + SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); + SDNode *In64 = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, + MVT::f64, SDValue(U64, 0), In, SubReg32); + SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, SDValue(In64, 0)); + SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, Shift32); + SDValue Out = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); + return Out; + } + llvm_unreachable("Unexpected bitcast combination"); +} + +SDValue SystemZTargetLowering::lowerVASTART(SDValue Op, + SelectionDAG &DAG) const { + MachineFunction &MF = DAG.getMachineFunction(); + SystemZMachineFunctionInfo *FuncInfo = + MF.getInfo<SystemZMachineFunctionInfo>(); + EVT PtrVT = getPointerTy(); + + SDValue Chain = Op.getOperand(0); + SDValue Addr = Op.getOperand(1); + const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); + DebugLoc DL = Op.getDebugLoc(); + + // The initial values of each field. + const unsigned NumFields = 4; + SDValue Fields[NumFields] = { + DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), PtrVT), + DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), PtrVT), + DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT), + DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT) + }; + + // Store each field into its respective slot. + SDValue MemOps[NumFields]; + unsigned Offset = 0; + for (unsigned I = 0; I < NumFields; ++I) { + SDValue FieldAddr = Addr; + if (Offset != 0) + FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, + DAG.getIntPtrConstant(Offset)); + MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, + MachinePointerInfo(SV, Offset), + false, false, 0); + Offset += 8; + } + return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps, NumFields); +} + +SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op, + SelectionDAG &DAG) const { + SDValue Chain = Op.getOperand(0); + SDValue DstPtr = Op.getOperand(1); + SDValue SrcPtr = Op.getOperand(2); + const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); + const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); + DebugLoc DL = Op.getDebugLoc(); + + return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32), + /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false, + MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); +} + +SDValue SystemZTargetLowering:: +lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { + SDValue Chain = Op.getOperand(0); + SDValue Size = Op.getOperand(1); + DebugLoc DL = Op.getDebugLoc(); + + unsigned SPReg = getStackPointerRegisterToSaveRestore(); + + // Get a reference to the stack pointer. + SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); + + // Get the new stack pointer value. + SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, Size); + + // Copy the new stack pointer back. + Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); + + // The allocated data lives above the 160 bytes allocated for the standard + // frame, plus any outgoing stack arguments. We don't know how much that + // amounts to yet, so emit a special ADJDYNALLOC placeholder. + SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); + SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); + + SDValue Ops[2] = { Result, Chain }; + return DAG.getMergeValues(Ops, 2, DL); +} + +SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op, + SelectionDAG &DAG) const { + EVT VT = Op.getValueType(); + DebugLoc DL = Op.getDebugLoc(); + assert(!is32Bit(VT) && "Only support 64-bit UMUL_LOHI"); + + // UMUL_LOHI64 returns the low result in the odd register and the high + // result in the even register. UMUL_LOHI is defined to return the + // low half first, so the results are in reverse order. + SDValue Ops[2]; + lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, + Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); + return DAG.getMergeValues(Ops, 2, DL); +} + +SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op, + SelectionDAG &DAG) const { + SDValue Op0 = Op.getOperand(0); + SDValue Op1 = Op.getOperand(1); + EVT VT = Op.getValueType(); + DebugLoc DL = Op.getDebugLoc(); + + // We use DSGF for 32-bit division. + if (is32Bit(VT)) { + Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); + Op1 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op1); + } + + // DSG(F) takes a 64-bit dividend, so the even register in the GR128 + // input is "don't care". The instruction returns the remainder in + // the even register and the quotient in the odd register. + SDValue Ops[2]; + lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::SDIVREM64, + Op0, Op1, Ops[1], Ops[0]); + return DAG.getMergeValues(Ops, 2, DL); +} + +SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op, + SelectionDAG &DAG) const { + EVT VT = Op.getValueType(); + DebugLoc DL = Op.getDebugLoc(); + + // DL(G) uses a double-width dividend, so we need to clear the even + // register in the GR128 input. The instruction returns the remainder + // in the even register and the quotient in the odd register. + SDValue Ops[2]; + if (is32Bit(VT)) + lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32, + Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); + else + lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64, + Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); + return DAG.getMergeValues(Ops, 2, DL); +} + +SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const { + assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation"); + + // Get the known-zero masks for each operand. + SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; + APInt KnownZero[2], KnownOne[2]; + DAG.ComputeMaskedBits(Ops[0], KnownZero[0], KnownOne[0]); + DAG.ComputeMaskedBits(Ops[1], KnownZero[1], KnownOne[1]); + + // See if the upper 32 bits of one operand and the lower 32 bits of the + // other are known zero. They are the low and high operands respectively. + uint64_t Masks[] = { KnownZero[0].getZExtValue(), + KnownZero[1].getZExtValue() }; + unsigned High, Low; + if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff) + High = 1, Low = 0; + else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff) + High = 0, Low = 1; + else + return Op; + + SDValue LowOp = Ops[Low]; + SDValue HighOp = Ops[High]; + + // If the high part is a constant, we're better off using IILH. + if (HighOp.getOpcode() == ISD::Constant) + return Op; + + // If the low part is a constant that is outside the range of LHI, + // then we're better off using IILF. + if (LowOp.getOpcode() == ISD::Constant) { + int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue()); + if (!isInt<16>(Value)) + return Op; + } + + // Check whether the high part is an AND that doesn't change the + // high 32 bits and just masks out low bits. We can skip it if so. + if (HighOp.getOpcode() == ISD::AND && + HighOp.getOperand(1).getOpcode() == ISD::Constant) { + ConstantSDNode *MaskNode = cast<ConstantSDNode>(HighOp.getOperand(1)); + uint64_t Mask = MaskNode->getZExtValue() | Masks[High]; + if ((Mask >> 32) == 0xffffffff) + HighOp = HighOp.getOperand(0); + } + + // Take advantage of the fact that all GR32 operations only change the + // low 32 bits by truncating Low to an i32 and inserting it directly + // using a subreg. The interesting cases are those where the truncation + // can be folded. + DebugLoc DL = Op.getDebugLoc(); + SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp); + SDValue SubReg32 = DAG.getTargetConstant(SystemZ::subreg_32bit, MVT::i64); + SDNode *Result = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, + MVT::i64, HighOp, Low32, SubReg32); + return SDValue(Result, 0); +} + +// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first +// two into the fullword ATOMIC_LOADW_* operation given by Opcode. +SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op, + SelectionDAG &DAG, + unsigned Opcode) const { + AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); + + // 32-bit operations need no code outside the main loop. + EVT NarrowVT = Node->getMemoryVT(); + EVT WideVT = MVT::i32; + if (NarrowVT == WideVT) + return Op; + + int64_t BitSize = NarrowVT.getSizeInBits(); + SDValue ChainIn = Node->getChain(); + SDValue Addr = Node->getBasePtr(); + SDValue Src2 = Node->getVal(); + MachineMemOperand *MMO = Node->getMemOperand(); + DebugLoc DL = Node->getDebugLoc(); + EVT PtrVT = Addr.getValueType(); + + // Convert atomic subtracts of constants into additions. + if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) + if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Src2)) { + Opcode = SystemZISD::ATOMIC_LOADW_ADD; + Src2 = DAG.getConstant(-Const->getSExtValue(), Src2.getValueType()); + } + + // Get the address of the containing word. + SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, + DAG.getConstant(-4, PtrVT)); + + // Get the number of bits that the word must be rotated left in order + // to bring the field to the top bits of a GR32. + SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, + DAG.getConstant(3, PtrVT)); + BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); + + // Get the complementing shift amount, for rotating a field in the top + // bits back to its proper position. + SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, + DAG.getConstant(0, WideVT), BitShift); + + // Extend the source operand to 32 bits and prepare it for the inner loop. + // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other + // operations require the source to be shifted in advance. (This shift + // can be folded if the source is constant.) For AND and NAND, the lower + // bits must be set, while for other opcodes they should be left clear. + if (Opcode != SystemZISD::ATOMIC_SWAPW) + Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2, + DAG.getConstant(32 - BitSize, WideVT)); + if (Opcode == SystemZISD::ATOMIC_LOADW_AND || + Opcode == SystemZISD::ATOMIC_LOADW_NAND) + Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2, + DAG.getConstant(uint32_t(-1) >> BitSize, WideVT)); + + // Construct the ATOMIC_LOADW_* node. + SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); + SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift, + DAG.getConstant(BitSize, WideVT) }; + SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, + array_lengthof(Ops), + NarrowVT, MMO); + + // Rotate the result of the final CS so that the field is in the lower + // bits of a GR32, then truncate it. + SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift, + DAG.getConstant(BitSize, WideVT)); + SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift); + + SDValue RetOps[2] = { Result, AtomicOp.getValue(1) }; + return DAG.getMergeValues(RetOps, 2, DL); +} + +// Node is an 8- or 16-bit ATOMIC_CMP_SWAP operation. Lower the first two +// into a fullword ATOMIC_CMP_SWAPW operation. +SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, + SelectionDAG &DAG) const { + AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); + + // We have native support for 32-bit compare and swap. + EVT NarrowVT = Node->getMemoryVT(); + EVT WideVT = MVT::i32; + if (NarrowVT == WideVT) + return Op; + + int64_t BitSize = NarrowVT.getSizeInBits(); + SDValue ChainIn = Node->getOperand(0); + SDValue Addr = Node->getOperand(1); + SDValue CmpVal = Node->getOperand(2); + SDValue SwapVal = Node->getOperand(3); + MachineMemOperand *MMO = Node->getMemOperand(); + DebugLoc DL = Node->getDebugLoc(); + EVT PtrVT = Addr.getValueType(); + + // Get the address of the containing word. + SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, + DAG.getConstant(-4, PtrVT)); + + // Get the number of bits that the word must be rotated left in order + // to bring the field to the top bits of a GR32. + SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, + DAG.getConstant(3, PtrVT)); + BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); + + // Get the complementing shift amount, for rotating a field in the top + // bits back to its proper position. + SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, + DAG.getConstant(0, WideVT), BitShift); + + // Construct the ATOMIC_CMP_SWAPW node. + SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); + SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift, + NegBitShift, DAG.getConstant(BitSize, WideVT) }; + SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL, + VTList, Ops, array_lengthof(Ops), + NarrowVT, MMO); + return AtomicOp; +} + +SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op, + SelectionDAG &DAG) const { + MachineFunction &MF = DAG.getMachineFunction(); + MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); + return DAG.getCopyFromReg(Op.getOperand(0), Op.getDebugLoc(), + SystemZ::R15D, Op.getValueType()); +} + +SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op, + SelectionDAG &DAG) const { + MachineFunction &MF = DAG.getMachineFunction(); + MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); + return DAG.getCopyToReg(Op.getOperand(0), Op.getDebugLoc(), + SystemZ::R15D, Op.getOperand(1)); +} + +SDValue SystemZTargetLowering::LowerOperation(SDValue Op, + SelectionDAG &DAG) const { + switch (Op.getOpcode()) { + case ISD::BR_CC: + return lowerBR_CC(Op, DAG); + case ISD::SELECT_CC: + return lowerSELECT_CC(Op, DAG); + case ISD::GlobalAddress: + return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG); + case ISD::GlobalTLSAddress: + return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG); + case ISD::BlockAddress: + return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG); + case ISD::JumpTable: + return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG); + case ISD::ConstantPool: + return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG); + case ISD::BITCAST: + return lowerBITCAST(Op, DAG); + case ISD::VASTART: + return lowerVASTART(Op, DAG); + case ISD::VACOPY: + return lowerVACOPY(Op, DAG); + case ISD::DYNAMIC_STACKALLOC: + return lowerDYNAMIC_STACKALLOC(Op, DAG); + case ISD::UMUL_LOHI: + return lowerUMUL_LOHI(Op, DAG); + case ISD::SDIVREM: + return lowerSDIVREM(Op, DAG); + case ISD::UDIVREM: + return lowerUDIVREM(Op, DAG); + case ISD::OR: + return lowerOR(Op, DAG); + case ISD::ATOMIC_SWAP: + return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_SWAPW); + case ISD::ATOMIC_LOAD_ADD: + return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD); + case ISD::ATOMIC_LOAD_SUB: + return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); + case ISD::ATOMIC_LOAD_AND: + return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_AND); + case ISD::ATOMIC_LOAD_OR: + return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_OR); + case ISD::ATOMIC_LOAD_XOR: + return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR); + case ISD::ATOMIC_LOAD_NAND: + return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND); + case ISD::ATOMIC_LOAD_MIN: + return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN); + case ISD::ATOMIC_LOAD_MAX: + return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX); + case ISD::ATOMIC_LOAD_UMIN: + return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN); + case ISD::ATOMIC_LOAD_UMAX: + return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX); + case ISD::ATOMIC_CMP_SWAP: + return lowerATOMIC_CMP_SWAP(Op, DAG); + case ISD::STACKSAVE: + return lowerSTACKSAVE(Op, DAG); + case ISD::STACKRESTORE: + return lowerSTACKRESTORE(Op, DAG); + default: + llvm_unreachable("Unexpected node to lower"); + } +} + +const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { +#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME + switch (Opcode) { + OPCODE(RET_FLAG); + OPCODE(CALL); + OPCODE(PCREL_WRAPPER); + OPCODE(CMP); + OPCODE(UCMP); + OPCODE(BR_CCMASK); + OPCODE(SELECT_CCMASK); + OPCODE(ADJDYNALLOC); + OPCODE(EXTRACT_ACCESS); + OPCODE(UMUL_LOHI64); + OPCODE(SDIVREM64); + OPCODE(UDIVREM32); + OPCODE(UDIVREM64); + OPCODE(ATOMIC_SWAPW); + OPCODE(ATOMIC_LOADW_ADD); + OPCODE(ATOMIC_LOADW_SUB); + OPCODE(ATOMIC_LOADW_AND); + OPCODE(ATOMIC_LOADW_OR); + OPCODE(ATOMIC_LOADW_XOR); + OPCODE(ATOMIC_LOADW_NAND); + OPCODE(ATOMIC_LOADW_MIN); + OPCODE(ATOMIC_LOADW_MAX); + OPCODE(ATOMIC_LOADW_UMIN); + OPCODE(ATOMIC_LOADW_UMAX); + OPCODE(ATOMIC_CMP_SWAPW); + } + return NULL; +#undef OPCODE +} + +//===----------------------------------------------------------------------===// +// Custom insertion +//===----------------------------------------------------------------------===// + +// Create a new basic block after MBB. +static MachineBasicBlock *emitBlockAfter(MachineBasicBlock *MBB) { + MachineFunction &MF = *MBB->getParent(); + MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); + MF.insert(llvm::next(MachineFunction::iterator(MBB)), NewMBB); + return NewMBB; +} + +// Split MBB after MI and return the new block (the one that contains +// instructions after MI). +static MachineBasicBlock *splitBlockAfter(MachineInstr *MI, + MachineBasicBlock *MBB) { + MachineBasicBlock *NewMBB = emitBlockAfter(MBB); + NewMBB->splice(NewMBB->begin(), MBB, + llvm::next(MachineBasicBlock::iterator(MI)), + MBB->end()); + NewMBB->transferSuccessorsAndUpdatePHIs(MBB); + return NewMBB; +} + +// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI. +MachineBasicBlock * +SystemZTargetLowering::emitSelect(MachineInstr *MI, + MachineBasicBlock *MBB) const { + const SystemZInstrInfo *TII = TM.getInstrInfo(); + + unsigned DestReg = MI->getOperand(0).getReg(); + unsigned TrueReg = MI->getOperand(1).getReg(); + unsigned FalseReg = MI->getOperand(2).getReg(); + unsigned CCMask = MI->getOperand(3).getImm(); + DebugLoc DL = MI->getDebugLoc(); + + MachineBasicBlock *StartMBB = MBB; + MachineBasicBlock *JoinMBB = splitBlockAfter(MI, MBB); + MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); + + // StartMBB: + // ... + // TrueVal = ... + // cmpTY ccX, r1, r2 + // jCC JoinMBB + // # fallthrough to FalseMBB + MBB = StartMBB; + BuildMI(MBB, DL, TII->get(SystemZ::BRCL)).addImm(CCMask).addMBB(JoinMBB); + MBB->addSuccessor(JoinMBB); + MBB->addSuccessor(FalseMBB); + + // FalseMBB: + // # fallthrough to JoinMBB + MBB = FalseMBB; + MBB->addSuccessor(JoinMBB); + + // JoinMBB: + // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ] + // ... + MBB = JoinMBB; + BuildMI(*MBB, MBB->begin(), DL, TII->get(SystemZ::PHI), DestReg) + .addReg(TrueReg).addMBB(StartMBB) + .addReg(FalseReg).addMBB(FalseMBB); + + MI->eraseFromParent(); + return JoinMBB; +} + +// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* +// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that +// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. +// BitSize is the width of the field in bits, or 0 if this is a partword +// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize +// is one of the operands. Invert says whether the field should be +// inverted after performing BinOpcode (e.g. for NAND). +MachineBasicBlock * +SystemZTargetLowering::emitAtomicLoadBinary(MachineInstr *MI, + MachineBasicBlock *MBB, + unsigned BinOpcode, + unsigned BitSize, + bool Invert) const { + const SystemZInstrInfo *TII = TM.getInstrInfo(); + MachineFunction &MF = *MBB->getParent(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + unsigned MaskNE = CCMaskForCondCode(ISD::SETNE); + bool IsSubWord = (BitSize < 32); + + // Extract the operands. Base can be a register or a frame index. + // Src2 can be a register or immediate. + unsigned Dest = MI->getOperand(0).getReg(); + MachineOperand Base = earlyUseOperand(MI->getOperand(1)); + int64_t Disp = MI->getOperand(2).getImm(); + MachineOperand Src2 = earlyUseOperand(MI->getOperand(3)); + unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); + unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); + DebugLoc DL = MI->getDebugLoc(); + if (IsSubWord) + BitSize = MI->getOperand(6).getImm(); + + // Subword operations use 32-bit registers. + const TargetRegisterClass *RC = (BitSize <= 32 ? + &SystemZ::GR32BitRegClass : + &SystemZ::GR64BitRegClass); + unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; + unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; + + // Get the right opcodes for the displacement. + LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); + CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); + assert(LOpcode && CSOpcode && "Displacement out of range"); + + // Create virtual registers for temporary results. + unsigned OrigVal = MRI.createVirtualRegister(RC); + unsigned OldVal = MRI.createVirtualRegister(RC); + unsigned NewVal = (BinOpcode || IsSubWord ? + MRI.createVirtualRegister(RC) : Src2.getReg()); + unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); + unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); + + // Insert a basic block for the main loop. + MachineBasicBlock *StartMBB = MBB; + MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); + MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); + + // StartMBB: + // ... + // %OrigVal = L Disp(%Base) + // # fall through to LoopMMB + MBB = StartMBB; + BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) + .addOperand(Base).addImm(Disp).addReg(0); + MBB->addSuccessor(LoopMBB); + + // LoopMBB: + // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ] + // %RotatedOldVal = RLL %OldVal, 0(%BitShift) + // %RotatedNewVal = OP %RotatedOldVal, %Src2 + // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) + // %Dest = CS %OldVal, %NewVal, Disp(%Base) + // JNE LoopMBB + // # fall through to DoneMMB + MBB = LoopMBB; + BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) + .addReg(OrigVal).addMBB(StartMBB) + .addReg(Dest).addMBB(LoopMBB); + if (IsSubWord) + BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) + .addReg(OldVal).addReg(BitShift).addImm(0); + if (Invert) { + // Perform the operation normally and then invert every bit of the field. + unsigned Tmp = MRI.createVirtualRegister(RC); + BuildMI(MBB, DL, TII->get(BinOpcode), Tmp) + .addReg(RotatedOldVal).addOperand(Src2); + if (BitSize < 32) + // XILF with the upper BitSize bits set. + BuildMI(MBB, DL, TII->get(SystemZ::XILF32), RotatedNewVal) + .addReg(Tmp).addImm(uint32_t(~0 << (32 - BitSize))); + else if (BitSize == 32) + // XILF with every bit set. + BuildMI(MBB, DL, TII->get(SystemZ::XILF32), RotatedNewVal) + .addReg(Tmp).addImm(~uint32_t(0)); + else { + // Use LCGR and add -1 to the result, which is more compact than + // an XILF, XILH pair. + unsigned Tmp2 = MRI.createVirtualRegister(RC); + BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); + BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) + .addReg(Tmp2).addImm(-1); + } + } else if (BinOpcode) + // A simply binary operation. + BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) + .addReg(RotatedOldVal).addOperand(Src2); + else if (IsSubWord) + // Use RISBG to rotate Src2 into position and use it to replace the + // field in RotatedOldVal. + BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) + .addReg(RotatedOldVal).addReg(Src2.getReg()) + .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); + if (IsSubWord) + BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) + .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); + BuildMI(MBB, DL, TII->get(CSOpcode), Dest) + .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); + BuildMI(MBB, DL, TII->get(SystemZ::BRCL)).addImm(MaskNE).addMBB(LoopMBB); + MBB->addSuccessor(LoopMBB); + MBB->addSuccessor(DoneMBB); + + MI->eraseFromParent(); + return DoneMBB; +} + +// Implement EmitInstrWithCustomInserter for pseudo +// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the +// instruction that should be used to compare the current field with the +// minimum or maximum value. KeepOldMask is the BRC condition-code mask +// for when the current field should be kept. BitSize is the width of +// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. +MachineBasicBlock * +SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI, + MachineBasicBlock *MBB, + unsigned CompareOpcode, + unsigned KeepOldMask, + unsigned BitSize) const { + const SystemZInstrInfo *TII = TM.getInstrInfo(); + MachineFunction &MF = *MBB->getParent(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + unsigned MaskNE = CCMaskForCondCode(ISD::SETNE); + bool IsSubWord = (BitSize < 32); + + // Extract the operands. Base can be a register or a frame index. + unsigned Dest = MI->getOperand(0).getReg(); + MachineOperand Base = earlyUseOperand(MI->getOperand(1)); + int64_t Disp = MI->getOperand(2).getImm(); + unsigned Src2 = MI->getOperand(3).getReg(); + unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); + unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); + DebugLoc DL = MI->getDebugLoc(); + if (IsSubWord) + BitSize = MI->getOperand(6).getImm(); + + // Subword operations use 32-bit registers. + const TargetRegisterClass *RC = (BitSize <= 32 ? + &SystemZ::GR32BitRegClass : + &SystemZ::GR64BitRegClass); + unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; + unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; + + // Get the right opcodes for the displacement. + LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); + CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); + assert(LOpcode && CSOpcode && "Displacement out of range"); + + // Create virtual registers for temporary results. + unsigned OrigVal = MRI.createVirtualRegister(RC); + unsigned OldVal = MRI.createVirtualRegister(RC); + unsigned NewVal = MRI.createVirtualRegister(RC); + unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); + unsigned RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); + unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); + + // Insert 3 basic blocks for the loop. + MachineBasicBlock *StartMBB = MBB; + MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); + MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); + MachineBasicBlock *UseAltMBB = emitBlockAfter(LoopMBB); + MachineBasicBlock *UpdateMBB = emitBlockAfter(UseAltMBB); + + // StartMBB: + // ... + // %OrigVal = L Disp(%Base) + // # fall through to LoopMMB + MBB = StartMBB; + BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) + .addOperand(Base).addImm(Disp).addReg(0); + MBB->addSuccessor(LoopMBB); + + // LoopMBB: + // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ] + // %RotatedOldVal = RLL %OldVal, 0(%BitShift) + // CompareOpcode %RotatedOldVal, %Src2 + // BRCL KeepOldMask, UpdateMBB + MBB = LoopMBB; + BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) + .addReg(OrigVal).addMBB(StartMBB) + .addReg(Dest).addMBB(UpdateMBB); + if (IsSubWord) + BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) + .addReg(OldVal).addReg(BitShift).addImm(0); + BuildMI(MBB, DL, TII->get(CompareOpcode)) + .addReg(RotatedOldVal).addReg(Src2); + BuildMI(MBB, DL, TII->get(SystemZ::BRCL)) + .addImm(KeepOldMask).addMBB(UpdateMBB); + MBB->addSuccessor(UpdateMBB); + MBB->addSuccessor(UseAltMBB); + + // UseAltMBB: + // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 + // # fall through to UpdateMMB + MBB = UseAltMBB; + if (IsSubWord) + BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) + .addReg(RotatedOldVal).addReg(Src2) + .addImm(32).addImm(31 + BitSize).addImm(0); + MBB->addSuccessor(UpdateMBB); + + // UpdateMBB: + // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ], + // [ %RotatedAltVal, UseAltMBB ] + // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) + // %Dest = CS %OldVal, %NewVal, Disp(%Base) + // JNE LoopMBB + // # fall through to DoneMMB + MBB = UpdateMBB; + BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) + .addReg(RotatedOldVal).addMBB(LoopMBB) + .addReg(RotatedAltVal).addMBB(UseAltMBB); + if (IsSubWord) + BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) + .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); + BuildMI(MBB, DL, TII->get(CSOpcode), Dest) + .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); + BuildMI(MBB, DL, TII->get(SystemZ::BRCL)).addImm(MaskNE).addMBB(LoopMBB); + MBB->addSuccessor(LoopMBB); + MBB->addSuccessor(DoneMBB); + + MI->eraseFromParent(); + return DoneMBB; +} + +// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW +// instruction MI. +MachineBasicBlock * +SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI, + MachineBasicBlock *MBB) const { + const SystemZInstrInfo *TII = TM.getInstrInfo(); + MachineFunction &MF = *MBB->getParent(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + unsigned MaskNE = CCMaskForCondCode(ISD::SETNE); + + // Extract the operands. Base can be a register or a frame index. + unsigned Dest = MI->getOperand(0).getReg(); + MachineOperand Base = earlyUseOperand(MI->getOperand(1)); + int64_t Disp = MI->getOperand(2).getImm(); + unsigned OrigCmpVal = MI->getOperand(3).getReg(); + unsigned OrigSwapVal = MI->getOperand(4).getReg(); + unsigned BitShift = MI->getOperand(5).getReg(); + unsigned NegBitShift = MI->getOperand(6).getReg(); + int64_t BitSize = MI->getOperand(7).getImm(); + DebugLoc DL = MI->getDebugLoc(); + + const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass; + + // Get the right opcodes for the displacement. + unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); + unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); + assert(LOpcode && CSOpcode && "Displacement out of range"); + + // Create virtual registers for temporary results. + unsigned OrigOldVal = MRI.createVirtualRegister(RC); + unsigned OldVal = MRI.createVirtualRegister(RC); + unsigned CmpVal = MRI.createVirtualRegister(RC); + unsigned SwapVal = MRI.createVirtualRegister(RC); + unsigned StoreVal = MRI.createVirtualRegister(RC); + unsigned RetryOldVal = MRI.createVirtualRegister(RC); + unsigned RetryCmpVal = MRI.createVirtualRegister(RC); + unsigned RetrySwapVal = MRI.createVirtualRegister(RC); + + // Insert 2 basic blocks for the loop. + MachineBasicBlock *StartMBB = MBB; + MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); + MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); + MachineBasicBlock *SetMBB = emitBlockAfter(LoopMBB); + + // StartMBB: + // ... + // %OrigOldVal = L Disp(%Base) + // # fall through to LoopMMB + MBB = StartMBB; + BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal) + .addOperand(Base).addImm(Disp).addReg(0); + MBB->addSuccessor(LoopMBB); + + // LoopMBB: + // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ] + // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ] + // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ] + // %Dest = RLL %OldVal, BitSize(%BitShift) + // ^^ The low BitSize bits contain the field + // of interest. + // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0 + // ^^ Replace the upper 32-BitSize bits of the + // comparison value with those that we loaded, + // so that we can use a full word comparison. + // CR %Dest, %RetryCmpVal + // JNE DoneMBB + // # Fall through to SetMBB + MBB = LoopMBB; + BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) + .addReg(OrigOldVal).addMBB(StartMBB) + .addReg(RetryOldVal).addMBB(SetMBB); + BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal) + .addReg(OrigCmpVal).addMBB(StartMBB) + .addReg(RetryCmpVal).addMBB(SetMBB); + BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) + .addReg(OrigSwapVal).addMBB(StartMBB) + .addReg(RetrySwapVal).addMBB(SetMBB); + BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest) + .addReg(OldVal).addReg(BitShift).addImm(BitSize); + BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal) + .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::CR)) + .addReg(Dest).addReg(RetryCmpVal); + BuildMI(MBB, DL, TII->get(SystemZ::BRCL)).addImm(MaskNE).addMBB(DoneMBB); + MBB->addSuccessor(DoneMBB); + MBB->addSuccessor(SetMBB); + + // SetMBB: + // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0 + // ^^ Replace the upper 32-BitSize bits of the new + // value with those that we loaded. + // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) + // ^^ Rotate the new field to its proper position. + // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base) + // JNE LoopMBB + // # fall through to ExitMMB + MBB = SetMBB; + BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) + .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) + .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize); + BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) + .addReg(OldVal).addReg(StoreVal).addOperand(Base).addImm(Disp); + BuildMI(MBB, DL, TII->get(SystemZ::BRCL)).addImm(MaskNE).addMBB(LoopMBB); + MBB->addSuccessor(LoopMBB); + MBB->addSuccessor(DoneMBB); + + MI->eraseFromParent(); + return DoneMBB; +} + +// Emit an extension from a GR32 or GR64 to a GR128. ClearEven is true +// if the high register of the GR128 value must be cleared or false if +// it's "don't care". SubReg is subreg_odd32 when extending a GR32 +// and subreg_odd when extending a GR64. +MachineBasicBlock * +SystemZTargetLowering::emitExt128(MachineInstr *MI, + MachineBasicBlock *MBB, + bool ClearEven, unsigned SubReg) const { + const SystemZInstrInfo *TII = TM.getInstrInfo(); + MachineFunction &MF = *MBB->getParent(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + DebugLoc DL = MI->getDebugLoc(); + + unsigned Dest = MI->getOperand(0).getReg(); + unsigned Src = MI->getOperand(1).getReg(); + unsigned In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); + + BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128); + if (ClearEven) { + unsigned NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); + unsigned Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); + + BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64) + .addImm(0); + BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128) + .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_high); + In128 = NewIn128; + } + BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) + .addReg(In128).addReg(Src).addImm(SubReg); + + MI->eraseFromParent(); + return MBB; +} + +MachineBasicBlock *SystemZTargetLowering:: +EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *MBB) const { + switch (MI->getOpcode()) { + case SystemZ::Select32: + case SystemZ::SelectF32: + case SystemZ::Select64: + case SystemZ::SelectF64: + case SystemZ::SelectF128: + return emitSelect(MI, MBB); + + case SystemZ::AEXT128_64: + return emitExt128(MI, MBB, false, SystemZ::subreg_low); + case SystemZ::ZEXT128_32: + return emitExt128(MI, MBB, true, SystemZ::subreg_low32); + case SystemZ::ZEXT128_64: + return emitExt128(MI, MBB, true, SystemZ::subreg_low); + + case SystemZ::ATOMIC_SWAPW: + return emitAtomicLoadBinary(MI, MBB, 0, 0); + case SystemZ::ATOMIC_SWAP_32: + return emitAtomicLoadBinary(MI, MBB, 0, 32); + case SystemZ::ATOMIC_SWAP_64: + return emitAtomicLoadBinary(MI, MBB, 0, 64); + + case SystemZ::ATOMIC_LOADW_AR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); + case SystemZ::ATOMIC_LOADW_AFI: + return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); + case SystemZ::ATOMIC_LOAD_AR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); + case SystemZ::ATOMIC_LOAD_AHI: + return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); + case SystemZ::ATOMIC_LOAD_AFI: + return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); + case SystemZ::ATOMIC_LOAD_AGR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); + case SystemZ::ATOMIC_LOAD_AGHI: + return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); + case SystemZ::ATOMIC_LOAD_AGFI: + return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); + + case SystemZ::ATOMIC_LOADW_SR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); + case SystemZ::ATOMIC_LOAD_SR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); + case SystemZ::ATOMIC_LOAD_SGR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); + + case SystemZ::ATOMIC_LOADW_NR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); + case SystemZ::ATOMIC_LOADW_NILH: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 0); + case SystemZ::ATOMIC_LOAD_NR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); + case SystemZ::ATOMIC_LOAD_NILL32: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL32, 32); + case SystemZ::ATOMIC_LOAD_NILH32: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 32); + case SystemZ::ATOMIC_LOAD_NILF32: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF32, 32); + case SystemZ::ATOMIC_LOAD_NGR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); + case SystemZ::ATOMIC_LOAD_NILL: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 64); + case SystemZ::ATOMIC_LOAD_NILH: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 64); + case SystemZ::ATOMIC_LOAD_NIHL: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64); + case SystemZ::ATOMIC_LOAD_NIHH: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64); + case SystemZ::ATOMIC_LOAD_NILF: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 64); + case SystemZ::ATOMIC_LOAD_NIHF: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64); + + case SystemZ::ATOMIC_LOADW_OR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); + case SystemZ::ATOMIC_LOADW_OILH: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH32, 0); + case SystemZ::ATOMIC_LOAD_OR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); + case SystemZ::ATOMIC_LOAD_OILL32: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL32, 32); + case SystemZ::ATOMIC_LOAD_OILH32: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH32, 32); + case SystemZ::ATOMIC_LOAD_OILF32: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF32, 32); + case SystemZ::ATOMIC_LOAD_OGR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); + case SystemZ::ATOMIC_LOAD_OILL: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 64); + case SystemZ::ATOMIC_LOAD_OILH: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 64); + case SystemZ::ATOMIC_LOAD_OIHL: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL, 64); + case SystemZ::ATOMIC_LOAD_OIHH: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH, 64); + case SystemZ::ATOMIC_LOAD_OILF: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 64); + case SystemZ::ATOMIC_LOAD_OIHF: + return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF, 64); + + case SystemZ::ATOMIC_LOADW_XR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); + case SystemZ::ATOMIC_LOADW_XILF: + return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF32, 0); + case SystemZ::ATOMIC_LOAD_XR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); + case SystemZ::ATOMIC_LOAD_XILF32: + return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF32, 32); + case SystemZ::ATOMIC_LOAD_XGR: + return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); + case SystemZ::ATOMIC_LOAD_XILF: + return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 64); + case SystemZ::ATOMIC_LOAD_XIHF: + return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF, 64); + + case SystemZ::ATOMIC_LOADW_NRi: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); + case SystemZ::ATOMIC_LOADW_NILHi: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 0, true); + case SystemZ::ATOMIC_LOAD_NRi: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); + case SystemZ::ATOMIC_LOAD_NILL32i: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL32, 32, true); + case SystemZ::ATOMIC_LOAD_NILH32i: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 32, true); + case SystemZ::ATOMIC_LOAD_NILF32i: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF32, 32, true); + case SystemZ::ATOMIC_LOAD_NGRi: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); + case SystemZ::ATOMIC_LOAD_NILLi: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 64, true); + case SystemZ::ATOMIC_LOAD_NILHi: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 64, true); + case SystemZ::ATOMIC_LOAD_NIHLi: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64, true); + case SystemZ::ATOMIC_LOAD_NIHHi: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64, true); + case SystemZ::ATOMIC_LOAD_NILFi: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 64, true); + case SystemZ::ATOMIC_LOAD_NIHFi: + return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64, true); + + case SystemZ::ATOMIC_LOADW_MIN: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, + SystemZ::CCMASK_CMP_LE, 0); + case SystemZ::ATOMIC_LOAD_MIN_32: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, + SystemZ::CCMASK_CMP_LE, 32); + case SystemZ::ATOMIC_LOAD_MIN_64: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, + SystemZ::CCMASK_CMP_LE, 64); + + case SystemZ::ATOMIC_LOADW_MAX: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, + SystemZ::CCMASK_CMP_GE, 0); + case SystemZ::ATOMIC_LOAD_MAX_32: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, + SystemZ::CCMASK_CMP_GE, 32); + case SystemZ::ATOMIC_LOAD_MAX_64: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, + SystemZ::CCMASK_CMP_GE, 64); + + case SystemZ::ATOMIC_LOADW_UMIN: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, + SystemZ::CCMASK_CMP_LE, 0); + case SystemZ::ATOMIC_LOAD_UMIN_32: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, + SystemZ::CCMASK_CMP_LE, 32); + case SystemZ::ATOMIC_LOAD_UMIN_64: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, + SystemZ::CCMASK_CMP_LE, 64); + + case SystemZ::ATOMIC_LOADW_UMAX: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, + SystemZ::CCMASK_CMP_GE, 0); + case SystemZ::ATOMIC_LOAD_UMAX_32: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, + SystemZ::CCMASK_CMP_GE, 32); + case SystemZ::ATOMIC_LOAD_UMAX_64: + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, + SystemZ::CCMASK_CMP_GE, 64); + + case SystemZ::ATOMIC_CMP_SWAPW: + return emitAtomicCmpSwapW(MI, MBB); + default: + llvm_unreachable("Unexpected instr type to insert"); + } +} |
