summaryrefslogtreecommitdiffstats
path: root/lib/CodeGen/SelectionDAG
diff options
context:
space:
mode:
Diffstat (limited to 'lib/CodeGen/SelectionDAG')
-rw-r--r--lib/CodeGen/SelectionDAG/DAGCombiner.cpp46
-rw-r--r--lib/CodeGen/SelectionDAG/FastISel.cpp7
-rw-r--r--lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp13
-rw-r--r--lib/CodeGen/SelectionDAG/InstrEmitter.cpp272
-rw-r--r--lib/CodeGen/SelectionDAG/InstrEmitter.h36
-rw-r--r--lib/CodeGen/SelectionDAG/SDNodeDbgValue.h14
-rw-r--r--lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp137
-rw-r--r--lib/CodeGen/SelectionDAG/SelectionDAG.cpp284
-rw-r--r--lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp29
-rw-r--r--lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp117
-rw-r--r--lib/CodeGen/SelectionDAG/TargetLowering.cpp21
11 files changed, 606 insertions, 370 deletions
diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index aa283ad..a336e0a 100644
--- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -5022,18 +5022,6 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
SDValue Chain = LD->getChain();
SDValue Ptr = LD->getBasePtr();
- // Try to infer better alignment information than the load already has.
- if (OptLevel != CodeGenOpt::None && LD->isUnindexed()) {
- if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
- if (Align > LD->getAlignment())
- return DAG.getExtLoad(LD->getExtensionType(), N->getDebugLoc(),
- LD->getValueType(0),
- Chain, Ptr, LD->getSrcValue(),
- LD->getSrcValueOffset(), LD->getMemoryVT(),
- LD->isVolatile(), LD->isNonTemporal(), Align);
- }
- }
-
// If load is not volatile and there are no uses of the loaded value (and
// the updated indexed value in case of indexed loads), change uses of the
// chain value into uses of the chain input (i.e. delete the dead load).
@@ -5099,6 +5087,18 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
}
}
+ // Try to infer better alignment information than the load already has.
+ if (OptLevel != CodeGenOpt::None && LD->isUnindexed()) {
+ if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
+ if (Align > LD->getAlignment())
+ return DAG.getExtLoad(LD->getExtensionType(), N->getDebugLoc(),
+ LD->getValueType(0),
+ Chain, Ptr, LD->getSrcValue(),
+ LD->getSrcValueOffset(), LD->getMemoryVT(),
+ LD->isVolatile(), LD->isNonTemporal(), Align);
+ }
+ }
+
if (CombinerAA) {
// Walk up chain skipping non-aliasing memory nodes.
SDValue BetterChain = FindBetterChain(N, Chain);
@@ -5250,17 +5250,6 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
SDValue Value = ST->getValue();
SDValue Ptr = ST->getBasePtr();
- // Try to infer better alignment information than the store already has.
- if (OptLevel != CodeGenOpt::None && ST->isUnindexed()) {
- if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
- if (Align > ST->getAlignment())
- return DAG.getTruncStore(Chain, N->getDebugLoc(), Value,
- Ptr, ST->getSrcValue(),
- ST->getSrcValueOffset(), ST->getMemoryVT(),
- ST->isVolatile(), ST->isNonTemporal(), Align);
- }
- }
-
// If this is a store of a bit convert, store the input value if the
// resultant store does not need a higher alignment than the original.
if (Value.getOpcode() == ISD::BIT_CONVERT && !ST->isTruncatingStore() &&
@@ -5351,6 +5340,17 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
}
}
+ // Try to infer better alignment information than the store already has.
+ if (OptLevel != CodeGenOpt::None && ST->isUnindexed()) {
+ if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
+ if (Align > ST->getAlignment())
+ return DAG.getTruncStore(Chain, N->getDebugLoc(), Value,
+ Ptr, ST->getSrcValue(),
+ ST->getSrcValueOffset(), ST->getMemoryVT(),
+ ST->isVolatile(), ST->isNonTemporal(), Align);
+ }
+ }
+
if (CombinerAA) {
// Walk up chain skipping non-aliasing memory nodes.
SDValue BetterChain = FindBetterChain(N, Chain);
diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp
index 3fc30ff..e4e9ef4 100644
--- a/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -340,10 +340,9 @@ bool FastISel::SelectCall(User *I) {
StaticAllocaMap.find(AI);
if (SI == StaticAllocaMap.end()) break; // VLAs.
int FI = SI->second;
- if (MMI) {
- if (MDNode *Dbg = DI->getMetadata("dbg"))
- MMI->setVariableDbgInfo(DI->getVariable(), FI, Dbg);
- }
+ if (MDNode *Dbg = DI->getDbgMetadata())
+ MMI->setVariableDbgInfo(DI->getVariable(), FI, Dbg);
+
// Building the map above is target independent. Generating DBG_VALUE
// inline is target dependent; do this now.
(void)TargetSelectInstruction(cast<Instruction>(I));
diff --git a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index 50f4c32..4fb2aa2 100644
--- a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -281,8 +281,17 @@ unsigned FunctionLoweringInfo::CreateRegForValue(const Value *V) {
GlobalVariable *llvm::ExtractTypeInfo(Value *V) {
V = V->stripPointerCasts();
GlobalVariable *GV = dyn_cast<GlobalVariable>(V);
- assert ((GV || isa<ConstantPointerNull>(V)) &&
- "TypeInfo must be a global variable or NULL");
+
+ if (GV && GV->getName() == ".llvm.eh.catch.all.value") {
+ assert(GV->hasInitializer() &&
+ "The EH catch-all value must have an initializer");
+ Value *Init = GV->getInitializer();
+ GV = dyn_cast<GlobalVariable>(Init);
+ if (!GV) V = cast<ConstantPointerNull>(Init);
+ }
+
+ assert((GV || isa<ConstantPointerNull>(V)) &&
+ "TypeInfo must be a global variable or NULL");
return GV;
}
diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index fda094d3..28ba343 100644
--- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -264,7 +264,8 @@ void
InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op,
unsigned IIOpNum,
const TargetInstrDesc *II,
- DenseMap<SDValue, unsigned> &VRBaseMap) {
+ DenseMap<SDValue, unsigned> &VRBaseMap,
+ bool IsDebug) {
assert(Op.getValueType() != MVT::Other &&
Op.getValueType() != MVT::Flag &&
"Chain and flag operands should occur at end of operand list!");
@@ -295,7 +296,11 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op,
}
}
- MI->addOperand(MachineOperand::CreateReg(VReg, isOptDef));
+ MI->addOperand(MachineOperand::CreateReg(VReg, isOptDef,
+ false/*isImp*/, false/*isKill*/,
+ false/*isDead*/, false/*isUndef*/,
+ false/*isEarlyClobber*/,
+ 0/*SubReg*/, IsDebug));
}
/// AddOperand - Add the specified operand to the specified machine instr. II
@@ -305,9 +310,10 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op,
void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op,
unsigned IIOpNum,
const TargetInstrDesc *II,
- DenseMap<SDValue, unsigned> &VRBaseMap) {
+ DenseMap<SDValue, unsigned> &VRBaseMap,
+ bool IsDebug) {
if (Op.isMachineOpcode()) {
- AddRegisterOperand(MI, Op, IIOpNum, II, VRBaseMap);
+ AddRegisterOperand(MI, Op, IIOpNum, II, VRBaseMap, IsDebug);
} else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
MI->addOperand(MachineOperand::CreateImm(C->getSExtValue()));
} else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Op)) {
@@ -356,7 +362,7 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op,
assert(Op.getValueType() != MVT::Other &&
Op.getValueType() != MVT::Flag &&
"Chain and flag operands should occur at end of operand list!");
- AddRegisterOperand(MI, Op, IIOpNum, II, VRBaseMap);
+ AddRegisterOperand(MI, Op, IIOpNum, II, VRBaseMap, IsDebug);
}
}
@@ -498,164 +504,156 @@ InstrEmitter::EmitCopyToRegClassNode(SDNode *Node,
assert(isNew && "Node emitted out of order - early");
}
-/// EmitDbgValue - Generate any debug info that refers to this Node. Constant
-/// dbg_value is not handled here.
-void
-InstrEmitter::EmitDbgValue(SDNode *Node,
- DenseMap<SDValue, unsigned> &VRBaseMap,
- SDDbgValue *sd) {
- if (!Node->getHasDebugValue())
- return;
- if (!sd)
- return;
- assert(sd->getKind() == SDDbgValue::SDNODE);
- unsigned VReg = getVR(SDValue(sd->getSDNode(), sd->getResNo()), VRBaseMap);
- const TargetInstrDesc &II = TII->get(TargetOpcode::DBG_VALUE);
- DebugLoc DL = sd->getDebugLoc();
- MachineInstr *MI;
- if (VReg) {
- MI = BuildMI(*MF, DL, II).addReg(VReg, RegState::Debug).
- addImm(sd->getOffset()).
- addMetadata(sd->getMDPtr());
- } else {
- // Insert an Undef so we can see what we dropped.
- MI = BuildMI(*MF, DL, II).addReg(0U).addImm(sd->getOffset()).
- addMetadata(sd->getMDPtr());
- }
- MBB->insert(InsertPos, MI);
-}
-
-/// EmitDbgValue - Generate debug info that does not refer to a SDNode.
-void
-InstrEmitter::EmitDbgValue(SDDbgValue *sd,
+/// EmitDbgValue - Generate machine instruction for a dbg_value node.
+///
+MachineInstr *InstrEmitter::EmitDbgValue(SDDbgValue *SD,
+ MachineBasicBlock *InsertBB,
+ DenseMap<SDValue, unsigned> &VRBaseMap,
DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) {
- if (!sd)
- return;
+ uint64_t Offset = SD->getOffset();
+ MDNode* MDPtr = SD->getMDPtr();
+ DebugLoc DL = SD->getDebugLoc();
+
const TargetInstrDesc &II = TII->get(TargetOpcode::DBG_VALUE);
- uint64_t Offset = sd->getOffset();
- MDNode* mdPtr = sd->getMDPtr();
- SDDbgValue::DbgValueKind kind = sd->getKind();
- DebugLoc DL = sd->getDebugLoc();
- MachineInstr* MI;
- if (kind == SDDbgValue::CONST) {
- Value *V = sd->getConst();
+ MachineInstrBuilder MIB = BuildMI(*MF, DL, II);
+ if (SD->getKind() == SDDbgValue::SDNODE) {
+ AddOperand(&*MIB, SDValue(SD->getSDNode(), SD->getResNo()),
+ (*MIB).getNumOperands(), &II, VRBaseMap, true /*IsDebug*/);
+ } else if (SD->getKind() == SDDbgValue::CONST) {
+ Value *V = SD->getConst();
if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) {
- MI = BuildMI(*MF, DL, II).addImm(CI->getZExtValue()).
- addImm(Offset).addMetadata(mdPtr);
+ MIB.addImm(CI->getSExtValue());
} else if (ConstantFP *CF = dyn_cast<ConstantFP>(V)) {
- MI = BuildMI(*MF, DL, II).addFPImm(CF).
- addImm(Offset).addMetadata(mdPtr);
+ MIB.addFPImm(CF);
} else {
// Could be an Undef. In any case insert an Undef so we can see what we
// dropped.
- MI = BuildMI(*MF, DL, II).addReg(0U).
- addImm(Offset).addMetadata(mdPtr);
+ MIB.addReg(0U);
}
- } else if (kind == SDDbgValue::FRAMEIX) {
- unsigned FrameIx = sd->getFrameIx();
+ } else if (SD->getKind() == SDDbgValue::FRAMEIX) {
+ unsigned FrameIx = SD->getFrameIx();
// Stack address; this needs to be lowered in target-dependent fashion.
// FIXME test that the target supports this somehow; if not emit Undef.
// Create a pseudo for EmitInstrWithCustomInserter's consumption.
- MI = BuildMI(*MF, DL, II).addImm(FrameIx).
- addImm(Offset).addMetadata(mdPtr);
- MBB = TLI->EmitInstrWithCustomInserter(MI, MBB, EM);
- InsertPos = MBB->end();
- return;
+ MIB.addImm(FrameIx).addImm(Offset).addMetadata(MDPtr);
+ abort();
+ TLI->EmitInstrWithCustomInserter(&*MIB, InsertBB, EM);
+ return 0;
} else {
// Insert an Undef so we can see what we dropped.
- MI = BuildMI(*MF, DL, II).addReg(0U).
- addImm(Offset).addMetadata(mdPtr);
+ MIB.addReg(0U);
}
- MBB->insert(InsertPos, MI);
+
+ MIB.addImm(Offset).addMetadata(MDPtr);
+ return &*MIB;
}
-/// EmitNode - Generate machine code for a node and needed dependencies.
+/// EmitMachineNode - Generate machine code for a target-specific node and
+/// needed dependencies.
///
-void InstrEmitter::EmitNode(SDNode *Node, bool IsClone, bool IsCloned,
- DenseMap<SDValue, unsigned> &VRBaseMap,
- DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) {
- // If machine instruction
- if (Node->isMachineOpcode()) {
- unsigned Opc = Node->getMachineOpcode();
-
- // Handle subreg insert/extract specially
- if (Opc == TargetOpcode::EXTRACT_SUBREG ||
- Opc == TargetOpcode::INSERT_SUBREG ||
- Opc == TargetOpcode::SUBREG_TO_REG) {
- EmitSubregNode(Node, VRBaseMap);
- return;
- }
+void InstrEmitter::
+EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
+ DenseMap<SDValue, unsigned> &VRBaseMap,
+ DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) {
+ unsigned Opc = Node->getMachineOpcode();
+
+ // Handle subreg insert/extract specially
+ if (Opc == TargetOpcode::EXTRACT_SUBREG ||
+ Opc == TargetOpcode::INSERT_SUBREG ||
+ Opc == TargetOpcode::SUBREG_TO_REG) {
+ EmitSubregNode(Node, VRBaseMap);
+ return;
+ }
- // Handle COPY_TO_REGCLASS specially.
- if (Opc == TargetOpcode::COPY_TO_REGCLASS) {
- EmitCopyToRegClassNode(Node, VRBaseMap);
- return;
- }
+ // Handle COPY_TO_REGCLASS specially.
+ if (Opc == TargetOpcode::COPY_TO_REGCLASS) {
+ EmitCopyToRegClassNode(Node, VRBaseMap);
+ return;
+ }
- if (Opc == TargetOpcode::IMPLICIT_DEF)
- // We want a unique VR for each IMPLICIT_DEF use.
- return;
-
- const TargetInstrDesc &II = TII->get(Opc);
- unsigned NumResults = CountResults(Node);
- unsigned NodeOperands = CountOperands(Node);
- bool HasPhysRegOuts = (NumResults > II.getNumDefs()) &&
- II.getImplicitDefs() != 0;
+ if (Opc == TargetOpcode::IMPLICIT_DEF)
+ // We want a unique VR for each IMPLICIT_DEF use.
+ return;
+
+ const TargetInstrDesc &II = TII->get(Opc);
+ unsigned NumResults = CountResults(Node);
+ unsigned NodeOperands = CountOperands(Node);
+ bool HasPhysRegOuts = NumResults > II.getNumDefs() && II.getImplicitDefs()!=0;
#ifndef NDEBUG
- unsigned NumMIOperands = NodeOperands + NumResults;
- assert((II.getNumOperands() == NumMIOperands ||
- HasPhysRegOuts || II.isVariadic()) &&
- "#operands for dag node doesn't match .td file!");
+ unsigned NumMIOperands = NodeOperands + NumResults;
+ if (II.isVariadic())
+ assert(NumMIOperands >= II.getNumOperands() &&
+ "Too few operands for a variadic node!");
+ else
+ assert(NumMIOperands >= II.getNumOperands() &&
+ NumMIOperands <= II.getNumOperands()+II.getNumImplicitDefs() &&
+ "#operands for dag node doesn't match .td file!");
#endif
- // Create the new machine instruction.
- MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), II);
-
- // Add result register values for things that are defined by this
- // instruction.
- if (NumResults)
- CreateVirtualRegisters(Node, MI, II, IsClone, IsCloned, VRBaseMap);
-
- // Emit all of the actual operands of this instruction, adding them to the
- // instruction as appropriate.
- bool HasOptPRefs = II.getNumDefs() > NumResults;
- assert((!HasOptPRefs || !HasPhysRegOuts) &&
- "Unable to cope with optional defs and phys regs defs!");
- unsigned NumSkip = HasOptPRefs ? II.getNumDefs() - NumResults : 0;
- for (unsigned i = NumSkip; i != NodeOperands; ++i)
- AddOperand(MI, Node->getOperand(i), i-NumSkip+II.getNumDefs(), &II,
- VRBaseMap);
-
- // Transfer all of the memory reference descriptions of this instruction.
- MI->setMemRefs(cast<MachineSDNode>(Node)->memoperands_begin(),
- cast<MachineSDNode>(Node)->memoperands_end());
-
- if (II.usesCustomInsertionHook()) {
- // Insert this instruction into the basic block using a target
- // specific inserter which may returns a new basic block.
- MBB = TLI->EmitInstrWithCustomInserter(MI, MBB, EM);
- InsertPos = MBB->end();
- } else {
- MBB->insert(InsertPos, MI);
- }
+ // Create the new machine instruction.
+ MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), II);
+
+ // Add result register values for things that are defined by this
+ // instruction.
+ if (NumResults)
+ CreateVirtualRegisters(Node, MI, II, IsClone, IsCloned, VRBaseMap);
+
+ // Emit all of the actual operands of this instruction, adding them to the
+ // instruction as appropriate.
+ bool HasOptPRefs = II.getNumDefs() > NumResults;
+ assert((!HasOptPRefs || !HasPhysRegOuts) &&
+ "Unable to cope with optional defs and phys regs defs!");
+ unsigned NumSkip = HasOptPRefs ? II.getNumDefs() - NumResults : 0;
+ for (unsigned i = NumSkip; i != NodeOperands; ++i)
+ AddOperand(MI, Node->getOperand(i), i-NumSkip+II.getNumDefs(), &II,
+ VRBaseMap);
+
+ // Transfer all of the memory reference descriptions of this instruction.
+ MI->setMemRefs(cast<MachineSDNode>(Node)->memoperands_begin(),
+ cast<MachineSDNode>(Node)->memoperands_end());
+
+ if (II.usesCustomInsertionHook()) {
+ // Insert this instruction into the basic block using a target
+ // specific inserter which may returns a new basic block.
+ MBB = TLI->EmitInstrWithCustomInserter(MI, MBB, EM);
+ InsertPos = MBB->end();
+ return;
+ }
+
+ MBB->insert(InsertPos, MI);
- // Additional results must be an physical register def.
- if (HasPhysRegOuts) {
- for (unsigned i = II.getNumDefs(); i < NumResults; ++i) {
- unsigned Reg = II.getImplicitDefs()[i - II.getNumDefs()];
- if (Node->hasAnyUseOfValue(i))
- EmitCopyFromReg(Node, i, IsClone, IsCloned, Reg, VRBaseMap);
- // If there are no uses, mark the register as dead now, so that
- // MachineLICM/Sink can see that it's dead. Don't do this if the
- // node has a Flag value, for the benefit of targets still using
- // Flag for values in physregs.
- else if (Node->getValueType(Node->getNumValues()-1) != MVT::Flag)
- MI->addRegisterDead(Reg, TRI);
- }
+ // Additional results must be an physical register def.
+ if (HasPhysRegOuts) {
+ for (unsigned i = II.getNumDefs(); i < NumResults; ++i) {
+ unsigned Reg = II.getImplicitDefs()[i - II.getNumDefs()];
+ if (Node->hasAnyUseOfValue(i))
+ EmitCopyFromReg(Node, i, IsClone, IsCloned, Reg, VRBaseMap);
+ // If there are no uses, mark the register as dead now, so that
+ // MachineLICM/Sink can see that it's dead. Don't do this if the
+ // node has a Flag value, for the benefit of targets still using
+ // Flag for values in physregs.
+ else if (Node->getValueType(Node->getNumValues()-1) != MVT::Flag)
+ MI->addRegisterDead(Reg, TRI);
}
- return;
}
+
+ // If the instruction has implicit defs and the node doesn't, mark the
+ // implicit def as dead. If the node has any flag outputs, we don't do this
+ // because we don't know what implicit defs are being used by flagged nodes.
+ if (Node->getValueType(Node->getNumValues()-1) != MVT::Flag)
+ if (const unsigned *IDList = II.getImplicitDefs()) {
+ for (unsigned i = NumResults, e = II.getNumDefs()+II.getNumImplicitDefs();
+ i != e; ++i)
+ MI->addRegisterDead(IDList[i-II.getNumDefs()], TRI);
+ }
+ return;
+}
+/// EmitSpecialNode - Generate machine code for a target-independent node and
+/// needed dependencies.
+void InstrEmitter::
+EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
+ DenseMap<SDValue, unsigned> &VRBaseMap) {
switch (Node->getOpcode()) {
default:
#ifndef NDEBUG
diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.h b/lib/CodeGen/SelectionDAG/InstrEmitter.h
index eefcd73..baabb75 100644
--- a/lib/CodeGen/SelectionDAG/InstrEmitter.h
+++ b/lib/CodeGen/SelectionDAG/InstrEmitter.h
@@ -64,7 +64,8 @@ class InstrEmitter {
void AddRegisterOperand(MachineInstr *MI, SDValue Op,
unsigned IIOpNum,
const TargetInstrDesc *II,
- DenseMap<SDValue, unsigned> &VRBaseMap);
+ DenseMap<SDValue, unsigned> &VRBaseMap,
+ bool IsDebug = false);
/// AddOperand - Add the specified operand to the specified machine instr. II
/// specifies the instruction information for the node, and IIOpNum is the
@@ -73,7 +74,8 @@ class InstrEmitter {
void AddOperand(MachineInstr *MI, SDValue Op,
unsigned IIOpNum,
const TargetInstrDesc *II,
- DenseMap<SDValue, unsigned> &VRBaseMap);
+ DenseMap<SDValue, unsigned> &VRBaseMap,
+ bool IsDebug = false);
/// EmitSubregNode - Generate machine code for subreg nodes.
///
@@ -98,22 +100,23 @@ public:
/// MachineInstr.
static unsigned CountOperands(SDNode *Node);
- /// EmitDbgValue - Generate any debug info that refers to this Node. Constant
- /// dbg_value is not handled here.
- void EmitDbgValue(SDNode *Node,
- DenseMap<SDValue, unsigned> &VRBaseMap,
- SDDbgValue* sd);
-
-
- /// EmitDbgValue - Generate a constant DBG_VALUE. No node is involved.
- void EmitDbgValue(SDDbgValue* sd,
- DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM);
+ /// EmitDbgValue - Generate machine instruction for a dbg_value node.
+ ///
+ MachineInstr *EmitDbgValue(SDDbgValue *SD,
+ MachineBasicBlock *InsertBB,
+ DenseMap<SDValue, unsigned> &VRBaseMap,
+ DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM);
/// EmitNode - Generate machine code for a node and needed dependencies.
///
void EmitNode(SDNode *Node, bool IsClone, bool IsCloned,
DenseMap<SDValue, unsigned> &VRBaseMap,
- DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM);
+ DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) {
+ if (Node->isMachineOpcode())
+ EmitMachineNode(Node, IsClone, IsCloned, VRBaseMap, EM);
+ else
+ EmitSpecialNode(Node, IsClone, IsCloned, VRBaseMap);
+ }
/// getBlock - Return the current basic block.
MachineBasicBlock *getBlock() { return MBB; }
@@ -124,6 +127,13 @@ public:
/// InstrEmitter - Construct an InstrEmitter and set it to start inserting
/// at the given position in the given block.
InstrEmitter(MachineBasicBlock *mbb, MachineBasicBlock::iterator insertpos);
+
+private:
+ void EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
+ DenseMap<SDValue, unsigned> &VRBaseMap,
+ DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM);
+ void EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
+ DenseMap<SDValue, unsigned> &VRBaseMap);
};
}
diff --git a/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h b/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h
index dbbd753..7638ea2 100644
--- a/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h
+++ b/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h
@@ -47,10 +47,12 @@ private:
uint64_t Offset;
DebugLoc DL;
unsigned Order;
+ bool Invalid;
public:
// Constructor for non-constants.
SDDbgValue(MDNode *mdP, SDNode *N, unsigned R, uint64_t off, DebugLoc dl,
- unsigned O) : mdPtr(mdP), Offset(off), DL(dl), Order(O) {
+ unsigned O) : mdPtr(mdP), Offset(off), DL(dl), Order(O),
+ Invalid(false) {
kind = SDNODE;
u.s.Node = N;
u.s.ResNo = R;
@@ -58,14 +60,14 @@ public:
// Constructor for constants.
SDDbgValue(MDNode *mdP, Value *C, uint64_t off, DebugLoc dl, unsigned O) :
- mdPtr(mdP), Offset(off), DL(dl), Order(O) {
+ mdPtr(mdP), Offset(off), DL(dl), Order(O), Invalid(false) {
kind = CONST;
u.Const = C;
}
// Constructor for frame indices.
SDDbgValue(MDNode *mdP, unsigned FI, uint64_t off, DebugLoc dl, unsigned O) :
- mdPtr(mdP), Offset(off), DL(dl), Order(O) {
+ mdPtr(mdP), Offset(off), DL(dl), Order(O), Invalid(false) {
kind = FRAMEIX;
u.FrameIx = FI;
}
@@ -97,6 +99,12 @@ public:
// Returns the SDNodeOrder. This is the order of the preceding node in the
// input.
unsigned getOrder() { return Order; }
+
+ // setIsInvalidated / isInvalidated - Setter / getter of the "Invalidated"
+ // property. A SDDbgValue is invalid if the SDNode that produces the value is
+ // deleted.
+ void setIsInvalidated() { Invalid = true; }
+ bool isInvalidated() { return Invalid; }
};
} // end llvm namespace
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index c13565a..e7ab2f0 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -23,6 +23,7 @@
#include "llvm/Target/TargetSubtarget.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Support/Debug.h"
@@ -407,19 +408,65 @@ void ScheduleDAGSDNodes::dumpNode(const SUnit *SU) const {
}
}
+namespace {
+ struct OrderSorter {
+ bool operator()(const std::pair<unsigned, MachineInstr*> &A,
+ const std::pair<unsigned, MachineInstr*> &B) {
+ return A.first < B.first;
+ }
+ };
+}
+
+// ProcessSourceNode - Process nodes with source order numbers. These are added
+// to a vector which EmitSchedule use to determine how to insert dbg_value
+// instructions in the right order.
+static void ProcessSourceNode(SDNode *N, SelectionDAG *DAG,
+ InstrEmitter &Emitter,
+ DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM,
+ DenseMap<SDValue, unsigned> &VRBaseMap,
+ SmallVector<std::pair<unsigned, MachineInstr*>, 32> &Orders,
+ SmallSet<unsigned, 8> &Seen) {
+ unsigned Order = DAG->GetOrdering(N);
+ if (!Order || !Seen.insert(Order))
+ return;
+
+ MachineBasicBlock *BB = Emitter.getBlock();
+ if (BB->empty() || BB->back().isPHI()) {
+ // Did not insert any instruction.
+ Orders.push_back(std::make_pair(Order, (MachineInstr*)0));
+ return;
+ }
+
+ Orders.push_back(std::make_pair(Order, &BB->back()));
+ if (!N->getHasDebugValue())
+ return;
+ // Opportunistically insert immediate dbg_value uses, i.e. those with source
+ // order number right after the N.
+ MachineBasicBlock::iterator InsertPos = Emitter.getInsertPos();
+ SmallVector<SDDbgValue*,2> &DVs = DAG->GetDbgValues(N);
+ for (unsigned i = 0, e = DVs.size(); i != e; ++i) {
+ if (DVs[i]->isInvalidated())
+ continue;
+ unsigned DVOrder = DVs[i]->getOrder();
+ if (DVOrder == ++Order) {
+ MachineInstr *DbgMI = Emitter.EmitDbgValue(DVs[i], BB, VRBaseMap, EM);
+ Orders.push_back(std::make_pair(DVOrder, DbgMI));
+ BB->insert(InsertPos, DbgMI);
+ DVs[i]->setIsInvalidated();
+ }
+ }
+}
+
+
/// EmitSchedule - Emit the machine code in scheduled order.
MachineBasicBlock *ScheduleDAGSDNodes::
EmitSchedule(DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) {
InstrEmitter Emitter(BB, InsertPos);
DenseMap<SDValue, unsigned> VRBaseMap;
DenseMap<SUnit*, unsigned> CopyVRBaseMap;
-
- // For now, any constant debug info nodes go at the beginning.
- for (SDDbgInfo::ConstDbgIterator I = DAG->DbgConstBegin(),
- E = DAG->DbgConstEnd(); I!=E; I++) {
- Emitter.EmitDbgValue(*I, EM);
- delete *I;
- }
+ SmallVector<std::pair<unsigned, MachineInstr*>, 32> Orders;
+ SmallSet<unsigned, 8> Seen;
+ bool HasDbg = DAG->hasDebugValues();
for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
SUnit *SU = Sequence[i];
@@ -442,22 +489,80 @@ EmitSchedule(DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) {
N = N->getFlaggedNode())
FlaggedNodes.push_back(N);
while (!FlaggedNodes.empty()) {
+ SDNode *N = FlaggedNodes.back();
Emitter.EmitNode(FlaggedNodes.back(), SU->OrigNode != SU, SU->isCloned,
VRBaseMap, EM);
- if (FlaggedNodes.back()->getHasDebugValue())
- if (SDDbgValue *sd = DAG->GetDbgInfo(FlaggedNodes.back())) {
- Emitter.EmitDbgValue(FlaggedNodes.back(), VRBaseMap, sd);
- delete sd;
- }
+ // Remember the the source order of the inserted instruction.
+ if (HasDbg)
+ ProcessSourceNode(N, DAG, Emitter, EM, VRBaseMap, Orders, Seen);
FlaggedNodes.pop_back();
}
Emitter.EmitNode(SU->getNode(), SU->OrigNode != SU, SU->isCloned,
VRBaseMap, EM);
- if (SU->getNode()->getHasDebugValue())
- if (SDDbgValue *sd = DAG->GetDbgInfo(SU->getNode())) {
- Emitter.EmitDbgValue(SU->getNode(), VRBaseMap, sd);
- delete sd;
+ // Remember the the source order of the inserted instruction.
+ if (HasDbg)
+ ProcessSourceNode(SU->getNode(), DAG, Emitter, EM, VRBaseMap, Orders,
+ Seen);
+ }
+
+ // Insert all the dbg_value which have not already been inserted in source
+ // order sequence.
+ if (HasDbg) {
+ MachineBasicBlock::iterator BBBegin = BB->empty() ? BB->end() : BB->begin();
+ while (BBBegin != BB->end() && BBBegin->isPHI())
+ ++BBBegin;
+
+ // Sort the source order instructions and use the order to insert debug
+ // values.
+ std::sort(Orders.begin(), Orders.end(), OrderSorter());
+
+ SDDbgInfo::DbgIterator DI = DAG->DbgBegin();
+ SDDbgInfo::DbgIterator DE = DAG->DbgEnd();
+ // Now emit the rest according to source order.
+ unsigned LastOrder = 0;
+ MachineInstr *LastMI = 0;
+ for (unsigned i = 0, e = Orders.size(); i != e && DI != DE; ++i) {
+ unsigned Order = Orders[i].first;
+ MachineInstr *MI = Orders[i].second;
+ // Insert all SDDbgValue's whose order(s) are before "Order".
+ if (!MI)
+ continue;
+ MachineBasicBlock *MIBB = MI->getParent();
+#ifndef NDEBUG
+ unsigned LastDIOrder = 0;
+#endif
+ for (; DI != DE &&
+ (*DI)->getOrder() >= LastOrder && (*DI)->getOrder() < Order; ++DI) {
+#ifndef NDEBUG
+ assert((*DI)->getOrder() >= LastDIOrder &&
+ "SDDbgValue nodes must be in source order!");
+ LastDIOrder = (*DI)->getOrder();
+#endif
+ if ((*DI)->isInvalidated())
+ continue;
+ MachineInstr *DbgMI = Emitter.EmitDbgValue(*DI, MIBB, VRBaseMap, EM);
+ if (!LastOrder)
+ // Insert to start of the BB (after PHIs).
+ BB->insert(BBBegin, DbgMI);
+ else {
+ MachineBasicBlock::iterator Pos = MI;
+ MIBB->insert(llvm::next(Pos), DbgMI);
+ }
}
+ LastOrder = Order;
+ LastMI = MI;
+ }
+ // Add trailing DbgValue's before the terminator. FIXME: May want to add
+ // some of them before one or more conditional branches?
+ while (DI != DE) {
+ MachineBasicBlock *InsertBB = Emitter.getBlock();
+ MachineBasicBlock::iterator Pos= Emitter.getBlock()->getFirstTerminator();
+ if (!(*DI)->isInvalidated()) {
+ MachineInstr *DbgMI= Emitter.EmitDbgValue(*DI, InsertBB, VRBaseMap, EM);
+ InsertBB->insert(Pos, DbgMI);
+ }
+ ++DI;
+ }
}
BB = Emitter.getBlock();
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index ed9146d..0ba65ab 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -598,8 +598,10 @@ void SelectionDAG::DeallocateNode(SDNode *N) {
// Remove the ordering of this node.
Ordering->remove(N);
- // And its entry in the debug info table, if any.
- DbgInfo->remove(N);
+ // If any of the SDDbgValue nodes refer to this SDNode, invalidate them.
+ SmallVector<SDDbgValue*, 2> &DbgVals = DbgInfo->getSDDbgValues(N);
+ for (unsigned i = 0, e = DbgVals.size(); i != e; ++i)
+ DbgVals[i]->setIsInvalidated();
}
/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
@@ -811,6 +813,7 @@ void SelectionDAG::init(MachineFunction &mf, MachineModuleInfo *mmi,
SelectionDAG::~SelectionDAG() {
allnodes_clear();
delete Ordering;
+ DbgInfo->clear();
delete DbgInfo;
}
@@ -839,6 +842,7 @@ void SelectionDAG::clear() {
Root = getEntryNode();
delete Ordering;
Ordering = new SDNodeOrdering();
+ DbgInfo->clear();
delete DbgInfo;
DbgInfo = new SDDbgInfo();
}
@@ -3128,11 +3132,17 @@ static SDValue getMemsetStringVal(EVT VT, DebugLoc dl, SelectionDAG &DAG,
if (Str.empty()) {
if (VT.isInteger())
return DAG.getConstant(0, VT);
- unsigned NumElts = VT.getVectorNumElements();
- MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
- return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
- DAG.getConstant(0,
- EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts)));
+ else if (VT.getSimpleVT().SimpleTy == MVT::f32 ||
+ VT.getSimpleVT().SimpleTy == MVT::f64)
+ return DAG.getConstantFP(0.0, VT);
+ else if (VT.isVector()) {
+ unsigned NumElts = VT.getVectorNumElements();
+ MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
+ return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
+ DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
+ EltVT, NumElts)));
+ } else
+ llvm_unreachable("Expected type!");
}
assert(!VT.isVector() && "Can't handle vector type here!");
@@ -3180,51 +3190,33 @@ static bool isMemSrcFromString(SDValue Src, std::string &Str) {
return false;
}
-/// MeetsMaxMemopRequirement - Determines if the number of memory ops required
-/// to replace the memset / memcpy is below the threshold. It also returns the
-/// types of the sequence of memory ops to perform memset / memcpy.
-static
-bool MeetsMaxMemopRequirement(std::vector<EVT> &MemOps,
- SDValue Dst, SDValue Src,
- unsigned Limit, uint64_t Size, unsigned &Align,
- std::string &Str, bool &isSrcStr,
- SelectionDAG &DAG,
- const TargetLowering &TLI) {
- isSrcStr = isMemSrcFromString(Src, Str);
- bool isSrcConst = isa<ConstantSDNode>(Src);
- EVT VT = TLI.getOptimalMemOpType(Size, Align, isSrcConst, isSrcStr, DAG);
- bool AllowUnalign = TLI.allowsUnalignedMemoryAccesses(VT);
- if (VT != MVT::Other) {
- const Type *Ty = VT.getTypeForEVT(*DAG.getContext());
- unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty);
- // If source is a string constant, this will require an unaligned load.
- if (NewAlign > Align && (isSrcConst || AllowUnalign)) {
- if (Dst.getOpcode() != ISD::FrameIndex) {
- // Can't change destination alignment. It requires a unaligned store.
- if (AllowUnalign)
- VT = MVT::Other;
- } else {
- int FI = cast<FrameIndexSDNode>(Dst)->getIndex();
- MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
- if (MFI->isFixedObjectIndex(FI)) {
- // Can't change destination alignment. It requires a unaligned store.
- if (AllowUnalign)
- VT = MVT::Other;
- } else {
- // Give the stack frame object a larger alignment if needed.
- if (MFI->getObjectAlignment(FI) < NewAlign)
- MFI->setObjectAlignment(FI, NewAlign);
- Align = NewAlign;
- }
- }
- }
- }
+/// FindOptimalMemOpLowering - Determines the optimial series memory ops
+/// to replace the memset / memcpy. Return true if the number of memory ops
+/// is below the threshold. It returns the types of the sequence of
+/// memory ops to perform memset / memcpy by reference.
+static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
+ unsigned Limit, uint64_t Size,
+ unsigned DstAlign, unsigned SrcAlign,
+ bool SafeToUseFP,
+ SelectionDAG &DAG,
+ const TargetLowering &TLI) {
+ assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
+ "Expecting memcpy / memset source to meet alignment requirement!");
+ // If 'SrcAlign' is zero, that means the memory operation does not need load
+ // the value, i.e. memset or memcpy from constant string. Otherwise, it's
+ // the inferred alignment of the source. 'DstAlign', on the other hand, is the
+ // specified alignment of the memory operation. If it is zero, that means
+ // it's possible to change the alignment of the destination.
+ EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, SafeToUseFP, DAG);
if (VT == MVT::Other) {
- if (TLI.allowsUnalignedMemoryAccesses(MVT::i64)) {
+ VT = TLI.getPointerTy();
+ const Type *Ty = VT.getTypeForEVT(*DAG.getContext());
+ if (DstAlign >= TLI.getTargetData()->getABITypeAlignment(Ty) ||
+ TLI.allowsUnalignedMemoryAccesses(VT)) {
VT = MVT::i64;
} else {
- switch (Align & 7) {
+ switch (DstAlign & 7) {
case 0: VT = MVT::i64; break;
case 4: VT = MVT::i32; break;
case 2: VT = MVT::i16; break;
@@ -3246,7 +3238,7 @@ bool MeetsMaxMemopRequirement(std::vector<EVT> &MemOps,
unsigned VTSize = VT.getSizeInBits() / 8;
while (VTSize > Size) {
// For now, only use non-vector load / store's for the left-over pieces.
- if (VT.isVector()) {
+ if (VT.isVector() || VT.isFloatingPoint()) {
VT = MVT::i64;
while (!TLI.isTypeLegal(VT))
VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
@@ -3269,11 +3261,11 @@ bool MeetsMaxMemopRequirement(std::vector<EVT> &MemOps,
}
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
- SDValue Chain, SDValue Dst,
- SDValue Src, uint64_t Size,
- unsigned Align, bool AlwaysInline,
- const Value *DstSV, uint64_t DstSVOff,
- const Value *SrcSV, uint64_t SrcSVOff){
+ SDValue Chain, SDValue Dst,
+ SDValue Src, uint64_t Size,
+ unsigned Align, bool AlwaysInline,
+ const Value *DstSV, uint64_t DstSVOff,
+ const Value *SrcSV, uint64_t SrcSVOff) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
// Expand memcpy to a series of load and store ops if the size operand falls
@@ -3282,15 +3274,33 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
uint64_t Limit = -1ULL;
if (!AlwaysInline)
Limit = TLI.getMaxStoresPerMemcpy();
- unsigned DstAlign = Align; // Destination alignment can change.
+ bool DstAlignCanChange = false;
+ MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
+ FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
+ if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
+ DstAlignCanChange = true;
+ unsigned SrcAlign = DAG.InferPtrAlignment(Src);
+ if (Align > SrcAlign)
+ SrcAlign = Align;
std::string Str;
- bool CopyFromStr;
- if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, Limit, Size, DstAlign,
- Str, CopyFromStr, DAG, TLI))
+ bool CopyFromStr = isMemSrcFromString(Src, Str);
+ bool isZeroStr = CopyFromStr && Str.empty();
+ if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
+ (DstAlignCanChange ? 0 : Align),
+ (isZeroStr ? 0 : SrcAlign), true, DAG, TLI))
return SDValue();
+ if (DstAlignCanChange) {
+ const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
+ unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty);
+ if (NewAlign > Align) {
+ // Give the stack frame object a larger alignment if needed.
+ if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
+ MFI->setObjectAlignment(FI->getIndex(), NewAlign);
+ Align = NewAlign;
+ }
+ }
- bool isZeroStr = CopyFromStr && Str.empty();
SmallVector<SDValue, 8> OutChains;
unsigned NumMemOps = MemOps.size();
uint64_t SrcOff = 0, DstOff = 0;
@@ -3299,16 +3309,17 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
unsigned VTSize = VT.getSizeInBits() / 8;
SDValue Value, Store;
- if (CopyFromStr && (isZeroStr || !VT.isVector())) {
+ if (CopyFromStr &&
+ (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
// It's unlikely a store of a vector immediate can be done in a single
// instruction. It would require a load from a constantpool first.
- // We also handle store a vector with all zero's.
+ // We only handle zero vectors here.
// FIXME: Handle other cases where store of vector immediate is done in
// a single instruction.
Value = getMemsetStringVal(VT, dl, DAG, TLI, Str, SrcOff);
Store = DAG.getStore(Chain, dl, Value,
getMemBasePlusOffset(Dst, DstOff, DAG),
- DstSV, DstSVOff + DstOff, false, false, DstAlign);
+ DstSV, DstSVOff + DstOff, false, false, Align);
} else {
// The type might not be legal for the target. This should only happen
// if the type is smaller than a legal type, as on PPC, so the right
@@ -3319,11 +3330,12 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
assert(NVT.bitsGE(VT));
Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
getMemBasePlusOffset(Src, SrcOff, DAG),
- SrcSV, SrcSVOff + SrcOff, VT, false, false, Align);
+ SrcSV, SrcSVOff + SrcOff, VT, false, false,
+ MinAlign(SrcAlign, SrcOff));
Store = DAG.getTruncStore(Chain, dl, Value,
getMemBasePlusOffset(Dst, DstOff, DAG),
DstSV, DstSVOff + DstOff, VT, false, false,
- DstAlign);
+ Align);
}
OutChains.push_back(Store);
SrcOff += VTSize;
@@ -3335,11 +3347,11 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
}
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
- SDValue Chain, SDValue Dst,
- SDValue Src, uint64_t Size,
- unsigned Align, bool AlwaysInline,
- const Value *DstSV, uint64_t DstSVOff,
- const Value *SrcSV, uint64_t SrcSVOff){
+ SDValue Chain, SDValue Dst,
+ SDValue Src, uint64_t Size,
+ unsigned Align,bool AlwaysInline,
+ const Value *DstSV, uint64_t DstSVOff,
+ const Value *SrcSV, uint64_t SrcSVOff) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
// Expand memmove to a series of load and store ops if the size operand falls
@@ -3348,15 +3360,32 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
uint64_t Limit = -1ULL;
if (!AlwaysInline)
Limit = TLI.getMaxStoresPerMemmove();
- unsigned DstAlign = Align; // Destination alignment can change.
- std::string Str;
- bool CopyFromStr;
- if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, Limit, Size, DstAlign,
- Str, CopyFromStr, DAG, TLI))
+ bool DstAlignCanChange = false;
+ MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
+ FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
+ if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
+ DstAlignCanChange = true;
+ unsigned SrcAlign = DAG.InferPtrAlignment(Src);
+ if (Align > SrcAlign)
+ SrcAlign = Align;
+
+ if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
+ (DstAlignCanChange ? 0 : Align),
+ SrcAlign, true, DAG, TLI))
return SDValue();
- uint64_t SrcOff = 0, DstOff = 0;
+ if (DstAlignCanChange) {
+ const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
+ unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty);
+ if (NewAlign > Align) {
+ // Give the stack frame object a larger alignment if needed.
+ if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
+ MFI->setObjectAlignment(FI->getIndex(), NewAlign);
+ Align = NewAlign;
+ }
+ }
+ uint64_t SrcOff = 0, DstOff = 0;
SmallVector<SDValue, 8> LoadValues;
SmallVector<SDValue, 8> LoadChains;
SmallVector<SDValue, 8> OutChains;
@@ -3368,7 +3397,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
Value = DAG.getLoad(VT, dl, Chain,
getMemBasePlusOffset(Src, SrcOff, DAG),
- SrcSV, SrcSVOff + SrcOff, false, false, Align);
+ SrcSV, SrcSVOff + SrcOff, false, false, SrcAlign);
LoadValues.push_back(Value);
LoadChains.push_back(Value.getValue(1));
SrcOff += VTSize;
@@ -3383,7 +3412,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
Store = DAG.getStore(Chain, dl, LoadValues[i],
getMemBasePlusOffset(Dst, DstOff, DAG),
- DstSV, DstSVOff + DstOff, false, false, DstAlign);
+ DstSV, DstSVOff + DstOff, false, false, Align);
OutChains.push_back(Store);
DstOff += VTSize;
}
@@ -3393,24 +3422,40 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
}
static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl,
- SDValue Chain, SDValue Dst,
- SDValue Src, uint64_t Size,
- unsigned Align,
- const Value *DstSV, uint64_t DstSVOff) {
+ SDValue Chain, SDValue Dst,
+ SDValue Src, uint64_t Size,
+ unsigned Align,
+ const Value *DstSV, uint64_t DstSVOff) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
// Expand memset to a series of load/store ops if the size operand
// falls below a certain threshold.
std::vector<EVT> MemOps;
- std::string Str;
- bool CopyFromStr;
- if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, TLI.getMaxStoresPerMemset(),
- Size, Align, Str, CopyFromStr, DAG, TLI))
+ bool DstAlignCanChange = false;
+ MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
+ FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
+ if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
+ DstAlignCanChange = true;
+ bool IsZero = isa<ConstantSDNode>(Src) &&
+ cast<ConstantSDNode>(Src)->isNullValue();
+ if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(),
+ Size, (DstAlignCanChange ? 0 : Align), 0,
+ IsZero, DAG, TLI))
return SDValue();
+ if (DstAlignCanChange) {
+ const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
+ unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty);
+ if (NewAlign > Align) {
+ // Give the stack frame object a larger alignment if needed.
+ if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
+ MFI->setObjectAlignment(FI->getIndex(), NewAlign);
+ Align = NewAlign;
+ }
+ }
+
SmallVector<SDValue, 8> OutChains;
uint64_t DstOff = 0;
-
unsigned NumMemOps = MemOps.size();
for (unsigned i = 0; i < NumMemOps; i++) {
EVT VT = MemOps[i];
@@ -3441,10 +3486,9 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst,
if (ConstantSize->isNullValue())
return Chain;
- SDValue Result =
- getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
- ConstantSize->getZExtValue(),
- Align, false, DstSV, DstSVOff, SrcSV, SrcSVOff);
+ SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
+ ConstantSize->getZExtValue(),Align,
+ false, DstSV, DstSVOff, SrcSV, SrcSVOff);
if (Result.getNode())
return Result;
}
@@ -4846,6 +4890,26 @@ SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
return NULL;
}
+/// getDbgValue - Creates a SDDbgValue node.
+///
+SDDbgValue *
+SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off,
+ DebugLoc DL, unsigned O) {
+ return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O);
+}
+
+SDDbgValue *
+SelectionDAG::getDbgValue(MDNode *MDPtr, Value *C, uint64_t Off,
+ DebugLoc DL, unsigned O) {
+ return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O);
+}
+
+SDDbgValue *
+SelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off,
+ DebugLoc DL, unsigned O) {
+ return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O);
+}
+
namespace {
/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
@@ -5241,24 +5305,12 @@ unsigned SelectionDAG::GetOrdering(const SDNode *SD) const {
return Ordering->getOrder(SD);
}
-/// AssignDbgInfo - Assign debug info to the SDNode.
-void SelectionDAG::AssignDbgInfo(SDNode* SD, SDDbgValue* db) {
- assert(SD && "Trying to assign dbg info to a null node!");
- DbgInfo->add(SD, db);
- SD->setHasDebugValue(true);
-}
-
-/// RememberDbgInfo - Remember debug info which is not assigned to an SDNode.
-void SelectionDAG::RememberDbgInfo(SDDbgValue* db) {
- DbgInfo->add(db);
-}
-
-/// GetDbgInfo - Get the debug info, if any, for the SDNode.
-SDDbgValue* SelectionDAG::GetDbgInfo(const SDNode *SD) {
- assert(SD && "Trying to get the order of a null node!");
- if (SD->getHasDebugValue())
- return DbgInfo->getSDDbgValue(SD);
- return 0;
+/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
+/// value is produced by SD.
+void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD) {
+ DbgInfo->add(DB, SD);
+ if (SD)
+ SD->setHasDebugValue(true);
}
//===----------------------------------------------------------------------===//
@@ -6094,8 +6146,20 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
// If this is a GlobalAddress + cst, return the alignment.
GlobalValue *GV;
int64_t GVOffset = 0;
- if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset))
- return MinAlign(GV->getAlignment(), GVOffset);
+ if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
+ // If GV has specified alignment, then use it. Otherwise, use the preferred
+ // alignment.
+ unsigned Align = GV->getAlignment();
+ if (!Align) {
+ if (GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV)) {
+ if (GVar->hasInitializer()) {
+ const TargetData *TD = TLI.getTargetData();
+ Align = TD->getPreferredAlignment(GVar);
+ }
+ }
+ }
+ return MinAlign(Align, GVOffset);
+ }
// If this is a direct reference to a stack slot, use information about the
// stack slot's alignment.
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 12096b9..922c6e8 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3800,7 +3800,7 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
int FI = SI->second;
if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo())
- if (MDNode *Dbg = DI.getMetadata("dbg"))
+ if (MDNode *Dbg = DI.getDbgMetadata())
MMI->setVariableDbgInfo(Variable, FI, Dbg);
return 0;
}
@@ -3824,22 +3824,19 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
// debug info exists.
++SDNodeOrder;
if (isa<ConstantInt>(V) || isa<ConstantFP>(V)) {
- SDDbgValue* dv = new SDDbgValue(Variable, V, Offset, dl, SDNodeOrder);
- DAG.RememberDbgInfo(dv);
+ DAG.AddDbgValue(DAG.getDbgValue(Variable, V, Offset, dl, SDNodeOrder));
} else {
SDValue &N = NodeMap[V];
- if (N.getNode()) {
- SDDbgValue *dv = new SDDbgValue(Variable, N.getNode(),
- N.getResNo(), Offset, dl, SDNodeOrder);
- DAG.AssignDbgInfo(N.getNode(), dv);
- } else {
+ if (N.getNode())
+ DAG.AddDbgValue(DAG.getDbgValue(Variable, N.getNode(),
+ N.getResNo(), Offset, dl, SDNodeOrder),
+ N.getNode());
+ else
// We may expand this to cover more cases. One case where we have no
// data available is an unreferenced parameter; we need this fallback.
- SDDbgValue* dv = new SDDbgValue(Variable,
+ DAG.AddDbgValue(DAG.getDbgValue(Variable,
UndefValue::get(V->getType()),
- Offset, dl, SDNodeOrder);
- DAG.RememberDbgInfo(dv);
- }
+ Offset, dl, SDNodeOrder));
}
// Build a debug info table entry.
@@ -3855,7 +3852,7 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
return 0; // VLAs.
int FI = SI->second;
if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo())
- if (MDNode *Dbg = DI.getMetadata("dbg"))
+ if (MDNode *Dbg = DI.getDbgMetadata())
MMI->setVariableDbgInfo(Variable, FI, Dbg);
return 0;
}
@@ -6054,8 +6051,10 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) {
}
if (!I->use_empty()) {
- SDValue Res = DAG.getMergeValues(&ArgValues[0], NumValues,
- SDB->getCurDebugLoc());
+ SDValue Res;
+ if (!ArgValues.empty())
+ Res = DAG.getMergeValues(&ArgValues[0], NumValues,
+ SDB->getCurDebugLoc());
SDB->setValue(I, Res);
// If this argument is live outside of the entry block, insert a copy from
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index cbbe431..ea96b21 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -61,6 +61,7 @@
using namespace llvm;
STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on");
+STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path");
static cl::opt<bool>
EnableFastISelVerbose("fast-isel-verbose", cl::Hidden,
@@ -365,23 +366,23 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
/// SetDebugLoc - Update MF's and SDB's DebugLocs if debug information is
/// attached with this instruction.
-static void SetDebugLoc(unsigned MDDbgKind, Instruction *I,
- SelectionDAGBuilder *SDB,
+static void SetDebugLoc(Instruction *I, SelectionDAGBuilder *SDB,
FastISel *FastIS, MachineFunction *MF) {
- if (MDNode *Dbg = I->getMetadata(MDDbgKind)) {
- DILocation DILoc(Dbg);
- DebugLoc Loc = ExtractDebugLocation(DILoc, MF->getDebugLocInfo());
+ MDNode *Dbg = I->getDbgMetadata();
+ if (Dbg == 0) return;
+
+ DILocation DILoc(Dbg);
+ DebugLoc Loc = ExtractDebugLocation(DILoc, MF->getDebugLocInfo());
- SDB->setCurDebugLoc(Loc);
+ SDB->setCurDebugLoc(Loc);
- if (FastIS)
- FastIS->setCurDebugLoc(Loc);
+ if (FastIS)
+ FastIS->setCurDebugLoc(Loc);
- // If the function doesn't have a default debug location yet, set
- // it. This is kind of a hack.
- if (MF->getDefaultDebugLoc().isUnknown())
- MF->setDefaultDebugLoc(Loc);
- }
+ // If the function doesn't have a default debug location yet, set
+ // it. This is kind of a hack.
+ if (MF->getDefaultDebugLoc().isUnknown())
+ MF->setDefaultDebugLoc(Loc);
}
/// ResetDebugLoc - Set MF's and SDB's DebugLocs to Unknown.
@@ -396,12 +397,11 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB,
BasicBlock::iterator End,
bool &HadTailCall) {
SDB->setCurrentBasicBlock(BB);
- unsigned MDDbgKind = LLVMBB->getContext().getMDKindID("dbg");
// Lower all of the non-terminator instructions. If a call is emitted
// as a tail call, cease emitting nodes for this block.
for (BasicBlock::iterator I = Begin; I != End && !SDB->HasTailCall; ++I) {
- SetDebugLoc(MDDbgKind, I, SDB, 0, MF);
+ SetDebugLoc(I, SDB, 0, MF);
if (!isa<TerminatorInst>(I)) {
SDB->visit(*I);
@@ -424,7 +424,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB,
HandlePHINodesInSuccessorBlocks(LLVMBB);
// Lower the terminator after the copies are emitted.
- SetDebugLoc(MDDbgKind, LLVMBB->getTerminator(), SDB, 0, MF);
+ SetDebugLoc(LLVMBB->getTerminator(), SDB, 0, MF);
SDB->visit(*LLVMBB->getTerminator());
ResetDebugLoc(SDB, 0);
}
@@ -842,9 +842,6 @@ void SelectionDAGISel::DoInstructionSelection() {
DEBUG(errs() << "===== Instruction selection ends:\n");
PostprocessISelDAG();
-
- // FIXME: This shouldn't be needed, remove it.
- CurDAG->RemoveDeadNodes();
}
@@ -865,8 +862,6 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn,
#endif
);
- unsigned MDDbgKind = Fn.getContext().getMDKindID("dbg");
-
// Iterate over all basic blocks in the function.
for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) {
BasicBlock *LLVMBB = &*I;
@@ -964,7 +959,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn,
break;
}
- SetDebugLoc(MDDbgKind, BI, SDB, FastIS, &MF);
+ SetDebugLoc(BI, SDB, FastIS, &MF);
// Try to select the instruction with FastISel.
if (FastIS->SelectInstruction(BI)) {
@@ -1592,8 +1587,9 @@ UpdateChainsAndFlags(SDNode *NodeToMatch, SDValue InputChain,
assert(ChainVal.getValueType() == MVT::Other && "Not a chain?");
CurDAG->ReplaceAllUsesOfValueWith(ChainVal, InputChain, &ISU);
- // If the node became dead, delete it.
- if (ChainNode->use_empty())
+ // If the node became dead and we haven't already seen it, delete it.
+ if (ChainNode->use_empty() &&
+ !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), ChainNode))
NowDeadNodes.push_back(ChainNode);
}
}
@@ -1614,8 +1610,9 @@ UpdateChainsAndFlags(SDNode *NodeToMatch, SDValue InputChain,
CurDAG->ReplaceAllUsesOfValueWith(SDValue(FRN, FRN->getNumValues()-1),
InputFlag, &ISU);
- // If the node became dead, delete it.
- if (FRN->use_empty())
+ // If the node became dead and we haven't already seen it, delete it.
+ if (FRN->use_empty() &&
+ !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), FRN))
NowDeadNodes.push_back(FRN);
}
}
@@ -1810,9 +1807,9 @@ MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
// It is possible we're using MorphNodeTo to replace a node with no
// normal results with one that has a normal result (or we could be
// adding a chain) and the input could have flags and chains as well.
- // In this case we need to shifting the operands down.
+ // In this case we need to shift the operands down.
// FIXME: This is a horrible hack and broken in obscure cases, no worse
- // than the old isel though. We should sink this into MorphNodeTo.
+ // than the old isel though.
int OldFlagResultNo = -1, OldChainResultNo = -1;
unsigned NTMNumResults = Node->getNumValues();
@@ -1888,7 +1885,9 @@ CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
ALWAYS_INLINE static bool
CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
SDNode *N) {
- return N->getOpcode() == MatcherTable[MatcherIndex++];
+ uint16_t Opc = MatcherTable[MatcherIndex++];
+ Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
+ return N->getOpcode() == Opc;
}
ALWAYS_INLINE static bool
@@ -2142,7 +2141,8 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
if (CaseSize == 0) break;
// Get the opcode, add the index to the table.
- unsigned Opc = MatcherTable[Idx++];
+ uint16_t Opc = MatcherTable[Idx++];
+ Opc |= (unsigned short)MatcherTable[Idx++] << 8;
if (Opc >= OpcodeOffset.size())
OpcodeOffset.resize((Opc+1)*2);
OpcodeOffset[Opc] = Idx;
@@ -2181,6 +2181,9 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
FailIndex = MatcherIndex+NumToSkip;
+ unsigned MatcherIndexOfPredicate = MatcherIndex;
+ (void)MatcherIndexOfPredicate; // silence warning.
+
// If we can't evaluate this predicate without pushing a scope (e.g. if
// it is a 'MoveParent') or if the predicate succeeds on this node, we
// push the scope and evaluate the full predicate chain.
@@ -2190,9 +2193,10 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
if (!Result)
break;
- DEBUG(errs() << " Skipped scope entry at index " << MatcherIndex
- << " continuing at " << FailIndex << "\n");
-
+ DEBUG(errs() << " Skipped scope entry (due to false predicate) at "
+ << "index " << MatcherIndexOfPredicate
+ << ", continuing at " << FailIndex << "\n");
+ ++NumDAGIselRetries;
// Otherwise, we know that this case of the Scope is guaranteed to fail,
// move to the next case.
@@ -2298,8 +2302,11 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
if (CaseSize == 0) break;
+ uint16_t Opc = MatcherTable[MatcherIndex++];
+ Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;
+
// If the opcode matches, then we will execute this case.
- if (CurNodeOpcode == MatcherTable[MatcherIndex++])
+ if (CurNodeOpcode == Opc)
break;
// Otherwise, skip over this case.
@@ -2428,6 +2435,35 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
continue;
}
+ case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0
+ case OPC_EmitMergeInputChains1_1: { // OPC_EmitMergeInputChains, 1, 1
+ // These are space-optimized forms of OPC_EmitMergeInputChains.
+ assert(InputChain.getNode() == 0 &&
+ "EmitMergeInputChains should be the first chain producing node");
+ assert(ChainNodesMatched.empty() &&
+ "Should only have one EmitMergeInputChains per match");
+
+ // Read all of the chained nodes.
+ unsigned RecNo = Opcode == OPC_EmitMergeInputChains1_1;
+ assert(RecNo < RecordedNodes.size() && "Invalid CheckSame");
+ ChainNodesMatched.push_back(RecordedNodes[RecNo].getNode());
+
+ // FIXME: What if other value results of the node have uses not matched
+ // by this pattern?
+ if (ChainNodesMatched.back() != NodeToMatch &&
+ !RecordedNodes[RecNo].hasOneUse()) {
+ ChainNodesMatched.clear();
+ break;
+ }
+
+ // Merge the input chains if they are not intra-pattern references.
+ InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);
+
+ if (InputChain.getNode() == 0)
+ break; // Failed to merge.
+ continue;
+ }
+
case OPC_EmitMergeInputChains: {
assert(InputChain.getNode() == 0 &&
"EmitMergeInputChains should be the first chain producing node");
@@ -2646,14 +2682,10 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
assert(ResSlot < RecordedNodes.size() && "Invalid CheckSame");
SDValue Res = RecordedNodes[ResSlot];
- // FIXME2: Eliminate this horrible hack by fixing the 'Gen' program
- // after (parallel) on input patterns are removed. This would also
- // allow us to stop encoding #results in OPC_CompleteMatch's table
- // entry.
- if (NodeToMatch->getNumValues() <= i ||
- NodeToMatch->getValueType(i) == MVT::Other ||
- NodeToMatch->getValueType(i) == MVT::Flag)
- break;
+ assert(i < NodeToMatch->getNumValues() &&
+ NodeToMatch->getValueType(i) != MVT::Other &&
+ NodeToMatch->getValueType(i) != MVT::Flag &&
+ "Invalid number of results to complete!");
assert((NodeToMatch->getValueType(i) == Res.getValueType() ||
NodeToMatch->getValueType(i) == MVT::iPTR ||
Res.getValueType() == MVT::iPTR ||
@@ -2685,6 +2717,7 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
// another child to try in the current 'Scope', otherwise pop it until we
// find a case to check.
DEBUG(errs() << " Match failed at index " << CurrentOpcodeIndex << "\n");
+ ++NumDAGIselRetries;
while (1) {
if (MatchScopes.empty()) {
CannotYetSelect(NodeToMatch);
diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index f7ef2d6..ea2ff2f 100644
--- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -186,11 +186,13 @@ static void InitLibcallNames(const char **Names) {
Names[RTLIB::FPROUND_PPCF128_F32] = "__trunctfsf2";
Names[RTLIB::FPROUND_F80_F64] = "__truncxfdf2";
Names[RTLIB::FPROUND_PPCF128_F64] = "__trunctfdf2";
- Names[RTLIB::FPTOSINT_F32_I8] = "__fixsfi8";
- Names[RTLIB::FPTOSINT_F32_I16] = "__fixsfi16";
+ Names[RTLIB::FPTOSINT_F32_I8] = "__fixsfqi";
+ Names[RTLIB::FPTOSINT_F32_I16] = "__fixsfhi";
Names[RTLIB::FPTOSINT_F32_I32] = "__fixsfsi";
Names[RTLIB::FPTOSINT_F32_I64] = "__fixsfdi";
Names[RTLIB::FPTOSINT_F32_I128] = "__fixsfti";
+ Names[RTLIB::FPTOSINT_F64_I8] = "__fixdfqi";
+ Names[RTLIB::FPTOSINT_F64_I16] = "__fixdfhi";
Names[RTLIB::FPTOSINT_F64_I32] = "__fixdfsi";
Names[RTLIB::FPTOSINT_F64_I64] = "__fixdfdi";
Names[RTLIB::FPTOSINT_F64_I128] = "__fixdfti";
@@ -200,11 +202,13 @@ static void InitLibcallNames(const char **Names) {
Names[RTLIB::FPTOSINT_PPCF128_I32] = "__fixtfsi";
Names[RTLIB::FPTOSINT_PPCF128_I64] = "__fixtfdi";
Names[RTLIB::FPTOSINT_PPCF128_I128] = "__fixtfti";
- Names[RTLIB::FPTOUINT_F32_I8] = "__fixunssfi8";
- Names[RTLIB::FPTOUINT_F32_I16] = "__fixunssfi16";
+ Names[RTLIB::FPTOUINT_F32_I8] = "__fixunssfqi";
+ Names[RTLIB::FPTOUINT_F32_I16] = "__fixunssfhi";
Names[RTLIB::FPTOUINT_F32_I32] = "__fixunssfsi";
Names[RTLIB::FPTOUINT_F32_I64] = "__fixunssfdi";
Names[RTLIB::FPTOUINT_F32_I128] = "__fixunssfti";
+ Names[RTLIB::FPTOUINT_F64_I8] = "__fixunsdfqi";
+ Names[RTLIB::FPTOUINT_F64_I16] = "__fixunsdfhi";
Names[RTLIB::FPTOUINT_F64_I32] = "__fixunsdfsi";
Names[RTLIB::FPTOUINT_F64_I64] = "__fixunsdfdi";
Names[RTLIB::FPTOUINT_F64_I128] = "__fixunsdfti";
@@ -314,6 +318,10 @@ RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) {
if (RetVT == MVT::i128)
return FPTOSINT_F32_I128;
} else if (OpVT == MVT::f64) {
+ if (RetVT == MVT::i8)
+ return FPTOSINT_F64_I8;
+ if (RetVT == MVT::i16)
+ return FPTOSINT_F64_I16;
if (RetVT == MVT::i32)
return FPTOSINT_F64_I32;
if (RetVT == MVT::i64)
@@ -353,6 +361,10 @@ RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) {
if (RetVT == MVT::i128)
return FPTOUINT_F32_I128;
} else if (OpVT == MVT::f64) {
+ if (RetVT == MVT::i8)
+ return FPTOUINT_F64_I8;
+ if (RetVT == MVT::i16)
+ return FPTOUINT_F64_I16;
if (RetVT == MVT::i32)
return FPTOUINT_F64_I32;
if (RetVT == MVT::i64)
@@ -475,7 +487,6 @@ TargetLowering::TargetLowering(TargetMachine &tm,TargetLoweringObjectFile *tlof)
memset(LoadExtActions, 0, sizeof(LoadExtActions));
memset(TruncStoreActions, 0, sizeof(TruncStoreActions));
memset(IndexedModeActions, 0, sizeof(IndexedModeActions));
- memset(ConvertActions, 0, sizeof(ConvertActions));
memset(CondCodeActions, 0, sizeof(CondCodeActions));
// Set default actions for various operations.
OpenPOWER on IntegriCloud