Vendor import of llvm trunk r161861:

http://llvm.org/svn/llvm-project/llvm/trunk@161861

37 files changed, 1527 insertions, 954 deletions

diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h
index 56a87f1..170a528 100644
--- a/include/llvm/CodeGen/AsmPrinter.h
+++ b/include/llvm/CodeGen/AsmPrinter.h
@@ -354,6 +354,13 @@ namespace llvm {
     void EmitLabelPlusOffset(const MCSymbol *Label, uint64_t Offset,
                                    unsigned Size) const;
 
+    /// EmitLabelReference - Emit something like ".long Label"
+    /// where the size in bytes of the directive is specified by Size and Label
+    /// specifies the label.
+    void EmitLabelReference(const MCSymbol *Label, unsigned Size) const {
+      EmitLabelPlusOffset(Label, 0, Size);
+    }
+
     //===------------------------------------------------------------------===//
     // Dwarf Emission Helper Routines
     //===------------------------------------------------------------------===//
diff --git a/include/llvm/CodeGen/DFAPacketizer.h b/include/llvm/CodeGen/DFAPacketizer.h
index ee1ed07..2d2db78 100644
--- a/include/llvm/CodeGen/DFAPacketizer.h
+++ b/include/llvm/CodeGen/DFAPacketizer.h
@@ -28,6 +28,7 @@
 
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/ADT/DenseMap.h"
+#include <map>
 
 namespace llvm {
 
@@ -36,7 +37,7 @@ class MachineInstr;
 class MachineLoopInfo;
 class MachineDominatorTree;
 class InstrItineraryData;
-class ScheduleDAGInstrs;
+class DefaultVLIWScheduler;
 class SUnit;
 
 class DFAPacketizer {
@@ -77,6 +78,8 @@ public:
   // reserveResources - Reserve the resources occupied by a machine
   // instruction and change the current state to reflect that change.
   void reserveResources(llvm::MachineInstr *MI);
+
+  const InstrItineraryData *getInstrItins() const { return InstrItins; }
 };
 
 // VLIWPacketizerList - Implements a simple VLIW packetizer using DFA. The
@@ -87,20 +90,21 @@ public:
 // and machine resource is marked as taken. If any dependency is found, a target
 // API call is made to prune the dependence.
 class VLIWPacketizerList {
+protected:
   const TargetMachine &TM;
   const MachineFunction &MF;
   const TargetInstrInfo *TII;
 
-  // Encapsulate data types not exposed to the target interface.
-  ScheduleDAGInstrs *SchedulerImpl;
+  // The VLIW Scheduler.
+  DefaultVLIWScheduler *VLIWScheduler;
 
-protected:
   // Vector of instructions assigned to the current packet.
   std::vector<MachineInstr*> CurrentPacketMIs;
   // DFA resource tracker.
   DFAPacketizer *ResourceTracker;
-  // Scheduling units.
-  std::vector<SUnit> SUnits;
+
+  // Generate MI -> SU map.
+  std::map<MachineInstr*, SUnit*> MIToSUnit;
 
 public:
   VLIWPacketizerList(
@@ -118,17 +122,32 @@ public:
   DFAPacketizer *getResourceTracker() {return ResourceTracker;}
 
   // addToPacket - Add MI to the current packet.
-  void addToPacket(MachineInstr *MI);
+  virtual MachineBasicBlock::iterator addToPacket(MachineInstr *MI) {
+    MachineBasicBlock::iterator MII = MI;
+    CurrentPacketMIs.push_back(MI);
+    ResourceTracker->reserveResources(MI);
+    return MII;
+  }
 
   // endPacket - End the current packet.
-  void endPacket(MachineBasicBlock *MBB, MachineInstr *I);
+  void endPacket(MachineBasicBlock *MBB, MachineInstr *MI);
+
+  // initPacketizerState - perform initialization before packetizing
+  // an instruction. This function is supposed to be overrided by
+  // the target dependent packetizer.
+  virtual void initPacketizerState(void) { return; }
 
   // ignorePseudoInstruction - Ignore bundling of pseudo instructions.
-  bool ignorePseudoInstruction(MachineInstr *I, MachineBasicBlock *MBB);
+  virtual bool ignorePseudoInstruction(MachineInstr *I,
+                                       MachineBasicBlock *MBB) {
+    return false;
+  }
 
-  // isSoloInstruction - return true if instruction I must end previous
-  // packet.
-  bool isSoloInstruction(MachineInstr *I);
+  // isSoloInstruction - return true if instruction MI can not be packetized
+  // with any other instruction, which means that MI itself is a packet.
+  virtual bool isSoloInstruction(MachineInstr *MI) {
+    return true;
+  }
 
   // isLegalToPacketizeTogether - Is it legal to packetize SUI and SUJ
   // together.
@@ -141,6 +160,7 @@ public:
   virtual bool isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) {
     return false;
   }
+
 };
 }
 
diff --git a/include/llvm/CodeGen/EdgeBundles.h b/include/llvm/CodeGen/EdgeBundles.h
index a1d29b1..e8a4a2d 100644
--- a/include/llvm/CodeGen/EdgeBundles.h
+++ b/include/llvm/CodeGen/EdgeBundles.h
@@ -46,7 +46,7 @@ public:
   unsigned getNumBundles() const { return EC.getNumClasses(); }
 
   /// getBlocks - Return an array of blocks that are connected to Bundle.
-  ArrayRef<unsigned> getBlocks(unsigned Bundle) { return Blocks[Bundle]; }
+  ArrayRef<unsigned> getBlocks(unsigned Bundle) const { return Blocks[Bundle]; }
 
   /// getMachineFunction - Return the last machine function computed.
   const MachineFunction *getMachineFunction() const { return MF; }
diff --git a/include/llvm/CodeGen/FastISel.h b/include/llvm/CodeGen/FastISel.h
index e57c8b1..7cb9695 100644
--- a/include/llvm/CodeGen/FastISel.h
+++ b/include/llvm/CodeGen/FastISel.h
@@ -34,6 +34,7 @@ class MachineFrameInfo;
 class MachineRegisterInfo;
 class TargetData;
 class TargetInstrInfo;
+class TargetLibraryInfo;
 class TargetLowering;
 class TargetMachine;
 class TargetRegisterClass;
@@ -57,6 +58,7 @@ protected:
   const TargetInstrInfo &TII;
   const TargetLowering &TLI;
   const TargetRegisterInfo &TRI;
+  const TargetLibraryInfo *LibInfo;
 
   /// The position of the last instruction for materializing constants
   /// for use in the current block. It resets to EmitStartPt when it
@@ -144,7 +146,8 @@ public:
   virtual ~FastISel();
 
 protected:
-  explicit FastISel(FunctionLoweringInfo &funcInfo);
+  explicit FastISel(FunctionLoweringInfo &funcInfo,
+                    const TargetLibraryInfo *libInfo);
 
   /// TargetSelectInstruction - This method is called by target-independent
   /// code when the normal FastISel process fails to select an instruction.
@@ -299,6 +302,15 @@ protected:
                             unsigned Op1, bool Op1IsKill,
                             uint64_t Imm);
 
+  /// FastEmitInst_rrii - Emit a MachineInstr with two register operands,
+  /// two immediates operands, and a result register in the given register
+  /// class.
+  unsigned FastEmitInst_rrii(unsigned MachineInstOpcode,
+                             const TargetRegisterClass *RC,
+                             unsigned Op0, bool Op0IsKill,
+                             unsigned Op1, bool Op1IsKill,
+                             uint64_t Imm1, uint64_t Imm2);
+
   /// FastEmitInst_i - Emit a MachineInstr with a single immediate
   /// operand, and a result register in the given register class.
   unsigned FastEmitInst_i(unsigned MachineInstrOpcode,
diff --git a/include/llvm/CodeGen/GCMetadata.h b/include/llvm/CodeGen/GCMetadata.h
index 45469ed..20e33f7 100644
--- a/include/llvm/CodeGen/GCMetadata.h
+++ b/include/llvm/CodeGen/GCMetadata.h
@@ -48,18 +48,18 @@ namespace llvm {
     /// PointKind - The type of a collector-safe point.
     ///
     enum PointKind {
-      Loop,    //< Instr is a loop (backwards branch).
-      Return,  //< Instr is a return instruction.
-      PreCall, //< Instr is a call instruction.
-      PostCall //< Instr is the return address of a call.
+      Loop,    ///< Instr is a loop (backwards branch).
+      Return,  ///< Instr is a return instruction.
+      PreCall, ///< Instr is a call instruction.
+      PostCall ///< Instr is the return address of a call.
     };
   }
 
   /// GCPoint - Metadata for a collector-safe point in machine code.
   ///
   struct GCPoint {
-    GC::PointKind Kind; //< The kind of the safe point.
-    MCSymbol *Label;    //< A label.
+    GC::PointKind Kind; ///< The kind of the safe point.
+    MCSymbol *Label;    ///< A label.
     DebugLoc Loc;
 
     GCPoint(GC::PointKind K, MCSymbol *L, DebugLoc DL)
@@ -69,9 +69,10 @@ namespace llvm {
   /// GCRoot - Metadata for a pointer to an object managed by the garbage
   /// collector.
   struct GCRoot {
-    int Num;            //< Usually a frame index.
-    int StackOffset;    //< Offset from the stack pointer.
-    const Constant *Metadata;//< Metadata straight from the call to llvm.gcroot.
+    int Num;            ///< Usually a frame index.
+    int StackOffset;    ///< Offset from the stack pointer.
+    const Constant *Metadata; ///< Metadata straight from the call
+                              ///< to llvm.gcroot.
 
     GCRoot(int N, const Constant *MD) : Num(N), StackOffset(-1), Metadata(MD) {}
   };
diff --git a/include/llvm/CodeGen/GCStrategy.h b/include/llvm/CodeGen/GCStrategy.h
index 1cbd36a..dfc26d7 100644
--- a/include/llvm/CodeGen/GCStrategy.h
+++ b/include/llvm/CodeGen/GCStrategy.h
@@ -65,14 +65,14 @@ namespace llvm {
     list_type Functions;
     
   protected:
-    unsigned NeededSafePoints; //< Bitmask of required safe points.
-    bool CustomReadBarriers;   //< Default is to insert loads.
-    bool CustomWriteBarriers;  //< Default is to insert stores.
-    bool CustomRoots;          //< Default is to pass through to backend.
-    bool CustomSafePoints;     //< Default is to use NeededSafePoints
-                               //  to find safe points.
-    bool InitRoots;            //< If set, roots are nulled during lowering.
-    bool UsesMetadata;         //< If set, backend must emit metadata tables.
+    unsigned NeededSafePoints; ///< Bitmask of required safe points.
+    bool CustomReadBarriers;   ///< Default is to insert loads.
+    bool CustomWriteBarriers;  ///< Default is to insert stores.
+    bool CustomRoots;          ///< Default is to pass through to backend.
+    bool CustomSafePoints;     ///< Default is to use NeededSafePoints
+                               ///< to find safe points.
+    bool InitRoots;            ///< If set, roots are nulled during lowering.
+    bool UsesMetadata;         ///< If set, backend must emit metadata tables.
     
   public:
     GCStrategy();
diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h
index ab8ab5d..f387bd5 100644
--- a/include/llvm/CodeGen/ISDOpcodes.h
+++ b/include/llvm/CodeGen/ISDOpcodes.h
@@ -37,87 +37,87 @@ namespace ISD {
   /// and getMachineOpcode() member functions of SDNode.
   ///
   enum NodeType {
-    // DELETED_NODE - This is an illegal value that is used to catch
-    // errors.  This opcode is not a legal opcode for any node.
+    /// DELETED_NODE - This is an illegal value that is used to catch
+    /// errors.  This opcode is not a legal opcode for any node.
     DELETED_NODE,
 
-    // EntryToken - This is the marker used to indicate the start of the region.
+    /// EntryToken - This is the marker used to indicate the start of a region.
     EntryToken,
 
-    // TokenFactor - This node takes multiple tokens as input and produces a
-    // single token result.  This is used to represent the fact that the operand
-    // operators are independent of each other.
+    /// TokenFactor - This node takes multiple tokens as input and produces a
+    /// single token result. This is used to represent the fact that the operand
+    /// operators are independent of each other.
     TokenFactor,
 
-    // AssertSext, AssertZext - These nodes record if a register contains a
-    // value that has already been zero or sign extended from a narrower type.
-    // These nodes take two operands.  The first is the node that has already
-    // been extended, and the second is a value type node indicating the width
-    // of the extension
+    /// AssertSext, AssertZext - These nodes record if a register contains a
+    /// value that has already been zero or sign extended from a narrower type.
+    /// These nodes take two operands.  The first is the node that has already
+    /// been extended, and the second is a value type node indicating the width
+    /// of the extension
     AssertSext, AssertZext,
 
-    // Various leaf nodes.
+    /// Various leaf nodes.
     BasicBlock, VALUETYPE, CONDCODE, Register, RegisterMask,
     Constant, ConstantFP,
     GlobalAddress, GlobalTLSAddress, FrameIndex,
     JumpTable, ConstantPool, ExternalSymbol, BlockAddress,
 
-    // The address of the GOT
+    /// The address of the GOT
     GLOBAL_OFFSET_TABLE,
 
-    // FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and
-    // llvm.returnaddress on the DAG.  These nodes take one operand, the index
-    // of the frame or return address to return.  An index of zero corresponds
-    // to the current function's frame or return address, an index of one to the
-    // parent's frame or return address, and so on.
+    /// FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and
+    /// llvm.returnaddress on the DAG.  These nodes take one operand, the index
+    /// of the frame or return address to return.  An index of zero corresponds
+    /// to the current function's frame or return address, an index of one to
+    /// the parent's frame or return address, and so on.
     FRAMEADDR, RETURNADDR,
 
-    // FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to
-    // first (possible) on-stack argument. This is needed for correct stack
-    // adjustment during unwind.
+    /// FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to
+    /// first (possible) on-stack argument. This is needed for correct stack
+    /// adjustment during unwind.
     FRAME_TO_ARGS_OFFSET,
 
-    // RESULT, OUTCHAIN = EXCEPTIONADDR(INCHAIN) - This node represents the
-    // address of the exception block on entry to an landing pad block.
+    /// RESULT, OUTCHAIN = EXCEPTIONADDR(INCHAIN) - This node represents the
+    /// address of the exception block on entry to an landing pad block.
     EXCEPTIONADDR,
 
-    // RESULT, OUTCHAIN = LSDAADDR(INCHAIN) - This node represents the
-    // address of the Language Specific Data Area for the enclosing function.
+    /// RESULT, OUTCHAIN = LSDAADDR(INCHAIN) - This node represents the
+    /// address of the Language Specific Data Area for the enclosing function.
     LSDAADDR,
 
-    // RESULT, OUTCHAIN = EHSELECTION(INCHAIN, EXCEPTION) - This node represents
-    // the selection index of the exception thrown.
+    /// RESULT, OUTCHAIN = EHSELECTION(INCHAIN, EXCEPTION) - This node
+    /// represents the selection index of the exception thrown.
     EHSELECTION,
 
-    // OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents
-    // 'eh_return' gcc dwarf builtin, which is used to return from
-    // exception. The general meaning is: adjust stack by OFFSET and pass
-    // execution to HANDLER. Many platform-related details also :)
+    /// OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents
+    /// 'eh_return' gcc dwarf builtin, which is used to return from
+    /// exception. The general meaning is: adjust stack by OFFSET and pass
+    /// execution to HANDLER. Many platform-related details also :)
     EH_RETURN,
 
-    // RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer)
-    // This corresponds to the eh.sjlj.setjmp intrinsic.
-    // It takes an input chain and a pointer to the jump buffer as inputs
-    // and returns an outchain.
+    /// RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer)
+    /// This corresponds to the eh.sjlj.setjmp intrinsic.
+    /// It takes an input chain and a pointer to the jump buffer as inputs
+    /// and returns an outchain.
     EH_SJLJ_SETJMP,
 
-    // OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer)
-    // This corresponds to the eh.sjlj.longjmp intrinsic.
-    // It takes an input chain and a pointer to the jump buffer as inputs
-    // and returns an outchain.
+    /// OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer)
+    /// This corresponds to the eh.sjlj.longjmp intrinsic.
+    /// It takes an input chain and a pointer to the jump buffer as inputs
+    /// and returns an outchain.
     EH_SJLJ_LONGJMP,
 
-    // TargetConstant* - Like Constant*, but the DAG does not do any folding,
-    // simplification, or lowering of the constant. They are used for constants
-    // which are known to fit in the immediate fields of their users, or for
-    // carrying magic numbers which are not values which need to be materialized
-    // in registers.
+    /// TargetConstant* - Like Constant*, but the DAG does not do any folding,
+    /// simplification, or lowering of the constant. They are used for constants
+    /// which are known to fit in the immediate fields of their users, or for
+    /// carrying magic numbers which are not values which need to be
+    /// materialized in registers.
     TargetConstant,
     TargetConstantFP,
 
-    // TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or
-    // anything else with this node, and this is valid in the target-specific
-    // dag, turning into a GlobalAddress operand.
+    /// TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or
+    /// anything else with this node, and this is valid in the target-specific
+    /// dag, turning into a GlobalAddress operand.
     TargetGlobalAddress,
     TargetGlobalTLSAddress,
     TargetFrameIndex,
@@ -126,6 +126,11 @@ namespace ISD {
     TargetExternalSymbol,
     TargetBlockAddress,
 
+    /// TargetIndex - Like a constant pool entry, but with completely
+    /// target-dependent semantics. Holds target flags, a 32-bit index, and a
+    /// 64-bit index. Targets can use this however they like.
+    TargetIndex,
+
     /// RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...)
     /// This node represents a target intrinsic function with no side effects.
     /// The first operand is the ID number of the intrinsic from the
@@ -148,93 +153,94 @@ namespace ISD {
     /// namespace.  The operands to the intrinsic follow.
     INTRINSIC_VOID,
 
-    // CopyToReg - This node has three operands: a chain, a register number to
-    // set to this value, and a value.
+    /// CopyToReg - This node has three operands: a chain, a register number to
+    /// set to this value, and a value.
     CopyToReg,
 
-    // CopyFromReg - This node indicates that the input value is a virtual or
-    // physical register that is defined outside of the scope of this
-    // SelectionDAG.  The register is available from the RegisterSDNode object.
+    /// CopyFromReg - This node indicates that the input value is a virtual or
+    /// physical register that is defined outside of the scope of this
+    /// SelectionDAG.  The register is available from the RegisterSDNode object.
     CopyFromReg,
 
-    // UNDEF - An undefined node
+    /// UNDEF - An undefined node.
     UNDEF,
 
-    // EXTRACT_ELEMENT - This is used to get the lower or upper (determined by
-    // a Constant, which is required to be operand #1) half of the integer or
-    // float value specified as operand #0.  This is only for use before
-    // legalization, for values that will be broken into multiple registers.
+    /// EXTRACT_ELEMENT - This is used to get the lower or upper (determined by
+    /// a Constant, which is required to be operand #1) half of the integer or
+    /// float value specified as operand #0.  This is only for use before
+    /// legalization, for values that will be broken into multiple registers.
     EXTRACT_ELEMENT,
 
-    // BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.  Given
-    // two values of the same integer value type, this produces a value twice as
-    // big.  Like EXTRACT_ELEMENT, this can only be used before legalization.
+    /// BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
+    /// Given two values of the same integer value type, this produces a value
+    /// twice as big.  Like EXTRACT_ELEMENT, this can only be used before
+    /// legalization.
     BUILD_PAIR,
 
-    // MERGE_VALUES - This node takes multiple discrete operands and returns
-    // them all as its individual results.  This nodes has exactly the same
-    // number of inputs and outputs. This node is useful for some pieces of the
-    // code generator that want to think about a single node with multiple
-    // results, not multiple nodes.
+    /// MERGE_VALUES - This node takes multiple discrete operands and returns
+    /// them all as its individual results.  This nodes has exactly the same
+    /// number of inputs and outputs. This node is useful for some pieces of the
+    /// code generator that want to think about a single node with multiple
+    /// results, not multiple nodes.
     MERGE_VALUES,
 
-    // Simple integer binary arithmetic operators.
+    /// Simple integer binary arithmetic operators.
     ADD, SUB, MUL, SDIV, UDIV, SREM, UREM,
 
-    // SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing
-    // a signed/unsigned value of type i[2*N], and return the full value as
-    // two results, each of type iN.
+    /// SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing
+    /// a signed/unsigned value of type i[2*N], and return the full value as
+    /// two results, each of type iN.
     SMUL_LOHI, UMUL_LOHI,
 
-    // SDIVREM/UDIVREM - Divide two integers and produce both a quotient and
-    // remainder result.
+    /// SDIVREM/UDIVREM - Divide two integers and produce both a quotient and
+    /// remainder result.
     SDIVREM, UDIVREM,
 
-    // CARRY_FALSE - This node is used when folding other nodes,
-    // like ADDC/SUBC, which indicate the carry result is always false.
+    /// CARRY_FALSE - This node is used when folding other nodes,
+    /// like ADDC/SUBC, which indicate the carry result is always false.
     CARRY_FALSE,
 
-    // Carry-setting nodes for multiple precision addition and subtraction.
-    // These nodes take two operands of the same value type, and produce two
-    // results.  The first result is the normal add or sub result, the second
-    // result is the carry flag result.
+    /// Carry-setting nodes for multiple precision addition and subtraction.
+    /// These nodes take two operands of the same value type, and produce two
+    /// results.  The first result is the normal add or sub result, the second
+    /// result is the carry flag result.
     ADDC, SUBC,
 
-    // Carry-using nodes for multiple precision addition and subtraction.  These
-    // nodes take three operands: The first two are the normal lhs and rhs to
-    // the add or sub, and the third is the input carry flag.  These nodes
-    // produce two results; the normal result of the add or sub, and the output
-    // carry flag.  These nodes both read and write a carry flag to allow them
-    // to them to be chained together for add and sub of arbitrarily large
-    // values.
+    /// Carry-using nodes for multiple precision addition and subtraction. These
+    /// nodes take three operands: The first two are the normal lhs and rhs to
+    /// the add or sub, and the third is the input carry flag.  These nodes
+    /// produce two results; the normal result of the add or sub, and the output
+    /// carry flag.  These nodes both read and write a carry flag to allow them
+    /// to them to be chained together for add and sub of arbitrarily large
+    /// values.
     ADDE, SUBE,
 
-    // RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
-    // These nodes take two operands: the normal LHS and RHS to the add. They
-    // produce two results: the normal result of the add, and a boolean that
-    // indicates if an overflow occurred (*not* a flag, because it may be stored
-    // to memory, etc.).  If the type of the boolean is not i1 then the high
-    // bits conform to getBooleanContents.
-    // These nodes are generated from the llvm.[su]add.with.overflow intrinsics.
+    /// RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
+    /// These nodes take two operands: the normal LHS and RHS to the add. They
+    /// produce two results: the normal result of the add, and a boolean that
+    /// indicates if an overflow occurred (*not* a flag, because it may be store
+    /// to memory, etc.).  If the type of the boolean is not i1 then the high
+    /// bits conform to getBooleanContents.
+    /// These nodes are generated from llvm.[su]add.with.overflow intrinsics.
     SADDO, UADDO,
 
-    // Same for subtraction
+    /// Same for subtraction.
     SSUBO, USUBO,
 
-    // Same for multiplication
+    /// Same for multiplication.
     SMULO, UMULO,
 
-    // Simple binary floating point operators.
+    /// Simple binary floating point operators.
     FADD, FSUB, FMUL, FMA, FDIV, FREM,
 
-    // FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.  NOTE: This
-    // DAG node does not require that X and Y have the same type, just that they
-    // are both floating point.  X and the result must have the same type.
-    // FCOPYSIGN(f32, f64) is allowed.
+    /// FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.  NOTE: This
+    /// DAG node does not require that X and Y have the same type, just that the
+    /// are both floating point.  X and the result must have the same type.
+    /// FCOPYSIGN(f32, f64) is allowed.
     FCOPYSIGN,
 
-    // INT = FGETSIGN(FP) - Return the sign bit of the specified floating point
-    // value as an integer 0/1 value.
+    /// INT = FGETSIGN(FP) - Return the sign bit of the specified floating point
+    /// value as an integer 0/1 value.
     FGETSIGN,
 
     /// BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the
@@ -292,13 +298,14 @@ namespace ISD {
     /// than the vector element type, and is implicitly truncated to it.
     SCALAR_TO_VECTOR,
 
-    // MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing
-    // an unsigned/signed value of type i[2*N], then return the top part.
+    /// MULHU/MULHS - Multiply high - Multiply two integers of type iN,
+    /// producing an unsigned/signed value of type i[2*N], then return the top
+    /// part.
     MULHU, MULHS,
 
     /// Bitwise operators - logical and, logical or, logical xor.
     AND, OR, XOR,
-    
+
     /// Shift and rotation operations.  After legalization, the type of the
     /// shift amount is known to be TLI.getShiftAmountTy().  Before legalization
     /// the shift amount can be any type, but care must be taken to ensure it is
@@ -306,7 +313,6 @@ namespace ISD {
     /// legalization, types like i1024 can occur and i8 doesn't have enough bits
     /// to represent the shift amount.  By convention, DAGCombine and
     /// SelectionDAGBuilder forces these shift amounts to i32 for simplicity.
-    ///
     SHL, SRA, SRL, ROTL, ROTR,
 
     /// Byte Swap and Counting operators.
@@ -315,67 +321,67 @@ namespace ISD {
     /// Bit counting operators with an undefined result for zero inputs.
     CTTZ_ZERO_UNDEF, CTLZ_ZERO_UNDEF,
 
-    // Select(COND, TRUEVAL, FALSEVAL).  If the type of the boolean COND is not
-    // i1 then the high bits must conform to getBooleanContents.
+    /// Select(COND, TRUEVAL, FALSEVAL).  If the type of the boolean COND is not
+    /// i1 then the high bits must conform to getBooleanContents.
     SELECT,
 
-    // Select with a vector condition (op #0) and two vector operands (ops #1
-    // and #2), returning a vector result.  All vectors have the same length.
-    // Much like the scalar select and setcc, each bit in the condition selects
-    // whether the corresponding result element is taken from op #1 or op #2.
-    // At first, the VSELECT condition is of vXi1 type. Later, targets may change
-    // the condition type in order to match the VSELECT node using a a pattern.
-    // The condition follows the BooleanContent format of the target.
+    /// Select with a vector condition (op #0) and two vector operands (ops #1
+    /// and #2), returning a vector result.  All vectors have the same length.
+    /// Much like the scalar select and setcc, each bit in the condition selects
+    /// whether the corresponding result element is taken from op #1 or op #2.
+    /// At first, the VSELECT condition is of vXi1 type. Later, targets may
+    /// change the condition type in order to match the VSELECT node using a
+    /// pattern. The condition follows the BooleanContent format of the target.
     VSELECT,
 
-    // Select with condition operator - This selects between a true value and
-    // a false value (ops #2 and #3) based on the boolean result of comparing
-    // the lhs and rhs (ops #0 and #1) of a conditional expression with the
-    // condition code in op #4, a CondCodeSDNode.
+    /// Select with condition operator - This selects between a true value and
+    /// a false value (ops #2 and #3) based on the boolean result of comparing
+    /// the lhs and rhs (ops #0 and #1) of a conditional expression with the
+    /// condition code in op #4, a CondCodeSDNode.
     SELECT_CC,
 
-    // SetCC operator - This evaluates to a true value iff the condition is
-    // true.  If the result value type is not i1 then the high bits conform
-    // to getBooleanContents.  The operands to this are the left and right
-    // operands to compare (ops #0, and #1) and the condition code to compare
-    // them with (op #2) as a CondCodeSDNode. If the operands are vector types
-    // then the result type must also be a vector type.
+    /// SetCC operator - This evaluates to a true value iff the condition is
+    /// true.  If the result value type is not i1 then the high bits conform
+    /// to getBooleanContents.  The operands to this are the left and right
+    /// operands to compare (ops #0, and #1) and the condition code to compare
+    /// them with (op #2) as a CondCodeSDNode. If the operands are vector types
+    /// then the result type must also be a vector type.
     SETCC,
 
-    // SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded
-    // integer shift operations, just like ADD/SUB_PARTS.  The operation
-    // ordering is:
-    //       [Lo,Hi] = op [LoLHS,HiLHS], Amt
+    /// SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded
+    /// integer shift operations, just like ADD/SUB_PARTS.  The operation
+    /// ordering is:
+    ///       [Lo,Hi] = op [LoLHS,HiLHS], Amt
     SHL_PARTS, SRA_PARTS, SRL_PARTS,
 
-    // Conversion operators.  These are all single input single output
-    // operations.  For all of these, the result type must be strictly
-    // wider or narrower (depending on the operation) than the source
-    // type.
+    /// Conversion operators.  These are all single input single output
+    /// operations.  For all of these, the result type must be strictly
+    /// wider or narrower (depending on the operation) than the source
+    /// type.
 
-    // SIGN_EXTEND - Used for integer types, replicating the sign bit
-    // into new bits.
+    /// SIGN_EXTEND - Used for integer types, replicating the sign bit
+    /// into new bits.
     SIGN_EXTEND,
 
-    // ZERO_EXTEND - Used for integer types, zeroing the new bits.
+    /// ZERO_EXTEND - Used for integer types, zeroing the new bits.
     ZERO_EXTEND,
 
-    // ANY_EXTEND - Used for integer types.  The high bits are undefined.
+    /// ANY_EXTEND - Used for integer types.  The high bits are undefined.
     ANY_EXTEND,
 
-    // TRUNCATE - Completely drop the high bits.
+    /// TRUNCATE - Completely drop the high bits.
     TRUNCATE,
 
-    // [SU]INT_TO_FP - These operators convert integers (whose interpreted sign
-    // depends on the first letter) to floating point.
+    /// [SU]INT_TO_FP - These operators convert integers (whose interpreted sign
+    /// depends on the first letter) to floating point.
     SINT_TO_FP,
     UINT_TO_FP,
 
-    // SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to
-    // sign extend a small value in a large integer register (e.g. sign
-    // extending the low 8 bits of a 32-bit register to fill the top 24 bits
-    // with the 7th bit).  The size of the smaller type is indicated by the 1th
-    // operand, a ValueType node.
+    /// SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to
+    /// sign extend a small value in a large integer register (e.g. sign
+    /// extending the low 8 bits of a 32-bit register to fill the top 24 bits
+    /// with the 7th bit).  The size of the smaller type is indicated by the 1th
+    /// operand, a ValueType node.
     SIGN_EXTEND_INREG,
 
     /// FP_TO_[US]INT - Convert a floating point value to a signed or unsigned
@@ -396,12 +402,12 @@ namespace ISD {
     /// FP_EXTEND(FP_ROUND(X,0)) because the extra bits aren't removed.
     FP_ROUND,
 
-    // FLT_ROUNDS_ - Returns current rounding mode:
-    // -1 Undefined
-    //  0 Round to 0
-    //  1 Round to nearest
-    //  2 Round to +inf
-    //  3 Round to -inf
+    /// FLT_ROUNDS_ - Returns current rounding mode:
+    /// -1 Undefined
+    ///  0 Round to 0
+    ///  1 Round to nearest
+    ///  2 Round to +inf
+    ///  3 Round to -inf
     FLT_ROUNDS_,
 
     /// X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and
@@ -414,208 +420,211 @@ namespace ISD {
     /// X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
     FP_EXTEND,
 
-    // BITCAST - This operator converts between integer, vector and FP
-    // values, as if the value was stored to memory with one type and loaded
-    // from the same address with the other type (or equivalently for vector
-    // format conversions, etc).  The source and result are required to have
-    // the same bit size (e.g.  f32 <-> i32).  This can also be used for
-    // int-to-int or fp-to-fp conversions, but that is a noop, deleted by
-    // getNode().
+    /// BITCAST - This operator converts between integer, vector and FP
+    /// values, as if the value was stored to memory with one type and loaded
+    /// from the same address with the other type (or equivalently for vector
+    /// format conversions, etc).  The source and result are required to have
+    /// the same bit size (e.g.  f32 <-> i32).  This can also be used for
+    /// int-to-int or fp-to-fp conversions, but that is a noop, deleted by
+    /// getNode().
     BITCAST,
 
-    // CONVERT_RNDSAT - This operator is used to support various conversions
-    // between various types (float, signed, unsigned and vectors of those
-    // types) with rounding and saturation. NOTE: Avoid using this operator as
-    // most target don't support it and the operator might be removed in the
-    // future. It takes the following arguments:
-    //   0) value
-    //   1) dest type (type to convert to)
-    //   2) src type (type to convert from)
-    //   3) rounding imm
-    //   4) saturation imm
-    //   5) ISD::CvtCode indicating the type of conversion to do
+    /// CONVERT_RNDSAT - This operator is used to support various conversions
+    /// between various types (float, signed, unsigned and vectors of those
+    /// types) with rounding and saturation. NOTE: Avoid using this operator as
+    /// most target don't support it and the operator might be removed in the
+    /// future. It takes the following arguments:
+    ///   0) value
+    ///   1) dest type (type to convert to)
+    ///   2) src type (type to convert from)
+    ///   3) rounding imm
+    ///   4) saturation imm
+    ///   5) ISD::CvtCode indicating the type of conversion to do
     CONVERT_RNDSAT,
 
-    // FP16_TO_FP32, FP32_TO_FP16 - These operators are used to perform
-    // promotions and truncation for half-precision (16 bit) floating
-    // numbers. We need special nodes since FP16 is a storage-only type with
-    // special semantics of operations.
+    /// FP16_TO_FP32, FP32_TO_FP16 - These operators are used to perform
+    /// promotions and truncation for half-precision (16 bit) floating
+    /// numbers. We need special nodes since FP16 is a storage-only type with
+    /// special semantics of operations.
     FP16_TO_FP32, FP32_TO_FP16,
 
-    // FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW,
-    // FLOG, FLOG2, FLOG10, FEXP, FEXP2,
-    // FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR - Perform various unary floating
-    // point operations. These are inspired by libm.
+    /// FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW,
+    /// FLOG, FLOG2, FLOG10, FEXP, FEXP2,
+    /// FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR - Perform various unary
+    /// floating point operations. These are inspired by libm.
     FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW,
     FLOG, FLOG2, FLOG10, FEXP, FEXP2,
     FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR,
 
-    // LOAD and STORE have token chains as their first operand, then the same
-    // operands as an LLVM load/store instruction, then an offset node that
-    // is added / subtracted from the base pointer to form the address (for
-    // indexed memory ops).
+    /// LOAD and STORE have token chains as their first operand, then the same
+    /// operands as an LLVM load/store instruction, then an offset node that
+    /// is added / subtracted from the base pointer to form the address (for
+    /// indexed memory ops).
     LOAD, STORE,
 
-    // DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned
-    // to a specified boundary.  This node always has two return values: a new
-    // stack pointer value and a chain. The first operand is the token chain,
-    // the second is the number of bytes to allocate, and the third is the
-    // alignment boundary.  The size is guaranteed to be a multiple of the stack
-    // alignment, and the alignment is guaranteed to be bigger than the stack
-    // alignment (if required) or 0 to get standard stack alignment.
+    /// DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned
+    /// to a specified boundary.  This node always has two return values: a new
+    /// stack pointer value and a chain. The first operand is the token chain,
+    /// the second is the number of bytes to allocate, and the third is the
+    /// alignment boundary.  The size is guaranteed to be a multiple of the
+    /// stack alignment, and the alignment is guaranteed to be bigger than the
+    /// stack alignment (if required) or 0 to get standard stack alignment.
     DYNAMIC_STACKALLOC,
 
-    // Control flow instructions.  These all have token chains.
+    /// Control flow instructions.  These all have token chains.
 
-    // BR - Unconditional branch.  The first operand is the chain
-    // operand, the second is the MBB to branch to.
+    /// BR - Unconditional branch.  The first operand is the chain
+    /// operand, the second is the MBB to branch to.
     BR,
 
-    // BRIND - Indirect branch.  The first operand is the chain, the second
-    // is the value to branch to, which must be of the same type as the target's
-    // pointer type.
+    /// BRIND - Indirect branch.  The first operand is the chain, the second
+    /// is the value to branch to, which must be of the same type as the
+    /// target's pointer type.
     BRIND,
 
-    // BR_JT - Jumptable branch. The first operand is the chain, the second
-    // is the jumptable index, the last one is the jumptable entry index.
+    /// BR_JT - Jumptable branch. The first operand is the chain, the second
+    /// is the jumptable index, the last one is the jumptable entry index.
     BR_JT,
 
-    // BRCOND - Conditional branch.  The first operand is the chain, the
-    // second is the condition, the third is the block to branch to if the
-    // condition is true.  If the type of the condition is not i1, then the
-    // high bits must conform to getBooleanContents.
+    /// BRCOND - Conditional branch.  The first operand is the chain, the
+    /// second is the condition, the third is the block to branch to if the
+    /// condition is true.  If the type of the condition is not i1, then the
+    /// high bits must conform to getBooleanContents.
     BRCOND,
 
-    // BR_CC - Conditional branch.  The behavior is like that of SELECT_CC, in
-    // that the condition is represented as condition code, and two nodes to
-    // compare, rather than as a combined SetCC node.  The operands in order are
-    // chain, cc, lhs, rhs, block to branch to if condition is true.
+    /// BR_CC - Conditional branch.  The behavior is like that of SELECT_CC, in
+    /// that the condition is represented as condition code, and two nodes to
+    /// compare, rather than as a combined SetCC node.  The operands in order
+    /// are chain, cc, lhs, rhs, block to branch to if condition is true.
     BR_CC,
 
-    // INLINEASM - Represents an inline asm block.  This node always has two
-    // return values: a chain and a flag result.  The inputs are as follows:
-    //   Operand #0   : Input chain.
-    //   Operand #1   : a ExternalSymbolSDNode with a pointer to the asm string.
-    //   Operand #2   : a MDNodeSDNode with the !srcloc metadata.
-    //   Operand #3   : HasSideEffect, IsAlignStack bits.
-    //   After this, it is followed by a list of operands with this format:
-    //     ConstantSDNode: Flags that encode whether it is a mem or not, the
-    //                     of operands that follow, etc.  See InlineAsm.h.
-    //     ... however many operands ...
-    //   Operand #last: Optional, an incoming flag.
-    //
-    // The variable width operands are required to represent target addressing
-    // modes as a single "operand", even though they may have multiple
-    // SDOperands.
+    /// INLINEASM - Represents an inline asm block.  This node always has two
+    /// return values: a chain and a flag result.  The inputs are as follows:
+    ///   Operand #0  : Input chain.
+    ///   Operand #1  : a ExternalSymbolSDNode with a pointer to the asm string.
+    ///   Operand #2  : a MDNodeSDNode with the !srcloc metadata.
+    ///   Operand #3  : HasSideEffect, IsAlignStack bits.
+    ///   After this, it is followed by a list of operands with this format:
+    ///     ConstantSDNode: Flags that encode whether it is a mem or not, the
+    ///                     of operands that follow, etc.  See InlineAsm.h.
+    ///     ... however many operands ...
+    ///   Operand #last: Optional, an incoming flag.
+    ///
+    /// The variable width operands are required to represent target addressing
+    /// modes as a single "operand", even though they may have multiple
+    /// SDOperands.
     INLINEASM,
 
-    // EH_LABEL - Represents a label in mid basic block used to track
-    // locations needed for debug and exception handling tables.  These nodes
-    // take a chain as input and return a chain.
+    /// EH_LABEL - Represents a label in mid basic block used to track
+    /// locations needed for debug and exception handling tables.  These nodes
+    /// take a chain as input and return a chain.
     EH_LABEL,
 
-    // STACKSAVE - STACKSAVE has one operand, an input chain.  It produces a
-    // value, the same type as the pointer type for the system, and an output
-    // chain.
+    /// STACKSAVE - STACKSAVE has one operand, an input chain.  It produces a
+    /// value, the same type as the pointer type for the system, and an output
+    /// chain.
     STACKSAVE,
 
-    // STACKRESTORE has two operands, an input chain and a pointer to restore to
-    // it returns an output chain.
+    /// STACKRESTORE has two operands, an input chain and a pointer to restore
+    /// to it returns an output chain.
     STACKRESTORE,
 
-    // CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of
-    // a call sequence, and carry arbitrary information that target might want
-    // to know.  The first operand is a chain, the rest are specified by the
-    // target and not touched by the DAG optimizers.
-    // CALLSEQ_START..CALLSEQ_END pairs may not be nested.
+    /// CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end
+    /// of a call sequence, and carry arbitrary information that target might
+    /// want to know.  The first operand is a chain, the rest are specified by
+    /// the target and not touched by the DAG optimizers.
+    /// CALLSEQ_START..CALLSEQ_END pairs may not be nested.
     CALLSEQ_START,  // Beginning of a call sequence
     CALLSEQ_END,    // End of a call sequence
 
-    // VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE,
-    // and the alignment. It returns a pair of values: the vaarg value and a
-    // new chain.
+    /// VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE,
+    /// and the alignment. It returns a pair of values: the vaarg value and a
+    /// new chain.
     VAARG,
 
-    // VACOPY - VACOPY has five operands: an input chain, a destination pointer,
-    // a source pointer, a SRCVALUE for the destination, and a SRCVALUE for the
-    // source.
+    /// VACOPY - VACOPY has 5 operands: an input chain, a destination pointer,
+    /// a source pointer, a SRCVALUE for the destination, and a SRCVALUE for the
+    /// source.
     VACOPY,
 
-    // VAEND, VASTART - VAEND and VASTART have three operands: an input chain, a
-    // pointer, and a SRCVALUE.
+    /// VAEND, VASTART - VAEND and VASTART have three operands: an input chain,
+    /// pointer, and a SRCVALUE.
     VAEND, VASTART,
 
-    // SRCVALUE - This is a node type that holds a Value* that is used to
-    // make reference to a value in the LLVM IR.
+    /// SRCVALUE - This is a node type that holds a Value* that is used to
+    /// make reference to a value in the LLVM IR.
     SRCVALUE,
 
-    // MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to
-    // reference metadata in the IR.
+    /// MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to
+    /// reference metadata in the IR.
     MDNODE_SDNODE,
 
-    // PCMARKER - This corresponds to the pcmarker intrinsic.
+    /// PCMARKER - This corresponds to the pcmarker intrinsic.
     PCMARKER,
 
-    // READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
-    // The only operand is a chain and a value and a chain are produced.  The
-    // value is the contents of the architecture specific cycle counter like
-    // register (or other high accuracy low latency clock source)
+    /// READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
+    /// The only operand is a chain and a value and a chain are produced.  The
+    /// value is the contents of the architecture specific cycle counter like
+    /// register (or other high accuracy low latency clock source)
     READCYCLECOUNTER,
 
-    // HANDLENODE node - Used as a handle for various purposes.
+    /// HANDLENODE node - Used as a handle for various purposes.
     HANDLENODE,
 
-    // INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.  It
-    // takes as input a token chain, the pointer to the trampoline, the pointer
-    // to the nested function, the pointer to pass for the 'nest' parameter, a
-    // SRCVALUE for the trampoline and another for the nested function (allowing
-    // targets to access the original Function*).  It produces a token chain as
-    // output.
+    /// INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.  It
+    /// takes as input a token chain, the pointer to the trampoline, the pointer
+    /// to the nested function, the pointer to pass for the 'nest' parameter, a
+    /// SRCVALUE for the trampoline and another for the nested function
+    /// (allowing targets to access the original Function*).
+    /// It produces a token chain as output.
     INIT_TRAMPOLINE,
 
-    // ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
-    // It takes a pointer to the trampoline and produces a (possibly) new
-    // pointer to the same trampoline with platform-specific adjustments
-    // applied.  The pointer it returns points to an executable block of code.
+    /// ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
+    /// It takes a pointer to the trampoline and produces a (possibly) new
+    /// pointer to the same trampoline with platform-specific adjustments
+    /// applied.  The pointer it returns points to an executable block of code.
     ADJUST_TRAMPOLINE,
 
-    // TRAP - Trapping instruction
+    /// TRAP - Trapping instruction
     TRAP,
 
-    // PREFETCH - This corresponds to a prefetch intrinsic. It takes chains are
-    // their first operand. The other operands are the address to prefetch,
-    // read / write specifier, locality specifier and instruction / data cache
-    // specifier.
+    /// DEBUGTRAP - Trap intended to get the attention of a debugger.
+    DEBUGTRAP,
+
+    /// PREFETCH - This corresponds to a prefetch intrinsic. The first operand
+    /// is the chain.  The other operands are the address to prefetch,
+    /// read / write specifier, locality specifier and instruction / data cache
+    /// specifier.
     PREFETCH,
 
-    // OUTCHAIN = MEMBARRIER(INCHAIN, load-load, load-store, store-load,
-    //                       store-store, device)
-    // This corresponds to the memory.barrier intrinsic.
-    // it takes an input chain, 4 operands to specify the type of barrier, an
-    // operand specifying if the barrier applies to device and uncached memory
-    // and produces an output chain.
+    /// OUTCHAIN = MEMBARRIER(INCHAIN, load-load, load-store, store-load,
+    ///                       store-store, device)
+    /// This corresponds to the memory.barrier intrinsic.
+    /// it takes an input chain, 4 operands to specify the type of barrier, an
+    /// operand specifying if the barrier applies to device and uncached memory
+    /// and produces an output chain.
     MEMBARRIER,
 
-    // OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope)
-    // This corresponds to the fence instruction. It takes an input chain, and
-    // two integer constants: an AtomicOrdering and a SynchronizationScope.
+    /// OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope)
+    /// This corresponds to the fence instruction. It takes an input chain, and
+    /// two integer constants: an AtomicOrdering and a SynchronizationScope.
     ATOMIC_FENCE,
 
-    // Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr)
-    // This corresponds to "load atomic" instruction.
+    /// Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr)
+    /// This corresponds to "load atomic" instruction.
     ATOMIC_LOAD,
 
-    // OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr, val)
-    // This corresponds to "store atomic" instruction.
+    /// OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr, val)
+    /// This corresponds to "store atomic" instruction.
     ATOMIC_STORE,
 
-    // Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
-    // This corresponds to the cmpxchg instruction.
+    /// Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
+    /// This corresponds to the cmpxchg instruction.
     ATOMIC_CMP_SWAP,
 
-    // Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt)
-    // Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt)
-    // These correspond to the atomicrmw instruction.
+    /// Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt)
+    /// Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt)
+    /// These correspond to the atomicrmw instruction.
     ATOMIC_SWAP,
     ATOMIC_LOAD_ADD,
     ATOMIC_LOAD_SUB,
@@ -790,16 +799,16 @@ namespace ISD {
   /// CvtCode enum - This enum defines the various converts CONVERT_RNDSAT
   /// supports.
   enum CvtCode {
-    CVT_FF,     // Float from Float
-    CVT_FS,     // Float from Signed
-    CVT_FU,     // Float from Unsigned
-    CVT_SF,     // Signed from Float
-    CVT_UF,     // Unsigned from Float
-    CVT_SS,     // Signed from Signed
-    CVT_SU,     // Signed from Unsigned
-    CVT_US,     // Unsigned from Signed
-    CVT_UU,     // Unsigned from Unsigned
-    CVT_INVALID // Marker - Invalid opcode
+    CVT_FF,     /// Float from Float
+    CVT_FS,     /// Float from Signed
+    CVT_FU,     /// Float from Unsigned
+    CVT_SF,     /// Signed from Float
+    CVT_UF,     /// Unsigned from Float
+    CVT_SS,     /// Signed from Signed
+    CVT_SU,     /// Signed from Unsigned
+    CVT_US,     /// Unsigned from Signed
+    CVT_UU,     /// Unsigned from Unsigned
+    CVT_INVALID /// Marker - Invalid opcode
   };
 
 } // end llvm::ISD namespace
diff --git a/include/llvm/CodeGen/LexicalScopes.h b/include/llvm/CodeGen/LexicalScopes.h
index eb01f66c..8414c64 100644
--- a/include/llvm/CodeGen/LexicalScopes.h
+++ b/include/llvm/CodeGen/LexicalScopes.h
@@ -158,7 +158,10 @@ class LexicalScope {
 public:
   LexicalScope(LexicalScope *P, const MDNode *D, const MDNode *I, bool A)
     : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(A),
-      LastInsn(0), FirstInsn(0), DFSIn(0), DFSOut(0), IndentLevel(0) {
+      LastInsn(0), FirstInsn(0), DFSIn(0), DFSOut(0) {
+#ifndef NDEBUG
+    IndentLevel = 0;
+#endif
     if (Parent)
       Parent->addChild(this);
   }
@@ -241,7 +244,9 @@ private:
   const MachineInstr *FirstInsn;      // First instruction of this scope.
   unsigned DFSIn, DFSOut;             // In & Out Depth use to determine
                                       // scope nesting.
+#ifndef NDEBUG
   mutable unsigned IndentLevel;       // Private state for dump()
+#endif
 };
 
 } // end llvm namespace
diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h
index a6008ab..a3ce47c 100644
--- a/include/llvm/CodeGen/LiveInterval.h
+++ b/include/llvm/CodeGen/LiveInterval.h
@@ -40,15 +40,6 @@ namespace llvm {
   /// definition and use points.
   ///
   class VNInfo {
-  private:
-    enum {
-      HAS_PHI_KILL    = 1,
-      IS_PHI_DEF      = 1 << 1,
-      IS_UNUSED       = 1 << 2
-    };
-
-    unsigned char flags;
-
   public:
     typedef BumpPtrAllocator Allocator;
 
@@ -60,60 +51,30 @@ namespace llvm {
 
     /// VNInfo constructor.
     VNInfo(unsigned i, SlotIndex d)
-      : flags(0), id(i), def(d)
+      : id(i), def(d)
     { }
 
     /// VNInfo construtor, copies values from orig, except for the value number.
     VNInfo(unsigned i, const VNInfo &orig)
-      : flags(orig.flags), id(i), def(orig.def)
+      : id(i), def(orig.def)
     { }
 
     /// Copy from the parameter into this VNInfo.
     void copyFrom(VNInfo &src) {
-      flags = src.flags;
       def = src.def;
     }
 
-    /// Used for copying value number info.
-    unsigned getFlags() const { return flags; }
-    void setFlags(unsigned flags) { this->flags = flags; }
-
-    /// Merge flags from another VNInfo
-    void mergeFlags(const VNInfo *VNI) {
-      flags = (flags | VNI->flags) & ~IS_UNUSED;
-    }
-
-    /// Returns true if one or more kills are PHI nodes.
-    /// Obsolete, do not use!
-    bool hasPHIKill() const { return flags & HAS_PHI_KILL; }
-    /// Set the PHI kill flag on this value.
-    void setHasPHIKill(bool hasKill) {
-      if (hasKill)
-        flags |= HAS_PHI_KILL;
-      else
-        flags &= ~HAS_PHI_KILL;
-    }
-
     /// Returns true if this value is defined by a PHI instruction (or was,
     /// PHI instrucions may have been eliminated).
-    bool isPHIDef() const { return flags & IS_PHI_DEF; }
-    /// Set the "phi def" flag on this value.
-    void setIsPHIDef(bool phiDef) {
-      if (phiDef)
-        flags |= IS_PHI_DEF;
-      else
-        flags &= ~IS_PHI_DEF;
-    }
+    /// PHI-defs begin at a block boundary, all other defs begin at register or
+    /// EC slots.
+    bool isPHIDef() const { return def.isBlock(); }
 
     /// Returns true if this value is unused.
-    bool isUnused() const { return flags & IS_UNUSED; }
-    /// Set the "is unused" flag on this value.
-    void setIsUnused(bool unused) {
-      if (unused)
-        flags |= IS_UNUSED;
-      else
-        flags &= ~IS_UNUSED;
-    }
+    bool isUnused() const { return !def.isValid(); }
+
+    /// Mark this value as unused.
+    void markUnused() { def = SlotIndex(); }
   };
 
   /// LiveRange structure - This represents a simple register range in the
@@ -274,6 +235,11 @@ namespace llvm {
       return VNI;
     }
 
+    /// createDeadDef - Make sure the interval has a value defined at Def.
+    /// If one already exists, return it. Otherwise allocate a new value and
+    /// add liveness for a dead def.
+    VNInfo *createDeadDef(SlotIndex Def, VNInfo::Allocator &VNInfoAllocator);
+
     /// Create a copy of the given value. The new value will be identical except
     /// for the Value number.
     VNInfo *createValueCopy(const VNInfo *orig,
@@ -288,17 +254,6 @@ namespace llvm {
     /// unused values.
     void RenumberValues(LiveIntervals &lis);
 
-    /// isOnlyLROfValNo - Return true if the specified live range is the only
-    /// one defined by the its val#.
-    bool isOnlyLROfValNo(const LiveRange *LR) {
-      for (const_iterator I = begin(), E = end(); I != E; ++I) {
-        const LiveRange *Tmp = I;
-        if (Tmp != LR && Tmp->valno == LR->valno)
-          return false;
-      }
-      return true;
-    }
-
     /// MergeValueNumberInto - This method is called when two value nubmers
     /// are found to be equivalent.  This eliminates V1, replacing all
     /// LiveRanges with the V1 value number with the V2 value number.  This can
@@ -377,14 +332,6 @@ namespace llvm {
       return I == end() ? 0 : &*I;
     }
 
-    const LiveRange *getLiveRangeBefore(SlotIndex Idx) const {
-      return getLiveRangeContaining(Idx.getPrevSlot());
-    }
-
-    LiveRange *getLiveRangeBefore(SlotIndex Idx) {
-      return getLiveRangeContaining(Idx.getPrevSlot());
-    }
-
     /// getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.
     VNInfo *getVNInfoAt(SlotIndex Idx) const {
       const_iterator I = FindLiveRangeContaining(Idx);
@@ -411,11 +358,6 @@ namespace llvm {
       return I != end() && I->start <= Idx ? I : end();
     }
 
-    /// findDefinedVNInfo - Find the by the specified
-    /// index (register interval) or defined
-    VNInfo *findDefinedVNInfoForRegInt(SlotIndex Idx) const;
-
-
     /// overlaps - Return true if the intersection of the two live intervals is
     /// not empty.
     bool overlaps(const LiveInterval& other) const {
@@ -498,10 +440,6 @@ namespace llvm {
       weight = HUGE_VALF;
     }
 
-    /// ComputeJoinedWeight - Set the weight of a live interval after
-    /// Other has been merged into it.
-    void ComputeJoinedWeight(const LiveInterval &Other);
-
     bool operator<(const LiveInterval& other) const {
       const SlotIndex &thisIndex = beginIndex();
       const SlotIndex &otherIndex = other.beginIndex();
@@ -509,15 +447,27 @@ namespace llvm {
               (thisIndex == otherIndex && reg < other.reg));
     }
 
-    void print(raw_ostream &OS, const TargetRegisterInfo *TRI = 0) const;
+    void print(raw_ostream &OS) const;
     void dump() const;
 
+    /// \brief Walk the interval and assert if any invariants fail to hold.
+    ///
+    /// Note that this is a no-op when asserts are disabled.
+#ifdef NDEBUG
+    void verify() const {}
+#else
+    void verify() const;
+#endif
+
   private:
 
     Ranges::iterator addRangeFrom(LiveRange LR, Ranges::iterator From);
     void extendIntervalEndTo(Ranges::iterator I, SlotIndex NewEnd);
     Ranges::iterator extendIntervalStartTo(Ranges::iterator I, SlotIndex NewStr);
     void markValNoForDeletion(VNInfo *V);
+    void mergeIntervalRanges(const LiveInterval &RHS,
+                             VNInfo *LHSValNo = 0,
+                             const VNInfo *RHSValNo = 0);
 
     LiveInterval& operator=(const LiveInterval& rhs); // DO NOT IMPLEMENT
 
@@ -528,6 +478,91 @@ namespace llvm {
     return OS;
   }
 
+  /// LiveRangeQuery - Query information about a live range around a given
+  /// instruction. This class hides the implementation details of live ranges,
+  /// and it should be used as the primary interface for examining live ranges
+  /// around instructions.
+  ///
+  class LiveRangeQuery {
+    VNInfo *EarlyVal;
+    VNInfo *LateVal;
+    SlotIndex EndPoint;
+    bool Kill;
+
+  public:
+    /// Create a LiveRangeQuery for the given live range and instruction index.
+    /// The sub-instruction slot of Idx doesn't matter, only the instruction it
+    /// refers to is considered.
+    LiveRangeQuery(const LiveInterval &LI, SlotIndex Idx)
+      : EarlyVal(0), LateVal(0), Kill(false) {
+      // Find the segment that enters the instruction.
+      LiveInterval::const_iterator I = LI.find(Idx.getBaseIndex());
+      LiveInterval::const_iterator E = LI.end();
+      if (I == E)
+        return;
+      // Is this an instruction live-in segment?
+      if (SlotIndex::isEarlierInstr(I->start, Idx)) {
+        EarlyVal = I->valno;
+        EndPoint = I->end;
+        // Move to the potentially live-out segment.
+        if (SlotIndex::isSameInstr(Idx, I->end)) {
+          Kill = true;
+          if (++I == E)
+            return;
+        }
+      }
+      // I now points to the segment that may be live-through, or defined by
+      // this instr. Ignore segments starting after the current instr.
+      if (SlotIndex::isEarlierInstr(Idx, I->start))
+        return;
+      LateVal = I->valno;
+      EndPoint = I->end;
+    }
+
+    /// Return the value that is live-in to the instruction. This is the value
+    /// that will be read by the instruction's use operands. Return NULL if no
+    /// value is live-in.
+    VNInfo *valueIn() const {
+      return EarlyVal;
+    }
+
+    /// Return true if the live-in value is killed by this instruction. This
+    /// means that either the live range ends at the instruction, or it changes
+    /// value.
+    bool isKill() const {
+      return Kill;
+    }
+
+    /// Return true if this instruction has a dead def.
+    bool isDeadDef() const {
+      return EndPoint.isDead();
+    }
+
+    /// Return the value leaving the instruction, if any. This can be a
+    /// live-through value, or a live def. A dead def returns NULL.
+    VNInfo *valueOut() const {
+      return isDeadDef() ? 0 : LateVal;
+    }
+
+    /// Return the value defined by this instruction, if any. This includes
+    /// dead defs, it is the value created by the instruction's def operands.
+    VNInfo *valueDefined() const {
+      return EarlyVal == LateVal ? 0 : LateVal;
+    }
+
+    /// Return the end point of the last live range segment to interact with
+    /// the instruction, if any.
+    ///
+    /// The end point is an invalid SlotIndex only if the live range doesn't
+    /// intersect the instruction at all.
+    ///
+    /// The end point may be at or past the end of the instruction's basic
+    /// block. That means the value was live out of the block.
+    SlotIndex endPoint() const {
+      return EndPoint;
+    }
+  };
+
   /// ConnectedVNInfoEqClasses - Helper class that can divide VNInfos in a
   /// LiveInterval into equivalence clases of connected components. A
   /// LiveInterval that has multiple connected components can be broken into
diff --git a/include/llvm/CodeGen/LiveIntervalAnalysis.h b/include/llvm/CodeGen/LiveIntervalAnalysis.h
index 76201c9..da521db 100644
--- a/include/llvm/CodeGen/LiveIntervalAnalysis.h
+++ b/include/llvm/CodeGen/LiveIntervalAnalysis.h
@@ -20,12 +20,13 @@
 #ifndef LLVM_CODEGEN_LIVEINTERVAL_ANALYSIS_H
 #define LLVM_CODEGEN_LIVEINTERVAL_ANALYSIS_H
 
+#include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/ADT/BitVector.h"
-#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Allocator.h"
@@ -35,7 +36,9 @@
 namespace llvm {
 
   class AliasAnalysis;
+  class LiveRangeCalc;
   class LiveVariables;
+  class MachineDominatorTree;
   class MachineLoopInfo;
   class TargetRegisterInfo;
   class MachineRegisterInfo;
@@ -44,27 +47,29 @@ namespace llvm {
   class VirtRegMap;
 
   class LiveIntervals : public MachineFunctionPass {
-    MachineFunction* mf_;
-    MachineRegisterInfo* mri_;
-    const TargetMachine* tm_;
-    const TargetRegisterInfo* tri_;
-    const TargetInstrInfo* tii_;
-    AliasAnalysis *aa_;
-    LiveVariables* lv_;
-    SlotIndexes* indexes_;
+    MachineFunction* MF;
+    MachineRegisterInfo* MRI;
+    const TargetMachine* TM;
+    const TargetRegisterInfo* TRI;
+    const TargetInstrInfo* TII;
+    AliasAnalysis *AA;
+    LiveVariables* LV;
+    SlotIndexes* Indexes;
+    MachineDominatorTree *DomTree;
+    LiveRangeCalc *LRCalc;
 
     /// Special pool allocator for VNInfo's (LiveInterval val#).
     ///
     VNInfo::Allocator VNInfoAllocator;
 
-    typedef DenseMap<unsigned, LiveInterval*> Reg2IntervalMap;
-    Reg2IntervalMap r2iMap_;
+    /// Live interval pointers for all the virtual registers.
+    IndexedMap<LiveInterval*, VirtReg2IndexFunctor> VirtRegIntervals;
 
-    /// allocatableRegs_ - A bit vector of allocatable registers.
-    BitVector allocatableRegs_;
+    /// AllocatableRegs - A bit vector of allocatable registers.
+    BitVector AllocatableRegs;
 
-    /// reservedRegs_ - A bit vector of reserved registers.
-    BitVector reservedRegs_;
+    /// ReservedRegs - A bit vector of reserved registers.
+    BitVector ReservedRegs;
 
     /// RegMaskSlots - Sorted list of instructions with register mask operands.
     /// Always use the 'r' slot, RegMasks are normal clobbers, not early
@@ -92,83 +97,59 @@ namespace llvm {
     /// block.
     SmallVector<std::pair<unsigned, unsigned>, 8> RegMaskBlocks;
 
+    /// RegUnitIntervals - Keep a live interval for each register unit as a way
+    /// of tracking fixed physreg interference.
+    SmallVector<LiveInterval*, 0> RegUnitIntervals;
+
   public:
     static char ID; // Pass identification, replacement for typeid
-    LiveIntervals() : MachineFunctionPass(ID) {
-      initializeLiveIntervalsPass(*PassRegistry::getPassRegistry());
-    }
+    LiveIntervals();
+    virtual ~LiveIntervals();
 
     // Calculate the spill weight to assign to a single instruction.
     static float getSpillWeight(bool isDef, bool isUse, unsigned loopDepth);
 
-    typedef Reg2IntervalMap::iterator iterator;
-    typedef Reg2IntervalMap::const_iterator const_iterator;
-    const_iterator begin() const { return r2iMap_.begin(); }
-    const_iterator end() const { return r2iMap_.end(); }
-    iterator begin() { return r2iMap_.begin(); }
-    iterator end() { return r2iMap_.end(); }
-    unsigned getNumIntervals() const { return (unsigned)r2iMap_.size(); }
-
-    LiveInterval &getInterval(unsigned reg) {
-      Reg2IntervalMap::iterator I = r2iMap_.find(reg);
-      assert(I != r2iMap_.end() && "Interval does not exist for register");
-      return *I->second;
+    LiveInterval &getInterval(unsigned Reg) {
+      LiveInterval *LI = VirtRegIntervals[Reg];
+      assert(LI && "Interval does not exist for virtual register");
+      return *LI;
     }
 
-    const LiveInterval &getInterval(unsigned reg) const {
-      Reg2IntervalMap::const_iterator I = r2iMap_.find(reg);
-      assert(I != r2iMap_.end() && "Interval does not exist for register");
-      return *I->second;
+    const LiveInterval &getInterval(unsigned Reg) const {
+      return const_cast<LiveIntervals*>(this)->getInterval(Reg);
     }
 
-    bool hasInterval(unsigned reg) const {
-      return r2iMap_.count(reg);
+    bool hasInterval(unsigned Reg) const {
+      return VirtRegIntervals.inBounds(Reg) && VirtRegIntervals[Reg];
     }
 
     /// isAllocatable - is the physical register reg allocatable in the current
     /// function?
     bool isAllocatable(unsigned reg) const {
-      return allocatableRegs_.test(reg);
+      return AllocatableRegs.test(reg);
     }
 
     /// isReserved - is the physical register reg reserved in the current
     /// function
     bool isReserved(unsigned reg) const {
-      return reservedRegs_.test(reg);
-    }
-
-    /// getScaledIntervalSize - get the size of an interval in "units,"
-    /// where every function is composed of one thousand units.  This
-    /// measure scales properly with empty index slots in the function.
-    double getScaledIntervalSize(LiveInterval& I) {
-      return (1000.0 * I.getSize()) / indexes_->getIndexesLength();
-    }
-
-    /// getFuncInstructionCount - Return the number of instructions in the
-    /// current function.
-    unsigned getFuncInstructionCount() {
-      return indexes_->getFunctionSize();
+      return ReservedRegs.test(reg);
     }
 
-    /// getApproximateInstructionCount - computes an estimate of the number
-    /// of instructions in a given LiveInterval.
-    unsigned getApproximateInstructionCount(LiveInterval& I) {
-      double IntervalPercentage = getScaledIntervalSize(I) / 1000.0;
-      return (unsigned)(IntervalPercentage * indexes_->getFunctionSize());
+    // Interval creation.
+    LiveInterval &getOrCreateInterval(unsigned Reg) {
+      if (!hasInterval(Reg)) {
+        VirtRegIntervals.grow(Reg);
+        VirtRegIntervals[Reg] = createInterval(Reg);
+      }
+      return getInterval(Reg);
     }
 
-    // Interval creation
-    LiveInterval &getOrCreateInterval(unsigned reg) {
-      Reg2IntervalMap::iterator I = r2iMap_.find(reg);
-      if (I == r2iMap_.end())
-        I = r2iMap_.insert(std::make_pair(reg, createInterval(reg))).first;
-      return *I->second;
+    // Interval removal.
+    void removeInterval(unsigned Reg) {
+      delete VirtRegIntervals[Reg];
+      VirtRegIntervals[Reg] = 0;
     }
 
-    /// dupInterval - Duplicate a live interval. The caller is responsible for
-    /// managing the allocated memory.
-    LiveInterval *dupInterval(LiveInterval *li);
-
     /// addLiveRangeToEndOfBlock - Given a register and an instruction,
     /// adds a live range from that instruction to the end of its MBB.
     LiveRange addLiveRangeToEndOfBlock(unsigned reg,
@@ -184,42 +165,38 @@ namespace llvm {
     bool shrinkToUses(LiveInterval *li,
                       SmallVectorImpl<MachineInstr*> *dead = 0);
 
-    // Interval removal
-
-    void removeInterval(unsigned Reg) {
-      DenseMap<unsigned, LiveInterval*>::iterator I = r2iMap_.find(Reg);
-      delete I->second;
-      r2iMap_.erase(I);
+    SlotIndexes *getSlotIndexes() const {
+      return Indexes;
     }
 
-    SlotIndexes *getSlotIndexes() const {
-      return indexes_;
+    AliasAnalysis *getAliasAnalysis() const {
+      return AA;
     }
 
     /// isNotInMIMap - returns true if the specified machine instr has been
     /// removed or was never entered in the map.
     bool isNotInMIMap(const MachineInstr* Instr) const {
-      return !indexes_->hasIndex(Instr);
+      return !Indexes->hasIndex(Instr);
     }
 
     /// Returns the base index of the given instruction.
     SlotIndex getInstructionIndex(const MachineInstr *instr) const {
-      return indexes_->getInstructionIndex(instr);
+      return Indexes->getInstructionIndex(instr);
     }
 
     /// Returns the instruction associated with the given index.
     MachineInstr* getInstructionFromIndex(SlotIndex index) const {
-      return indexes_->getInstructionFromIndex(index);
+      return Indexes->getInstructionFromIndex(index);
     }
 
     /// Return the first index in the given basic block.
     SlotIndex getMBBStartIdx(const MachineBasicBlock *mbb) const {
-      return indexes_->getMBBStartIdx(mbb);
+      return Indexes->getMBBStartIdx(mbb);
     }
 
     /// Return the last index in the given basic block.
     SlotIndex getMBBEndIdx(const MachineBasicBlock *mbb) const {
-      return indexes_->getMBBEndIdx(mbb);
+      return Indexes->getMBBEndIdx(mbb);
     }
 
     bool isLiveInToMBB(const LiveInterval &li,
@@ -233,24 +210,24 @@ namespace llvm {
     }
 
     MachineBasicBlock* getMBBFromIndex(SlotIndex index) const {
-      return indexes_->getMBBFromIndex(index);
+      return Indexes->getMBBFromIndex(index);
     }
 
     SlotIndex InsertMachineInstrInMaps(MachineInstr *MI) {
-      return indexes_->insertMachineInstrInMaps(MI);
+      return Indexes->insertMachineInstrInMaps(MI);
     }
 
     void RemoveMachineInstrFromMaps(MachineInstr *MI) {
-      indexes_->removeMachineInstrFromMaps(MI);
+      Indexes->removeMachineInstrFromMaps(MI);
     }
 
     void ReplaceMachineInstrInMaps(MachineInstr *MI, MachineInstr *NewMI) {
-      indexes_->replaceMachineInstrInMaps(MI, NewMI);
+      Indexes->replaceMachineInstrInMaps(MI, NewMI);
     }
 
     bool findLiveInMBBs(SlotIndex Start, SlotIndex End,
                         SmallVectorImpl<MachineBasicBlock*> &MBBs) const {
-      return indexes_->findLiveInMBBs(Start, End, MBBs);
+      return Indexes->findLiveInMBBs(Start, End, MBBs);
     }
 
     VNInfo::Allocator& getVNInfoAllocator() { return VNInfoAllocator; }
@@ -264,18 +241,15 @@ namespace llvm {
     /// print - Implement the dump method.
     virtual void print(raw_ostream &O, const Module* = 0) const;
 
-    /// isReMaterializable - Returns true if every definition of MI of every
-    /// val# of the specified interval is re-materializable. Also returns true
-    /// by reference if all of the defs are load instructions.
-    bool isReMaterializable(const LiveInterval &li,
-                            const SmallVectorImpl<LiveInterval*> *SpillIs,
-                            bool &isLoad);
-
     /// intervalIsInOneMBB - If LI is confined to a single basic block, return
     /// a pointer to that block.  If LI is live in to or out of any block,
     /// return NULL.
     MachineBasicBlock *intervalIsInOneMBB(const LiveInterval &LI) const;
 
+    /// Returns true if VNI is killed by any PHI-def values in LI.
+    /// This may conservatively return true to avoid expensive computations.
+    bool hasPHIKill(const LiveInterval &LI, const VNInfo *VNI) const;
+
     /// addKillFlags - Add kill flags to any instruction that kills a virtual
     /// register.
     void addKillFlags();
@@ -337,13 +311,47 @@ namespace llvm {
     bool checkRegMaskInterference(LiveInterval &LI,
                                   BitVector &UsableRegs);
 
+    // Register unit functions.
+    //
+    // Fixed interference occurs when MachineInstrs use physregs directly
+    // instead of virtual registers. This typically happens when passing
+    // arguments to a function call, or when instructions require operands in
+    // fixed registers.
+    //
+    // Each physreg has one or more register units, see MCRegisterInfo. We
+    // track liveness per register unit to handle aliasing registers more
+    // efficiently.
+
+    /// getRegUnit - Return the live range for Unit.
+    /// It will be computed if it doesn't exist.
+    LiveInterval &getRegUnit(unsigned Unit) {
+      LiveInterval *LI = RegUnitIntervals[Unit];
+      if (!LI) {
+        // Compute missing ranges on demand.
+        RegUnitIntervals[Unit] = LI = new LiveInterval(Unit, HUGE_VALF);
+        computeRegUnitInterval(LI);
+      }
+      return *LI;
+    }
+
+    /// getCachedRegUnit - Return the live range for Unit if it has already
+    /// been computed, or NULL if it hasn't been computed yet.
+    LiveInterval *getCachedRegUnit(unsigned Unit) {
+      return RegUnitIntervals[Unit];
+    }
+
   private:
     /// computeIntervals - Compute live intervals.
     void computeIntervals();
 
+    /// Compute live intervals for all virtual registers.
+    void computeVirtRegs();
+
+    /// Compute RegMaskSlots and RegMaskBits.
+    void computeRegMasks();
+
     /// handleRegisterDef - update intervals for a register def
-    /// (calls handlePhysicalRegisterDef and
-    /// handleVirtualRegisterDef)
+    /// (calls handleVirtualRegisterDef)
     void handleRegisterDef(MachineBasicBlock *MBB,
                            MachineBasicBlock::iterator MI,
                            SlotIndex MIIdx,
@@ -363,43 +371,15 @@ namespace llvm {
                                   unsigned MOIdx,
                                   LiveInterval& interval);
 
-    /// handlePhysicalRegisterDef - update intervals for a physical register
-    /// def.
-    void handlePhysicalRegisterDef(MachineBasicBlock* mbb,
-                                   MachineBasicBlock::iterator mi,
-                                   SlotIndex MIIdx, MachineOperand& MO,
-                                   LiveInterval &interval);
-
-    /// handleLiveInRegister - Create interval for a livein register.
-    void handleLiveInRegister(MachineBasicBlock* mbb,
-                              SlotIndex MIIdx,
-                              LiveInterval &interval);
-
-    /// getReMatImplicitUse - If the remat definition MI has one (for now, we
-    /// only allow one) virtual register operand, then its uses are implicitly
-    /// using the register. Returns the virtual register.
-    unsigned getReMatImplicitUse(const LiveInterval &li,
-                                 MachineInstr *MI) const;
-
-    /// isValNoAvailableAt - Return true if the val# of the specified interval
-    /// which reaches the given instruction also reaches the specified use
-    /// index.
-    bool isValNoAvailableAt(const LiveInterval &li, MachineInstr *MI,
-                            SlotIndex UseIdx) const;
-
-    /// isReMaterializable - Returns true if the definition MI of the specified
-    /// val# of the specified interval is re-materializable. Also returns true
-    /// by reference if the def is a load.
-    bool isReMaterializable(const LiveInterval &li, const VNInfo *ValNo,
-                            MachineInstr *MI,
-                            const SmallVectorImpl<LiveInterval*> *SpillIs,
-                            bool &isLoad);
-
     static LiveInterval* createInterval(unsigned Reg);
 
     void printInstrs(raw_ostream &O) const;
     void dumpInstrs() const;
 
+    void computeLiveInRegUnits();
+    void computeRegUnitInterval(LiveInterval*);
+    void computeVirtRegInterval(LiveInterval*);
+
     class HMEditor;
   };
 } // End llvm namespace
diff --git a/include/llvm/CodeGen/LiveRangeEdit.h b/include/llvm/CodeGen/LiveRangeEdit.h
index 57a6193..def7b00 100644
--- a/include/llvm/CodeGen/LiveRangeEdit.h
+++ b/include/llvm/CodeGen/LiveRangeEdit.h
@@ -55,29 +55,29 @@ public:
   };
 
 private:
-  LiveInterval &parent_;
-  SmallVectorImpl<LiveInterval*> &newRegs_;
+  LiveInterval *Parent;
+  SmallVectorImpl<LiveInterval*> &NewRegs;
   MachineRegisterInfo &MRI;
   LiveIntervals &LIS;
   VirtRegMap *VRM;
   const TargetInstrInfo &TII;
-  Delegate *const delegate_;
+  Delegate *const TheDelegate;
 
-  /// firstNew_ - Index of the first register added to newRegs_.
-  const unsigned firstNew_;
+  /// FirstNew - Index of the first register added to NewRegs.
+  const unsigned FirstNew;
 
-  /// scannedRemattable_ - true when remattable values have been identified.
-  bool scannedRemattable_;
+  /// ScannedRemattable - true when remattable values have been identified.
+  bool ScannedRemattable;
 
-  /// remattable_ - Values defined by remattable instructions as identified by
+  /// Remattable - Values defined by remattable instructions as identified by
   /// tii.isTriviallyReMaterializable().
-  SmallPtrSet<const VNInfo*,4> remattable_;
+  SmallPtrSet<const VNInfo*,4> Remattable;
 
-  /// rematted_ - Values that were actually rematted, and so need to have their
+  /// Rematted - Values that were actually rematted, and so need to have their
   /// live range trimmed or entirely removed.
-  SmallPtrSet<const VNInfo*,4> rematted_;
+  SmallPtrSet<const VNInfo*,4> Rematted;
 
-  /// scanRemattable - Identify the parent_ values that may rematerialize.
+  /// scanRemattable - Identify the Parent values that may rematerialize.
   void scanRemattable(AliasAnalysis *aa);
 
   /// allUsesAvailableAt - Return true if all registers used by OrigMI at
@@ -99,32 +99,35 @@ public:
   /// @param vrm Map of virtual registers to physical registers for this
   ///            function.  If NULL, no virtual register map updates will
   ///            be done.  This could be the case if called before Regalloc.
-  LiveRangeEdit(LiveInterval &parent,
+  LiveRangeEdit(LiveInterval *parent,
                 SmallVectorImpl<LiveInterval*> &newRegs,
                 MachineFunction &MF,
                 LiveIntervals &lis,
                 VirtRegMap *vrm,
                 Delegate *delegate = 0)
-    : parent_(parent), newRegs_(newRegs),
+    : Parent(parent), NewRegs(newRegs),
       MRI(MF.getRegInfo()), LIS(lis), VRM(vrm),
       TII(*MF.getTarget().getInstrInfo()),
-      delegate_(delegate),
-      firstNew_(newRegs.size()),
-      scannedRemattable_(false) {}
+      TheDelegate(delegate),
+      FirstNew(newRegs.size()),
+      ScannedRemattable(false) {}
 
-  LiveInterval &getParent() const { return parent_; }
-  unsigned getReg() const { return parent_.reg; }
+  LiveInterval &getParent() const {
+   assert(Parent && "No parent LiveInterval");
+   return *Parent;
+  }
+  unsigned getReg() const { return getParent().reg; }
 
   /// Iterator for accessing the new registers added by this edit.
   typedef SmallVectorImpl<LiveInterval*>::const_iterator iterator;
-  iterator begin() const { return newRegs_.begin()+firstNew_; }
-  iterator end() const { return newRegs_.end(); }
-  unsigned size() const { return newRegs_.size()-firstNew_; }
+  iterator begin() const { return NewRegs.begin()+FirstNew; }
+  iterator end() const { return NewRegs.end(); }
+  unsigned size() const { return NewRegs.size()-FirstNew; }
   bool empty() const { return size() == 0; }
-  LiveInterval *get(unsigned idx) const { return newRegs_[idx+firstNew_]; }
+  LiveInterval *get(unsigned idx) const { return NewRegs[idx+FirstNew]; }
 
   ArrayRef<LiveInterval*> regs() const {
-    return makeArrayRef(newRegs_).slice(firstNew_);
+    return makeArrayRef(NewRegs).slice(FirstNew);
   }
 
   /// createFrom - Create a new virtual register based on OldReg.
@@ -174,12 +177,12 @@ public:
   /// markRematerialized - explicitly mark a value as rematerialized after doing
   /// it manually.
   void markRematerialized(const VNInfo *ParentVNI) {
-    rematted_.insert(ParentVNI);
+    Rematted.insert(ParentVNI);
   }
 
   /// didRematerialize - Return true if ParentVNI was rematerialized anywhere.
   bool didRematerialize(const VNInfo *ParentVNI) const {
-    return rematted_.count(ParentVNI);
+    return Rematted.count(ParentVNI);
   }
 
   /// eraseVirtReg - Notify the delegate that Reg is no longer in use, and try
diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h
index ef9c0c2..c917bd8 100644
--- a/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/include/llvm/CodeGen/MachineBasicBlock.h
@@ -143,10 +143,7 @@ public:
     IterTy MII;
 
   public:
-    bundle_iterator(IterTy mii) : MII(mii) {
-      assert(!MII->isInsideBundle() &&
-             "It's not legal to initialize bundle_iterator with a bundled MI");
-    }
+    bundle_iterator(IterTy mii) : MII(mii) {}
 
     bundle_iterator(Ty &mi) : MII(mi) {
       assert(!mi.isInsideBundle() &&
@@ -156,7 +153,10 @@ public:
       assert((!mi || !mi->isInsideBundle()) &&
              "It's not legal to initialize bundle_iterator with a bundled MI");
     }
-    bundle_iterator(const bundle_iterator &I) : MII(I.MII) {}
+    // Template allows conversion from const to nonconst.
+    template<class OtherTy, class OtherIterTy>
+    bundle_iterator(const bundle_iterator<OtherTy, OtherIterTy> &I)
+      : MII(I.getInstrIterator()) {}
     bundle_iterator() : MII(0) {}
 
     Ty &operator*() const { return *MII; }
@@ -173,29 +173,24 @@ public:
 
     // Increment and decrement operators...
     bundle_iterator &operator--() {      // predecrement - Back up
-      do {
-        --MII;
-      } while (MII->isInsideBundle());
+      do --MII;
+      while (MII->isInsideBundle());
       return *this;
     }
     bundle_iterator &operator++() {      // preincrement - Advance
-      do {
-        ++MII;
-      } while (MII->isInsideBundle());
+      IterTy E = MII->getParent()->instr_end();
+      do ++MII;
+      while (MII != E && MII->isInsideBundle());
       return *this;
     }
     bundle_iterator operator--(int) {    // postdecrement operators...
       bundle_iterator tmp = *this;
-      do {
-        --MII;
-      } while (MII->isInsideBundle());
+      --*this;
       return tmp;
     }
     bundle_iterator operator++(int) {    // postincrement operators...
       bundle_iterator tmp = *this;
-      do {
-        ++MII;
-      } while (MII->isInsideBundle());
+      ++*this;
       return tmp;
     }
 
@@ -235,42 +230,14 @@ public:
   reverse_instr_iterator       instr_rend  ()       { return Insts.rend();   }
   const_reverse_instr_iterator instr_rend  () const { return Insts.rend();   }
 
-  iterator                begin()       { return Insts.begin();  }
-  const_iterator          begin() const { return Insts.begin();  }
-  iterator                  end()       {
-    instr_iterator II = instr_end();
-    if (II != instr_begin()) {
-      while (II->isInsideBundle())
-        --II;
-    }
-    return II;
-  }
-  const_iterator            end() const {
-    const_instr_iterator II = instr_end();
-    if (II != instr_begin()) {
-      while (II->isInsideBundle())
-        --II;
-    }
-    return II;
-  }
-  reverse_iterator       rbegin()       {
-    reverse_instr_iterator II = instr_rbegin();
-    if (II != instr_rend()) {
-      while (II->isInsideBundle())
-        ++II;
-    }
-    return II;
-  }
-  const_reverse_iterator rbegin() const {
-    const_reverse_instr_iterator II = instr_rbegin();
-    if (II != instr_rend()) {
-      while (II->isInsideBundle())
-        ++II;
-    }
-    return II;
-  }
-  reverse_iterator       rend  ()       { return Insts.rend();   }
-  const_reverse_iterator rend  () const { return Insts.rend();   }
+  iterator                begin()       { return instr_begin();  }
+  const_iterator          begin() const { return instr_begin();  }
+  iterator                end  ()       { return instr_end();    }
+  const_iterator          end  () const { return instr_end();    }
+  reverse_iterator       rbegin()       { return instr_rbegin(); }
+  const_reverse_iterator rbegin() const { return instr_rbegin(); }
+  reverse_iterator       rend  ()       { return instr_rend();   }
+  const_reverse_iterator rend  () const { return instr_rend();   }
 
 
   // Machine-CFG iterators
@@ -412,6 +379,10 @@ public:
   /// which refer to fromMBB to refer to this.
   void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *fromMBB);
 
+  /// isPredecessor - Return true if the specified MBB is a predecessor of this
+  /// block.
+  bool isPredecessor(const MachineBasicBlock *MBB) const;
+
   /// isSuccessor - Return true if the specified MBB is a successor of this
   /// block.
   bool isSuccessor(const MachineBasicBlock *MBB) const;
diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h
index 44402a9..8b958e4 100644
--- a/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/include/llvm/CodeGen/MachineFrameInfo.h
@@ -359,7 +359,7 @@ public:
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
     Objects[ObjectIdx+NumFixedObjects].Alignment = Align;
-    MaxAlignment = std::max(MaxAlignment, Align);
+    ensureMaxAlignment(Align);
   }
 
   /// NeedsStackProtector - Returns true if the object may need stack
@@ -416,9 +416,11 @@ public:
   ///
   unsigned getMaxAlignment() const { return MaxAlignment; }
 
-  /// setMaxAlignment - Set the preferred alignment.
-  ///
-  void setMaxAlignment(unsigned Align) { MaxAlignment = Align; }
+  /// ensureMaxAlignment - Make sure the function is at least Align bytes
+  /// aligned.
+  void ensureMaxAlignment(unsigned Align) {
+    if (MaxAlignment < Align) MaxAlignment = Align;
+  }
 
   /// AdjustsStack - Return true if this function adjusts the stack -- e.g.,
   /// when calling another function. This is only valid during and after
@@ -485,7 +487,7 @@ public:
     Objects.push_back(StackObject(Size, Alignment, 0, false, isSS, MayNeedSP));
     int Index = (int)Objects.size() - NumFixedObjects - 1;
     assert(Index >= 0 && "Bad frame index!");
-    MaxAlignment = std::max(MaxAlignment, Alignment);
+    ensureMaxAlignment(Alignment);
     return Index;
   }
 
@@ -496,7 +498,7 @@ public:
   int CreateSpillStackObject(uint64_t Size, unsigned Alignment) {
     CreateStackObject(Size, Alignment, true, false);
     int Index = (int)Objects.size() - NumFixedObjects - 1;
-    MaxAlignment = std::max(MaxAlignment, Alignment);
+    ensureMaxAlignment(Alignment);
     return Index;
   }
 
@@ -515,7 +517,7 @@ public:
   int CreateVariableSizedObject(unsigned Alignment) {
     HasVarSizedObjects = true;
     Objects.push_back(StackObject(0, Alignment, 0, false, false, true));
-    MaxAlignment = std::max(MaxAlignment, Alignment);
+    ensureMaxAlignment(Alignment);
     return (int)Objects.size()-NumFixedObjects-1;
   }
 
diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h
index dda2dc7..062c750 100644
--- a/include/llvm/CodeGen/MachineFunction.h
+++ b/include/llvm/CodeGen/MachineFunction.h
@@ -189,8 +189,8 @@ public:
   ///
   void setAlignment(unsigned A) { Alignment = A; }
 
-  /// EnsureAlignment - Make sure the function is at least 1 << A bytes aligned.
-  void EnsureAlignment(unsigned A) {
+  /// ensureAlignment - Make sure the function is at least 1 << A bytes aligned.
+  void ensureAlignment(unsigned A) {
     if (Alignment < A) Alignment = A;
   }
 
diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h
index 65093d7..4c5eb8b 100644
--- a/include/llvm/CodeGen/MachineInstr.h
+++ b/include/llvm/CodeGen/MachineInstr.h
@@ -635,6 +635,30 @@ public:
       getOperand(0).getSubReg() == getOperand(1).getSubReg();
   }
 
+  /// isTransient - Return true if this is a transient instruction that is
+  /// either very likely to be eliminated during register allocation (such as
+  /// copy-like instructions), or if this instruction doesn't have an
+  /// execution-time cost.
+  bool isTransient() const {
+    switch(getOpcode()) {
+    default: return false;
+    // Copy-like instructions are usually eliminated during register allocation.
+    case TargetOpcode::PHI:
+    case TargetOpcode::COPY:
+    case TargetOpcode::INSERT_SUBREG:
+    case TargetOpcode::SUBREG_TO_REG:
+    case TargetOpcode::REG_SEQUENCE:
+    // Pseudo-instructions that don't produce any real output.
+    case TargetOpcode::IMPLICIT_DEF:
+    case TargetOpcode::KILL:
+    case TargetOpcode::PROLOG_LABEL:
+    case TargetOpcode::EH_LABEL:
+    case TargetOpcode::GC_LABEL:
+    case TargetOpcode::DBG_VALUE:
+      return true;
+    }
+  }
+
   /// getBundleSize - Return the number of instructions inside the MI bundle.
   unsigned getBundleSize() const;
 
@@ -912,12 +936,12 @@ private:
   /// RemoveRegOperandsFromUseLists - Unlink all of the register operands in
   /// this instruction from their respective use lists.  This requires that the
   /// operands already be on their use lists.
-  void RemoveRegOperandsFromUseLists();
+  void RemoveRegOperandsFromUseLists(MachineRegisterInfo&);
 
   /// AddRegOperandsToUseLists - Add all of the register operands in
   /// this instruction from their respective use lists.  This requires that the
   /// operands not be on their use lists yet.
-  void AddRegOperandsToUseLists(MachineRegisterInfo &RegInfo);
+  void AddRegOperandsToUseLists(MachineRegisterInfo&);
 
   /// hasPropertyInBundle - Slow path for hasProperty when we're dealing with a
   /// bundle.
diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h
index 99849a6..654361f 100644
--- a/include/llvm/CodeGen/MachineInstrBuilder.h
+++ b/include/llvm/CodeGen/MachineInstrBuilder.h
@@ -34,6 +34,7 @@ namespace RegState {
     Undef          = 0x20,
     EarlyClobber   = 0x40,
     Debug          = 0x80,
+    InternalRead   = 0x100,
     DefineNoRead   = Define | Undef,
     ImplicitDefine = Implicit | Define,
     ImplicitKill   = Implicit | Kill
@@ -67,7 +68,8 @@ public:
                                              flags & RegState::Undef,
                                              flags & RegState::EarlyClobber,
                                              SubReg,
-                                             flags & RegState::Debug));
+                                             flags & RegState::Debug,
+                                             flags & RegState::InternalRead));
     return *this;
   }
 
@@ -106,6 +108,12 @@ public:
     return *this;
   }
 
+  const MachineInstrBuilder &addTargetIndex(unsigned Idx, int64_t Offset = 0,
+                                          unsigned char TargetFlags = 0) const {
+    MI->addOperand(MachineOperand::CreateTargetIndex(Idx, Offset, TargetFlags));
+    return *this;
+  }
+
   const MachineInstrBuilder &addJumpTableIndex(unsigned Idx,
                                           unsigned char TargetFlags = 0) const {
     MI->addOperand(MachineOperand::CreateJTI(Idx, TargetFlags));
@@ -310,6 +318,9 @@ inline unsigned getDeadRegState(bool B) {
 inline unsigned getUndefRegState(bool B) {
   return B ? RegState::Undef : 0;
 }
+inline unsigned getInternalReadRegState(bool B) {
+  return B ? RegState::InternalRead : 0;
+}
 
 } // End llvm namespace
 
diff --git a/include/llvm/CodeGen/MachineInstrBundle.h b/include/llvm/CodeGen/MachineInstrBundle.h
index 0fb4969..dc5f9a6 100644
--- a/include/llvm/CodeGen/MachineInstrBundle.h
+++ b/include/llvm/CodeGen/MachineInstrBundle.h
@@ -43,14 +43,14 @@ bool finalizeBundles(MachineFunction &MF);
 
 /// getBundleStart - Returns the first instruction in the bundle containing MI.
 ///
-static inline MachineInstr *getBundleStart(MachineInstr *MI) {
+inline MachineInstr *getBundleStart(MachineInstr *MI) {
   MachineBasicBlock::instr_iterator I = MI;
   while (I->isInsideBundle())
     --I;
   return I;
 }
 
-static inline const MachineInstr *getBundleStart(const MachineInstr *MI) {
+inline const MachineInstr *getBundleStart(const MachineInstr *MI) {
   MachineBasicBlock::const_instr_iterator I = MI;
   while (I->isInsideBundle())
     --I;
diff --git a/include/llvm/CodeGen/MachineJumpTableInfo.h b/include/llvm/CodeGen/MachineJumpTableInfo.h
index 6bd6682..f7c4e86 100644
--- a/include/llvm/CodeGen/MachineJumpTableInfo.h
+++ b/include/llvm/CodeGen/MachineJumpTableInfo.h
@@ -10,9 +10,9 @@
 // The MachineJumpTableInfo class keeps track of jump tables referenced by
 // lowered switch instructions in the MachineFunction.
 //
-// Instructions reference the address of these jump tables through the use of 
-// MO_JumpTableIndex values.  When emitting assembly or machine code, these 
-// virtual address references are converted to refer to the address of the 
+// Instructions reference the address of these jump tables through the use of
+// MO_JumpTableIndex values.  When emitting assembly or machine code, these
+// virtual address references are converted to refer to the address of the
 // function jump tables.
 //
 //===----------------------------------------------------------------------===//
@@ -34,11 +34,11 @@ class raw_ostream;
 struct MachineJumpTableEntry {
   /// MBBs - The vector of basic blocks from which to create the jump table.
   std::vector<MachineBasicBlock*> MBBs;
-  
+
   explicit MachineJumpTableEntry(const std::vector<MachineBasicBlock*> &M)
   : MBBs(M) {}
 };
-  
+
 class MachineJumpTableInfo {
 public:
   /// JTEntryKind - This enum indicates how each entry of the jump table is
@@ -57,7 +57,7 @@ public:
     /// with a relocation as gp-relative, e.g.:
     ///     .gprel32 LBB123
     EK_GPRel32BlockAddress,
-    
+
     /// EK_LabelDifference32 - Each entry is the address of the block minus
     /// the address of the jump table.  This is used for PIC jump tables where
     /// gprel32 is not supported.  e.g.:
@@ -80,18 +80,18 @@ private:
   std::vector<MachineJumpTableEntry> JumpTables;
 public:
   explicit MachineJumpTableInfo(JTEntryKind Kind): EntryKind(Kind) {}
-    
+
   JTEntryKind getEntryKind() const { return EntryKind; }
 
   /// getEntrySize - Return the size of each entry in the jump table.
   unsigned getEntrySize(const TargetData &TD) const;
   /// getEntryAlignment - Return the alignment of each entry in the jump table.
   unsigned getEntryAlignment(const TargetData &TD) const;
-  
+
   /// createJumpTableIndex - Create a new jump table.
   ///
   unsigned createJumpTableIndex(const std::vector<MachineBasicBlock*> &DestBBs);
-  
+
   /// isEmpty - Return true if there are no jump tables.
   ///
   bool isEmpty() const { return JumpTables.empty(); }
@@ -105,7 +105,7 @@ public:
   void RemoveJumpTable(unsigned Idx) {
     JumpTables[Idx].MBBs.clear();
   }
-  
+
   /// ReplaceMBBInJumpTables - If Old is the target of any jump tables, update
   /// the jump tables to branch to New instead.
   bool ReplaceMBBInJumpTables(MachineBasicBlock *Old, MachineBasicBlock *New);
diff --git a/include/llvm/CodeGen/MachineLoopInfo.h b/include/llvm/CodeGen/MachineLoopInfo.h
index 6dd9440..3e204be 100644
--- a/include/llvm/CodeGen/MachineLoopInfo.h
+++ b/include/llvm/CodeGen/MachineLoopInfo.h
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file defines the MachineLoopInfo class that is used to identify natural 
+// This file defines the MachineLoopInfo class that is used to identify natural
 // loops and determine the loop depth of various nodes of the CFG.  Note that
 // natural loops may actually be several loops that share the same header node.
 //
@@ -35,6 +35,12 @@
 
 namespace llvm {
 
+// Implementation in LoopInfoImpl.h
+#ifdef __GNUC__
+class MachineLoop;
+__extension__ extern template class LoopBase<MachineBasicBlock, MachineLoop>;
+#endif
+
 class MachineLoop : public LoopBase<MachineBasicBlock, MachineLoop> {
 public:
   MachineLoop();
@@ -57,6 +63,12 @@ private:
     : LoopBase<MachineBasicBlock, MachineLoop>(MBB) {}
 };
 
+// Implementation in LoopInfoImpl.h
+#ifdef __GNUC__
+__extension__ extern template
+class LoopInfoBase<MachineBasicBlock, MachineLoop>;
+#endif
+
 class MachineLoopInfo : public MachineFunctionPass {
   LoopInfoBase<MachineBasicBlock, MachineLoop> LI;
   friend class LoopBase<MachineBasicBlock, MachineLoop>;
diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h
index d244dd9..37d42b3 100644
--- a/include/llvm/CodeGen/MachineOperand.h
+++ b/include/llvm/CodeGen/MachineOperand.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CODEGEN_MACHINEOPERAND_H
 #define LLVM_CODEGEN_MACHINEOPERAND_H
 
+#include "llvm/ADT/Hashing.h"
 #include "llvm/Support/DataTypes.h"
 #include <cassert>
 
@@ -44,6 +45,7 @@ public:
     MO_MachineBasicBlock,      ///< MachineBasicBlock reference
     MO_FrameIndex,             ///< Abstract Stack Frame Index
     MO_ConstantPoolIndex,      ///< Address of indexed Constant in Constant Pool
+    MO_TargetIndex,            ///< Target-dependent index+offset operand.
     MO_JumpTableIndex,         ///< Address of indexed Jump Table for switch
     MO_ExternalSymbol,         ///< Name of external global symbol
     MO_GlobalAddress,          ///< Address of a global value
@@ -148,7 +150,7 @@ private:
 
     struct {                  // For MO_Register.
       // Register number is in SmallContents.RegNo.
-      MachineOperand **Prev;  // Access list for register.
+      MachineOperand *Prev;   // Access list for register. See MRI.
       MachineOperand *Next;
     } Reg;
 
@@ -214,6 +216,8 @@ public:
   bool isFI() const { return OpKind == MO_FrameIndex; }
   /// isCPI - Tests if this is a MO_ConstantPoolIndex operand.
   bool isCPI() const { return OpKind == MO_ConstantPoolIndex; }
+  /// isTargetIndex - Tests if this is a MO_TargetIndex operand.
+  bool isTargetIndex() const { return OpKind == MO_TargetIndex; }
   /// isJTI - Tests if this is a MO_JumpTableIndex operand.
   bool isJTI() const { return OpKind == MO_JumpTableIndex; }
   /// isGlobal - Tests if this is a MO_GlobalAddress operand.
@@ -301,13 +305,6 @@ public:
     return !isUndef() && !isInternalRead() && (isUse() || getSubReg());
   }
 
-  /// getNextOperandForReg - Return the next MachineOperand in the function that
-  /// uses or defines this register.
-  MachineOperand *getNextOperandForReg() const {
-    assert(isReg() && "This is not a register operand!");
-    return Contents.Reg.Next;
-  }
-
   //===--------------------------------------------------------------------===//
   // Mutators for Register Operands
   //===--------------------------------------------------------------------===//
@@ -334,17 +331,9 @@ public:
   ///
   void substPhysReg(unsigned Reg, const TargetRegisterInfo&);
 
-  void setIsUse(bool Val = true) {
-    assert(isReg() && "Wrong MachineOperand accessor");
-    assert((Val || !isDebug()) && "Marking a debug operation as def");
-    IsDef = !Val;
-  }
+  void setIsUse(bool Val = true) { setIsDef(!Val); }
 
-  void setIsDef(bool Val = true) {
-    assert(isReg() && "Wrong MachineOperand accessor");
-    assert((!Val || !isDebug()) && "Marking a debug operation as def");
-    IsDef = Val;
-  }
+  void setIsDef(bool Val = true);
 
   void setImplicit(bool Val = true) {
     assert(isReg() && "Wrong MachineOperand accessor");
@@ -407,7 +396,7 @@ public:
   }
 
   int getIndex() const {
-    assert((isFI() || isCPI() || isJTI()) &&
+    assert((isFI() || isCPI() || isTargetIndex() || isJTI()) &&
            "Wrong MachineOperand accessor");
     return Contents.OffsetedInfo.Val.Index;
   }
@@ -430,8 +419,8 @@ public:
   /// getOffset - Return the offset from the symbol in this operand. This always
   /// returns 0 for ExternalSymbol operands.
   int64_t getOffset() const {
-    assert((isGlobal() || isSymbol() || isCPI() || isBlockAddress()) &&
-           "Wrong MachineOperand accessor");
+    assert((isGlobal() || isSymbol() || isCPI() || isTargetIndex() ||
+            isBlockAddress()) && "Wrong MachineOperand accessor");
     return (int64_t(Contents.OffsetedInfo.OffsetHi) << 32) |
            SmallContents.OffsetLo;
   }
@@ -478,14 +467,14 @@ public:
   }
 
   void setOffset(int64_t Offset) {
-    assert((isGlobal() || isSymbol() || isCPI() || isBlockAddress()) &&
-        "Wrong MachineOperand accessor");
+    assert((isGlobal() || isSymbol() || isCPI() || isTargetIndex() ||
+            isBlockAddress()) && "Wrong MachineOperand accessor");
     SmallContents.OffsetLo = unsigned(Offset);
     Contents.OffsetedInfo.OffsetHi = int(Offset >> 32);
   }
 
   void setIndex(int Idx) {
-    assert((isFI() || isCPI() || isJTI()) &&
+    assert((isFI() || isCPI() || isTargetIndex() || isJTI()) &&
            "Wrong MachineOperand accessor");
     Contents.OffsetedInfo.Val.Index = Idx;
   }
@@ -503,6 +492,13 @@ public:
   /// operand. Note: This method ignores isKill and isDead properties.
   bool isIdenticalTo(const MachineOperand &Other) const;
 
+  /// \brief MachineOperand hash_value overload.
+  ///
+  /// Note that this includes the same information in the hash that
+  /// isIdenticalTo uses for comparison. It is thus suited for use in hash
+  /// tables which use that function for equality comparisons only.
+  friend hash_code hash_value(const MachineOperand &MO);
+
   /// ChangeToImmediate - Replace this operand with a new immediate operand of
   /// the specified value.  If an operand is known to be an immediate already,
   /// the setImm method should be used.
@@ -542,14 +538,15 @@ public:
                                   bool isUndef = false,
                                   bool isEarlyClobber = false,
                                   unsigned SubReg = 0,
-                                  bool isDebug = false) {
+                                  bool isDebug = false,
+                                  bool isInternalRead = false) {
     MachineOperand Op(MachineOperand::MO_Register);
     Op.IsDef = isDef;
     Op.IsImp = isImp;
     Op.IsKill = isKill;
     Op.IsDead = isDead;
     Op.IsUndef = isUndef;
-    Op.IsInternalRead = false;
+    Op.IsInternalRead = isInternalRead;
     Op.IsEarlyClobber = isEarlyClobber;
     Op.IsDebug = isDebug;
     Op.SmallContents.RegNo = Reg;
@@ -578,6 +575,14 @@ public:
     Op.setTargetFlags(TargetFlags);
     return Op;
   }
+  static MachineOperand CreateTargetIndex(unsigned Idx, int64_t Offset,
+                                          unsigned char TargetFlags = 0) {
+    MachineOperand Op(MachineOperand::MO_TargetIndex);
+    Op.setIndex(Idx);
+    Op.setOffset(Offset);
+    Op.setTargetFlags(TargetFlags);
+    return Op;
+  }
   static MachineOperand CreateJTI(unsigned Idx,
                                   unsigned char TargetFlags = 0) {
     MachineOperand Op(MachineOperand::MO_JumpTableIndex);
@@ -653,15 +658,6 @@ private:
     assert(isReg() && "Can only add reg operand to use lists");
     return Contents.Reg.Prev != 0;
   }
-
-  /// AddRegOperandToRegInfo - Add this register operand to the specified
-  /// MachineRegisterInfo.  If it is null, then the next/prev fields should be
-  /// explicitly nulled out.
-  void AddRegOperandToRegInfo(MachineRegisterInfo *RegInfo);
-
-  /// RemoveRegOperandFromRegInfo - Remove this register operand from the
-  /// MachineRegisterInfo it is linked with.
-  void RemoveRegOperandFromRegInfo();
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const MachineOperand& MO) {
diff --git a/include/llvm/CodeGen/MachinePassRegistry.h b/include/llvm/CodeGen/MachinePassRegistry.h
index c41e8e26..90ee7f4 100644
--- a/include/llvm/CodeGen/MachinePassRegistry.h
+++ b/include/llvm/CodeGen/MachinePassRegistry.h
@@ -26,7 +26,7 @@ namespace llvm {
 typedef void *(*MachinePassCtor)();
 
 
-//===----------------------------------------------------------------------===// 
+//===----------------------------------------------------------------------===//
 ///
 /// MachinePassRegistryListener - Listener to adds and removals of nodes in
 /// registration list.
@@ -42,7 +42,7 @@ public:
 };
 
 
-//===----------------------------------------------------------------------===// 
+//===----------------------------------------------------------------------===//
 ///
 /// MachinePassRegistryNode - Machine pass node stored in registration list.
 ///
@@ -55,7 +55,7 @@ private:
   const char *Name;                     // Name of function pass.
   const char *Description;              // Description string.
   MachinePassCtor Ctor;                 // Function pass creator.
-  
+
 public:
 
   MachinePassRegistryNode(const char *N, const char *D, MachinePassCtor C)
@@ -72,11 +72,11 @@ public:
   const char *getDescription()            const { return Description; }
   MachinePassCtor getCtor()               const { return Ctor; }
   void setNext(MachinePassRegistryNode *N)      { Next = N; }
-  
+
 };
 
 
-//===----------------------------------------------------------------------===// 
+//===----------------------------------------------------------------------===//
 ///
 /// MachinePassRegistry - Track the registration of machine passes.
 ///
@@ -88,7 +88,7 @@ private:
   MachinePassRegistryNode *List;        // List of registry nodes.
   MachinePassCtor Default;              // Default function pass creator.
   MachinePassRegistryListener* Listener;// Listener for list adds are removes.
-  
+
 public:
 
   // NO CONSTRUCTOR - we don't want static constructor ordering to mess
@@ -99,6 +99,7 @@ public:
   MachinePassRegistryNode *getList()                    { return List; }
   MachinePassCtor getDefault()                          { return Default; }
   void setDefault(MachinePassCtor C)                    { Default = C; }
+  void setDefault(StringRef Name);
   void setListener(MachinePassRegistryListener *L)      { Listener = L; }
 
   /// Add - Adds a function pass to the registration list.
@@ -126,7 +127,7 @@ public:
 
   void initialize(cl::Option &O) {
     cl::parser<typename RegistryClass::FunctionPassCtor>::initialize(O);
-    
+
     // Add existing passes to option.
     for (RegistryClass *Node = RegistryClass::getList();
          Node; Node = Node->getNext()) {
@@ -134,7 +135,7 @@ public:
                       (typename RegistryClass::FunctionPassCtor)Node->getCtor(),
                              Node->getDescription());
     }
-    
+
     // Make sure we listen for list changes.
     RegistryClass::setListener(this);
   }
diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h
index 3272fbd..42a8aa4 100644
--- a/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -57,6 +57,26 @@ class MachineRegisterInfo {
   /// physical registers.
   MachineOperand **PhysRegUseDefLists;
 
+  /// getRegUseDefListHead - Return the head pointer for the register use/def
+  /// list for the specified virtual or physical register.
+  MachineOperand *&getRegUseDefListHead(unsigned RegNo) {
+    if (TargetRegisterInfo::isVirtualRegister(RegNo))
+      return VRegInfo[RegNo].second;
+    return PhysRegUseDefLists[RegNo];
+  }
+
+  MachineOperand *getRegUseDefListHead(unsigned RegNo) const {
+    if (TargetRegisterInfo::isVirtualRegister(RegNo))
+      return VRegInfo[RegNo].second;
+    return PhysRegUseDefLists[RegNo];
+  }
+
+  /// Get the next element in the use-def chain.
+  static MachineOperand *getNextOperandForReg(const MachineOperand *MO) {
+    assert(MO && MO->isReg() && "This is not a register operand!");
+    return MO->Contents.Reg.Next;
+  }
+
   /// UsedPhysRegs - This is a bit vector that is computed and set by the
   /// register allocator, and must be kept up to date by passes that run after
   /// register allocation (though most don't modify this).  This is used
@@ -129,12 +149,21 @@ public:
   // Register Info
   //===--------------------------------------------------------------------===//
 
+  // Strictly for use by MachineInstr.cpp.
+  void addRegOperandToUseList(MachineOperand *MO);
+
+  // Strictly for use by MachineInstr.cpp.
+  void removeRegOperandFromUseList(MachineOperand *MO);
+
   /// reg_begin/reg_end - Provide iteration support to walk over all definitions
   /// and uses of a register within the MachineFunction that corresponds to this
   /// MachineRegisterInfo object.
   template<bool Uses, bool Defs, bool SkipDebug>
   class defusechain_iterator;
 
+  // Make it a friend so it can access getNextOperandForReg().
+  template<bool, bool, bool> friend class defusechain_iterator;
+
   /// reg_iterator/reg_begin/reg_end - Walk all defs and uses of the specified
   /// register.
   typedef defusechain_iterator<true,true,false> reg_iterator;
@@ -172,6 +201,15 @@ public:
   /// specified register (it may be live-in).
   bool def_empty(unsigned RegNo) const { return def_begin(RegNo) == def_end(); }
 
+  /// hasOneDef - Return true if there is exactly one instruction defining the
+  /// specified register.
+  bool hasOneDef(unsigned RegNo) const {
+    def_iterator DI = def_begin(RegNo);
+    if (DI == def_end())
+      return false;
+    return ++DI == def_end();
+  }
+
   /// use_iterator/use_begin/use_end - Walk all uses of the specified register.
   typedef defusechain_iterator<true,false,false> use_iterator;
   use_iterator use_begin(unsigned RegNo) const {
@@ -185,7 +223,12 @@ public:
 
   /// hasOneUse - Return true if there is exactly one instruction using the
   /// specified register.
-  bool hasOneUse(unsigned RegNo) const;
+  bool hasOneUse(unsigned RegNo) const {
+    use_iterator UI = use_begin(RegNo);
+    if (UI == use_end())
+      return false;
+    return ++UI == use_end();
+  }
 
   /// use_nodbg_iterator/use_nodbg_begin/use_nodbg_end - Walk all uses of the
   /// specified register, skipping those marked as Debug.
@@ -218,25 +261,16 @@ public:
   /// constraints.
   void replaceRegWith(unsigned FromReg, unsigned ToReg);
 
-  /// getRegUseDefListHead - Return the head pointer for the register use/def
-  /// list for the specified virtual or physical register.
-  MachineOperand *&getRegUseDefListHead(unsigned RegNo) {
-    if (TargetRegisterInfo::isVirtualRegister(RegNo))
-      return VRegInfo[RegNo].second;
-    return PhysRegUseDefLists[RegNo];
-  }
-
-  MachineOperand *getRegUseDefListHead(unsigned RegNo) const {
-    if (TargetRegisterInfo::isVirtualRegister(RegNo))
-      return VRegInfo[RegNo].second;
-    return PhysRegUseDefLists[RegNo];
-  }
-
   /// getVRegDef - Return the machine instr that defines the specified virtual
   /// register or null if none is found.  This assumes that the code is in SSA
   /// form, so there should only be one definition.
   MachineInstr *getVRegDef(unsigned Reg) const;
 
+  /// getUniqueVRegDef - Return the unique machine instr that defines the
+  /// specified virtual register or null if none is found.  If there are
+  /// multiple definitions or no definition, return null.
+  MachineInstr *getUniqueVRegDef(unsigned Reg) const;
+
   /// clearKillFlags - Iterate over all the uses of the given register and
   /// clear the kill flag from the MachineOperand. This function is used by
   /// optimization passes which extend register lifetimes and need only
@@ -336,7 +370,7 @@ public:
   bool isPhysRegOrOverlapUsed(unsigned Reg) const {
     if (UsedPhysRegMask.test(Reg))
       return true;
-    for (const uint16_t *AI = TRI->getOverlaps(Reg); *AI; ++AI)
+    for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
       if (UsedPhysRegs.test(*AI))
         return true;
     return false;
@@ -434,10 +468,6 @@ public:
                         const TargetRegisterInfo &TRI,
                         const TargetInstrInfo &TII);
 
-private:
-  void HandleVRegListReallocation();
-
-public:
   /// defusechain_iterator - This class provides iterator support for machine
   /// operands in the function that use or define a specific register.  If
   /// ReturnUses is true it returns uses of registers, if ReturnDefs is true it
@@ -481,13 +511,22 @@ public:
     // Iterator traversal: forward iteration only
     defusechain_iterator &operator++() {          // Preincrement
       assert(Op && "Cannot increment end iterator!");
-      Op = Op->getNextOperandForReg();
-
-      // If this is an operand we don't care about, skip it.
-      while (Op && ((!ReturnUses && Op->isUse()) ||
-                    (!ReturnDefs && Op->isDef()) ||
-                    (SkipDebug && Op->isDebug())))
-        Op = Op->getNextOperandForReg();
+      Op = getNextOperandForReg(Op);
+
+      // All defs come before the uses, so stop def_iterator early.
+      if (!ReturnUses) {
+        if (Op) {
+          if (Op->isUse())
+            Op = 0;
+          else
+            assert(!Op->isDebug() && "Can't have debug defs");
+        }
+      } else {
+        // If this is an operand we don't care about, skip it.
+        while (Op && ((!ReturnDefs && Op->isDef()) ||
+                      (SkipDebug && Op->isDebug())))
+          Op = getNextOperandForReg(Op);
+      }
 
       return *this;
     }
diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h
index e852009..8da2045 100644
--- a/include/llvm/CodeGen/MachineScheduler.h
+++ b/include/llvm/CodeGen/MachineScheduler.h
@@ -19,7 +19,7 @@
 //                     createCustomMachineSched);
 //
 // Inside <Target>PassConfig:
-//   enablePass(MachineSchedulerID);
+//   enablePass(&MachineSchedulerID);
 //   MachineSchedRegistry::setDefault(createCustomMachineSched);
 //
 //===----------------------------------------------------------------------===//
@@ -35,6 +35,7 @@ class AliasAnalysis;
 class LiveIntervals;
 class MachineDominatorTree;
 class MachineLoopInfo;
+class RegisterClassInfo;
 class ScheduleDAGInstrs;
 
 /// MachineSchedContext provides enough context from the MachineScheduler pass
@@ -47,7 +48,10 @@ struct MachineSchedContext {
   AliasAnalysis *AA;
   LiveIntervals *LIS;
 
-  MachineSchedContext(): MF(0), MLI(0), MDT(0), PassConfig(0), AA(0), LIS(0) {}
+  RegisterClassInfo *RegClassInfo;
+
+  MachineSchedContext();
+  virtual ~MachineSchedContext();
 };
 
 /// MachineSchedRegistry provides a selection of available machine instruction
@@ -81,6 +85,9 @@ public:
   static void setDefault(ScheduleDAGCtor C) {
     Registry.setDefault((MachinePassCtor)C);
   }
+  static void setDefault(StringRef Name) {
+    Registry.setDefault(Name);
+  }
   static void setListener(MachinePassRegistryListener *L) {
     Registry.setListener(L);
   }
diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h
index e76fe99..07b3b45 100644
--- a/include/llvm/CodeGen/Passes.h
+++ b/include/llvm/CodeGen/Passes.h
@@ -24,6 +24,7 @@ namespace llvm {
   class FunctionPass;
   class MachineFunctionPass;
   class PassInfo;
+  class PassManagerBase;
   class TargetLowering;
   class TargetRegisterClass;
   class raw_ostream;
@@ -31,8 +32,6 @@ namespace llvm {
 
 namespace llvm {
 
-extern char &NoPassID; // Allow targets to choose not to run a pass.
-
 class PassConfigImpl;
 
 /// Target-Independent Code Generator Pass Configuration Options.
@@ -54,9 +53,15 @@ public:
   /// optimization after regalloc.
   static char PostRAMachineLICMID;
 
+private:
+  PassManagerBase *PM;
+  AnalysisID StartAfter;
+  AnalysisID StopAfter;
+  bool Started;
+  bool Stopped;
+
 protected:
   TargetMachine *TM;
-  PassManagerBase *PM;
   PassConfigImpl *Impl; // Internal data structures
   bool Initialized;     // Flagged after all passes are configured.
 
@@ -91,6 +96,18 @@ public:
 
   CodeGenOpt::Level getOptLevel() const { return TM->getOptLevel(); }
 
+  /// setStartStopPasses - Set the StartAfter and StopAfter passes to allow
+  /// running only a portion of the normal code-gen pass sequence.  If the
+  /// Start pass ID is zero, then compilation will begin at the normal point;
+  /// otherwise, clear the Started flag to indicate that passes should not be
+  /// added until the starting pass is seen.  If the Stop pass ID is zero,
+  /// then compilation will continue to the end.
+  void setStartStopPasses(AnalysisID Start, AnalysisID Stop) {
+    StartAfter = Start;
+    StopAfter = Stop;
+    Started = (StartAfter == 0);
+  }
+
   void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); }
 
   bool getEnableTailMerge() const { return EnableTailMerge; }
@@ -98,16 +115,19 @@ public:
 
   /// Allow the target to override a specific pass without overriding the pass
   /// pipeline. When passes are added to the standard pipeline at the
-  /// point where StadardID is expected, add TargetID in its place.
-  void substitutePass(char &StandardID, char &TargetID);
+  /// point where StandardID is expected, add TargetID in its place.
+  void substitutePass(AnalysisID StandardID, AnalysisID TargetID);
+
+  /// Insert InsertedPassID pass after TargetPassID pass.
+  void insertPass(AnalysisID TargetPassID, AnalysisID InsertedPassID);
 
   /// Allow the target to enable a specific standard pass by default.
-  void enablePass(char &ID) { substitutePass(ID, ID); }
+  void enablePass(AnalysisID PassID) { substitutePass(PassID, PassID); }
 
   /// Allow the target to disable a specific standard pass by default.
-  void disablePass(char &ID) { substitutePass(ID, NoPassID); }
+  void disablePass(AnalysisID PassID) { substitutePass(PassID, 0); }
 
-  /// Return the pass ssubtituted for StandardID by the target.
+  /// Return the pass substituted for StandardID by the target.
   /// If no substitution exists, return StandardID.
   AnalysisID getPassSubstitution(AnalysisID StandardID) const;
 
@@ -118,6 +138,9 @@ public:
   /// transforms following machine independent optimization.
   virtual void addIRPasses();
 
+  /// Add passes to lower exception handling for the code generator.
+  void addPassesToHandleExceptions();
+
   /// Add common passes that perform LLVM IR to IR transforms in preparation for
   /// instruction selection.
   virtual void addISelPrepare();
@@ -172,6 +195,18 @@ protected:
   /// LLVMTargetMachine provides standard regalloc passes for most targets.
   virtual void addOptimizedRegAlloc(FunctionPass *RegAllocPass);
 
+  /// addPreRewrite - Add passes to the optimized register allocation pipeline
+  /// after register allocation is complete, but before virtual registers are
+  /// rewritten to physical registers.
+  ///
+  /// These passes must preserve VirtRegMap and LiveIntervals, and when running
+  /// after RABasic or RAGreedy, they should take advantage of LiveRegMatrix.
+  /// When these passes run, VirtRegMap contains legal physreg assignments for
+  /// all virtual registers.
+  virtual bool addPreRewrite() {
+    return false;
+  }
+
   /// addFinalizeRegAlloc - This method may be implemented by targets that want
   /// to run passes within the regalloc pipeline, immediately after the register
   /// allocation pass itself. These passes run as soon as virtual regisiters
@@ -216,8 +251,12 @@ protected:
   ///
 
   /// Add a CodeGen pass at this point in the pipeline after checking overrides.
-  /// Return the pass that was added, or NoPassID.
-  AnalysisID addPass(char &ID);
+  /// Return the pass that was added, or zero if no pass was added.
+  AnalysisID addPass(AnalysisID PassID);
+
+  /// Add a pass to the PassManager if that pass is supposed to be run, as
+  /// determined by the StartAfter and StopAfter options.
+  void addPass(Pass *P);
 
   /// addMachinePasses helper to create the target-selected or overriden
   /// regalloc pass.
@@ -226,7 +265,7 @@ protected:
   /// printAndVerify - Add a pass to dump then verify the machine function, if
   /// those steps are enabled.
   ///
-  void printAndVerify(const char *Banner) const;
+  void printAndVerify(const char *Banner);
 };
 } // namespace llvm
 
@@ -276,6 +315,10 @@ namespace llvm {
   ///  This pass is still in development
   extern char &StrongPHIEliminationID;
 
+  /// LiveIntervals - This analysis keeps track of the live ranges of virtual
+  /// and physical registers.
+  extern char &LiveIntervalsID;
+
   /// LiveStacks pass. An analysis keeping track of the liveness of stack slots.
   extern char &LiveStacksID;
 
@@ -297,6 +340,10 @@ namespace llvm {
   /// basic blocks.
   extern char &SpillPlacementID;
 
+  /// VirtRegRewriter pass. Rewrite virtual registers to physical registers as
+  /// assigned in VirtRegMap.
+  extern char &VirtRegRewriterID;
+
   /// UnreachableMachineBlockElimination - This pass removes unreachable
   /// machine basic blocks.
   extern char &UnreachableMachineBlockElimID;
@@ -342,10 +389,21 @@ namespace llvm {
   /// branches.
   extern char &BranchFolderPassID;
 
+  /// MachineFunctionPrinterPass - This pass prints out MachineInstr's.
+  extern char &MachineFunctionPrinterPassID;
+
   /// TailDuplicate - Duplicate blocks with unconditional branches
   /// into tails of their predecessors.
   extern char &TailDuplicateID;
 
+  /// MachineTraceMetrics - This pass computes critical path and CPU resource
+  /// usage in an ensemble of traces.
+  extern char &MachineTraceMetricsID;
+
+  /// EarlyIfConverter - This pass performs if-conversion on SSA form by
+  /// inserting cmov instructions.
+  extern char &EarlyIfConverterID;
+
   /// IfConverter - This pass performs machine code if conversion.
   extern char &IfConverterID;
 
diff --git a/include/llvm/CodeGen/ProcessImplicitDefs.h b/include/llvm/CodeGen/ProcessImplicitDefs.h
deleted file mode 100644
index 6ab57f0..0000000
--- a/include/llvm/CodeGen/ProcessImplicitDefs.h
+++ /dev/null
@@ -1,51 +0,0 @@
-//===-------------- llvm/CodeGen/ProcessImplicitDefs.h ----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#ifndef LLVM_CODEGEN_PROCESSIMPLICITDEFS_H
-#define LLVM_CODEGEN_PROCESSIMPLICITDEFS_H
-
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/ADT/SmallSet.h"
-
-namespace llvm {
-
-  class MachineInstr;
-  class TargetInstrInfo;
-  class TargetRegisterInfo;
-  class MachineRegisterInfo;
-  class LiveVariables;
-
-  /// Process IMPLICIT_DEF instructions and make sure there is one implicit_def
-  /// for each use. Add isUndef marker to implicit_def defs and their uses.
-  class ProcessImplicitDefs : public MachineFunctionPass {
-    const TargetInstrInfo *TII;
-    const TargetRegisterInfo *TRI;
-    MachineRegisterInfo *MRI;
-    LiveVariables *LV;
-
-    bool CanTurnIntoImplicitDef(MachineInstr *MI, unsigned Reg,
-                                unsigned OpIdx,
-                                SmallSet<unsigned, 8> &ImpDefRegs);
-
-  public:
-    static char ID;
-
-    ProcessImplicitDefs() : MachineFunctionPass(ID) {
-      initializeProcessImplicitDefsPass(*PassRegistry::getPassRegistry());
-    }
-
-    virtual void getAnalysisUsage(AnalysisUsage &au) const;
-
-    virtual bool runOnMachineFunction(MachineFunction &fn);
-  };
-
-}
-
-#endif // LLVM_CODEGEN_PROCESSIMPLICITDEFS_H
diff --git a/include/llvm/CodeGen/RegisterClassInfo.h b/include/llvm/CodeGen/RegisterClassInfo.h
new file mode 100644
index 0000000..400e1f4
--- /dev/null
+++ b/include/llvm/CodeGen/RegisterClassInfo.h
@@ -0,0 +1,132 @@
+//===-- RegisterClassInfo.h - Dynamic Register Class Info -*- C++ -*-------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the RegisterClassInfo class which provides dynamic
+// information about target register classes. Callee saved and reserved
+// registers depends on calling conventions and other dynamic information, so
+// some things cannot be determined statically.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_REGISTERCLASSINFO_H
+#define LLVM_CODEGEN_REGISTERCLASSINFO_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+
+namespace llvm {
+
+class RegisterClassInfo {
+  struct RCInfo {
+    unsigned Tag;
+    unsigned NumRegs;
+    bool ProperSubClass;
+    OwningArrayPtr<unsigned> Order;
+
+    RCInfo() : Tag(0), NumRegs(0), ProperSubClass(false) {}
+    operator ArrayRef<unsigned>() const {
+      return makeArrayRef(Order.get(), NumRegs);
+    }
+  };
+
+  // Brief cached information for each register class.
+  OwningArrayPtr<RCInfo> RegClass;
+
+  // Tag changes whenever cached information needs to be recomputed. An RCInfo
+  // entry is valid when its tag matches.
+  unsigned Tag;
+
+  const MachineFunction *MF;
+  const TargetRegisterInfo *TRI;
+
+  // Callee saved registers of last MF. Assumed to be valid until the next
+  // runOnFunction() call.
+  const uint16_t *CalleeSaved;
+
+  // Map register number to CalleeSaved index + 1;
+  SmallVector<uint8_t, 4> CSRNum;
+
+  // Reserved registers in the current MF.
+  BitVector Reserved;
+
+  // Compute all information about RC.
+  void compute(const TargetRegisterClass *RC) const;
+
+  // Return an up-to-date RCInfo for RC.
+  const RCInfo &get(const TargetRegisterClass *RC) const {
+    const RCInfo &RCI = RegClass[RC->getID()];
+    if (Tag != RCI.Tag)
+      compute(RC);
+    return RCI;
+  }
+
+public:
+  RegisterClassInfo();
+
+  /// runOnFunction - Prepare to answer questions about MF. This must be called
+  /// before any other methods are used.
+  void runOnMachineFunction(const MachineFunction &MF);
+
+  /// getNumAllocatableRegs - Returns the number of actually allocatable
+  /// registers in RC in the current function.
+  unsigned getNumAllocatableRegs(const TargetRegisterClass *RC) const {
+    return get(RC).NumRegs;
+  }
+
+  /// getOrder - Returns the preferred allocation order for RC. The order
+  /// contains no reserved registers, and registers that alias callee saved
+  /// registers come last.
+  ArrayRef<unsigned> getOrder(const TargetRegisterClass *RC) const {
+    return get(RC);
+  }
+
+  /// isProperSubClass - Returns true if RC has a legal super-class with more
+  /// allocatable registers.
+  ///
+  /// Register classes like GR32_NOSP are not proper sub-classes because %esp
+  /// is not allocatable.  Similarly, tGPR is not a proper sub-class in Thumb
+  /// mode because the GPR super-class is not legal.
+  bool isProperSubClass(const TargetRegisterClass *RC) const {
+    return get(RC).ProperSubClass;
+  }
+
+  /// getLastCalleeSavedAlias - Returns the last callee saved register that
+  /// overlaps PhysReg, or 0 if Reg doesn't overlap a CSR.
+  unsigned getLastCalleeSavedAlias(unsigned PhysReg) const {
+    assert(TargetRegisterInfo::isPhysicalRegister(PhysReg));
+    if (unsigned N = CSRNum[PhysReg])
+      return CalleeSaved[N-1];
+    return 0;
+  }
+
+  /// isReserved - Returns true when PhysReg is a reserved register.
+  ///
+  /// Reserved registers may belong to an allocatable register class, but the
+  /// target has explicitly requested that they are not used.
+  ///
+  bool isReserved(unsigned PhysReg) const {
+    return Reserved.test(PhysReg);
+  }
+
+  /// isAllocatable - Returns true when PhysReg belongs to an allocatable
+  /// register class and it hasn't been reserved.
+  ///
+  /// Allocatable registers may show up in the allocation order of some virtual
+  /// register, so a register allocator needs to track its liveness and
+  /// availability.
+  bool isAllocatable(unsigned PhysReg) const {
+    return TRI->isInAllocatableClass(PhysReg) && !isReserved(PhysReg);
+  }
+};
+} // end namespace llvm
+
+#endif
+
diff --git a/include/llvm/CodeGen/RegisterPressure.h b/include/llvm/CodeGen/RegisterPressure.h
new file mode 100644
index 0000000..2043155
--- /dev/null
+++ b/include/llvm/CodeGen/RegisterPressure.h
@@ -0,0 +1,282 @@
+//===-- RegisterPressure.h - Dynamic Register Pressure -*- C++ -*-------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the RegisterPressure class which can be used to track
+// MachineInstr level register pressure.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_REGISTERPRESSURE_H
+#define LLVM_CODEGEN_REGISTERPRESSURE_H
+
+#include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/ADT/SparseSet.h"
+
+namespace llvm {
+
+class LiveIntervals;
+class RegisterClassInfo;
+class MachineInstr;
+
+/// Base class for register pressure results.
+struct RegisterPressure {
+  /// Map of max reg pressure indexed by pressure set ID, not class ID.
+  std::vector<unsigned> MaxSetPressure;
+
+  /// List of live in registers.
+  SmallVector<unsigned,8> LiveInRegs;
+  SmallVector<unsigned,8> LiveOutRegs;
+
+  /// Increase register pressure for each pressure set impacted by this register
+  /// class. Normally called by RegPressureTracker, but may be called manually
+  /// to account for live through (global liveness).
+  void increase(const TargetRegisterClass *RC, const TargetRegisterInfo *TRI);
+
+  /// Decrease register pressure for each pressure set impacted by this register
+  /// class. This is only useful to account for spilling or rematerialization.
+  void decrease(const TargetRegisterClass *RC, const TargetRegisterInfo *TRI);
+
+  void dump(const TargetRegisterInfo *TRI);
+};
+
+/// RegisterPressure computed within a region of instructions delimited by
+/// TopIdx and BottomIdx.  During pressure computation, the maximum pressure per
+/// register pressure set is increased. Once pressure within a region is fully
+/// computed, the live-in and live-out sets are recorded.
+///
+/// This is preferable to RegionPressure when LiveIntervals are available,
+/// because delimiting regions by SlotIndex is more robust and convenient than
+/// holding block iterators. The block contents can change without invalidating
+/// the pressure result.
+struct IntervalPressure : RegisterPressure {
+  /// Record the boundary of the region being tracked.
+  SlotIndex TopIdx;
+  SlotIndex BottomIdx;
+
+  void reset();
+
+  void openTop(SlotIndex NextTop);
+
+  void openBottom(SlotIndex PrevBottom);
+};
+
+/// RegisterPressure computed within a region of instructions delimited by
+/// TopPos and BottomPos. This is a less precise version of IntervalPressure for
+/// use when LiveIntervals are unavailable.
+struct RegionPressure : RegisterPressure {
+  /// Record the boundary of the region being tracked.
+  MachineBasicBlock::const_iterator TopPos;
+  MachineBasicBlock::const_iterator BottomPos;
+
+  void reset();
+
+  void openTop(MachineBasicBlock::const_iterator PrevTop);
+
+  void openBottom(MachineBasicBlock::const_iterator PrevBottom);
+};
+
+/// An element of pressure difference that identifies the pressure set and
+/// amount of increase or decrease in units of pressure.
+struct PressureElement {
+  unsigned PSetID;
+  int UnitIncrease;
+
+  PressureElement(): PSetID(~0U), UnitIncrease(0) {}
+  PressureElement(unsigned id, int inc): PSetID(id), UnitIncrease(inc) {}
+
+  bool isValid() const { return PSetID != ~0U; }
+};
+
+/// Store the effects of a change in pressure on things that MI scheduler cares
+/// about.
+///
+/// Excess records the value of the largest difference in register units beyond
+/// the target's pressure limits across the affected pressure sets, where
+/// largest is defined as the absolute value of the difference. Negative
+/// ExcessUnits indicates a reduction in pressure that had already exceeded the
+/// target's limits.
+///
+/// CriticalMax records the largest increase in the tracker's max pressure that
+/// exceeds the critical limit for some pressure set determined by the client.
+///
+/// CurrentMax records the largest increase in the tracker's max pressure that
+/// exceeds the current limit for some pressure set determined by the client.
+struct RegPressureDelta {
+  PressureElement Excess;
+  PressureElement CriticalMax;
+  PressureElement CurrentMax;
+
+  RegPressureDelta() {}
+};
+
+/// Track the current register pressure at some position in the instruction
+/// stream, and remember the high water mark within the region traversed. This
+/// does not automatically consider live-through ranges. The client may
+/// independently adjust for global liveness.
+///
+/// Each RegPressureTracker only works within a MachineBasicBlock. Pressure can
+/// be tracked across a larger region by storing a RegisterPressure result at
+/// each block boundary and explicitly adjusting pressure to account for block
+/// live-in and live-out register sets.
+///
+/// RegPressureTracker holds a reference to a RegisterPressure result that it
+/// computes incrementally. During downward tracking, P.BottomIdx or P.BottomPos
+/// is invalid until it reaches the end of the block or closeRegion() is
+/// explicitly called. Similarly, P.TopIdx is invalid during upward
+/// tracking. Changing direction has the side effect of closing region, and
+/// traversing past TopIdx or BottomIdx reopens it.
+class RegPressureTracker {
+  const MachineFunction     *MF;
+  const TargetRegisterInfo  *TRI;
+  const RegisterClassInfo   *RCI;
+  const MachineRegisterInfo *MRI;
+  const LiveIntervals       *LIS;
+
+  /// We currently only allow pressure tracking within a block.
+  const MachineBasicBlock *MBB;
+
+  /// Track the max pressure within the region traversed so far.
+  RegisterPressure &P;
+
+  /// Run in two modes dependending on whether constructed with IntervalPressure
+  /// or RegisterPressure. If requireIntervals is false, LIS are ignored.
+  bool RequireIntervals;
+
+  /// Register pressure corresponds to liveness before this instruction
+  /// iterator. It may point to the end of the block rather than an instruction.
+  MachineBasicBlock::const_iterator CurrPos;
+
+  /// Pressure map indexed by pressure set ID, not class ID.
+  std::vector<unsigned> CurrSetPressure;
+
+  /// List of live registers.
+  SparseSet<unsigned> LivePhysRegs;
+  SparseSet<unsigned, VirtReg2IndexFunctor> LiveVirtRegs;
+
+public:
+  RegPressureTracker(IntervalPressure &rp) :
+    MF(0), TRI(0), RCI(0), LIS(0), MBB(0), P(rp), RequireIntervals(true) {}
+
+  RegPressureTracker(RegionPressure &rp) :
+    MF(0), TRI(0), RCI(0), LIS(0), MBB(0), P(rp), RequireIntervals(false) {}
+
+  void init(const MachineFunction *mf, const RegisterClassInfo *rci,
+            const LiveIntervals *lis, const MachineBasicBlock *mbb,
+            MachineBasicBlock::const_iterator pos);
+
+  /// Force liveness of registers. Particularly useful to initialize the
+  /// livein/out state of the tracker before the first call to advance/recede.
+  void addLiveRegs(ArrayRef<unsigned> Regs);
+
+  /// Get the MI position corresponding to this register pressure.
+  MachineBasicBlock::const_iterator getPos() const { return CurrPos; }
+
+  // Reset the MI position corresponding to the register pressure. This allows
+  // schedulers to move instructions above the RegPressureTracker's
+  // CurrPos. Since the pressure is computed before CurrPos, the iterator
+  // position changes while pressure does not.
+  void setPos(MachineBasicBlock::const_iterator Pos) { CurrPos = Pos; }
+
+  /// Recede across the previous instruction.
+  bool recede();
+
+  /// Advance across the current instruction.
+  bool advance();
+
+  /// Finalize the region boundaries and recored live ins and live outs.
+  void closeRegion();
+
+  /// Get the resulting register pressure over the traversed region.
+  /// This result is complete if either advance() or recede() has returned true,
+  /// or if closeRegion() was explicitly invoked.
+  RegisterPressure &getPressure() { return P; }
+
+  /// Get the register set pressure at the current position, which may be less
+  /// than the pressure across the traversed region.
+  std::vector<unsigned> &getRegSetPressureAtPos() { return CurrSetPressure; }
+
+  void discoverPhysLiveIn(unsigned Reg);
+  void discoverPhysLiveOut(unsigned Reg);
+
+  void discoverVirtLiveIn(unsigned Reg);
+  void discoverVirtLiveOut(unsigned Reg);
+
+  bool isTopClosed() const;
+  bool isBottomClosed() const;
+
+  void closeTop();
+  void closeBottom();
+
+  /// Consider the pressure increase caused by traversing this instruction
+  /// bottom-up. Find the pressure set with the most change beyond its pressure
+  /// limit based on the tracker's current pressure, and record the number of
+  /// excess register units of that pressure set introduced by this instruction.
+  void getMaxUpwardPressureDelta(const MachineInstr *MI,
+                                 RegPressureDelta &Delta,
+                                 ArrayRef<PressureElement> CriticalPSets,
+                                 ArrayRef<unsigned> MaxPressureLimit);
+
+  /// Consider the pressure increase caused by traversing this instruction
+  /// top-down. Find the pressure set with the most change beyond its pressure
+  /// limit based on the tracker's current pressure, and record the number of
+  /// excess register units of that pressure set introduced by this instruction.
+  void getMaxDownwardPressureDelta(const MachineInstr *MI,
+                                   RegPressureDelta &Delta,
+                                   ArrayRef<PressureElement> CriticalPSets,
+                                   ArrayRef<unsigned> MaxPressureLimit);
+
+  /// Find the pressure set with the most change beyond its pressure limit after
+  /// traversing this instruction either upward or downward depending on the
+  /// closed end of the current region.
+  void getMaxPressureDelta(const MachineInstr *MI, RegPressureDelta &Delta,
+                           ArrayRef<PressureElement> CriticalPSets,
+                           ArrayRef<unsigned> MaxPressureLimit) {
+    if (isTopClosed())
+      return getMaxDownwardPressureDelta(MI, Delta, CriticalPSets,
+                                         MaxPressureLimit);
+
+    assert(isBottomClosed() && "Uninitialized pressure tracker");
+    return getMaxUpwardPressureDelta(MI, Delta, CriticalPSets,
+                                     MaxPressureLimit);
+  }
+
+  /// Get the pressure of each PSet after traversing this instruction bottom-up.
+  void getUpwardPressure(const MachineInstr *MI,
+                         std::vector<unsigned> &PressureResult,
+                         std::vector<unsigned> &MaxPressureResult);
+
+  /// Get the pressure of each PSet after traversing this instruction top-down.
+  void getDownwardPressure(const MachineInstr *MI,
+                           std::vector<unsigned> &PressureResult,
+                           std::vector<unsigned> &MaxPressureResult);
+
+  void getPressureAfterInst(const MachineInstr *MI,
+                            std::vector<unsigned> &PressureResult,
+                            std::vector<unsigned> &MaxPressureResult) {
+    if (isTopClosed())
+      return getUpwardPressure(MI, PressureResult, MaxPressureResult);
+
+    assert(isBottomClosed() && "Uninitialized pressure tracker");
+    return getDownwardPressure(MI, PressureResult, MaxPressureResult);
+  }
+
+protected:
+  void increasePhysRegPressure(ArrayRef<unsigned> Regs);
+  void decreasePhysRegPressure(ArrayRef<unsigned> Regs);
+
+  void increaseVirtRegPressure(ArrayRef<unsigned> Regs);
+  void decreaseVirtRegPressure(ArrayRef<unsigned> Regs);
+
+  void bumpUpwardPressure(const MachineInstr *MI);
+  void bumpDownwardPressure(const MachineInstr *MI);
+};
+} // end namespace llvm
+
+#endif
diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h
index f4de693..85ab47b 100644
--- a/include/llvm/CodeGen/ScheduleDAG.h
+++ b/include/llvm/CodeGen/ScheduleDAG.h
@@ -117,8 +117,9 @@ namespace llvm {
       }
     }
 
-    bool operator==(const SDep &Other) const {
-      if (Dep != Other.Dep || Latency != Other.Latency) return false;
+    /// Return true if the specified SDep is equivalent except for latency.
+    bool overlaps(const SDep &Other) const {
+      if (Dep != Other.Dep) return false;
       switch (Dep.getInt()) {
       case Data:
       case Anti:
@@ -133,6 +134,10 @@ namespace llvm {
       llvm_unreachable("Invalid dependency kind!");
     }
 
+    bool operator==(const SDep &Other) const {
+      return overlaps(Other) && Latency == Other.Latency;
+    }
+
     bool operator!=(const SDep &Other) const {
       return !operator==(Other);
     }
@@ -272,6 +277,9 @@ namespace llvm {
     unsigned Depth;                     // Node depth.
     unsigned Height;                    // Node height.
   public:
+    unsigned TopReadyCycle; // Cycle relative to start when node is ready.
+    unsigned BotReadyCycle; // Cycle relative to end when node is ready.
+
     const TargetRegisterClass *CopyDstRC; // Is a special copy node if not null.
     const TargetRegisterClass *CopySrcRC;
 
@@ -287,7 +295,7 @@ namespace llvm {
         isScheduleHigh(false), isScheduleLow(false), isCloned(false),
         SchedulingPref(Sched::None),
         isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0),
-        CopyDstRC(NULL), CopySrcRC(NULL) {}
+        TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}
 
     /// SUnit - Construct an SUnit for post-regalloc scheduling to represent
     /// a MachineInstr.
@@ -301,7 +309,7 @@ namespace llvm {
         isScheduleHigh(false), isScheduleLow(false), isCloned(false),
         SchedulingPref(Sched::None),
         isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0),
-        CopyDstRC(NULL), CopySrcRC(NULL) {}
+        TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}
 
     /// SUnit - Construct a placeholder SUnit.
     SUnit()
@@ -314,7 +322,7 @@ namespace llvm {
         isScheduleHigh(false), isScheduleLow(false), isCloned(false),
         SchedulingPref(Sched::None),
         isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0),
-        CopyDstRC(NULL), CopySrcRC(NULL) {}
+        TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}
 
     /// setNode - Assign the representative SDNode for this SUnit.
     /// This may be used during pre-regalloc scheduling.
@@ -552,12 +560,6 @@ namespace llvm {
     ///
     virtual void computeLatency(SUnit *SU) = 0;
 
-    /// ComputeOperandLatency - Override dependence edge latency using
-    /// operand use/def information
-    ///
-    virtual void computeOperandLatency(SUnit *, SUnit *,
-                                       SDep&) const { }
-
     /// ForceUnitLatencies - Return true if all scheduling edges should be given
     /// a latency value of one.  The default is to return false; schedulers may
     /// override this as needed.
diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h
index 4fee108..1bde942 100644
--- a/include/llvm/CodeGen/ScheduleDAGInstrs.h
+++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h
@@ -28,6 +28,7 @@ namespace llvm {
   class MachineLoopInfo;
   class MachineDominatorTree;
   class LiveIntervals;
+  class RegPressureTracker;
 
   /// LoopDependencies - This class analyzes loop-oriented register
   /// dependencies, which are used to guide scheduling decisions.
@@ -35,7 +36,6 @@ namespace llvm {
   /// scheduled as soon as possible after the variable's last use.
   ///
   class LoopDependencies {
-    const MachineLoopInfo &MLI;
     const MachineDominatorTree &MDT;
 
   public:
@@ -43,9 +43,7 @@ namespace llvm {
       LoopDeps;
     LoopDeps Deps;
 
-    LoopDependencies(const MachineLoopInfo &mli,
-                     const MachineDominatorTree &mdt) :
-      MLI(mli), MDT(mdt) {}
+    LoopDependencies(const MachineDominatorTree &mdt) : MDT(mdt) {}
 
     /// VisitLoop - Clear out any previous state and analyze the given loop.
     ///
@@ -105,7 +103,7 @@ namespace llvm {
 
     VReg2SUnit(unsigned reg, SUnit *su): VirtReg(reg), SU(su) {}
 
-    unsigned getSparseSetKey() const {
+    unsigned getSparseSetIndex() const {
       return TargetRegisterInfo::virtReg2Index(VirtReg);
     }
   };
@@ -160,7 +158,7 @@ namespace llvm {
   /// compares ValueT's, only unsigned keys. This allows the set to be cleared
   /// between scheduling regions in constant time as long as ValueT does not
   /// require a destructor.
-  typedef SparseSet<VReg2SUnit> VReg2SUnitMap;
+  typedef SparseSet<VReg2SUnit, VirtReg2IndexFunctor> VReg2SUnitMap;
 
   /// ScheduleDAGInstrs - A ScheduleDAG subclass for scheduling lists of
   /// MachineInstrs.
@@ -229,7 +227,7 @@ namespace llvm {
     ///
     LoopDependencies LoopRegs;
 
-    /// DbgValues - Remember instruction that preceeds DBG_VALUE.
+    /// DbgValues - Remember instruction that precedes DBG_VALUE.
     /// These are generated by buildSchedGraph but persist so they can be
     /// referenced when emitting the final schedule.
     typedef std::vector<std::pair<MachineInstr *, MachineInstr *> >
@@ -275,7 +273,7 @@ namespace llvm {
 
     /// buildSchedGraph - Build SUnits from the MachineBasicBlock that we are
     /// input.
-    void buildSchedGraph(AliasAnalysis *AA);
+    void buildSchedGraph(AliasAnalysis *AA, RegPressureTracker *RPTracker = 0);
 
     /// addSchedBarrierDeps - Add dependencies from instructions in the current
     /// list of instructions being scheduled to scheduling barrier. We want to
@@ -290,11 +288,15 @@ namespace llvm {
     ///
     virtual void computeLatency(SUnit *SU);
 
-    /// computeOperandLatency - Override dependence edge latency using
+    /// computeOperandLatency - Return dependence edge latency using
     /// operand use/def information
     ///
-    virtual void computeOperandLatency(SUnit *Def, SUnit *Use,
-                                       SDep& dep) const;
+    /// FindMin may be set to get the minimum vs. expected latency. Minimum
+    /// latency is used for scheduling groups, while expected latency is for
+    /// instruction cost and critical path.
+    virtual unsigned computeOperandLatency(SUnit *Def, SUnit *Use,
+                                           const SDep& dep,
+                                           bool FindMin = false) const;
 
     /// schedule - Order nodes according to selected style, filling
     /// in the Sequence member.
@@ -321,10 +323,6 @@ namespace llvm {
     void addPhysRegDeps(SUnit *SU, unsigned OperIdx);
     void addVRegDefDeps(SUnit *SU, unsigned OperIdx);
     void addVRegUseDeps(SUnit *SU, unsigned OperIdx);
-
-    VReg2SUnitMap::iterator findVRegDef(unsigned VirtReg) {
-      return VRegDefs.find(TargetRegisterInfo::virtReg2Index(VirtReg));
-    }
   };
 
   /// newSUnit - Creates a new SUnit and return a ptr to it.
diff --git a/include/llvm/CodeGen/ScheduleHazardRecognizer.h b/include/llvm/CodeGen/ScheduleHazardRecognizer.h
index 2f53baa..9dfa344 100644
--- a/include/llvm/CodeGen/ScheduleHazardRecognizer.h
+++ b/include/llvm/CodeGen/ScheduleHazardRecognizer.h
@@ -46,6 +46,8 @@ public:
 
   /// atIssueLimit - Return true if no more instructions may be issued in this
   /// cycle.
+  ///
+  /// FIXME: remove this once MachineScheduler is the only client.
   virtual bool atIssueLimit() const { return false; }
 
   /// getHazardType - Return the hazard type of emitting this node.  There are
@@ -55,7 +57,7 @@ public:
   ///     other instruction is available, issue it first.
   ///  * NoopHazard: issuing this instruction would break the program.  If
   ///     some other instruction can be issued, do so, otherwise issue a noop.
-  virtual HazardType getHazardType(SUnit *m, int Stalls) {
+  virtual HazardType getHazardType(SUnit *m, int Stalls = 0) {
     return NoHazard;
   }
 
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index 6a7a87e..1ccfe54 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -177,6 +177,44 @@ class SelectionDAG {
   /// DbgInfo - Tracks dbg_value information through SDISel.
   SDDbgInfo *DbgInfo;
 
+public:
+  /// DAGUpdateListener - Clients of various APIs that cause global effects on
+  /// the DAG can optionally implement this interface.  This allows the clients
+  /// to handle the various sorts of updates that happen.
+  ///
+  /// A DAGUpdateListener automatically registers itself with DAG when it is
+  /// constructed, and removes itself when destroyed in RAII fashion.
+  struct DAGUpdateListener {
+    DAGUpdateListener *const Next;
+    SelectionDAG &DAG;
+
+    explicit DAGUpdateListener(SelectionDAG &D)
+      : Next(D.UpdateListeners), DAG(D) {
+      DAG.UpdateListeners = this;
+    }
+
+    virtual ~DAGUpdateListener() {
+      assert(DAG.UpdateListeners == this &&
+             "DAGUpdateListeners must be destroyed in LIFO order");
+      DAG.UpdateListeners = Next;
+    }
+
+    /// NodeDeleted - The node N that was deleted and, if E is not null, an
+    /// equivalent node E that replaced it.
+    virtual void NodeDeleted(SDNode *N, SDNode *E);
+
+    /// NodeUpdated - The node N that was updated.
+    virtual void NodeUpdated(SDNode *N);
+  };
+
+private:
+  /// DAGUpdateListener is a friend so it can manipulate the listener stack.
+  friend struct DAGUpdateListener;
+
+  /// UpdateListeners - Linked list of registered DAGUpdateListener instances.
+  /// This stack is maintained by DAGUpdateListener RAII.
+  DAGUpdateListener *UpdateListeners;
+
   /// setGraphColorHelper - Implementation of setSubgraphColor.
   /// Return whether we had to truncate the search.
   ///
@@ -384,6 +422,8 @@ public:
                                   int Offset = 0, unsigned char TargetFlags=0) {
     return getConstantPool(C, VT, Align, Offset, true, TargetFlags);
   }
+  SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0,
+                         unsigned char TargetFlags = 0);
   // When generating a branch to a BB, we don't in general know enough
   // to provide debug info for the BB at that time, so keep this one around.
   SDValue getBasicBlock(MachineBasicBlock *MBB);
@@ -817,30 +857,14 @@ public:
   SDDbgValue *getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off,
                           DebugLoc DL, unsigned O);
 
-  /// DAGUpdateListener - Clients of various APIs that cause global effects on
-  /// the DAG can optionally implement this interface.  This allows the clients
-  /// to handle the various sorts of updates that happen.
-  class DAGUpdateListener {
-  public:
-    virtual ~DAGUpdateListener();
-
-    /// NodeDeleted - The node N that was deleted and, if E is not null, an
-    /// equivalent node E that replaced it.
-    virtual void NodeDeleted(SDNode *N, SDNode *E) = 0;
-
-    /// NodeUpdated - The node N that was updated.
-    virtual void NodeUpdated(SDNode *N) = 0;
-  };
-
   /// RemoveDeadNode - Remove the specified node from the system. If any of its
   /// operands then becomes dead, remove them as well. Inform UpdateListener
   /// for each node deleted.
-  void RemoveDeadNode(SDNode *N, DAGUpdateListener *UpdateListener = 0);
+  void RemoveDeadNode(SDNode *N);
 
   /// RemoveDeadNodes - This method deletes the unreachable nodes in the
   /// given list, and any nodes that become unreachable as a result.
-  void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes,
-                       DAGUpdateListener *UpdateListener = 0);
+  void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes);
 
   /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
   /// This can cause recursive merging of nodes in the DAG.  Use the first
@@ -857,24 +881,19 @@ public:
   /// to be given new uses. These new uses of From are left in place, and
   /// not automatically transferred to To.
   ///
-  void ReplaceAllUsesWith(SDValue From, SDValue Op,
-                          DAGUpdateListener *UpdateListener = 0);
-  void ReplaceAllUsesWith(SDNode *From, SDNode *To,
-                          DAGUpdateListener *UpdateListener = 0);
-  void ReplaceAllUsesWith(SDNode *From, const SDValue *To,
-                          DAGUpdateListener *UpdateListener = 0);
+  void ReplaceAllUsesWith(SDValue From, SDValue Op);
+  void ReplaceAllUsesWith(SDNode *From, SDNode *To);
+  void ReplaceAllUsesWith(SDNode *From, const SDValue *To);
 
   /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
   /// uses of other values produced by From.Val alone.
-  void ReplaceAllUsesOfValueWith(SDValue From, SDValue To,
-                                 DAGUpdateListener *UpdateListener = 0);
+  void ReplaceAllUsesOfValueWith(SDValue From, SDValue To);
 
   /// ReplaceAllUsesOfValuesWith - Like ReplaceAllUsesOfValueWith, but
   /// for multiple values at once. This correctly handles the case where
   /// there is an overlap between the From values and the To values.
   void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To,
-                                  unsigned Num,
-                                  DAGUpdateListener *UpdateListener = 0);
+                                  unsigned Num);
 
   /// AssignTopologicalOrder - Topological-sort the AllNodes list and a
   /// assign a unique node id for each node in the DAG based on their
@@ -1031,7 +1050,7 @@ public:
 
 private:
   bool RemoveNodeFromCSEMaps(SDNode *N);
-  void AddModifiedNodeToCSEMaps(SDNode *N, DAGUpdateListener *UpdateListener);
+  void AddModifiedNodeToCSEMaps(SDNode *N);
   SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos);
   SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
                                void *&InsertPos);
diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h
index ee3f231..c42f655 100644
--- a/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/include/llvm/CodeGen/SelectionDAGISel.h
@@ -172,53 +172,22 @@ protected:
   ///
   unsigned DAGSize;
 
-  /// ISelPosition - Node iterator marking the current position of
-  /// instruction selection as it procedes through the topologically-sorted
-  /// node list.
-  SelectionDAG::allnodes_iterator ISelPosition;
-
-
-  /// ISelUpdater - helper class to handle updates of the
-  /// instruction selection graph.
-  class ISelUpdater : public SelectionDAG::DAGUpdateListener {
-    virtual void anchor();
-    SelectionDAG::allnodes_iterator &ISelPosition;
-  public:
-    explicit ISelUpdater(SelectionDAG::allnodes_iterator &isp)
-      : ISelPosition(isp) {}
-
-    /// NodeDeleted - Handle nodes deleted from the graph. If the
-    /// node being deleted is the current ISelPosition node, update
-    /// ISelPosition.
-    ///
-    virtual void NodeDeleted(SDNode *N, SDNode *E) {
-      if (ISelPosition == SelectionDAG::allnodes_iterator(N))
-        ++ISelPosition;
-    }
-
-    /// NodeUpdated - Ignore updates for now.
-    virtual void NodeUpdated(SDNode *N) {}
-  };
-
   /// ReplaceUses - replace all uses of the old node F with the use
   /// of the new node T.
   void ReplaceUses(SDValue F, SDValue T) {
-    ISelUpdater ISU(ISelPosition);
-    CurDAG->ReplaceAllUsesOfValueWith(F, T, &ISU);
+    CurDAG->ReplaceAllUsesOfValueWith(F, T);
   }
 
   /// ReplaceUses - replace all uses of the old nodes F with the use
   /// of the new nodes T.
   void ReplaceUses(const SDValue *F, const SDValue *T, unsigned Num) {
-    ISelUpdater ISU(ISelPosition);
-    CurDAG->ReplaceAllUsesOfValuesWith(F, T, Num, &ISU);
+    CurDAG->ReplaceAllUsesOfValuesWith(F, T, Num);
   }
 
   /// ReplaceUses - replace all uses of the old node F with the use
   /// of the new node T.
   void ReplaceUses(SDNode *F, SDNode *T) {
-    ISelUpdater ISU(ISelPosition);
-    CurDAG->ReplaceAllUsesWith(F, T, &ISU);
+    CurDAG->ReplaceAllUsesWith(F, T);
   }
 
 
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index f8248b8..0dfb394 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -74,6 +74,10 @@ namespace ISD {
   /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
   /// element is not an undef.
   bool isScalarToVector(const SDNode *N);
+
+  /// allOperandsUndef - Return true if the node has at least one operand
+  /// and all operands of the specified node are ISD::UNDEF.
+  bool allOperandsUndef(const SDNode *N);
 }  // end llvm:ISD namespace
 
 //===----------------------------------------------------------------------===//
@@ -1339,6 +1343,29 @@ public:
   }
 };
 
+/// Completely target-dependent object reference.
+class TargetIndexSDNode : public SDNode {
+  unsigned char TargetFlags;
+  int Index;
+  int64_t Offset;
+  friend class SelectionDAG;
+public:
+
+  TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned char TF)
+    : SDNode(ISD::TargetIndex, DebugLoc(), getSDVTList(VT)),
+      TargetFlags(TF), Index(Idx), Offset(Ofs) {}
+public:
+
+  unsigned char getTargetFlags() const { return TargetFlags; }
+  int getIndex() const { return Index; }
+  int64_t getOffset() const { return Offset; }
+
+  static bool classof(const TargetIndexSDNode*) { return true; }
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::TargetIndex;
+  }
+};
+
 class BasicBlockSDNode : public SDNode {
   MachineBasicBlock *MBB;
   friend class SelectionDAG;
diff --git a/include/llvm/CodeGen/SlotIndexes.h b/include/llvm/CodeGen/SlotIndexes.h
index 0457e43..c52599b 100644
--- a/include/llvm/CodeGen/SlotIndexes.h
+++ b/include/llvm/CodeGen/SlotIndexes.h
@@ -76,7 +76,6 @@ namespace llvm {
   /// SlotIndex - An opaque wrapper around machine indexes.
   class SlotIndex {
     friend class SlotIndexes;
-    friend struct DenseMapInfo<SlotIndex>;
 
     enum Slot {
       /// Basic block boundary.  Used for live ranges entering and leaving a
@@ -121,11 +120,6 @@ namespace llvm {
       return static_cast<Slot>(lie.getInt());
     }
 
-    static inline unsigned getHashValue(const SlotIndex &v) {
-      void *ptrVal = v.lie.getOpaqueValue();
-      return (unsigned((intptr_t)ptrVal)) ^ (unsigned((intptr_t)ptrVal) >> 9);
-    }
-
   public:
     enum {
       /// The default distance between instructions as returned by distance().
@@ -133,14 +127,6 @@ namespace llvm {
       InstrDist = 4 * Slot_Count
     };
 
-    static inline SlotIndex getEmptyKey() {
-      return SlotIndex(0, 1);
-    }
-
-    static inline SlotIndex getTombstoneKey() {
-      return SlotIndex(0, 2);
-    }
-
     /// Construct an invalid index.
     SlotIndex() : lie(0, 0) {}
 
@@ -293,23 +279,6 @@ namespace llvm {
 
   };
 
-  /// DenseMapInfo specialization for SlotIndex.
-  template <>
-  struct DenseMapInfo<SlotIndex> {
-    static inline SlotIndex getEmptyKey() {
-      return SlotIndex::getEmptyKey();
-    }
-    static inline SlotIndex getTombstoneKey() {
-      return SlotIndex::getTombstoneKey();
-    }
-    static inline unsigned getHashValue(const SlotIndex &v) {
-      return SlotIndex::getHashValue(v);
-    }
-    static inline bool isEqual(const SlotIndex &LHS, const SlotIndex &RHS) {
-      return (LHS == RHS);
-    }
-  };
-
   template <> struct isPodLike<SlotIndex> { static const bool value = true; };
 
 
@@ -344,7 +313,6 @@ namespace llvm {
     IndexList indexList;
 
     MachineFunction *mf;
-    unsigned functionSize;
 
     typedef DenseMap<const MachineInstr*, SlotIndex> Mi2IndexMap;
     Mi2IndexMap mi2iMap;
@@ -402,19 +370,6 @@ namespace llvm {
       return SlotIndex(&indexList.back(), 0);
     }
 
-    /// Returns the distance between the highest and lowest indexes allocated
-    /// so far.
-    unsigned getIndexesLength() const {
-      assert(indexList.front().getIndex() == 0 &&
-             "Initial index isn't zero?");
-      return indexList.back().getIndex();
-    }
-
-    /// Returns the number of instructions in the function.
-    unsigned getFunctionSize() const {
-      return functionSize;
-    }
-
     /// Returns true if the given machine instr is mapped to an index,
     /// otherwise returns false.
     bool hasIndex(const MachineInstr *instr) const {
@@ -444,7 +399,7 @@ namespace llvm {
     }
 
     /// getIndexBefore - Returns the index of the last indexed instruction
-    /// before MI, or the the start index of its basic block.
+    /// before MI, or the start index of its basic block.
     /// MI is not required to have an index.
     SlotIndex getIndexBefore(const MachineInstr *MI) const {
       const MachineBasicBlock *MBB = MI->getParent();
@@ -590,7 +545,7 @@ namespace llvm {
         nextItr = getIndexAfter(mi).listEntry();
         prevItr = prior(nextItr);
       } else {
-        // Insert mi's index immediately after the preceeding instruction.
+        // Insert mi's index immediately after the preceding instruction.
         prevItr = getIndexBefore(mi).listEntry();
         nextItr = llvm::next(prevItr);
       }
diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index 5a42136..9849e92 100644
--- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -33,6 +33,8 @@ namespace llvm {
 
 
 class TargetLoweringObjectFileELF : public TargetLoweringObjectFile {
+  bool UseInitArray;
+
 public:
   virtual ~TargetLoweringObjectFileELF() {}
 
@@ -66,6 +68,7 @@ public:
   getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang,
                           MachineModuleInfo *MMI) const;
 
+  void InitializeELF(bool UseInitArray_);
   virtual const MCSection *
   getStaticCtorSection(unsigned Priority = 65535) const;
   virtual const MCSection *
diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h
index 76c2357..eb38cd3 100644
--- a/include/llvm/CodeGen/ValueTypes.h
+++ b/include/llvm/CodeGen/ValueTypes.h
@@ -68,34 +68,38 @@ namespace llvm {
       v2i32          =  22,   //  2 x i32
       v4i32          =  23,   //  4 x i32
       v8i32          =  24,   //  8 x i32
-      v1i64          =  25,   //  1 x i64
-      v2i64          =  26,   //  2 x i64
-      v4i64          =  27,   //  4 x i64
-      v8i64          =  28,   //  8 x i64
-
-      v2f16          =  29,   //  2 x f16
-      v2f32          =  30,   //  2 x f32
-      v4f32          =  31,   //  4 x f32
-      v8f32          =  32,   //  8 x f32
-      v2f64          =  33,   //  2 x f64
-      v4f64          =  34,   //  4 x f64
+      v16i32         =  25,   // 16 x i32
+      v1i64          =  26,   //  1 x i64
+      v2i64          =  27,   //  2 x i64
+      v4i64          =  28,   //  4 x i64
+      v8i64          =  29,   //  8 x i64
+      v16i64         =  30,   // 16 x i64
+
+      v2f16          =  31,   //  2 x f16
+      v2f32          =  32,   //  2 x f32
+      v4f32          =  33,   //  4 x f32
+      v8f32          =  34,   //  8 x f32
+      v2f64          =  35,   //  2 x f64
+      v4f64          =  36,   //  4 x f64
 
       FIRST_VECTOR_VALUETYPE = v2i8,
       LAST_VECTOR_VALUETYPE  = v4f64,
+      FIRST_INTEGER_VECTOR_VALUETYPE = v2i8,
+      LAST_INTEGER_VECTOR_VALUETYPE = v16i64,
       FIRST_FP_VECTOR_VALUETYPE = v2f16,
       LAST_FP_VECTOR_VALUETYPE = v4f64,
 
-      x86mmx         =  35,   // This is an X86 MMX value
+      x86mmx         =  37,   // This is an X86 MMX value
 
-      Glue           =  36,   // This glues nodes together during pre-RA sched
+      Glue           =  38,   // This glues nodes together during pre-RA sched
 
-      isVoid         =  37,   // This has no value
+      isVoid         =  39,   // This has no value
 
-      Untyped        =  38,   // This value takes a register, but has
+      Untyped        =  40,   // This value takes a register, but has
                               // unspecified type.  The register class
                               // will be determined by the opcode.
 
-      LAST_VALUETYPE =  39,   // This always remains at the end of the list.
+      LAST_VALUETYPE =  41,   // This always remains at the end of the list.
 
       // This is the current maximum for LAST_VALUETYPE.
       // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors
@@ -153,15 +157,16 @@ namespace llvm {
     bool isFloatingPoint() const {
       return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE &&
                SimpleTy <= MVT::LAST_FP_VALUETYPE) ||
-	      (SimpleTy >= MVT::FIRST_FP_VECTOR_VALUETYPE &&
-         SimpleTy <= MVT::LAST_FP_VECTOR_VALUETYPE));
+              (SimpleTy >= MVT::FIRST_FP_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_FP_VECTOR_VALUETYPE));
     }
 
     /// isInteger - Return true if this is an integer, or a vector integer type.
     bool isInteger() const {
       return ((SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE &&
                SimpleTy <= MVT::LAST_INTEGER_VALUETYPE) ||
-	      (SimpleTy >= MVT::v2i8 && SimpleTy <= MVT::v8i64));
+              (SimpleTy >= MVT::FIRST_INTEGER_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_INTEGER_VECTOR_VALUETYPE));
     }
 
     /// isVector - Return true if this is a vector value type.
@@ -170,6 +175,37 @@ namespace llvm {
               SimpleTy <= MVT::LAST_VECTOR_VALUETYPE);
     }
 
+    /// is64BitVector - Return true if this is a 64-bit vector type.
+    bool is64BitVector() const {
+      return (SimpleTy == MVT::v8i8  || SimpleTy == MVT::v4i16 ||
+              SimpleTy == MVT::v2i32 || SimpleTy == MVT::v1i64 ||
+              SimpleTy == MVT::v2f32);
+    }
+
+    /// is128BitVector - Return true if this is a 128-bit vector type.
+    bool is128BitVector() const {
+      return (SimpleTy == MVT::v16i8 || SimpleTy == MVT::v8i16 ||
+              SimpleTy == MVT::v4i32 || SimpleTy == MVT::v2i64 ||
+              SimpleTy == MVT::v4f32 || SimpleTy == MVT::v2f64);
+    }
+
+    /// is256BitVector - Return true if this is a 256-bit vector type.
+    bool is256BitVector() const {
+      return (SimpleTy == MVT::v8f32 || SimpleTy == MVT::v4f64  ||
+              SimpleTy == MVT::v32i8 || SimpleTy == MVT::v16i16 ||
+              SimpleTy == MVT::v8i32 || SimpleTy == MVT::v4i64);
+    }
+
+    /// is512BitVector - Return true if this is a 512-bit vector type.
+    bool is512BitVector() const {
+      return (SimpleTy == MVT::v8i64 || SimpleTy == MVT::v16i32);
+    }
+
+    /// is1024BitVector - Return true if this is a 1024-bit vector type.
+    bool is1024BitVector() const {
+      return (SimpleTy == MVT::v16i64);
+    }
+
     /// isPow2VectorType - Returns true if the given vector is a power of 2.
     bool isPow2VectorType() const {
       unsigned NElts = getVectorNumElements();
@@ -196,7 +232,7 @@ namespace llvm {
     MVT getVectorElementType() const {
       switch (SimpleTy) {
       default:
-        return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
+        llvm_unreachable("Not a vector MVT!");
       case v2i8 :
       case v4i8 :
       case v8i8 :
@@ -208,11 +244,13 @@ namespace llvm {
       case v16i16: return i16;
       case v2i32:
       case v4i32:
-      case v8i32: return i32;
+      case v8i32:
+      case v16i32: return i32;
       case v1i64:
       case v2i64:
       case v4i64:
-      case v8i64: return i64;
+      case v8i64:
+      case v16i64: return i64;
       case v2f16: return f16;
       case v2f32:
       case v4f32:
@@ -225,10 +263,12 @@ namespace llvm {
     unsigned getVectorNumElements() const {
       switch (SimpleTy) {
       default:
-        return ~0U;
+        llvm_unreachable("Not a vector MVT!");
       case v32i8: return 32;
       case v16i8:
-      case v16i16: return 16;
+      case v16i16:
+      case v16i32:
+      case v16i64:return 16;
       case v8i8 :
       case v8i16:
       case v8i32:
@@ -295,7 +335,9 @@ namespace llvm {
       case v4i64:
       case v8f32:
       case v4f64: return 256;
+      case v16i32:
       case v8i64: return 512;
+      case v16i64:return 1024;
       }
     }
 
@@ -368,12 +410,14 @@ namespace llvm {
         if (NumElements == 2)  return MVT::v2i32;
         if (NumElements == 4)  return MVT::v4i32;
         if (NumElements == 8)  return MVT::v8i32;
+        if (NumElements == 16) return MVT::v16i32;
         break;
       case MVT::i64:
         if (NumElements == 1)  return MVT::v1i64;
         if (NumElements == 2)  return MVT::v2i64;
         if (NumElements == 4)  return MVT::v4i64;
         if (NumElements == 8)  return MVT::v8i64;
+        if (NumElements == 16) return MVT::v16i64;
         break;
       case MVT::f16:
         if (NumElements == 2)  return MVT::v2f16;
@@ -487,32 +531,27 @@ namespace llvm {
 
     /// is64BitVector - Return true if this is a 64-bit vector type.
     bool is64BitVector() const {
-      if (!isSimple())
-        return isExtended64BitVector();
-
-      return (V == MVT::v8i8  || V==MVT::v4i16 || V==MVT::v2i32 ||
-              V == MVT::v1i64 || V==MVT::v2f32);
+      return isSimple() ? V.is64BitVector() : isExtended64BitVector();
     }
 
     /// is128BitVector - Return true if this is a 128-bit vector type.
     bool is128BitVector() const {
-      if (!isSimple())
-        return isExtended128BitVector();
-      return (V==MVT::v16i8 || V==MVT::v8i16 || V==MVT::v4i32 ||
-              V==MVT::v2i64 || V==MVT::v4f32 || V==MVT::v2f64);
+      return isSimple() ? V.is128BitVector() : isExtended128BitVector();
     }
 
     /// is256BitVector - Return true if this is a 256-bit vector type.
-    inline bool is256BitVector() const {
-      if (!isSimple())
-        return isExtended256BitVector();
-      return (V == MVT::v8f32  || V == MVT::v4f64 || V == MVT::v32i8 ||
-              V == MVT::v16i16 || V == MVT::v8i32 || V == MVT::v4i64);
+    bool is256BitVector() const {
+      return isSimple() ? V.is256BitVector() : isExtended256BitVector();
     }
 
     /// is512BitVector - Return true if this is a 512-bit vector type.
-    inline bool is512BitVector() const {
-      return isSimple() ? (V == MVT::v8i64) : isExtended512BitVector();
+    bool is512BitVector() const {
+      return isSimple() ? V.is512BitVector() : isExtended512BitVector();
+    }
+
+    /// is1024BitVector - Return true if this is a 1024-bit vector type.
+    bool is1024BitVector() const {
+      return isSimple() ? V.is1024BitVector() : isExtended1024BitVector();
     }
 
     /// isOverloaded - Return true if this is an overloaded type for TableGen.
@@ -705,6 +744,7 @@ namespace llvm {
     bool isExtended128BitVector() const;
     bool isExtended256BitVector() const;
     bool isExtended512BitVector() const;
+    bool isExtended1024BitVector() const;
     EVT getExtendedVectorElementType() const;
     unsigned getExtendedVectorNumElements() const;
     unsigned getExtendedSizeInBits() const;
diff --git a/include/llvm/CodeGen/ValueTypes.td b/include/llvm/CodeGen/ValueTypes.td
index 6c22690..f4b75bd 100644
--- a/include/llvm/CodeGen/ValueTypes.td
+++ b/include/llvm/CodeGen/ValueTypes.td
@@ -45,22 +45,24 @@ def v16i16 : ValueType<256, 21>;   // 16 x i16 vector value
 def v2i32  : ValueType<64 , 22>;   //  2 x i32 vector value
 def v4i32  : ValueType<128, 23>;   //  4 x i32 vector value
 def v8i32  : ValueType<256, 24>;   //  8 x i32 vector value
-def v1i64  : ValueType<64 , 25>;   //  1 x i64 vector value
-def v2i64  : ValueType<128, 26>;   //  2 x i64 vector value
-def v4i64  : ValueType<256, 27>;   //  4 x i64 vector value
-def v8i64  : ValueType<512, 28>;   //  8 x i64 vector value
+def v16i32 : ValueType<512, 25>;   // 16 x i32 vector value
+def v1i64  : ValueType<64 , 26>;   //  1 x i64 vector value
+def v2i64  : ValueType<128, 27>;   //  2 x i64 vector value
+def v4i64  : ValueType<256, 28>;   //  4 x i64 vector value
+def v8i64  : ValueType<512, 29>;   //  8 x i64 vector value
+def v16i64 : ValueType<1024,30>;  // 16 x i64 vector value
 
-def v2f16  : ValueType<32 , 29>;   //  2 x f16 vector value
-def v2f32  : ValueType<64 , 30>;   //  2 x f32 vector value
-def v4f32  : ValueType<128, 31>;   //  4 x f32 vector value
-def v8f32  : ValueType<256, 32>;   //  8 x f32 vector value
-def v2f64  : ValueType<128, 33>;   //  2 x f64 vector value
-def v4f64  : ValueType<256, 34>;   //  4 x f64 vector value
+def v2f16  : ValueType<32 , 31>;   //  2 x f16 vector value
+def v2f32  : ValueType<64 , 32>;   //  2 x f32 vector value
+def v4f32  : ValueType<128, 33>;   //  4 x f32 vector value
+def v8f32  : ValueType<256, 34>;   //  8 x f32 vector value
+def v2f64  : ValueType<128, 35>;   //  2 x f64 vector value
+def v4f64  : ValueType<256, 36>;   //  4 x f64 vector value
 
-def x86mmx : ValueType<64 , 35>;   // X86 MMX value
-def FlagVT : ValueType<0  , 36>;   // Pre-RA sched glue
-def isVoid : ValueType<0  , 37>;   // Produces no value
-def untyped: ValueType<8  , 38>;   // Produces an untyped value
+def x86mmx : ValueType<64 , 37>;   // X86 MMX value
+def FlagVT : ValueType<0  , 38>;   // Pre-RA sched glue
+def isVoid : ValueType<0  , 39>;   // Produces no value
+def untyped: ValueType<8  , 40>;   // Produces an untyped value
 
 def MetadataVT: ValueType<0, 250>; // Metadata