Vendor import of llvm trunk r154661:

http://llvm.org/svn/llvm-project/llvm/trunk@r154661

54 files changed, 2877 insertions, 1216 deletions

diff --git a/include/llvm/CodeGen/Analysis.h b/include/llvm/CodeGen/Analysis.h
index d8e6407..0b609ed 100644
--- a/include/llvm/CodeGen/Analysis.h
+++ b/include/llvm/CodeGen/Analysis.h
@@ -27,6 +27,7 @@ namespace llvm {
 class GlobalVariable;
 class TargetLowering;
 class SDNode;
+class SDValue;
 class SelectionDAG;
 
 /// ComputeLinearIndex - Given an LLVM IR aggregate type and a sequence
@@ -70,6 +71,10 @@ bool hasInlineAsmMemConstraint(InlineAsm::ConstraintInfoVector &CInfos,
 ///
 ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred);
 
+/// getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats,
+/// return the equivalent code if we're allowed to assume that NaNs won't occur.
+ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC);
+
 /// getICmpCondCode - Return the ISD condition code corresponding to
 /// the given LLVM IR integer condition code.
 ///
@@ -85,7 +90,7 @@ bool isInTailCallPosition(ImmutableCallSite CS, Attributes CalleeRetAttr,
                           const TargetLowering &TLI);
 
 bool isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
-                          const TargetLowering &TLI);
+                          SDValue &Chain, const TargetLowering &TLI);
 
 } // End llvm namespace
 
diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h
index 06c5c83..56a87f1 100644
--- a/include/llvm/CodeGen/AsmPrinter.h
+++ b/include/llvm/CodeGen/AsmPrinter.h
@@ -18,16 +18,12 @@
 
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/ErrorHandling.h"
 
 namespace llvm {
   class BlockAddress;
   class GCStrategy;
   class Constant;
-  class ConstantArray;
-  class ConstantFP;
-  class ConstantInt;
-  class ConstantStruct;
-  class ConstantVector;
   class GCMetadataPrinter;
   class GlobalValue;
   class GlobalVariable;
@@ -37,14 +33,11 @@ namespace llvm {
   class MachineLocation;
   class MachineLoopInfo;
   class MachineLoop;
-  class MachineConstantPool;
-  class MachineConstantPoolEntry;
   class MachineConstantPoolValue;
   class MachineJumpTableInfo;
   class MachineModuleInfo;
   class MachineMove;
   class MCAsmInfo;
-  class MCInst;
   class MCContext;
   class MCSection;
   class MCStreamer;
@@ -56,8 +49,6 @@ namespace llvm {
   class TargetLoweringObjectFile;
   class TargetData;
   class TargetMachine;
-  class Twine;
-  class Type;
 
   /// AsmPrinter - This class is intended to be used as a driving class for all
   /// asm writers.
@@ -97,6 +88,11 @@ namespace llvm {
     ///
     MCSymbol *CurrentFnSym;
 
+    /// The symbol used to represent the start of the current function for the
+    /// purpose of calculating its size (e.g. using the .size directive). By
+    /// default, this is equal to CurrentFnSym.
+    MCSymbol *CurrentFnSymForSize;
+
   private:
     // GCMetadataPrinters - The garbage collection metadata printer table.
     void *GCMetadataPrinters;  // Really a DenseMap.
@@ -194,6 +190,11 @@ namespace llvm {
 
     bool needsSEHMoves();
 
+    /// needsRelocationsForDwarfStringPool - Specifies whether the object format
+    /// expects to use relocations to refer to debug entries. Alternatively we
+    /// emit section offsets in bytes from the start of the string pool.
+    bool needsRelocationsForDwarfStringPool() const;
+
     /// EmitConstantPool - Print to the current output stream assembly
     /// representations of the constants in the constant pool MCP. This is
     /// used to print out constants which have been "spilled to memory" by
@@ -256,13 +257,20 @@ namespace llvm {
 
     /// EmitInstruction - Targets should implement this to emit instructions.
     virtual void EmitInstruction(const MachineInstr *) {
-      assert(0 && "EmitInstruction not implemented");
+      llvm_unreachable("EmitInstruction not implemented");
     }
 
     virtual void EmitFunctionEntryLabel();
 
     virtual void EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV);
 
+    /// EmitXXStructor - Targets can override this to change how global
+    /// constants that are part of a C++ static/global constructor list are
+    /// emitted.
+    virtual void EmitXXStructor(const Constant *CV) {
+      EmitGlobalConstant(CV);
+    }
+
     /// isBlockOnlyReachableByFallthough - Return true if the basic block has
     /// exactly one predecessor and the control transfer mechanism between
     /// the predecessor and this block is a fall-through.
@@ -466,7 +474,7 @@ namespace llvm {
                             const MachineBasicBlock *MBB,
                             unsigned uid) const;
     void EmitLLVMUsedList(const Constant *List);
-    void EmitXXStructorList(const Constant *List);
+    void EmitXXStructorList(const Constant *List, bool isCtor);
     GCMetadataPrinter *GetOrCreateGCPrinter(GCStrategy *C);
   };
 }
diff --git a/include/llvm/CodeGen/BinaryObject.h b/include/llvm/CodeGen/BinaryObject.h
deleted file mode 100644
index 8c1431f..0000000
--- a/include/llvm/CodeGen/BinaryObject.h
+++ /dev/null
@@ -1,353 +0,0 @@
-//===-- llvm/CodeGen/BinaryObject.h - Binary Object. -----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines a Binary Object Aka. "blob" for holding data from code
-// generators, ready for data to the object module code writters.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_BINARYOBJECT_H
-#define LLVM_CODEGEN_BINARYOBJECT_H
-
-#include "llvm/CodeGen/MachineRelocation.h"
-#include "llvm/Support/DataTypes.h"
-
-#include <string>
-#include <vector>
-
-namespace llvm {
-
-typedef std::vector<uint8_t> BinaryData;
-
-class BinaryObject {
-protected:
-  std::string Name;
-  bool IsLittleEndian;
-  bool Is64Bit;
-  BinaryData Data;
-  std::vector<MachineRelocation> Relocations;
-
-public:
-  /// Constructors and destructor
-  BinaryObject() {}
-
-  BinaryObject(bool isLittleEndian, bool is64Bit)
-    : IsLittleEndian(isLittleEndian), Is64Bit(is64Bit) {}
-
-  BinaryObject(const std::string &name, bool isLittleEndian, bool is64Bit)
-    : Name(name), IsLittleEndian(isLittleEndian), Is64Bit(is64Bit) {}
-
-  ~BinaryObject() {}
-
-  /// getName - get name of BinaryObject
-  inline std::string getName() const { return Name; }
-
-  /// get size of binary data
-  size_t size() const {
-    return Data.size();
-  }
-
-  /// get binary data
-  BinaryData& getData() {
-    return Data;
-  }
-
-  /// get machine relocations
-  const std::vector<MachineRelocation>& getRelocations() const {
-    return Relocations;
-  }
-
-  /// hasRelocations - Return true if 'Relocations' is not empty
-  bool hasRelocations() const {
-    return !Relocations.empty();
-  }
-
-  /// emitZeros - This callback is invoked to emit a arbitrary number 
-  /// of zero bytes to the data stream.
-  inline void emitZeros(unsigned Size) {
-    for (unsigned i=0; i < Size; ++i)
-      emitByte(0);
-  }
-
-  /// emitByte - This callback is invoked when a byte needs to be
-  /// written to the data stream.
-  inline void emitByte(uint8_t B) {
-    Data.push_back(B);
-  }
-
-  /// emitWord16 - This callback is invoked when a 16-bit word needs to be
-  /// written to the data stream in correct endian format and correct size.
-  inline void emitWord16(uint16_t W) {
-    if (IsLittleEndian)
-      emitWord16LE(W);
-    else
-      emitWord16BE(W);
-  }
-
-  /// emitWord16LE - This callback is invoked when a 16-bit word needs to be
-  /// written to the data stream in correct endian format and correct size.
-  inline void emitWord16LE(uint16_t W) {
-    Data.push_back((uint8_t)(W >> 0));
-    Data.push_back((uint8_t)(W >> 8));
-  }
-
-  /// emitWord16BE - This callback is invoked when a 16-bit word needs to be
-  /// written to the data stream in correct endian format and correct size.
-  inline void emitWord16BE(uint16_t W) {
-    Data.push_back((uint8_t)(W >> 8));
-    Data.push_back((uint8_t)(W >> 0));
-  }
-
-  /// emitWord - This callback is invoked when a word needs to be
-  /// written to the data stream in correct endian format and correct size.
-  inline void emitWord(uint64_t W) {
-    if (!Is64Bit)
-      emitWord32(W);
-    else
-      emitWord64(W);
-  }
-
-  /// emitWord32 - This callback is invoked when a 32-bit word needs to be
-  /// written to the data stream in correct endian format.
-  inline void emitWord32(uint32_t W) {
-    if (IsLittleEndian)
-      emitWordLE(W);
-    else
-      emitWordBE(W);
-  }
-
-  /// emitWord64 - This callback is invoked when a 32-bit word needs to be
-  /// written to the data stream in correct endian format.
-  inline void emitWord64(uint64_t W) {
-    if (IsLittleEndian)
-      emitDWordLE(W);
-    else
-      emitDWordBE(W);
-  }
-
-  /// emitWord64 - This callback is invoked when a x86_fp80 needs to be
-  /// written to the data stream in correct endian format.
-  inline void emitWordFP80(const uint64_t *W, unsigned PadSize) {
-    if (IsLittleEndian) {
-      emitWord64(W[0]);
-      emitWord16(W[1]);  
-    } else {
-      emitWord16(W[1]);  
-      emitWord64(W[0]);
-    }
-    emitZeros(PadSize);
-  }
-
-  /// emitWordLE - This callback is invoked when a 32-bit word needs to be
-  /// written to the data stream in little-endian format.
-  inline void emitWordLE(uint32_t W) {
-    Data.push_back((uint8_t)(W >>  0));
-    Data.push_back((uint8_t)(W >>  8));
-    Data.push_back((uint8_t)(W >> 16));
-    Data.push_back((uint8_t)(W >> 24));
-  }
-
-  /// emitWordBE - This callback is invoked when a 32-bit word needs to be
-  /// written to the data stream in big-endian format.
-  ///
-  inline void emitWordBE(uint32_t W) {
-    Data.push_back((uint8_t)(W >> 24));
-    Data.push_back((uint8_t)(W >> 16));
-    Data.push_back((uint8_t)(W >>  8));
-    Data.push_back((uint8_t)(W >>  0));
-  }
-
-  /// emitDWordLE - This callback is invoked when a 64-bit word needs to be
-  /// written to the data stream in little-endian format.
-  inline void emitDWordLE(uint64_t W) {
-    Data.push_back((uint8_t)(W >>  0));
-    Data.push_back((uint8_t)(W >>  8));
-    Data.push_back((uint8_t)(W >> 16));
-    Data.push_back((uint8_t)(W >> 24));
-    Data.push_back((uint8_t)(W >> 32));
-    Data.push_back((uint8_t)(W >> 40));
-    Data.push_back((uint8_t)(W >> 48));
-    Data.push_back((uint8_t)(W >> 56));
-  }
-
-  /// emitDWordBE - This callback is invoked when a 64-bit word needs to be
-  /// written to the data stream in big-endian format.
-  inline void emitDWordBE(uint64_t W) {
-    Data.push_back((uint8_t)(W >> 56));
-    Data.push_back((uint8_t)(W >> 48));
-    Data.push_back((uint8_t)(W >> 40));
-    Data.push_back((uint8_t)(W >> 32));
-    Data.push_back((uint8_t)(W >> 24));
-    Data.push_back((uint8_t)(W >> 16));
-    Data.push_back((uint8_t)(W >>  8));
-    Data.push_back((uint8_t)(W >>  0));
-  }
-
-  /// fixByte - This callback is invoked when a byte needs to be
-  /// fixup the buffer.
-  inline void fixByte(uint8_t B, uint32_t offset) {
-    Data[offset] = B;
-  }
-
-  /// fixWord16 - This callback is invoked when a 16-bit word needs to
-  /// fixup the data stream in correct endian format.
-  inline void fixWord16(uint16_t W, uint32_t offset) {
-    if (IsLittleEndian)
-      fixWord16LE(W, offset);
-    else
-      fixWord16BE(W, offset);
-  }
-
-  /// emitWord16LE - This callback is invoked when a 16-bit word needs to
-  /// fixup the data stream in little endian format.
-  inline void fixWord16LE(uint16_t W, uint32_t offset) {
-    Data[offset]   = (uint8_t)(W >> 0);
-    Data[++offset] = (uint8_t)(W >> 8);
-  }
-
-  /// fixWord16BE - This callback is invoked when a 16-bit word needs to
-  /// fixup data stream in big endian format.
-  inline void fixWord16BE(uint16_t W, uint32_t offset) {
-    Data[offset]   = (uint8_t)(W >> 8);
-    Data[++offset] = (uint8_t)(W >> 0);
-  }
-
-  /// emitWord - This callback is invoked when a word needs to
-  /// fixup the data in correct endian format and correct size.
-  inline void fixWord(uint64_t W, uint32_t offset) {
-    if (!Is64Bit)
-      fixWord32(W, offset);
-    else
-      fixWord64(W, offset);
-  }
-
-  /// fixWord32 - This callback is invoked when a 32-bit word needs to
-  /// fixup the data in correct endian format.
-  inline void fixWord32(uint32_t W, uint32_t offset) {
-    if (IsLittleEndian)
-      fixWord32LE(W, offset);
-    else
-      fixWord32BE(W, offset);
-  }
-
-  /// fixWord32LE - This callback is invoked when a 32-bit word needs to
-  /// fixup the data in little endian format.
-  inline void fixWord32LE(uint32_t W, uint32_t offset) {
-    Data[offset]   = (uint8_t)(W >>  0);
-    Data[++offset] = (uint8_t)(W >>  8);
-    Data[++offset] = (uint8_t)(W >> 16);
-    Data[++offset] = (uint8_t)(W >> 24);
-  }
-
-  /// fixWord32BE - This callback is invoked when a 32-bit word needs to
-  /// fixup the data in big endian format.
-  inline void fixWord32BE(uint32_t W, uint32_t offset) {
-    Data[offset]   = (uint8_t)(W >> 24);
-    Data[++offset] = (uint8_t)(W >> 16);
-    Data[++offset] = (uint8_t)(W >>  8);
-    Data[++offset] = (uint8_t)(W >>  0);
-  }
-
-  /// fixWord64 - This callback is invoked when a 64-bit word needs to
-  /// fixup the data in correct endian format.
-  inline void fixWord64(uint64_t W, uint32_t offset) {
-    if (IsLittleEndian)
-      fixWord64LE(W, offset);
-    else
-      fixWord64BE(W, offset);
-  }
-
-  /// fixWord64BE - This callback is invoked when a 64-bit word needs to
-  /// fixup the data in little endian format.
-  inline void fixWord64LE(uint64_t W, uint32_t offset) {
-    Data[offset]   = (uint8_t)(W >>  0);
-    Data[++offset] = (uint8_t)(W >>  8);
-    Data[++offset] = (uint8_t)(W >> 16);
-    Data[++offset] = (uint8_t)(W >> 24);
-    Data[++offset] = (uint8_t)(W >> 32);
-    Data[++offset] = (uint8_t)(W >> 40);
-    Data[++offset] = (uint8_t)(W >> 48);
-    Data[++offset] = (uint8_t)(W >> 56);
-  }
-
-  /// fixWord64BE - This callback is invoked when a 64-bit word needs to
-  /// fixup the data in big endian format.
-  inline void fixWord64BE(uint64_t W, uint32_t offset) {
-    Data[offset]   = (uint8_t)(W >> 56);
-    Data[++offset] = (uint8_t)(W >> 48);
-    Data[++offset] = (uint8_t)(W >> 40);
-    Data[++offset] = (uint8_t)(W >> 32);
-    Data[++offset] = (uint8_t)(W >> 24);
-    Data[++offset] = (uint8_t)(W >> 16);
-    Data[++offset] = (uint8_t)(W >>  8);
-    Data[++offset] = (uint8_t)(W >>  0);
-  }
-
-  /// emitAlignment - Pad the data to the specified alignment.
-  void emitAlignment(unsigned Alignment, uint8_t fill = 0) {
-    if (Alignment <= 1) return;
-    unsigned PadSize = -Data.size() & (Alignment-1);
-    for (unsigned i = 0; i<PadSize; ++i)
-      Data.push_back(fill);
-  }
-
-  /// emitULEB128Bytes - This callback is invoked when a ULEB128 needs to be
-  /// written to the data stream.
-  void emitULEB128Bytes(uint64_t Value) {
-    do {
-      uint8_t Byte = (uint8_t)(Value & 0x7f);
-      Value >>= 7;
-      if (Value) Byte |= 0x80;
-      emitByte(Byte);
-    } while (Value);
-  }
-
-  /// emitSLEB128Bytes - This callback is invoked when a SLEB128 needs to be
-  /// written to the data stream.
-  void emitSLEB128Bytes(int64_t Value) {
-    int Sign = Value >> (8 * sizeof(Value) - 1);
-    bool IsMore;
-
-    do {
-      uint8_t Byte = (uint8_t)(Value & 0x7f);
-      Value >>= 7;
-      IsMore = Value != Sign || ((Byte ^ Sign) & 0x40) != 0;
-      if (IsMore) Byte |= 0x80;
-      emitByte(Byte);
-    } while (IsMore);
-  }
-
-  /// emitString - This callback is invoked when a String needs to be
-  /// written to the data stream.
-  void emitString(const std::string &String) {
-    for (unsigned i = 0, N = static_cast<unsigned>(String.size()); i<N; ++i) {
-      unsigned char C = String[i];
-      emitByte(C);
-    }
-    emitByte(0);
-  }
-
-  /// getCurrentPCOffset - Return the offset from the start of the emitted
-  /// buffer that we are currently writing to.
-  uintptr_t getCurrentPCOffset() const {
-    return Data.size();
-  }
-
-  /// addRelocation - Whenever a relocatable address is needed, it should be
-  /// noted with this interface.
-  void addRelocation(const MachineRelocation& relocation) {
-    Relocations.push_back(relocation);
-  }
-
-};
-
-} // end namespace llvm
-
-#endif
-
diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h
index 77dc644..3afe309 100644
--- a/include/llvm/CodeGen/CallingConvLower.h
+++ b/include/llvm/CodeGen/CallingConvLower.h
@@ -229,7 +229,7 @@ public:
 
   /// getFirstUnallocated - Return the first unallocated register in the set, or
   /// NumRegs if they are all allocated.
-  unsigned getFirstUnallocated(const unsigned *Regs, unsigned NumRegs) const {
+  unsigned getFirstUnallocated(const uint16_t *Regs, unsigned NumRegs) const {
     for (unsigned i = 0; i != NumRegs; ++i)
       if (!isAllocated(Regs[i]))
         return i;
@@ -256,7 +256,7 @@ public:
   /// AllocateReg - Attempt to allocate one of the specified registers.  If none
   /// are available, return zero.  Otherwise, return the first one available,
   /// marking it and any aliases as allocated.
-  unsigned AllocateReg(const unsigned *Regs, unsigned NumRegs) {
+  unsigned AllocateReg(const uint16_t *Regs, unsigned NumRegs) {
     unsigned FirstUnalloc = getFirstUnallocated(Regs, NumRegs);
     if (FirstUnalloc == NumRegs)
       return 0;    // Didn't find the reg.
@@ -268,7 +268,7 @@ public:
   }
 
   /// Version of AllocateReg with list of registers to be shadowed.
-  unsigned AllocateReg(const unsigned *Regs, const unsigned *ShadowRegs,
+  unsigned AllocateReg(const uint16_t *Regs, const uint16_t *ShadowRegs,
                        unsigned NumRegs) {
     unsigned FirstUnalloc = getFirstUnallocated(Regs, NumRegs);
     if (FirstUnalloc == NumRegs)
@@ -306,12 +306,12 @@ public:
 
   // First GPR that carries part of a byval aggregate that's split
   // between registers and memory.
-  unsigned getFirstByValReg() { return FirstByValRegValid ? FirstByValReg : 0; }
+  unsigned getFirstByValReg() const { return FirstByValRegValid ? FirstByValReg : 0; }
   void setFirstByValReg(unsigned r) { FirstByValReg = r; FirstByValRegValid = true; }
   void clearFirstByValReg() { FirstByValReg = 0; FirstByValRegValid = false; }
-  bool isFirstByValRegValid() { return FirstByValRegValid; }
+  bool isFirstByValRegValid() const { return FirstByValRegValid; }
 
-  ParmContext getCallOrPrologue() { return CallOrPrologue; }
+  ParmContext getCallOrPrologue() const { return CallOrPrologue; }
 
 private:
   /// MarkAllocated - Mark a register and all of its aliases as allocated.
diff --git a/include/llvm/CodeGen/DFAPacketizer.h b/include/llvm/CodeGen/DFAPacketizer.h
new file mode 100644
index 0000000..2d2db78
--- /dev/null
+++ b/include/llvm/CodeGen/DFAPacketizer.h
@@ -0,0 +1,167 @@
+//=- llvm/CodeGen/DFAPacketizer.h - DFA Packetizer for VLIW ---*- C++ -*-=====//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This class implements a deterministic finite automaton (DFA) based
+// packetizing mechanism for VLIW architectures. It provides APIs to
+// determine whether there exists a legal mapping of instructions to
+// functional unit assignments in a packet. The DFA is auto-generated from
+// the target's Schedule.td file.
+//
+// A DFA consists of 3 major elements: states, inputs, and transitions. For
+// the packetizing mechanism, the input is the set of instruction classes for
+// a target. The state models all possible combinations of functional unit
+// consumption for a given set of instructions in a packet. A transition
+// models the addition of an instruction to a packet. In the DFA constructed
+// by this class, if an instruction can be added to a packet, then a valid
+// transition exists from the corresponding state. Invalid transitions
+// indicate that the instruction cannot be added to the current packet.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_DFAPACKETIZER_H
+#define LLVM_CODEGEN_DFAPACKETIZER_H
+
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/ADT/DenseMap.h"
+#include <map>
+
+namespace llvm {
+
+class MCInstrDesc;
+class MachineInstr;
+class MachineLoopInfo;
+class MachineDominatorTree;
+class InstrItineraryData;
+class DefaultVLIWScheduler;
+class SUnit;
+
+class DFAPacketizer {
+private:
+  typedef std::pair<unsigned, unsigned> UnsignPair;
+  const InstrItineraryData *InstrItins;
+  int CurrentState;
+  const int (*DFAStateInputTable)[2];
+  const unsigned *DFAStateEntryTable;
+
+  // CachedTable is a map from <FromState, Input> to ToState.
+  DenseMap<UnsignPair, unsigned> CachedTable;
+
+  // ReadTable - Read the DFA transition table and update CachedTable.
+  void ReadTable(unsigned int state);
+
+public:
+  DFAPacketizer(const InstrItineraryData *I, const int (*SIT)[2],
+                const unsigned *SET);
+
+  // Reset the current state to make all resources available.
+  void clearResources() {
+    CurrentState = 0;
+  }
+
+  // canReserveResources - Check if the resources occupied by a MCInstrDesc
+  // are available in the current state.
+  bool canReserveResources(const llvm::MCInstrDesc *MID);
+
+  // reserveResources - Reserve the resources occupied by a MCInstrDesc and
+  // change the current state to reflect that change.
+  void reserveResources(const llvm::MCInstrDesc *MID);
+
+  // canReserveResources - Check if the resources occupied by a machine
+  // instruction are available in the current state.
+  bool canReserveResources(llvm::MachineInstr *MI);
+
+  // reserveResources - Reserve the resources occupied by a machine
+  // instruction and change the current state to reflect that change.
+  void reserveResources(llvm::MachineInstr *MI);
+
+  const InstrItineraryData *getInstrItins() const { return InstrItins; }
+};
+
+// VLIWPacketizerList - Implements a simple VLIW packetizer using DFA. The
+// packetizer works on machine basic blocks. For each instruction I in BB, the
+// packetizer consults the DFA to see if machine resources are available to
+// execute I. If so, the packetizer checks if I depends on any instruction J in
+// the current packet. If no dependency is found, I is added to current packet
+// and machine resource is marked as taken. If any dependency is found, a target
+// API call is made to prune the dependence.
+class VLIWPacketizerList {
+protected:
+  const TargetMachine &TM;
+  const MachineFunction &MF;
+  const TargetInstrInfo *TII;
+
+  // The VLIW Scheduler.
+  DefaultVLIWScheduler *VLIWScheduler;
+
+  // Vector of instructions assigned to the current packet.
+  std::vector<MachineInstr*> CurrentPacketMIs;
+  // DFA resource tracker.
+  DFAPacketizer *ResourceTracker;
+
+  // Generate MI -> SU map.
+  std::map<MachineInstr*, SUnit*> MIToSUnit;
+
+public:
+  VLIWPacketizerList(
+    MachineFunction &MF, MachineLoopInfo &MLI, MachineDominatorTree &MDT,
+    bool IsPostRA);
+
+  virtual ~VLIWPacketizerList();
+
+  // PacketizeMIs - Implement this API in the backend to bundle instructions.
+  void PacketizeMIs(MachineBasicBlock *MBB,
+                    MachineBasicBlock::iterator BeginItr,
+                    MachineBasicBlock::iterator EndItr);
+
+  // getResourceTracker - return ResourceTracker
+  DFAPacketizer *getResourceTracker() {return ResourceTracker;}
+
+  // addToPacket - Add MI to the current packet.
+  virtual MachineBasicBlock::iterator addToPacket(MachineInstr *MI) {
+    MachineBasicBlock::iterator MII = MI;
+    CurrentPacketMIs.push_back(MI);
+    ResourceTracker->reserveResources(MI);
+    return MII;
+  }
+
+  // endPacket - End the current packet.
+  void endPacket(MachineBasicBlock *MBB, MachineInstr *MI);
+
+  // initPacketizerState - perform initialization before packetizing
+  // an instruction. This function is supposed to be overrided by
+  // the target dependent packetizer.
+  virtual void initPacketizerState(void) { return; }
+
+  // ignorePseudoInstruction - Ignore bundling of pseudo instructions.
+  virtual bool ignorePseudoInstruction(MachineInstr *I,
+                                       MachineBasicBlock *MBB) {
+    return false;
+  }
+
+  // isSoloInstruction - return true if instruction MI can not be packetized
+  // with any other instruction, which means that MI itself is a packet.
+  virtual bool isSoloInstruction(MachineInstr *MI) {
+    return true;
+  }
+
+  // isLegalToPacketizeTogether - Is it legal to packetize SUI and SUJ
+  // together.
+  virtual bool isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) {
+    return false;
+  }
+
+  // isLegalToPruneDependencies - Is it legal to prune dependece between SUI
+  // and SUJ.
+  virtual bool isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) {
+    return false;
+  }
+
+};
+}
+
+#endif
diff --git a/include/llvm/CodeGen/EdgeBundles.h b/include/llvm/CodeGen/EdgeBundles.h
index 8aab3c6..a1d29b1 100644
--- a/include/llvm/CodeGen/EdgeBundles.h
+++ b/include/llvm/CodeGen/EdgeBundles.h
@@ -18,6 +18,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/IntEqClasses.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 
 namespace llvm {
@@ -61,7 +62,7 @@ private:
 /// Specialize WriteGraph, the standard implementation won't work.
 raw_ostream &WriteGraph(raw_ostream &O, const EdgeBundles &G,
                         bool ShortNames = false,
-                        const std::string &Title = "");
+                        const Twine &Title = "");
 
 } // end namespace llvm
 
diff --git a/include/llvm/CodeGen/FastISel.h b/include/llvm/CodeGen/FastISel.h
index 18202d9..e57c8b1 100644
--- a/include/llvm/CodeGen/FastISel.h
+++ b/include/llvm/CodeGen/FastISel.h
@@ -21,9 +21,11 @@
 namespace llvm {
 
 class AllocaInst;
+class Constant;
 class ConstantFP;
 class FunctionLoweringInfo;
 class Instruction;
+class LoadInst;
 class MachineBasicBlock;
 class MachineConstantPool;
 class MachineFunction;
@@ -36,7 +38,8 @@ class TargetLowering;
 class TargetMachine;
 class TargetRegisterClass;
 class TargetRegisterInfo;
-class LoadInst;
+class User;
+class Value;
 
 /// FastISel - This is a fast-path instruction selection class that
 /// generates poor code and doesn't support illegal types or non-trivial
@@ -358,6 +361,8 @@ private:
 
   bool SelectExtractValue(const User *I);
 
+  bool SelectInsertValue(const User *I);
+
   /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
   /// Emit code to ensure constants are copied into registers when needed.
   /// Remember the virtual registers that need to be added to the Machine PHI
@@ -378,6 +383,10 @@ private:
 
   /// hasTrivialKill - Test whether the given value has exactly one use.
   bool hasTrivialKill(const Value *V) const;
+
+  /// removeDeadCode - Remove all dead instructions between the I and E.
+  void removeDeadCode(MachineBasicBlock::iterator I,
+                      MachineBasicBlock::iterator E);
 };
 
 }
diff --git a/include/llvm/CodeGen/FunctionLoweringInfo.h b/include/llvm/CodeGen/FunctionLoweringInfo.h
index 09dac85..8cf22ec 100644
--- a/include/llvm/CodeGen/FunctionLoweringInfo.h
+++ b/include/llvm/CodeGen/FunctionLoweringInfo.h
@@ -21,10 +21,8 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IndexedMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
-#ifndef NDEBUG
-#include "llvm/ADT/SmallSet.h"
-#endif
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
@@ -98,8 +96,8 @@ public:
   MachineBasicBlock::iterator InsertPt;
 
 #ifndef NDEBUG
-  SmallSet<const Instruction *, 8> CatchInfoLost;
-  SmallSet<const Instruction *, 8> CatchInfoFound;
+  SmallPtrSet<const Instruction *, 8> CatchInfoLost;
+  SmallPtrSet<const Instruction *, 8> CatchInfoFound;
 #endif
 
   struct LiveOutInfo {
@@ -112,7 +110,7 @@ public:
 
   /// VisitedBBs - The set of basic blocks visited thus far by instruction
   /// selection.
-  DenseSet<const BasicBlock*> VisitedBBs;
+  SmallPtrSet<const BasicBlock*, 4> VisitedBBs;
 
   /// PHINodesToUpdate - A list of phi instructions whose operand list will
   /// be updated after processing the current basic block.
@@ -202,7 +200,7 @@ public:
   /// setArgumentFrameIndex - Record frame index for the byval
   /// argument.
   void setArgumentFrameIndex(const Argument *A, int FI);
-  
+
   /// getArgumentFrameIndex - Get frame index for the byval argument.
   int getArgumentFrameIndex(const Argument *A);
 
@@ -211,16 +209,18 @@ private:
   IndexedMap<LiveOutInfo, VirtReg2IndexFunctor> LiveOutRegInfo;
 };
 
+/// ComputeUsesVAFloatArgument - Determine if any floating-point values are
+/// being passed to this variadic function, and set the MachineModuleInfo's
+/// usesVAFloatArgument flag if so. This flag is used to emit an undefined
+/// reference to _fltused on Windows, which will link in MSVCRT's
+/// floating-point support.
+void ComputeUsesVAFloatArgument(const CallInst &I, MachineModuleInfo *MMI);
+
 /// AddCatchInfo - Extract the personality and type infos from an eh.selector
 /// call, and add them to the specified machine basic block.
 void AddCatchInfo(const CallInst &I,
                   MachineModuleInfo *MMI, MachineBasicBlock *MBB);
 
-/// CopyCatchInfo - Copy catch information from SuccBB (or one of its
-/// successors) to LPad.
-void CopyCatchInfo(const BasicBlock *SuccBB, const BasicBlock *LPad,
-                   MachineModuleInfo *MMI, FunctionLoweringInfo &FLI);
-
 /// AddLandingPadInfo - Extract the exception handling information from the
 /// landingpad instruction and add them to the specified machine module info.
 void AddLandingPadInfo(const LandingPadInst &I, MachineModuleInfo &MMI,
diff --git a/include/llvm/CodeGen/GCStrategy.h b/include/llvm/CodeGen/GCStrategy.h
index cd760db..1cbd36a 100644
--- a/include/llvm/CodeGen/GCStrategy.h
+++ b/include/llvm/CodeGen/GCStrategy.h
@@ -37,6 +37,7 @@
 #define LLVM_CODEGEN_GCSTRATEGY_H
 
 #include "llvm/CodeGen/GCMetadata.h"
+#include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/Support/Registry.h"
 #include <string>
 
@@ -68,6 +69,8 @@ namespace llvm {
     bool CustomReadBarriers;   //< Default is to insert loads.
     bool CustomWriteBarriers;  //< Default is to insert stores.
     bool CustomRoots;          //< Default is to pass through to backend.
+    bool CustomSafePoints;     //< Default is to use NeededSafePoints
+                               //  to find safe points.
     bool InitRoots;            //< If set, roots are nulled during lowering.
     bool UsesMetadata;         //< If set, backend must emit metadata tables.
     
@@ -87,7 +90,9 @@ namespace llvm {
 
     /// needsSafePoitns - True if safe points of any kind are required. By
     //                    default, none are recorded.
-    bool needsSafePoints() const { return NeededSafePoints != 0; }
+    bool needsSafePoints() const {
+      return CustomSafePoints || NeededSafePoints != 0;
+    }
     
     /// needsSafePoint(Kind) - True if the given kind of safe point is
     //                          required. By default, none are recorded.
@@ -109,6 +114,11 @@ namespace llvm {
     ///               can generate a stack map. If true, then
     //                performCustomLowering must delete them.
     bool customRoots() const { return CustomRoots; }
+
+    /// customSafePoints - By default, the GC analysis will find safe
+    ///                    points according to NeededSafePoints. If true,
+    ///                    then findCustomSafePoints must create them.
+    bool customSafePoints() const { return CustomSafePoints; }
     
     /// initializeRoots - If set, gcroot intrinsics should initialize their
     //                    allocas to null before the first use. This is
@@ -135,6 +145,7 @@ namespace llvm {
     /// which the LLVM IR can be modified.
     virtual bool initializeCustomLowering(Module &F);
     virtual bool performCustomLowering(Function &F);
+    virtual bool findCustomSafePoints(GCFunctionInfo& FI, MachineFunction& MF);
   };
   
 }
diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h
index 184e96d..ab8ab5d 100644
--- a/include/llvm/CodeGen/ISDOpcodes.h
+++ b/include/llvm/CodeGen/ISDOpcodes.h
@@ -57,7 +57,7 @@ namespace ISD {
     AssertSext, AssertZext,
 
     // Various leaf nodes.
-    BasicBlock, VALUETYPE, CONDCODE, Register,
+    BasicBlock, VALUETYPE, CONDCODE, Register, RegisterMask,
     Constant, ConstantFP,
     GlobalAddress, GlobalTLSAddress, FrameIndex,
     JumpTable, ConstantPool, ExternalSymbol, BlockAddress,
@@ -107,13 +107,6 @@ namespace ISD {
     // and returns an outchain.
     EH_SJLJ_LONGJMP,
 
-    // OUTCHAIN = EH_SJLJ_DISPATCHSETUP(INCHAIN, setjmpval)
-    // This corresponds to the eh.sjlj.dispatchsetup intrinsic. It takes an
-    // input chain and the value returning from setjmp as inputs and returns an
-    // outchain. By default, this does nothing. Targets can lower this to unwind
-    // setup code if needed.
-    EH_SJLJ_DISPATCHSETUP,
-
     // TargetConstant* - Like Constant*, but the DAG does not do any folding,
     // simplification, or lowering of the constant. They are used for constants
     // which are known to fit in the immediate fields of their users, or for
@@ -319,6 +312,9 @@ namespace ISD {
     /// Byte Swap and Counting operators.
     BSWAP, CTTZ, CTLZ, CTPOP,
 
+    /// Bit counting operators with an undefined result for zero inputs.
+    CTTZ_ZERO_UNDEF, CTLZ_ZERO_UNDEF,
+
     // Select(COND, TRUEVAL, FALSEVAL).  If the type of the boolean COND is not
     // i1 then the high bits must conform to getBooleanContents.
     SELECT,
@@ -327,6 +323,9 @@ namespace ISD {
     // and #2), returning a vector result.  All vectors have the same length.
     // Much like the scalar select and setcc, each bit in the condition selects
     // whether the corresponding result element is taken from op #1 or op #2.
+    // At first, the VSELECT condition is of vXi1 type. Later, targets may change
+    // the condition type in order to match the VSELECT node using a a pattern.
+    // The condition follows the BooleanContent format of the target.
     VSELECT,
 
     // Select with condition operator - This selects between a true value and
diff --git a/include/llvm/CodeGen/JITCodeEmitter.h b/include/llvm/CodeGen/JITCodeEmitter.h
index 88e22d6..89f00e9 100644
--- a/include/llvm/CodeGen/JITCodeEmitter.h
+++ b/include/llvm/CodeGen/JITCodeEmitter.h
@@ -51,6 +51,7 @@ class Function;
 /// occurred, more memory is allocated, and we reemit the code into it.
 /// 
 class JITCodeEmitter : public MachineCodeEmitter {
+  virtual void anchor();
 public:
   virtual ~JITCodeEmitter() {}
 
diff --git a/include/llvm/CodeGen/LatencyPriorityQueue.h b/include/llvm/CodeGen/LatencyPriorityQueue.h
index 1ed2547..8fb31aa 100644
--- a/include/llvm/CodeGen/LatencyPriorityQueue.h
+++ b/include/llvm/CodeGen/LatencyPriorityQueue.h
@@ -85,11 +85,11 @@ namespace llvm {
 
     virtual void dump(ScheduleDAG* DAG) const;
 
-    // ScheduledNode - As nodes are scheduled, we look to see if there are any
+    // scheduledNode - As nodes are scheduled, we look to see if there are any
     // successor nodes that have a single unscheduled predecessor.  If so, that
     // single predecessor has a higher priority, since scheduling it will make
     // the node available.
-    void ScheduledNode(SUnit *Node);
+    void scheduledNode(SUnit *Node);
 
 private:
     void AdjustPriorityOfUnscheduledPreds(SUnit *SU);
diff --git a/include/llvm/CodeGen/LexicalScopes.h b/include/llvm/CodeGen/LexicalScopes.h
index 0271c5d..eb01f66c 100644
--- a/include/llvm/CodeGen/LexicalScopes.h
+++ b/include/llvm/CodeGen/LexicalScopes.h
@@ -153,6 +153,7 @@ private:
 /// LexicalScope - This class is used to track scope information.
 ///
 class LexicalScope {
+  virtual void anchor();
 
 public:
   LexicalScope(LexicalScope *P, const MDNode *D, const MDNode *I, bool A)
@@ -208,7 +209,7 @@ public:
       Parent->closeInsnRange(NewScope);
   }
 
-  /// dominates - Return true if current scope dominsates given lexical scope.
+  /// dominates - Return true if current scope dominates given lexical scope.
   bool dominates(const LexicalScope *S) const {
     if (S == this)
       return true;
diff --git a/include/llvm/CodeGen/LinkAllCodegenComponents.h b/include/llvm/CodeGen/LinkAllCodegenComponents.h
index 098dd0b..46dd004 100644
--- a/include/llvm/CodeGen/LinkAllCodegenComponents.h
+++ b/include/llvm/CodeGen/LinkAllCodegenComponents.h
@@ -31,24 +31,20 @@ namespace {
       if (std::getenv("bar") != (char*) -1)
         return;
 
-      (void) llvm::createDeadMachineInstructionElimPass();
-
       (void) llvm::createFastRegisterAllocator();
       (void) llvm::createBasicRegisterAllocator();
-      (void) llvm::createLinearScanRegisterAllocator();
       (void) llvm::createGreedyRegisterAllocator();
       (void) llvm::createDefaultPBQPRegisterAllocator();
 
       llvm::linkOcamlGC();
       llvm::linkShadowStackGC();
-      
+
       (void) llvm::createBURRListDAGScheduler(NULL, llvm::CodeGenOpt::Default);
-      (void) llvm::createTDRRListDAGScheduler(NULL, llvm::CodeGenOpt::Default);
       (void) llvm::createSourceListDAGScheduler(NULL,llvm::CodeGenOpt::Default);
       (void) llvm::createHybridListDAGScheduler(NULL,llvm::CodeGenOpt::Default);
-      (void) llvm::createTDListDAGScheduler(NULL, llvm::CodeGenOpt::Default);
       (void) llvm::createFastDAGScheduler(NULL, llvm::CodeGenOpt::Default);
       (void) llvm::createDefaultScheduler(NULL, llvm::CodeGenOpt::Default);
+      (void) llvm::createVLIWDAGScheduler(NULL, llvm::CodeGenOpt::Default);
 
     }
   } ForceCodegenLinking; // Force link by creating a global definition.
diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h
index 2288c1a..a6008ab 100644
--- a/include/llvm/CodeGen/LiveInterval.h
+++ b/include/llvm/CodeGen/LiveInterval.h
@@ -43,12 +43,10 @@ namespace llvm {
   private:
     enum {
       HAS_PHI_KILL    = 1,
-      REDEF_BY_EC     = 1 << 1,
-      IS_PHI_DEF      = 1 << 2,
-      IS_UNUSED       = 1 << 3
+      IS_PHI_DEF      = 1 << 1,
+      IS_UNUSED       = 1 << 2
     };
 
-    MachineInstr *copy;
     unsigned char flags;
 
   public:
@@ -57,23 +55,22 @@ namespace llvm {
     /// The ID number of this value.
     unsigned id;
 
-    /// The index of the defining instruction (if isDefAccurate() returns true).
+    /// The index of the defining instruction.
     SlotIndex def;
 
     /// VNInfo constructor.
-    VNInfo(unsigned i, SlotIndex d, MachineInstr *c)
-      : copy(c), flags(0), id(i), def(d)
+    VNInfo(unsigned i, SlotIndex d)
+      : flags(0), id(i), def(d)
     { }
 
     /// VNInfo construtor, copies values from orig, except for the value number.
     VNInfo(unsigned i, const VNInfo &orig)
-      : copy(orig.copy), flags(orig.flags), id(i), def(orig.def)
+      : flags(orig.flags), id(i), def(orig.def)
     { }
 
     /// Copy from the parameter into this VNInfo.
     void copyFrom(VNInfo &src) {
       flags = src.flags;
-      copy = src.copy;
       def = src.def;
     }
 
@@ -86,19 +83,6 @@ namespace llvm {
       flags = (flags | VNI->flags) & ~IS_UNUSED;
     }
 
-    /// For a register interval, if this VN was definied by a copy instr
-    /// getCopy() returns a pointer to it, otherwise returns 0.
-    /// For a stack interval the behaviour of this method is undefined.
-    MachineInstr* getCopy() const { return copy; }
-    /// For a register interval, set the copy member.
-    /// This method should not be called on stack intervals as it may lead to
-    /// undefined behavior.
-    void setCopy(MachineInstr *c) { copy = c; }
-
-    /// isDefByCopy - Return true when this value was defined by a copy-like
-    /// instruction as determined by MachineInstr::isCopyLike.
-    bool isDefByCopy() const { return copy != 0; }
-
     /// Returns true if one or more kills are PHI nodes.
     /// Obsolete, do not use!
     bool hasPHIKill() const { return flags & HAS_PHI_KILL; }
@@ -110,17 +94,6 @@ namespace llvm {
         flags &= ~HAS_PHI_KILL;
     }
 
-    /// Returns true if this value is re-defined by an early clobber somewhere
-    /// during the live range.
-    bool hasRedefByEC() const { return flags & REDEF_BY_EC; }
-    /// Set the "redef by early clobber" flag on this value.
-    void setHasRedefByEC(bool hasRedef) {
-      if (hasRedef)
-        flags |= REDEF_BY_EC;
-      else
-        flags &= ~REDEF_BY_EC;
-    }
-
     /// Returns true if this value is defined by a PHI instruction (or was,
     /// PHI instrucions may have been eliminated).
     bool isPHIDef() const { return flags & IS_PHI_DEF; }
@@ -294,10 +267,9 @@ namespace llvm {
 
     /// getNextValue - Create a new value number and return it.  MIIdx specifies
     /// the instruction that defines the value number.
-    VNInfo *getNextValue(SlotIndex def, MachineInstr *CopyMI,
-                         VNInfo::Allocator &VNInfoAllocator) {
+    VNInfo *getNextValue(SlotIndex def, VNInfo::Allocator &VNInfoAllocator) {
       VNInfo *VNI =
-        new (VNInfoAllocator) VNInfo((unsigned)valnos.size(), def, CopyMI);
+        new (VNInfoAllocator) VNInfo((unsigned)valnos.size(), def);
       valnos.push_back(VNI);
       return VNI;
     }
@@ -381,7 +353,7 @@ namespace llvm {
     /// point is not contained in the half-open live range. It is usually the
     /// getDefIndex() slot following its last use.
     bool killedAt(SlotIndex index) const {
-      const_iterator r = find(index.getUseIndex());
+      const_iterator r = find(index.getRegSlot(true));
       return r != end() && r->end == index;
     }
 
@@ -405,6 +377,14 @@ namespace llvm {
       return I == end() ? 0 : &*I;
     }
 
+    const LiveRange *getLiveRangeBefore(SlotIndex Idx) const {
+      return getLiveRangeContaining(Idx.getPrevSlot());
+    }
+
+    LiveRange *getLiveRangeBefore(SlotIndex Idx) {
+      return getLiveRangeContaining(Idx.getPrevSlot());
+    }
+
     /// getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.
     VNInfo *getVNInfoAt(SlotIndex Idx) const {
       const_iterator I = FindLiveRangeContaining(Idx);
diff --git a/include/llvm/CodeGen/LiveIntervalAnalysis.h b/include/llvm/CodeGen/LiveIntervalAnalysis.h
index 8ca58b8..76201c9 100644
--- a/include/llvm/CodeGen/LiveIntervalAnalysis.h
+++ b/include/llvm/CodeGen/LiveIntervalAnalysis.h
@@ -63,8 +63,34 @@ namespace llvm {
     /// allocatableRegs_ - A bit vector of allocatable registers.
     BitVector allocatableRegs_;
 
-    /// CloneMIs - A list of clones as result of re-materialization.
-    std::vector<MachineInstr*> CloneMIs;
+    /// reservedRegs_ - A bit vector of reserved registers.
+    BitVector reservedRegs_;
+
+    /// RegMaskSlots - Sorted list of instructions with register mask operands.
+    /// Always use the 'r' slot, RegMasks are normal clobbers, not early
+    /// clobbers.
+    SmallVector<SlotIndex, 8> RegMaskSlots;
+
+    /// RegMaskBits - This vector is parallel to RegMaskSlots, it holds a
+    /// pointer to the corresponding register mask.  This pointer can be
+    /// recomputed as:
+    ///
+    ///   MI = Indexes->getInstructionFromIndex(RegMaskSlot[N]);
+    ///   unsigned OpNum = findRegMaskOperand(MI);
+    ///   RegMaskBits[N] = MI->getOperand(OpNum).getRegMask();
+    ///
+    /// This is kept in a separate vector partly because some standard
+    /// libraries don't support lower_bound() with mixed objects, partly to
+    /// improve locality when searching in RegMaskSlots.
+    /// Also see the comment in LiveInterval::find().
+    SmallVector<const uint32_t*, 8> RegMaskBits;
+
+    /// For each basic block number, keep (begin, size) pairs indexing into the
+    /// RegMaskSlots and RegMaskBits arrays.
+    /// Note that basic block numbers may not be layout contiguous, that's why
+    /// we can't just keep track of the first register mask in each basic
+    /// block.
+    SmallVector<std::pair<unsigned, unsigned>, 8> RegMaskBlocks;
 
   public:
     static char ID; // Pass identification, replacement for typeid
@@ -105,6 +131,12 @@ namespace llvm {
       return allocatableRegs_.test(reg);
     }
 
+    /// isReserved - is the physical register reg reserved in the current
+    /// function
+    bool isReserved(unsigned reg) const {
+      return reservedRegs_.test(reg);
+    }
+
     /// getScaledIntervalSize - get the size of an interval in "units,"
     /// where every function is composed of one thousand units.  This
     /// measure scales properly with empty index slots in the function.
@@ -125,19 +157,6 @@ namespace llvm {
       return (unsigned)(IntervalPercentage * indexes_->getFunctionSize());
     }
 
-    /// conflictsWithPhysReg - Returns true if the specified register is used or
-    /// defined during the duration of the specified interval. Copies to and
-    /// from li.reg are allowed. This method is only able to analyze simple
-    /// ranges that stay within a single basic block. Anything else is
-    /// considered a conflict.
-    bool conflictsWithPhysReg(const LiveInterval &li, VirtRegMap &vrm,
-                              unsigned reg);
-
-    /// conflictsWithAliasRef - Similar to conflictsWithPhysRegRef except
-    /// it checks for alias uses and defs.
-    bool conflictsWithAliasRef(LiveInterval &li, unsigned Reg,
-                                   SmallPtrSet<MachineInstr*,32> &JoinedCopies);
-
     // Interval creation
     LiveInterval &getOrCreateInterval(unsigned reg) {
       Reg2IntervalMap::iterator I = r2iMap_.find(reg);
@@ -177,14 +196,6 @@ namespace llvm {
       return indexes_;
     }
 
-    SlotIndex getZeroIndex() const {
-      return indexes_->getZeroIndex();
-    }
-
-    SlotIndex getInvalidIndex() const {
-      return indexes_->getInvalidIndex();
-    }
-
     /// isNotInMIMap - returns true if the specified machine instr has been
     /// removed or was never entered in the map.
     bool isNotInMIMap(const MachineInstr* Instr) const {
@@ -216,21 +227,11 @@ namespace llvm {
       return li.liveAt(getMBBStartIdx(mbb));
     }
 
-    LiveRange* findEnteringRange(LiveInterval &li,
-                                 const MachineBasicBlock *mbb) {
-      return li.getLiveRangeContaining(getMBBStartIdx(mbb));
-    }
-
     bool isLiveOutOfMBB(const LiveInterval &li,
                         const MachineBasicBlock *mbb) const {
       return li.liveAt(getMBBEndIdx(mbb).getPrevSlot());
     }
 
-    LiveRange* findExitingRange(LiveInterval &li,
-                                const MachineBasicBlock *mbb) {
-      return li.getLiveRangeContaining(getMBBEndIdx(mbb).getPrevSlot());
-    }
-
     MachineBasicBlock* getMBBFromIndex(SlotIndex index) const {
       return indexes_->getMBBFromIndex(index);
     }
@@ -247,19 +248,11 @@ namespace llvm {
       indexes_->replaceMachineInstrInMaps(MI, NewMI);
     }
 
-    void InsertMBBInMaps(MachineBasicBlock *MBB) {
-      indexes_->insertMBBInMaps(MBB);
-    }
-
     bool findLiveInMBBs(SlotIndex Start, SlotIndex End,
                         SmallVectorImpl<MachineBasicBlock*> &MBBs) const {
       return indexes_->findLiveInMBBs(Start, End, MBBs);
     }
 
-    void renumber() {
-      indexes_->renumberIndexes();
-    }
-
     VNInfo::Allocator& getVNInfoAllocator() { return VNInfoAllocator; }
 
     virtual void getAnalysisUsage(AnalysisUsage &AU) const;
@@ -271,20 +264,6 @@ namespace llvm {
     /// print - Implement the dump method.
     virtual void print(raw_ostream &O, const Module* = 0) const;
 
-    /// addIntervalsForSpills - Create new intervals for spilled defs / uses of
-    /// the given interval. FIXME: It also returns the weight of the spill slot
-    /// (if any is created) by reference. This is temporary.
-    std::vector<LiveInterval*>
-    addIntervalsForSpills(const LiveInterval& i,
-                          const SmallVectorImpl<LiveInterval*> *SpillIs,
-                          const MachineLoopInfo *loopInfo, VirtRegMap& vrm);
-
-    /// spillPhysRegAroundRegDefsUses - Spill the specified physical register
-    /// around all defs and uses of the specified interval. Return true if it
-    /// was able to cut its interval.
-    bool spillPhysRegAroundRegDefsUses(const LiveInterval &li,
-                                       unsigned PhysReg, VirtRegMap &vrm);
-
     /// isReMaterializable - Returns true if every definition of MI of every
     /// val# of the specified interval is re-materializable. Also returns true
     /// by reference if all of the defs are load instructions.
@@ -292,33 +271,71 @@ namespace llvm {
                             const SmallVectorImpl<LiveInterval*> *SpillIs,
                             bool &isLoad);
 
-    /// isReMaterializable - Returns true if the definition MI of the specified
-    /// val# of the specified interval is re-materializable.
-    bool isReMaterializable(const LiveInterval &li, const VNInfo *ValNo,
-                            MachineInstr *MI);
+    /// intervalIsInOneMBB - If LI is confined to a single basic block, return
+    /// a pointer to that block.  If LI is live in to or out of any block,
+    /// return NULL.
+    MachineBasicBlock *intervalIsInOneMBB(const LiveInterval &LI) const;
+
+    /// addKillFlags - Add kill flags to any instruction that kills a virtual
+    /// register.
+    void addKillFlags();
 
-    /// getRepresentativeReg - Find the largest super register of the specified
-    /// physical register.
-    unsigned getRepresentativeReg(unsigned Reg) const;
+    /// handleMove - call this method to notify LiveIntervals that
+    /// instruction 'mi' has been moved within a basic block. This will update
+    /// the live intervals for all operands of mi. Moves between basic blocks
+    /// are not supported.
+    void handleMove(MachineInstr* MI);
 
-    /// getNumConflictsWithPhysReg - Return the number of uses and defs of the
-    /// specified interval that conflicts with the specified physical register.
-    unsigned getNumConflictsWithPhysReg(const LiveInterval &li,
-                                        unsigned PhysReg) const;
+    /// moveIntoBundle - Update intervals for operands of MI so that they
+    /// begin/end on the SlotIndex for BundleStart.
+    ///
+    /// Requires MI and BundleStart to have SlotIndexes, and assumes
+    /// existing liveness is accurate. BundleStart should be the first
+    /// instruction in the Bundle.
+    void handleMoveIntoBundle(MachineInstr* MI, MachineInstr* BundleStart);
+
+    // Register mask functions.
+    //
+    // Machine instructions may use a register mask operand to indicate that a
+    // large number of registers are clobbered by the instruction.  This is
+    // typically used for calls.
+    //
+    // For compile time performance reasons, these clobbers are not recorded in
+    // the live intervals for individual physical registers.  Instead,
+    // LiveIntervalAnalysis maintains a sorted list of instructions with
+    // register mask operands.
+
+    /// getRegMaskSlots - Returns a sorted array of slot indices of all
+    /// instructions with register mask operands.
+    ArrayRef<SlotIndex> getRegMaskSlots() const { return RegMaskSlots; }
+
+    /// getRegMaskSlotsInBlock - Returns a sorted array of slot indices of all
+    /// instructions with register mask operands in the basic block numbered
+    /// MBBNum.
+    ArrayRef<SlotIndex> getRegMaskSlotsInBlock(unsigned MBBNum) const {
+      std::pair<unsigned, unsigned> P = RegMaskBlocks[MBBNum];
+      return getRegMaskSlots().slice(P.first, P.second);
+    }
 
-    /// intervalIsInOneMBB - Returns true if the specified interval is entirely
-    /// within a single basic block.
-    bool intervalIsInOneMBB(const LiveInterval &li) const;
+    /// getRegMaskBits() - Returns an array of register mask pointers
+    /// corresponding to getRegMaskSlots().
+    ArrayRef<const uint32_t*> getRegMaskBits() const { return RegMaskBits; }
 
-    /// getLastSplitPoint - Return the last possible insertion point in mbb for
-    /// spilling and splitting code. This is the first terminator, or the call
-    /// instruction if li is live into a landing pad successor.
-    MachineBasicBlock::iterator getLastSplitPoint(const LiveInterval &li,
-                                                  MachineBasicBlock *mbb) const;
+    /// getRegMaskBitsInBlock - Returns an array of mask pointers corresponding
+    /// to getRegMaskSlotsInBlock(MBBNum).
+    ArrayRef<const uint32_t*> getRegMaskBitsInBlock(unsigned MBBNum) const {
+      std::pair<unsigned, unsigned> P = RegMaskBlocks[MBBNum];
+      return getRegMaskBits().slice(P.first, P.second);
+    }
 
-    /// addKillFlags - Add kill flags to any instruction that kills a virtual
-    /// register.
-    void addKillFlags();
+    /// checkRegMaskInterference - Test if LI is live across any register mask
+    /// instructions, and compute a bit mask of physical registers that are not
+    /// clobbered by any of them.
+    ///
+    /// Returns false if LI doesn't cross any register mask instructions. In
+    /// that case, the bit vector is not filled in.
+    bool checkRegMaskInterference(LiveInterval &LI,
+                                  BitVector &UsableRegs);
 
   private:
     /// computeIntervals - Compute live intervals.
@@ -351,13 +368,12 @@ namespace llvm {
     void handlePhysicalRegisterDef(MachineBasicBlock* mbb,
                                    MachineBasicBlock::iterator mi,
                                    SlotIndex MIIdx, MachineOperand& MO,
-                                   LiveInterval &interval,
-                                   MachineInstr *CopyMI);
+                                   LiveInterval &interval);
 
     /// handleLiveInRegister - Create interval for a livein register.
     void handleLiveInRegister(MachineBasicBlock* mbb,
                               SlotIndex MIIdx,
-                              LiveInterval &interval, bool isAlias = false);
+                              LiveInterval &interval);
 
     /// getReMatImplicitUse - If the remat definition MI has one (for now, we
     /// only allow one) virtual register operand, then its uses are implicitly
@@ -379,88 +395,12 @@ namespace llvm {
                             const SmallVectorImpl<LiveInterval*> *SpillIs,
                             bool &isLoad);
 
-    /// tryFoldMemoryOperand - Attempts to fold either a spill / restore from
-    /// slot / to reg or any rematerialized load into ith operand of specified
-    /// MI. If it is successul, MI is updated with the newly created MI and
-    /// returns true.
-    bool tryFoldMemoryOperand(MachineInstr* &MI, VirtRegMap &vrm,
-                              MachineInstr *DefMI, SlotIndex InstrIdx,
-                              SmallVector<unsigned, 2> &Ops,
-                              bool isSS, int FrameIndex, unsigned Reg);
-
-    /// canFoldMemoryOperand - Return true if the specified load / store
-    /// folding is possible.
-    bool canFoldMemoryOperand(MachineInstr *MI,
-                              SmallVector<unsigned, 2> &Ops,
-                              bool ReMatLoadSS) const;
-
-    /// anyKillInMBBAfterIdx - Returns true if there is a kill of the specified
-    /// VNInfo that's after the specified index but is within the basic block.
-    bool anyKillInMBBAfterIdx(const LiveInterval &li, const VNInfo *VNI,
-                              MachineBasicBlock *MBB,
-                              SlotIndex Idx) const;
-
-    /// hasAllocatableSuperReg - Return true if the specified physical register
-    /// has any super register that's allocatable.
-    bool hasAllocatableSuperReg(unsigned Reg) const;
-
-    /// SRInfo - Spill / restore info.
-    struct SRInfo {
-      SlotIndex index;
-      unsigned vreg;
-      bool canFold;
-      SRInfo(SlotIndex i, unsigned vr, bool f)
-        : index(i), vreg(vr), canFold(f) {}
-    };
-
-    bool alsoFoldARestore(int Id, SlotIndex index, unsigned vr,
-                          BitVector &RestoreMBBs,
-                          DenseMap<unsigned,std::vector<SRInfo> >&RestoreIdxes);
-    void eraseRestoreInfo(int Id, SlotIndex index, unsigned vr,
-                          BitVector &RestoreMBBs,
-                          DenseMap<unsigned,std::vector<SRInfo> >&RestoreIdxes);
-
-    /// handleSpilledImpDefs - Remove IMPLICIT_DEF instructions which are being
-    /// spilled and create empty intervals for their uses.
-    void handleSpilledImpDefs(const LiveInterval &li, VirtRegMap &vrm,
-                              const TargetRegisterClass* rc,
-                              std::vector<LiveInterval*> &NewLIs);
-
-    /// rewriteImplicitOps - Rewrite implicit use operands of MI (i.e. uses of
-    /// interval on to-be re-materialized operands of MI) with new register.
-    void rewriteImplicitOps(const LiveInterval &li,
-                           MachineInstr *MI, unsigned NewVReg, VirtRegMap &vrm);
-
-    /// rewriteInstructionForSpills, rewriteInstructionsForSpills - Helper
-    /// functions for addIntervalsForSpills to rewrite uses / defs for the given
-    /// live range.
-    bool rewriteInstructionForSpills(const LiveInterval &li, const VNInfo *VNI,
-        bool TrySplit, SlotIndex index, SlotIndex end,
-        MachineInstr *MI, MachineInstr *OrigDefMI, MachineInstr *DefMI,
-        unsigned Slot, int LdSlot,
-        bool isLoad, bool isLoadSS, bool DefIsReMat, bool CanDelete,
-        VirtRegMap &vrm, const TargetRegisterClass* rc,
-        SmallVector<int, 4> &ReMatIds, const MachineLoopInfo *loopInfo,
-        unsigned &NewVReg, unsigned ImpUse, bool &HasDef, bool &HasUse,
-        DenseMap<unsigned,unsigned> &MBBVRegsMap,
-        std::vector<LiveInterval*> &NewLIs);
-    void rewriteInstructionsForSpills(const LiveInterval &li, bool TrySplit,
-        LiveInterval::Ranges::const_iterator &I,
-        MachineInstr *OrigDefMI, MachineInstr *DefMI, unsigned Slot, int LdSlot,
-        bool isLoad, bool isLoadSS, bool DefIsReMat, bool CanDelete,
-        VirtRegMap &vrm, const TargetRegisterClass* rc,
-        SmallVector<int, 4> &ReMatIds, const MachineLoopInfo *loopInfo,
-        BitVector &SpillMBBs,
-        DenseMap<unsigned,std::vector<SRInfo> > &SpillIdxes,
-        BitVector &RestoreMBBs,
-        DenseMap<unsigned,std::vector<SRInfo> > &RestoreIdxes,
-        DenseMap<unsigned,unsigned> &MBBVRegsMap,
-        std::vector<LiveInterval*> &NewLIs);
-
     static LiveInterval* createInterval(unsigned Reg);
 
     void printInstrs(raw_ostream &O) const;
     void dumpInstrs() const;
+
+    class HMEditor;
   };
 } // End llvm namespace
 
diff --git a/include/llvm/CodeGen/LiveRangeEdit.h b/include/llvm/CodeGen/LiveRangeEdit.h
new file mode 100644
index 0000000..57a6193
--- /dev/null
+++ b/include/llvm/CodeGen/LiveRangeEdit.h
@@ -0,0 +1,207 @@
+//===---- LiveRangeEdit.h - Basic tools for split and spill -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// The LiveRangeEdit class represents changes done to a virtual register when it
+// is spilled or split.
+//
+// The parent register is never changed. Instead, a number of new virtual
+// registers are created and added to the newRegs vector.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_LIVERANGEEDIT_H
+#define LLVM_CODEGEN_LIVERANGEEDIT_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/CodeGen/LiveInterval.h"
+#include "llvm/Target/TargetMachine.h"
+
+namespace llvm {
+
+class AliasAnalysis;
+class LiveIntervals;
+class MachineLoopInfo;
+class MachineRegisterInfo;
+class VirtRegMap;
+
+class LiveRangeEdit {
+public:
+  /// Callback methods for LiveRangeEdit owners.
+  class Delegate {
+    virtual void anchor();
+  public:
+    /// Called immediately before erasing a dead machine instruction.
+    virtual void LRE_WillEraseInstruction(MachineInstr *MI) {}
+
+    /// Called when a virtual register is no longer used. Return false to defer
+    /// its deletion from LiveIntervals.
+    virtual bool LRE_CanEraseVirtReg(unsigned) { return true; }
+
+    /// Called before shrinking the live range of a virtual register.
+    virtual void LRE_WillShrinkVirtReg(unsigned) {}
+
+    /// Called after cloning a virtual register.
+    /// This is used for new registers representing connected components of Old.
+    virtual void LRE_DidCloneVirtReg(unsigned New, unsigned Old) {}
+
+    virtual ~Delegate() {}
+  };
+
+private:
+  LiveInterval &parent_;
+  SmallVectorImpl<LiveInterval*> &newRegs_;
+  MachineRegisterInfo &MRI;
+  LiveIntervals &LIS;
+  VirtRegMap *VRM;
+  const TargetInstrInfo &TII;
+  Delegate *const delegate_;
+
+  /// firstNew_ - Index of the first register added to newRegs_.
+  const unsigned firstNew_;
+
+  /// scannedRemattable_ - true when remattable values have been identified.
+  bool scannedRemattable_;
+
+  /// remattable_ - Values defined by remattable instructions as identified by
+  /// tii.isTriviallyReMaterializable().
+  SmallPtrSet<const VNInfo*,4> remattable_;
+
+  /// rematted_ - Values that were actually rematted, and so need to have their
+  /// live range trimmed or entirely removed.
+  SmallPtrSet<const VNInfo*,4> rematted_;
+
+  /// scanRemattable - Identify the parent_ values that may rematerialize.
+  void scanRemattable(AliasAnalysis *aa);
+
+  /// allUsesAvailableAt - Return true if all registers used by OrigMI at
+  /// OrigIdx are also available with the same value at UseIdx.
+  bool allUsesAvailableAt(const MachineInstr *OrigMI, SlotIndex OrigIdx,
+                          SlotIndex UseIdx);
+
+  /// foldAsLoad - If LI has a single use and a single def that can be folded as
+  /// a load, eliminate the register by folding the def into the use.
+  bool foldAsLoad(LiveInterval *LI, SmallVectorImpl<MachineInstr*> &Dead);
+
+public:
+  /// Create a LiveRangeEdit for breaking down parent into smaller pieces.
+  /// @param parent The register being spilled or split.
+  /// @param newRegs List to receive any new registers created. This needn't be
+  ///                empty initially, any existing registers are ignored.
+  /// @param MF The MachineFunction the live range edit is taking place in.
+  /// @param lis The collection of all live intervals in this function.
+  /// @param vrm Map of virtual registers to physical registers for this
+  ///            function.  If NULL, no virtual register map updates will
+  ///            be done.  This could be the case if called before Regalloc.
+  LiveRangeEdit(LiveInterval &parent,
+                SmallVectorImpl<LiveInterval*> &newRegs,
+                MachineFunction &MF,
+                LiveIntervals &lis,
+                VirtRegMap *vrm,
+                Delegate *delegate = 0)
+    : parent_(parent), newRegs_(newRegs),
+      MRI(MF.getRegInfo()), LIS(lis), VRM(vrm),
+      TII(*MF.getTarget().getInstrInfo()),
+      delegate_(delegate),
+      firstNew_(newRegs.size()),
+      scannedRemattable_(false) {}
+
+  LiveInterval &getParent() const { return parent_; }
+  unsigned getReg() const { return parent_.reg; }
+
+  /// Iterator for accessing the new registers added by this edit.
+  typedef SmallVectorImpl<LiveInterval*>::const_iterator iterator;
+  iterator begin() const { return newRegs_.begin()+firstNew_; }
+  iterator end() const { return newRegs_.end(); }
+  unsigned size() const { return newRegs_.size()-firstNew_; }
+  bool empty() const { return size() == 0; }
+  LiveInterval *get(unsigned idx) const { return newRegs_[idx+firstNew_]; }
+
+  ArrayRef<LiveInterval*> regs() const {
+    return makeArrayRef(newRegs_).slice(firstNew_);
+  }
+
+  /// createFrom - Create a new virtual register based on OldReg.
+  LiveInterval &createFrom(unsigned OldReg);
+
+  /// create - Create a new register with the same class and original slot as
+  /// parent.
+  LiveInterval &create() {
+    return createFrom(getReg());
+  }
+
+  /// anyRematerializable - Return true if any parent values may be
+  /// rematerializable.
+  /// This function must be called before any rematerialization is attempted.
+  bool anyRematerializable(AliasAnalysis*);
+
+  /// checkRematerializable - Manually add VNI to the list of rematerializable
+  /// values if DefMI may be rematerializable.
+  bool checkRematerializable(VNInfo *VNI, const MachineInstr *DefMI,
+                             AliasAnalysis*);
+
+  /// Remat - Information needed to rematerialize at a specific location.
+  struct Remat {
+    VNInfo *ParentVNI;      // parent_'s value at the remat location.
+    MachineInstr *OrigMI;   // Instruction defining ParentVNI.
+    explicit Remat(VNInfo *ParentVNI) : ParentVNI(ParentVNI), OrigMI(0) {}
+  };
+
+  /// canRematerializeAt - Determine if ParentVNI can be rematerialized at
+  /// UseIdx. It is assumed that parent_.getVNINfoAt(UseIdx) == ParentVNI.
+  /// When cheapAsAMove is set, only cheap remats are allowed.
+  bool canRematerializeAt(Remat &RM,
+                          SlotIndex UseIdx,
+                          bool cheapAsAMove);
+
+  /// rematerializeAt - Rematerialize RM.ParentVNI into DestReg by inserting an
+  /// instruction into MBB before MI. The new instruction is mapped, but
+  /// liveness is not updated.
+  /// Return the SlotIndex of the new instruction.
+  SlotIndex rematerializeAt(MachineBasicBlock &MBB,
+                            MachineBasicBlock::iterator MI,
+                            unsigned DestReg,
+                            const Remat &RM,
+                            const TargetRegisterInfo&,
+                            bool Late = false);
+
+  /// markRematerialized - explicitly mark a value as rematerialized after doing
+  /// it manually.
+  void markRematerialized(const VNInfo *ParentVNI) {
+    rematted_.insert(ParentVNI);
+  }
+
+  /// didRematerialize - Return true if ParentVNI was rematerialized anywhere.
+  bool didRematerialize(const VNInfo *ParentVNI) const {
+    return rematted_.count(ParentVNI);
+  }
+
+  /// eraseVirtReg - Notify the delegate that Reg is no longer in use, and try
+  /// to erase it from LIS.
+  void eraseVirtReg(unsigned Reg);
+
+  /// eliminateDeadDefs - Try to delete machine instructions that are now dead
+  /// (allDefsAreDead returns true). This may cause live intervals to be trimmed
+  /// and further dead efs to be eliminated.
+  /// RegsBeingSpilled lists registers currently being spilled by the register
+  /// allocator.  These registers should not be split into new intervals
+  /// as currently those new intervals are not guaranteed to spill.
+  void eliminateDeadDefs(SmallVectorImpl<MachineInstr*> &Dead,
+                         ArrayRef<unsigned> RegsBeingSpilled 
+                          = ArrayRef<unsigned>());
+
+  /// calculateRegClassAndHint - Recompute register class and hint for each new
+  /// register.
+  void calculateRegClassAndHint(MachineFunction&,
+                                const MachineLoopInfo&);
+};
+
+}
+
+#endif
diff --git a/include/llvm/CodeGen/LiveVariables.h b/include/llvm/CodeGen/LiveVariables.h
index 7ba901f..d4bb409 100644
--- a/include/llvm/CodeGen/LiveVariables.h
+++ b/include/llvm/CodeGen/LiveVariables.h
@@ -85,17 +85,11 @@ public:
     ///
     SparseBitVector<> AliveBlocks;
 
-    /// NumUses - Number of uses of this register across the entire function.
-    ///
-    unsigned NumUses;
-
     /// Kills - List of MachineInstruction's which are the last use of this
     /// virtual register (kill it) in their basic block.
     ///
     std::vector<MachineInstr*> Kills;
 
-    VarInfo() : NumUses(0) {}
-
     /// removeKill - Delete a kill corresponding to the specified
     /// machine instruction. Returns true if there was a kill
     /// corresponding to this instruction, false otherwise.
@@ -166,6 +160,9 @@ private:   // Intermediate data structures
   /// the last use of the whole register.
   bool HandlePhysRegKill(unsigned Reg, MachineInstr *MI);
 
+  /// HandleRegMask - Call HandlePhysRegKill for all registers clobbered by Mask.
+  void HandleRegMask(const MachineOperand&);
+
   void HandlePhysRegUse(unsigned Reg, MachineInstr *MI);
   void HandlePhysRegDef(unsigned Reg, MachineInstr *MI,
                         SmallVector<unsigned, 4> &Defs);
diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h
index 5a20e95..ef9c0c2 100644
--- a/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/include/llvm/CodeGen/MachineBasicBlock.h
@@ -77,6 +77,7 @@ class MachineBasicBlock : public ilist_node<MachineBasicBlock> {
   /// (disable optimization).
   std::vector<uint32_t> Weights;
   typedef std::vector<uint32_t>::iterator weight_iterator;
+  typedef std::vector<uint32_t>::const_iterator const_weight_iterator;
 
   /// LiveIns - Keep track of the physical registers that are livein of
   /// the basicblock.
@@ -84,8 +85,9 @@ class MachineBasicBlock : public ilist_node<MachineBasicBlock> {
 
   /// Alignment - Alignment of the basic block. Zero if the basic block does
   /// not need to be aligned.
+  /// The alignment is specified as log2(bytes).
   unsigned Alignment;
-  
+
   /// IsLandingPad - Indicate that this basic block is entered via an
   /// exception handler.
   bool IsLandingPad;
@@ -115,6 +117,10 @@ public:
   /// "(null)".
   StringRef getName() const;
 
+  /// getFullName - Return a formatted string to identify this block and its
+  /// parent function.
+  std::string getFullName() const;
+
   /// hasAddressTaken - Test whether this block is potentially the target
   /// of an indirect branch.
   bool hasAddressTaken() const { return AddressTaken; }
@@ -128,10 +134,89 @@ public:
   const MachineFunction *getParent() const { return xParent; }
   MachineFunction *getParent() { return xParent; }
 
-  typedef Instructions::iterator                              iterator;
-  typedef Instructions::const_iterator                  const_iterator;
-  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
-  typedef std::reverse_iterator<iterator>             reverse_iterator;
+
+  /// bundle_iterator - MachineBasicBlock iterator that automatically skips over
+  /// MIs that are inside bundles (i.e. walk top level MIs only).
+  template<typename Ty, typename IterTy>
+  class bundle_iterator
+    : public std::iterator<std::bidirectional_iterator_tag, Ty, ptrdiff_t> {
+    IterTy MII;
+
+  public:
+    bundle_iterator(IterTy mii) : MII(mii) {
+      assert(!MII->isInsideBundle() &&
+             "It's not legal to initialize bundle_iterator with a bundled MI");
+    }
+
+    bundle_iterator(Ty &mi) : MII(mi) {
+      assert(!mi.isInsideBundle() &&
+             "It's not legal to initialize bundle_iterator with a bundled MI");
+    }
+    bundle_iterator(Ty *mi) : MII(mi) {
+      assert((!mi || !mi->isInsideBundle()) &&
+             "It's not legal to initialize bundle_iterator with a bundled MI");
+    }
+    bundle_iterator(const bundle_iterator &I) : MII(I.MII) {}
+    bundle_iterator() : MII(0) {}
+
+    Ty &operator*() const { return *MII; }
+    Ty *operator->() const { return &operator*(); }
+
+    operator Ty*() const { return MII; }
+
+    bool operator==(const bundle_iterator &x) const {
+      return MII == x.MII;
+    }
+    bool operator!=(const bundle_iterator &x) const {
+      return !operator==(x);
+    }
+
+    // Increment and decrement operators...
+    bundle_iterator &operator--() {      // predecrement - Back up
+      do {
+        --MII;
+      } while (MII->isInsideBundle());
+      return *this;
+    }
+    bundle_iterator &operator++() {      // preincrement - Advance
+      do {
+        ++MII;
+      } while (MII->isInsideBundle());
+      return *this;
+    }
+    bundle_iterator operator--(int) {    // postdecrement operators...
+      bundle_iterator tmp = *this;
+      do {
+        --MII;
+      } while (MII->isInsideBundle());
+      return tmp;
+    }
+    bundle_iterator operator++(int) {    // postincrement operators...
+      bundle_iterator tmp = *this;
+      do {
+        ++MII;
+      } while (MII->isInsideBundle());
+      return tmp;
+    }
+
+    IterTy getInstrIterator() const {
+      return MII;
+    }
+  };
+
+  typedef Instructions::iterator                                 instr_iterator;
+  typedef Instructions::const_iterator                     const_instr_iterator;
+  typedef std::reverse_iterator<instr_iterator>          reverse_instr_iterator;
+  typedef
+  std::reverse_iterator<const_instr_iterator>      const_reverse_instr_iterator;
+
+  typedef
+  bundle_iterator<MachineInstr,instr_iterator>                         iterator;
+  typedef
+  bundle_iterator<const MachineInstr,const_instr_iterator>       const_iterator;
+  typedef std::reverse_iterator<const_iterator>          const_reverse_iterator;
+  typedef std::reverse_iterator<iterator>                      reverse_iterator;
+
 
   unsigned size() const { return (unsigned)Insts.size(); }
   bool empty() const { return Insts.empty(); }
@@ -141,15 +226,53 @@ public:
   const MachineInstr& front() const { return Insts.front(); }
   const MachineInstr& back()  const { return Insts.back(); }
 
+  instr_iterator                instr_begin()       { return Insts.begin();  }
+  const_instr_iterator          instr_begin() const { return Insts.begin();  }
+  instr_iterator                  instr_end()       { return Insts.end();    }
+  const_instr_iterator            instr_end() const { return Insts.end();    }
+  reverse_instr_iterator       instr_rbegin()       { return Insts.rbegin(); }
+  const_reverse_instr_iterator instr_rbegin() const { return Insts.rbegin(); }
+  reverse_instr_iterator       instr_rend  ()       { return Insts.rend();   }
+  const_reverse_instr_iterator instr_rend  () const { return Insts.rend();   }
+
   iterator                begin()       { return Insts.begin();  }
   const_iterator          begin() const { return Insts.begin();  }
-  iterator                  end()       { return Insts.end();    }
-  const_iterator            end() const { return Insts.end();    }
-  reverse_iterator       rbegin()       { return Insts.rbegin(); }
-  const_reverse_iterator rbegin() const { return Insts.rbegin(); }
+  iterator                  end()       {
+    instr_iterator II = instr_end();
+    if (II != instr_begin()) {
+      while (II->isInsideBundle())
+        --II;
+    }
+    return II;
+  }
+  const_iterator            end() const {
+    const_instr_iterator II = instr_end();
+    if (II != instr_begin()) {
+      while (II->isInsideBundle())
+        --II;
+    }
+    return II;
+  }
+  reverse_iterator       rbegin()       {
+    reverse_instr_iterator II = instr_rbegin();
+    if (II != instr_rend()) {
+      while (II->isInsideBundle())
+        ++II;
+    }
+    return II;
+  }
+  const_reverse_iterator rbegin() const {
+    const_reverse_instr_iterator II = instr_rbegin();
+    if (II != instr_rend()) {
+      while (II->isInsideBundle())
+        ++II;
+    }
+    return II;
+  }
   reverse_iterator       rend  ()       { return Insts.rend();   }
   const_reverse_iterator rend  () const { return Insts.rend();   }
 
+
   // Machine-CFG iterators
   typedef std::vector<MachineBasicBlock *>::iterator       pred_iterator;
   typedef std::vector<MachineBasicBlock *>::const_iterator const_pred_iterator;
@@ -219,10 +342,12 @@ public:
   bool            livein_empty() const { return LiveIns.empty(); }
 
   /// getAlignment - Return alignment of the basic block.
+  /// The alignment is specified as log2(bytes).
   ///
   unsigned getAlignment() const { return Alignment; }
 
   /// setAlignment - Set alignment of the basic block.
+  /// The alignment is specified as log2(bytes).
   ///
   void setAlignment(unsigned Align) { Alignment = Align; }
 
@@ -239,7 +364,7 @@ public:
   const MachineBasicBlock *getLandingPadSuccessor() const;
 
   // Code Layout methods.
-  
+
   /// moveBefore/moveAfter - move 'this' block before or after the specified
   /// block.  This only moves the block, it does not modify the CFG or adjust
   /// potential fall-throughs at the end of the block.
@@ -286,7 +411,7 @@ public:
   /// in transferSuccessors, and update PHI operands in the successor blocks
   /// which refer to fromMBB to refer to this.
   void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *fromMBB);
-  
+
   /// isSuccessor - Return true if the specified MBB is a successor of this
   /// block.
   bool isSuccessor(const MachineBasicBlock *MBB) const;
@@ -304,7 +429,7 @@ public:
   /// branch to do so (e.g., a table jump).  True is a conservative answer.
   bool canFallThrough();
 
-  /// Returns a pointer to the first instructon in this block that is not a 
+  /// Returns a pointer to the first instructon in this block that is not a
   /// PHINode instruction. When adding instruction to the beginning of the
   /// basic block, they should be added before the returned value, not before
   /// the first instruction, which might be PHI.
@@ -320,18 +445,16 @@ public:
   /// instruction of this basic block. If a terminator does not exist,
   /// it returns end()
   iterator getFirstTerminator();
+  const_iterator getFirstTerminator() const;
 
-  const_iterator getFirstTerminator() const {
-    return const_cast<MachineBasicBlock*>(this)->getFirstTerminator();
-  }
+  /// getFirstInstrTerminator - Same getFirstTerminator but it ignores bundles
+  /// and return an instr_iterator instead.
+  instr_iterator getFirstInstrTerminator();
 
   /// getLastNonDebugInstr - returns an iterator to the last non-debug
   /// instruction in the basic block, or end()
   iterator getLastNonDebugInstr();
-
-  const_iterator getLastNonDebugInstr() const {
-    return const_cast<MachineBasicBlock*>(this)->getLastNonDebugInstr();
-  }
+  const_iterator getLastNonDebugInstr() const;
 
   /// SplitCriticalEdge - Split the critical edge from this block to the
   /// given successor block, and return the newly created block, or null
@@ -344,38 +467,88 @@ public:
   void pop_front() { Insts.pop_front(); }
   void pop_back() { Insts.pop_back(); }
   void push_back(MachineInstr *MI) { Insts.push_back(MI); }
+
   template<typename IT>
-  void insert(iterator I, IT S, IT E) { Insts.insert(I, S, E); }
-  iterator insert(iterator I, MachineInstr *M) { return Insts.insert(I, M); }
-  iterator insertAfter(iterator I, MachineInstr *M) { 
-    return Insts.insertAfter(I, M); 
+  void insert(instr_iterator I, IT S, IT E) {
+    Insts.insert(I, S, E);
+  }
+  instr_iterator insert(instr_iterator I, MachineInstr *M) {
+    return Insts.insert(I, M);
+  }
+  instr_iterator insertAfter(instr_iterator I, MachineInstr *M) {
+    return Insts.insertAfter(I, M);
   }
 
-  // erase - Remove the specified element or range from the instruction list.
-  // These functions delete any instructions removed.
-  //
-  iterator erase(iterator I)             { return Insts.erase(I); }
-  iterator erase(iterator I, iterator E) { return Insts.erase(I, E); }
-  MachineInstr *remove(MachineInstr *I)  { return Insts.remove(I); }
-  void clear()                           { Insts.clear(); }
+  template<typename IT>
+  void insert(iterator I, IT S, IT E) {
+    Insts.insert(I.getInstrIterator(), S, E);
+  }
+  iterator insert(iterator I, MachineInstr *M) {
+    return Insts.insert(I.getInstrIterator(), M);
+  }
+  iterator insertAfter(iterator I, MachineInstr *M) {
+    return Insts.insertAfter(I.getInstrIterator(), M);
+  }
+
+  /// erase - Remove the specified element or range from the instruction list.
+  /// These functions delete any instructions removed.
+  ///
+  instr_iterator erase(instr_iterator I) {
+    return Insts.erase(I);
+  }
+  instr_iterator erase(instr_iterator I, instr_iterator E) {
+    return Insts.erase(I, E);
+  }
+  instr_iterator erase_instr(MachineInstr *I) {
+    instr_iterator MII(I);
+    return erase(MII);
+  }
+
+  iterator erase(iterator I);
+  iterator erase(iterator I, iterator E) {
+    return Insts.erase(I.getInstrIterator(), E.getInstrIterator());
+  }
+  iterator erase(MachineInstr *I) {
+    iterator MII(I);
+    return erase(MII);
+  }
+
+  /// remove - Remove the instruction from the instruction list. This function
+  /// does not delete the instruction. WARNING: Note, if the specified
+  /// instruction is a bundle this function will remove all the bundled
+  /// instructions as well. It is up to the caller to keep a list of the
+  /// bundled instructions and re-insert them if desired. This function is
+  /// *not recommended* for manipulating instructions with bundles. Use
+  /// splice instead.
+  MachineInstr *remove(MachineInstr *I);
+  void clear() {
+    Insts.clear();
+  }
 
   /// splice - Take an instruction from MBB 'Other' at the position From,
   /// and insert it into this MBB right before 'where'.
-  void splice(iterator where, MachineBasicBlock *Other, iterator From) {
+  void splice(instr_iterator where, MachineBasicBlock *Other,
+              instr_iterator From) {
     Insts.splice(where, Other->Insts, From);
   }
+  void splice(iterator where, MachineBasicBlock *Other, iterator From);
 
   /// splice - Take a block of instructions from MBB 'Other' in the range [From,
   /// To), and insert them into this MBB right before 'where'.
+  void splice(instr_iterator where, MachineBasicBlock *Other, instr_iterator From,
+              instr_iterator To) {
+    Insts.splice(where, Other->Insts, From, To);
+  }
   void splice(iterator where, MachineBasicBlock *Other, iterator From,
               iterator To) {
-    Insts.splice(where, Other->Insts, From, To);
+    Insts.splice(where.getInstrIterator(), Other->Insts,
+                 From.getInstrIterator(), To.getInstrIterator());
   }
 
   /// removeFromParent - This method unlinks 'this' from the containing
   /// function, and returns it, but does not delete it.
   MachineBasicBlock *removeFromParent();
-  
+
   /// eraseFromParent - This method unlinks 'this' from the containing
   /// function and deletes it.
   void eraseFromParent();
@@ -396,7 +569,10 @@ public:
 
   /// findDebugLoc - find the next valid DebugLoc starting at MBBI, skipping
   /// any DBG_VALUE instructions.  Return UnknownLoc if there is none.
-  DebugLoc findDebugLoc(MachineBasicBlock::iterator &MBBI);
+  DebugLoc findDebugLoc(instr_iterator MBBI);
+  DebugLoc findDebugLoc(iterator MBBI) {
+    return findDebugLoc(MBBI.getInstrIterator());
+  }
 
   // Debugging methods.
   void dump() const;
@@ -418,13 +594,14 @@ private:
   /// getWeightIterator - Return weight iterator corresponding to the I
   /// successor iterator.
   weight_iterator getWeightIterator(succ_iterator I);
+  const_weight_iterator getWeightIterator(const_succ_iterator I) const;
 
   friend class MachineBranchProbabilityInfo;
 
   /// getSuccWeight - Return weight of the edge from this block to MBB. This
   /// method should NOT be called directly, but by using getEdgeWeight method
   /// from MachineBranchProbabilityInfo class.
-  uint32_t getSuccWeight(MachineBasicBlock *succ);
+  uint32_t getSuccWeight(const MachineBasicBlock *succ) const;
 
 
   // Methods used to maintain doubly linked list of blocks...
diff --git a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index 416d40b..a9c7bf7 100644
--- a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -20,6 +20,7 @@
 
 namespace llvm {
 
+class MachineBasicBlock;
 class MachineBranchProbabilityInfo;
 template<class BlockT, class FunctionT, class BranchProbInfoT>
 class BlockFrequencyImpl;
@@ -28,7 +29,8 @@ class BlockFrequencyImpl;
 /// machine basic block frequencies.
 class MachineBlockFrequencyInfo : public MachineFunctionPass {
 
-  BlockFrequencyImpl<MachineBasicBlock, MachineFunction, MachineBranchProbabilityInfo> *MBFI;
+  BlockFrequencyImpl<MachineBasicBlock, MachineFunction,
+                     MachineBranchProbabilityInfo> *MBFI;
 
 public:
   static char ID;
@@ -46,7 +48,7 @@ public:
   /// that we should not rely on the value itself, but only on the comparison to
   /// the other block frequencies. We do this to avoid using of floating points.
   ///
-  BlockFrequency getBlockFreq(MachineBasicBlock *MBB) const;
+  BlockFrequency getBlockFreq(const MachineBasicBlock *MBB) const;
 };
 
 }
diff --git a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h
index d9673e2..af4db7d 100644
--- a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h
+++ b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h
@@ -25,6 +25,7 @@ class raw_ostream;
 class MachineBasicBlock;
 
 class MachineBranchProbabilityInfo : public ImmutablePass {
+  virtual void anchor();
 
   // Default weight value. Used when we don't have information about the edge.
   // TODO: DEFAULT_WEIGHT makes sense during static predication, when none of
@@ -34,9 +35,6 @@ class MachineBranchProbabilityInfo : public ImmutablePass {
   // weight to just "inherit" the non-zero weight of an adjacent successor.
   static const uint32_t DEFAULT_WEIGHT = 16;
 
-  // Get sum of the block successors' weights.
-  uint32_t getSumForBlock(MachineBasicBlock *MBB) const;
-
 public:
   static char ID;
 
@@ -51,17 +49,27 @@ public:
 
   // Return edge weight. If we don't have any informations about it - return
   // DEFAULT_WEIGHT.
-  uint32_t getEdgeWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst) const;
+  uint32_t getEdgeWeight(const MachineBasicBlock *Src,
+                         const MachineBasicBlock *Dst) const;
+
+  // Get sum of the block successors' weights, potentially scaling them to fit
+  // within 32-bits. If scaling is required, sets Scale based on the necessary
+  // adjustment. Any edge weights used with the sum should be divided by Scale.
+  uint32_t getSumForBlock(const MachineBasicBlock *MBB, uint32_t &Scale) const;
 
   // A 'Hot' edge is an edge which probability is >= 80%.
   bool isEdgeHot(MachineBasicBlock *Src, MachineBasicBlock *Dst) const;
 
   // Return a hot successor for the block BB or null if there isn't one.
+  // NB: This routine's complexity is linear on the number of successors.
   MachineBasicBlock *getHotSucc(MachineBasicBlock *MBB) const;
 
   // Return a probability as a fraction between 0 (0% probability) and
   // 1 (100% probability), however the value is never equal to 0, and can be 1
   // only iff SRC block has only one successor.
+  // NB: This routine's complexity is linear on the number of successors of
+  // Src. Querying sequentially for each successor's probability is a quadratic
+  // query pattern.
   BranchProbability getEdgeProbability(MachineBasicBlock *Src,
                                        MachineBasicBlock *Dst) const;
 
diff --git a/include/llvm/CodeGen/MachineCodeEmitter.h b/include/llvm/CodeGen/MachineCodeEmitter.h
index 428aada..86e8f27 100644
--- a/include/llvm/CodeGen/MachineCodeEmitter.h
+++ b/include/llvm/CodeGen/MachineCodeEmitter.h
@@ -20,6 +20,8 @@
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/DebugLoc.h"
 
+#include <string>
+
 namespace llvm {
 
 class MachineBasicBlock;
@@ -49,6 +51,7 @@ class MCSymbol;
 /// occurred, more memory is allocated, and we reemit the code into it.
 /// 
 class MachineCodeEmitter {
+  virtual void anchor();
 protected:
   /// BufferBegin/BufferEnd - Pointers to the start and end of the memory
   /// allocated for this code buffer.
diff --git a/include/llvm/CodeGen/MachineConstantPool.h b/include/llvm/CodeGen/MachineConstantPool.h
index 29f4f44..d6d65a2 100644
--- a/include/llvm/CodeGen/MachineConstantPool.h
+++ b/include/llvm/CodeGen/MachineConstantPool.h
@@ -34,6 +34,7 @@ class raw_ostream;
 /// Abstract base class for all machine specific constantpool value subclasses.
 ///
 class MachineConstantPoolValue {
+  virtual void anchor();
   Type *Ty;
 
 public:
diff --git a/include/llvm/CodeGen/MachineDominators.h b/include/llvm/CodeGen/MachineDominators.h
index ab944a2..82a4ac8 100644
--- a/include/llvm/CodeGen/MachineDominators.h
+++ b/include/llvm/CodeGen/MachineDominators.h
@@ -84,7 +84,8 @@ public:
 
     // Loop through the basic block until we find A or B.
     MachineBasicBlock::iterator I = BBA->begin();
-    for (; &*I != A && &*I != B; ++I) /*empty*/;
+    for (; &*I != A && &*I != B; ++I)
+      /*empty*/ ;
 
     //if(!DT.IsPostDominators) {
       // A dominates B if it is found first in the basic block.
diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h
index b347ca8..44402a9 100644
--- a/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/include/llvm/CodeGen/MachineFrameInfo.h
@@ -465,7 +465,7 @@ public:
   bool isSpillSlotObjectIndex(int ObjectIdx) const {
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
-    return Objects[ObjectIdx+NumFixedObjects].isSpillSlot;;
+    return Objects[ObjectIdx+NumFixedObjects].isSpillSlot;
   }
 
   /// isDeadObjectIndex - Returns true if the specified index corresponds to
diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h
index 6e08f7b..dda2dc7 100644
--- a/include/llvm/CodeGen/MachineFunction.h
+++ b/include/llvm/CodeGen/MachineFunction.h
@@ -120,10 +120,12 @@ class MachineFunction {
   /// Alignment - The alignment of the function.
   unsigned Alignment;
 
-  /// CallsSetJmp - True if the function calls setjmp or sigsetjmp. This is used
-  /// to limit optimizations which cannot reason about the control flow of
-  /// setjmp.
-  bool CallsSetJmp;
+  /// ExposesReturnsTwice - True if the function calls setjmp or related
+  /// functions with attribute "returns twice", but doesn't have
+  /// the attribute itself.
+  /// This is used to limit optimizations which cannot reason
+  /// about the control flow of such functions.
+  bool ExposesReturnsTwice;
 
   MachineFunction(const MachineFunction &); // DO NOT IMPLEMENT
   void operator=(const MachineFunction&);   // DO NOT IMPLEMENT
@@ -187,20 +189,22 @@ public:
   ///
   void setAlignment(unsigned A) { Alignment = A; }
 
-  /// EnsureAlignment - Make sure the function is at least 'A' bits aligned.
+  /// EnsureAlignment - Make sure the function is at least 1 << A bytes aligned.
   void EnsureAlignment(unsigned A) {
     if (Alignment < A) Alignment = A;
   }
 
-  /// callsSetJmp - Returns true if the function calls setjmp or sigsetjmp.
-  bool callsSetJmp() const {
-    return CallsSetJmp;
+  /// exposesReturnsTwice - Returns true if the function calls setjmp or
+  /// any other similar functions with attribute "returns twice" without
+  /// having the attribute itself.
+  bool exposesReturnsTwice() const {
+    return ExposesReturnsTwice;
   }
 
-  /// setCallsSetJmp - Set a flag that indicates if there's a call to setjmp or
-  /// sigsetjmp.
-  void setCallsSetJmp(bool B) {
-    CallsSetJmp = B;
+  /// setCallsSetJmp - Set a flag that indicates if there's a call to
+  /// a "returns twice" function.
+  void setExposesReturnsTwice(bool B) {
+    ExposesReturnsTwice = B;
   }
   
   /// getInfo - Keep track of various per-function pieces of information for
@@ -376,7 +380,8 @@ public:
   MachineMemOperand *getMachineMemOperand(MachinePointerInfo PtrInfo,
                                           unsigned f, uint64_t s,
                                           unsigned base_alignment,
-                                          const MDNode *TBAAInfo = 0);
+                                          const MDNode *TBAAInfo = 0,
+                                          const MDNode *Ranges = 0);
   
   /// getMachineMemOperand - Allocate a new MachineMemOperand by copying
   /// an existing one, adjusting by an offset and using the given size.
@@ -437,6 +442,7 @@ template <> struct GraphTraits<MachineFunction*> :
   typedef MachineFunction::iterator nodes_iterator;
   static nodes_iterator nodes_begin(MachineFunction *F) { return F->begin(); }
   static nodes_iterator nodes_end  (MachineFunction *F) { return F->end(); }
+  static unsigned       size       (MachineFunction *F) { return F->size(); }
 };
 template <> struct GraphTraits<const MachineFunction*> :
   public GraphTraits<const MachineBasicBlock*> {
@@ -452,6 +458,9 @@ template <> struct GraphTraits<const MachineFunction*> :
   static nodes_iterator nodes_end  (const MachineFunction *F) {
     return F->end();
   }
+  static unsigned       size       (const MachineFunction *F)  {
+    return F->size();
+  }
 };
 
 
diff --git a/include/llvm/CodeGen/MachineFunctionAnalysis.h b/include/llvm/CodeGen/MachineFunctionAnalysis.h
index 50676ad..50ea206 100644
--- a/include/llvm/CodeGen/MachineFunctionAnalysis.h
+++ b/include/llvm/CodeGen/MachineFunctionAnalysis.h
@@ -26,17 +26,14 @@ class MachineFunction;
 struct MachineFunctionAnalysis : public FunctionPass {
 private:
   const TargetMachine &TM;
-  CodeGenOpt::Level OptLevel;
   MachineFunction *MF;
   unsigned NextFnNum;
 public:
   static char ID;
-  explicit MachineFunctionAnalysis(const TargetMachine &tm,
-                                   CodeGenOpt::Level OL = CodeGenOpt::Default);
+  explicit MachineFunctionAnalysis(const TargetMachine &tm);
   ~MachineFunctionAnalysis();
 
   MachineFunction &getMF() const { return *MF; }
-  CodeGenOpt::Level getOptLevel() const { return OptLevel; }
   
   virtual const char* getPassName() const {
     return "Machine Function Analysis";
diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h
index cae38f3..65093d7 100644
--- a/include/llvm/CodeGen/MachineInstr.h
+++ b/include/llvm/CodeGen/MachineInstr.h
@@ -19,6 +19,7 @@
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/Target/TargetOpcodes.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
 #include "llvm/ADT/STLExtras.h"
@@ -53,9 +54,11 @@ public:
   };
 
   enum MIFlag {
-    NoFlags    = 0,
-    FrameSetup = 1 << 0                 // Instruction is used as a part of
+    NoFlags      = 0,
+    FrameSetup   = 1 << 0,              // Instruction is used as a part of
                                         // function frame setup code.
+    InsideBundle = 1 << 1               // Instruction is inside a bundle (not
+                                        // the first MI in a bundle)
   };
 private:
   const MCInstrDesc *MCID;              // Instruction descriptor.
@@ -71,9 +74,10 @@ private:
                                         // anything other than to convey comment
                                         // information to AsmPrinter.
 
+  uint16_t NumMemRefs;                  // information on memory references
+  mmo_iterator MemRefs;
+
   std::vector<MachineOperand> Operands; // the operands
-  mmo_iterator MemRefs;                 // information on memory references
-  mmo_iterator MemRefsEnd;
   MachineBasicBlock *Parent;            // Pointer to the owning basic block.
   DebugLoc debugLoc;                    // Source line information.
 
@@ -148,6 +152,12 @@ public:
     AsmPrinterFlags |= (uint8_t)Flag;
   }
 
+  /// clearAsmPrinterFlag - clear specific AsmPrinter flags
+  ///
+  void clearAsmPrinterFlag(CommentFlag Flag) {
+    AsmPrinterFlags &= ~Flag;
+  }
+
   /// getFlags - Return the MI flags bitvector.
   uint8_t getFlags() const {
     return Flags;
@@ -167,12 +177,64 @@ public:
     Flags = flags;
   }
 
-  /// clearAsmPrinterFlag - clear specific AsmPrinter flags
+  /// clearFlag - Clear a MI flag.
+  void clearFlag(MIFlag Flag) {
+    Flags &= ~((uint8_t)Flag);
+  }
+
+  /// isInsideBundle - Return true if MI is in a bundle (but not the first MI
+  /// in a bundle).
   ///
-  void clearAsmPrinterFlag(CommentFlag Flag) {
-    AsmPrinterFlags &= ~Flag;
+  /// A bundle looks like this before it's finalized:
+  ///   ----------------
+  ///   |      MI      |
+  ///   ----------------
+  ///          |
+  ///   ----------------
+  ///   |      MI    * |
+  ///   ----------------
+  ///          |
+  ///   ----------------
+  ///   |      MI    * |
+  ///   ----------------
+  /// In this case, the first MI starts a bundle but is not inside a bundle, the
+  /// next 2 MIs are considered "inside" the bundle.
+  ///
+  /// After a bundle is finalized, it looks like this:
+  ///   ----------------
+  ///   |    Bundle    |
+  ///   ----------------
+  ///          |
+  ///   ----------------
+  ///   |      MI    * |
+  ///   ----------------
+  ///          |
+  ///   ----------------
+  ///   |      MI    * |
+  ///   ----------------
+  ///          |
+  ///   ----------------
+  ///   |      MI    * |
+  ///   ----------------
+  /// The first instruction has the special opcode "BUNDLE". It's not "inside"
+  /// a bundle, but the next three MIs are.
+  bool isInsideBundle() const {
+    return getFlag(InsideBundle);
   }
 
+  /// setIsInsideBundle - Set InsideBundle bit.
+  ///
+  void setIsInsideBundle(bool Val = true) {
+    if (Val)
+      setFlag(InsideBundle);
+    else
+      clearFlag(InsideBundle);
+  }
+
+  /// isBundled - Return true if this instruction part of a bundle. This is true
+  /// if either itself or its following instruction is marked "InsideBundle".
+  bool isBundled() const;
+
   /// getDebugLoc - Returns the debug location id of this MachineInstr.
   ///
   DebugLoc getDebugLoc() const { return debugLoc; }
@@ -223,15 +285,285 @@ public:
 
   /// Access to memory operands of the instruction
   mmo_iterator memoperands_begin() const { return MemRefs; }
-  mmo_iterator memoperands_end() const { return MemRefsEnd; }
-  bool memoperands_empty() const { return MemRefsEnd == MemRefs; }
+  mmo_iterator memoperands_end() const { return MemRefs + NumMemRefs; }
+  bool memoperands_empty() const { return NumMemRefs == 0; }
 
   /// hasOneMemOperand - Return true if this instruction has exactly one
   /// MachineMemOperand.
   bool hasOneMemOperand() const {
-    return MemRefsEnd - MemRefs == 1;
+    return NumMemRefs == 1;
+  }
+
+  /// API for querying MachineInstr properties. They are the same as MCInstrDesc
+  /// queries but they are bundle aware.
+
+  enum QueryType {
+    IgnoreBundle,    // Ignore bundles
+    AnyInBundle,     // Return true if any instruction in bundle has property
+    AllInBundle      // Return true if all instructions in bundle have property
+  };
+
+  /// hasProperty - Return true if the instruction (or in the case of a bundle,
+  /// the instructions inside the bundle) has the specified property.
+  /// The first argument is the property being queried.
+  /// The second argument indicates whether the query should look inside
+  /// instruction bundles.
+  bool hasProperty(unsigned MCFlag, QueryType Type = AnyInBundle) const {
+    // Inline the fast path.
+    if (Type == IgnoreBundle || !isBundle())
+      return getDesc().getFlags() & (1 << MCFlag);
+
+    // If we have a bundle, take the slow path.
+    return hasPropertyInBundle(1 << MCFlag, Type);
+  }
+
+  /// isVariadic - Return true if this instruction can have a variable number of
+  /// operands.  In this case, the variable operands will be after the normal
+  /// operands but before the implicit definitions and uses (if any are
+  /// present).
+  bool isVariadic(QueryType Type = IgnoreBundle) const {
+    return hasProperty(MCID::Variadic, Type);
+  }
+
+  /// hasOptionalDef - Set if this instruction has an optional definition, e.g.
+  /// ARM instructions which can set condition code if 's' bit is set.
+  bool hasOptionalDef(QueryType Type = IgnoreBundle) const {
+    return hasProperty(MCID::HasOptionalDef, Type);
+  }
+
+  /// isPseudo - Return true if this is a pseudo instruction that doesn't
+  /// correspond to a real machine instruction.
+  ///
+  bool isPseudo(QueryType Type = IgnoreBundle) const {
+    return hasProperty(MCID::Pseudo, Type);
+  }
+
+  bool isReturn(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::Return, Type);
+  }
+
+  bool isCall(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::Call, Type);
+  }
+
+  /// isBarrier - Returns true if the specified instruction stops control flow
+  /// from executing the instruction immediately following it.  Examples include
+  /// unconditional branches and return instructions.
+  bool isBarrier(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::Barrier, Type);
+  }
+
+  /// isTerminator - Returns true if this instruction part of the terminator for
+  /// a basic block.  Typically this is things like return and branch
+  /// instructions.
+  ///
+  /// Various passes use this to insert code into the bottom of a basic block,
+  /// but before control flow occurs.
+  bool isTerminator(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::Terminator, Type);
+  }
+
+  /// isBranch - Returns true if this is a conditional, unconditional, or
+  /// indirect branch.  Predicates below can be used to discriminate between
+  /// these cases, and the TargetInstrInfo::AnalyzeBranch method can be used to
+  /// get more information.
+  bool isBranch(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::Branch, Type);
+  }
+
+  /// isIndirectBranch - Return true if this is an indirect branch, such as a
+  /// branch through a register.
+  bool isIndirectBranch(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::IndirectBranch, Type);
+  }
+
+  /// isConditionalBranch - Return true if this is a branch which may fall
+  /// through to the next instruction or may transfer control flow to some other
+  /// block.  The TargetInstrInfo::AnalyzeBranch method can be used to get more
+  /// information about this branch.
+  bool isConditionalBranch(QueryType Type = AnyInBundle) const {
+    return isBranch(Type) & !isBarrier(Type) & !isIndirectBranch(Type);
+  }
+
+  /// isUnconditionalBranch - Return true if this is a branch which always
+  /// transfers control flow to some other block.  The
+  /// TargetInstrInfo::AnalyzeBranch method can be used to get more information
+  /// about this branch.
+  bool isUnconditionalBranch(QueryType Type = AnyInBundle) const {
+    return isBranch(Type) & isBarrier(Type) & !isIndirectBranch(Type);
+  }
+
+  // isPredicable - Return true if this instruction has a predicate operand that
+  // controls execution.  It may be set to 'always', or may be set to other
+  /// values.   There are various methods in TargetInstrInfo that can be used to
+  /// control and modify the predicate in this instruction.
+  bool isPredicable(QueryType Type = AllInBundle) const {
+    // If it's a bundle than all bundled instructions must be predicable for this
+    // to return true.
+    return hasProperty(MCID::Predicable, Type);
+  }
+
+  /// isCompare - Return true if this instruction is a comparison.
+  bool isCompare(QueryType Type = IgnoreBundle) const {
+    return hasProperty(MCID::Compare, Type);
+  }
+
+  /// isMoveImmediate - Return true if this instruction is a move immediate
+  /// (including conditional moves) instruction.
+  bool isMoveImmediate(QueryType Type = IgnoreBundle) const {
+    return hasProperty(MCID::MoveImm, Type);
   }
 
+  /// isBitcast - Return true if this instruction is a bitcast instruction.
+  ///
+  bool isBitcast(QueryType Type = IgnoreBundle) const {
+    return hasProperty(MCID::Bitcast, Type);
+  }
+
+  /// isNotDuplicable - Return true if this instruction cannot be safely
+  /// duplicated.  For example, if the instruction has a unique labels attached
+  /// to it, duplicating it would cause multiple definition errors.
+  bool isNotDuplicable(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::NotDuplicable, Type);
+  }
+
+  /// hasDelaySlot - Returns true if the specified instruction has a delay slot
+  /// which must be filled by the code generator.
+  bool hasDelaySlot(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::DelaySlot, Type);
+  }
+
+  /// canFoldAsLoad - Return true for instructions that can be folded as
+  /// memory operands in other instructions. The most common use for this
+  /// is instructions that are simple loads from memory that don't modify
+  /// the loaded value in any way, but it can also be used for instructions
+  /// that can be expressed as constant-pool loads, such as V_SETALLONES
+  /// on x86, to allow them to be folded when it is beneficial.
+  /// This should only be set on instructions that return a value in their
+  /// only virtual register definition.
+  bool canFoldAsLoad(QueryType Type = IgnoreBundle) const {
+    return hasProperty(MCID::FoldableAsLoad, Type);
+  }
+
+  //===--------------------------------------------------------------------===//
+  // Side Effect Analysis
+  //===--------------------------------------------------------------------===//
+
+  /// mayLoad - Return true if this instruction could possibly read memory.
+  /// Instructions with this flag set are not necessarily simple load
+  /// instructions, they may load a value and modify it, for example.
+  bool mayLoad(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::MayLoad, Type);
+  }
+
+
+  /// mayStore - Return true if this instruction could possibly modify memory.
+  /// Instructions with this flag set are not necessarily simple store
+  /// instructions, they may store a modified value based on their operands, or
+  /// may not actually modify anything, for example.
+  bool mayStore(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::MayStore, Type);
+  }
+
+  //===--------------------------------------------------------------------===//
+  // Flags that indicate whether an instruction can be modified by a method.
+  //===--------------------------------------------------------------------===//
+
+  /// isCommutable - Return true if this may be a 2- or 3-address
+  /// instruction (of the form "X = op Y, Z, ..."), which produces the same
+  /// result if Y and Z are exchanged.  If this flag is set, then the
+  /// TargetInstrInfo::commuteInstruction method may be used to hack on the
+  /// instruction.
+  ///
+  /// Note that this flag may be set on instructions that are only commutable
+  /// sometimes.  In these cases, the call to commuteInstruction will fail.
+  /// Also note that some instructions require non-trivial modification to
+  /// commute them.
+  bool isCommutable(QueryType Type = IgnoreBundle) const {
+    return hasProperty(MCID::Commutable, Type);
+  }
+
+  /// isConvertibleTo3Addr - Return true if this is a 2-address instruction
+  /// which can be changed into a 3-address instruction if needed.  Doing this
+  /// transformation can be profitable in the register allocator, because it
+  /// means that the instruction can use a 2-address form if possible, but
+  /// degrade into a less efficient form if the source and dest register cannot
+  /// be assigned to the same register.  For example, this allows the x86
+  /// backend to turn a "shl reg, 3" instruction into an LEA instruction, which
+  /// is the same speed as the shift but has bigger code size.
+  ///
+  /// If this returns true, then the target must implement the
+  /// TargetInstrInfo::convertToThreeAddress method for this instruction, which
+  /// is allowed to fail if the transformation isn't valid for this specific
+  /// instruction (e.g. shl reg, 4 on x86).
+  ///
+  bool isConvertibleTo3Addr(QueryType Type = IgnoreBundle) const {
+    return hasProperty(MCID::ConvertibleTo3Addr, Type);
+  }
+
+  /// usesCustomInsertionHook - Return true if this instruction requires
+  /// custom insertion support when the DAG scheduler is inserting it into a
+  /// machine basic block.  If this is true for the instruction, it basically
+  /// means that it is a pseudo instruction used at SelectionDAG time that is
+  /// expanded out into magic code by the target when MachineInstrs are formed.
+  ///
+  /// If this is true, the TargetLoweringInfo::InsertAtEndOfBasicBlock method
+  /// is used to insert this into the MachineBasicBlock.
+  bool usesCustomInsertionHook(QueryType Type = IgnoreBundle) const {
+    return hasProperty(MCID::UsesCustomInserter, Type);
+  }
+
+  /// hasPostISelHook - Return true if this instruction requires *adjustment*
+  /// after instruction selection by calling a target hook. For example, this
+  /// can be used to fill in ARM 's' optional operand depending on whether
+  /// the conditional flag register is used.
+  bool hasPostISelHook(QueryType Type = IgnoreBundle) const {
+    return hasProperty(MCID::HasPostISelHook, Type);
+  }
+
+  /// isRematerializable - Returns true if this instruction is a candidate for
+  /// remat.  This flag is deprecated, please don't use it anymore.  If this
+  /// flag is set, the isReallyTriviallyReMaterializable() method is called to
+  /// verify the instruction is really rematable.
+  bool isRematerializable(QueryType Type = AllInBundle) const {
+    // It's only possible to re-mat a bundle if all bundled instructions are
+    // re-materializable.
+    return hasProperty(MCID::Rematerializable, Type);
+  }
+
+  /// isAsCheapAsAMove - Returns true if this instruction has the same cost (or
+  /// less) than a move instruction. This is useful during certain types of
+  /// optimizations (e.g., remat during two-address conversion or machine licm)
+  /// where we would like to remat or hoist the instruction, but not if it costs
+  /// more than moving the instruction into the appropriate register. Note, we
+  /// are not marking copies from and to the same register class with this flag.
+  bool isAsCheapAsAMove(QueryType Type = AllInBundle) const {
+    // Only returns true for a bundle if all bundled instructions are cheap.
+    // FIXME: This probably requires a target hook.
+    return hasProperty(MCID::CheapAsAMove, Type);
+  }
+
+  /// hasExtraSrcRegAllocReq - Returns true if this instruction source operands
+  /// have special register allocation requirements that are not captured by the
+  /// operand register classes. e.g. ARM::STRD's two source registers must be an
+  /// even / odd pair, ARM::STM registers have to be in ascending order.
+  /// Post-register allocation passes should not attempt to change allocations
+  /// for sources of instructions with this flag.
+  bool hasExtraSrcRegAllocReq(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::ExtraSrcRegAllocReq, Type);
+  }
+
+  /// hasExtraDefRegAllocReq - Returns true if this instruction def operands
+  /// have special register allocation requirements that are not captured by the
+  /// operand register classes. e.g. ARM::LDRD's two def registers must be an
+  /// even / odd pair, ARM::LDM registers have to be in ascending order.
+  /// Post-register allocation passes should not attempt to change allocations
+  /// for definitions of instructions with this flag.
+  bool hasExtraDefRegAllocReq(QueryType Type = AnyInBundle) const {
+    return hasProperty(MCID::ExtraDefRegAllocReq, Type);
+  }
+
+
   enum MICheckType {
     CheckDefs,      // Check all operands for equality
     CheckKillDead,  // Check all operands including kill / dead markers
@@ -281,6 +613,9 @@ public:
   bool isRegSequence() const {
     return getOpcode() == TargetOpcode::REG_SEQUENCE;
   }
+  bool isBundle() const {
+    return getOpcode() == TargetOpcode::BUNDLE;
+  }
   bool isCopy() const {
     return getOpcode() == TargetOpcode::COPY;
   }
@@ -300,6 +635,9 @@ public:
       getOperand(0).getSubReg() == getOperand(1).getSubReg();
   }
 
+  /// getBundleSize - Return the number of instructions inside the MI bundle.
+  unsigned getBundleSize() const;
+
   /// readsRegister - Return true if the MachineInstr reads the specified
   /// register. If TargetRegisterInfo is passed, then it also checks if there
   /// is a read of a super-register.
@@ -372,6 +710,7 @@ public:
   /// that are not dead are skipped. If Overlap is true, then it also looks for
   /// defs that merely overlap the specified register. If TargetRegisterInfo is
   /// non-null, then it also checks if there is a def of a super-register.
+  /// This may also return a register mask operand when Overlap is true.
   int findRegisterDefOperandIdx(unsigned Reg,
                                 bool isDead = false, bool Overlap = false,
                                 const TargetRegisterInfo *TRI = NULL) const;
@@ -416,7 +755,7 @@ public:
   /// isRegTiedToUseOperand - Given the index of a register def operand,
   /// check if the register def is tied to a source operand, due to either
   /// two-address elimination or inline assembly constraints. Returns the
-  /// first tied use operand index by reference is UseOpIdx is not null.
+  /// first tied use operand index by reference if UseOpIdx is not null.
   bool isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx = 0) const;
 
   /// isRegTiedToDefOperand - Return true if the use operand of the specified
@@ -448,6 +787,10 @@ public:
                          const TargetRegisterInfo *RegInfo,
                          bool AddIfNotFound = false);
 
+  /// clearRegisterKills - Clear all kill flags affecting Reg.  If RegInfo is
+  /// provided, this includes super-register kills.
+  void clearRegisterKills(unsigned Reg, const TargetRegisterInfo *RegInfo);
+
   /// addRegisterDead - We have determined MI defined a register without a use.
   /// Look for the operand that defines it and mark it as IsDead. If
   /// AddIfNotFound is true, add a implicit operand if it's not found. Returns
@@ -462,7 +805,10 @@ public:
 
   /// setPhysRegsDeadExcept - Mark every physreg used by this instruction as
   /// dead except those in the UsedRegs list.
-  void setPhysRegsDeadExcept(const SmallVectorImpl<unsigned> &UsedRegs,
+  ///
+  /// On instructions with register mask operands, also add implicit-def
+  /// operands for all registers in UsedRegs.
+  void setPhysRegsDeadExcept(ArrayRef<unsigned> UsedRegs,
                              const TargetRegisterInfo &TRI);
 
   /// isSafeToMove - Return true if it is safe to move this instruction. If
@@ -550,7 +896,7 @@ public:
   /// list. This does not transfer ownership.
   void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) {
     MemRefs = NewMemRefs;
-    MemRefsEnd = NewMemRefsEnd;
+    NumMemRefs = NewMemRefsEnd - NewMemRefs;
   }
 
 private:
@@ -572,6 +918,10 @@ private:
   /// this instruction from their respective use lists.  This requires that the
   /// operands not be on their use lists yet.
   void AddRegOperandsToUseLists(MachineRegisterInfo &RegInfo);
+
+  /// hasPropertyInBundle - Slow path for hasProperty when we're dealing with a
+  /// bundle.
+  bool hasPropertyInBundle(unsigned Mask, QueryType Type) const;
 };
 
 /// MachineInstrExpressionTrait - Special DenseMapInfo traits to compare
diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h
index b989027..99849a6 100644
--- a/include/llvm/CodeGen/MachineInstrBuilder.h
+++ b/include/llvm/CodeGen/MachineInstrBuilder.h
@@ -34,6 +34,7 @@ namespace RegState {
     Undef          = 0x20,
     EarlyClobber   = 0x40,
     Debug          = 0x80,
+    DefineNoRead   = Define | Undef,
     ImplicitDefine = Implicit | Define,
     ImplicitKill   = Implicit | Kill
   };
@@ -124,6 +125,11 @@ public:
     return *this;
   }
 
+  const MachineInstrBuilder &addRegMask(const uint32_t *Mask) const {
+    MI->addOperand(MachineOperand::CreateRegMask(Mask));
+    return *this;
+  }
+
   const MachineInstrBuilder &addMemOperand(MachineMemOperand *MMO) const {
     MI->addMemOperand(*MI->getParent()->getParent(), MMO);
     return *this;
@@ -209,6 +215,30 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
   return MachineInstrBuilder(MI).addReg(DestReg, RegState::Define);
 }
 
+inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
+                                   MachineBasicBlock::instr_iterator I,
+                                   DebugLoc DL,
+                                   const MCInstrDesc &MCID,
+                                   unsigned DestReg) {
+  MachineInstr *MI = BB.getParent()->CreateMachineInstr(MCID, DL);
+  BB.insert(I, MI);
+  return MachineInstrBuilder(MI).addReg(DestReg, RegState::Define);
+}
+
+inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
+                                   MachineInstr *I,
+                                   DebugLoc DL,
+                                   const MCInstrDesc &MCID,
+                                   unsigned DestReg) {
+  if (I->isInsideBundle()) {
+    MachineBasicBlock::instr_iterator MII = I;
+    return BuildMI(BB, MII, DL, MCID, DestReg);
+  }
+
+  MachineBasicBlock::iterator MII = I;
+  return BuildMI(BB, MII, DL, MCID, DestReg);
+}
+
 /// BuildMI - This version of the builder inserts the newly-built
 /// instruction before the given position in the given MachineBasicBlock, and
 /// does NOT take a destination register.
@@ -222,6 +252,28 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
   return MachineInstrBuilder(MI);
 }
 
+inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
+                                   MachineBasicBlock::instr_iterator I,
+                                   DebugLoc DL,
+                                   const MCInstrDesc &MCID) {
+  MachineInstr *MI = BB.getParent()->CreateMachineInstr(MCID, DL);
+  BB.insert(I, MI);
+  return MachineInstrBuilder(MI);
+}
+
+inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
+                                   MachineInstr *I,
+                                   DebugLoc DL,
+                                   const MCInstrDesc &MCID) {
+  if (I->isInsideBundle()) {
+    MachineBasicBlock::instr_iterator MII = I;
+    return BuildMI(BB, MII, DL, MCID);
+  }
+
+  MachineBasicBlock::iterator MII = I;
+  return BuildMI(BB, MII, DL, MCID);
+}
+
 /// BuildMI - This version of the builder inserts the newly-built
 /// instruction at the end of the given MachineBasicBlock, and does NOT take a
 /// destination register.
diff --git a/include/llvm/CodeGen/MachineInstrBundle.h b/include/llvm/CodeGen/MachineInstrBundle.h
new file mode 100644
index 0000000..0fb4969
--- /dev/null
+++ b/include/llvm/CodeGen/MachineInstrBundle.h
@@ -0,0 +1,203 @@
+//===-- CodeGen/MachineInstBundle.h - MI bundle utilities -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provide utility functions to manipulate machine instruction
+// bundles.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MACHINEINSTRBUNDLE_H
+#define LLVM_CODEGEN_MACHINEINSTRBUNDLE_H
+
+#include "llvm/CodeGen/MachineBasicBlock.h"
+
+namespace llvm {
+
+/// finalizeBundle - Finalize a machine instruction bundle which includes
+/// a sequence of instructions starting from FirstMI to LastMI (exclusive).
+/// This routine adds a BUNDLE instruction to represent the bundle, it adds
+/// IsInternalRead markers to MachineOperands which are defined inside the
+/// bundle, and it copies externally visible defs and uses to the BUNDLE
+/// instruction.
+void finalizeBundle(MachineBasicBlock &MBB,
+                    MachineBasicBlock::instr_iterator FirstMI,
+                    MachineBasicBlock::instr_iterator LastMI);
+  
+/// finalizeBundle - Same functionality as the previous finalizeBundle except
+/// the last instruction in the bundle is not provided as an input. This is
+/// used in cases where bundles are pre-determined by marking instructions
+/// with 'InsideBundle' marker. It returns the MBB instruction iterator that
+/// points to the end of the bundle.
+MachineBasicBlock::instr_iterator finalizeBundle(MachineBasicBlock &MBB,
+                    MachineBasicBlock::instr_iterator FirstMI);
+
+/// finalizeBundles - Finalize instruction bundles in the specified
+/// MachineFunction. Return true if any bundles are finalized.
+bool finalizeBundles(MachineFunction &MF);
+
+/// getBundleStart - Returns the first instruction in the bundle containing MI.
+///
+static inline MachineInstr *getBundleStart(MachineInstr *MI) {
+  MachineBasicBlock::instr_iterator I = MI;
+  while (I->isInsideBundle())
+    --I;
+  return I;
+}
+
+static inline const MachineInstr *getBundleStart(const MachineInstr *MI) {
+  MachineBasicBlock::const_instr_iterator I = MI;
+  while (I->isInsideBundle())
+    --I;
+  return I;
+}
+
+//===----------------------------------------------------------------------===//
+// MachineOperand iterator
+//
+
+/// MachineOperandIteratorBase - Iterator that can visit all operands on a
+/// MachineInstr, or all operands on a bundle of MachineInstrs.  This class is
+/// not intended to be used directly, use one of the sub-classes instead.
+///
+/// Intended use:
+///
+///   for (MIBundleOperands MIO(MI); MIO.isValid(); ++MIO) {
+///     if (!MIO->isReg())
+///       continue;
+///     ...
+///   }
+///
+class MachineOperandIteratorBase {
+  MachineBasicBlock::instr_iterator InstrI, InstrE;
+  MachineInstr::mop_iterator OpI, OpE;
+
+  // If the operands on InstrI are exhausted, advance InstrI to the next
+  // bundled instruction with operands.
+  void advance() {
+    while (OpI == OpE) {
+      // Don't advance off the basic block, or into a new bundle.
+      if (++InstrI == InstrE || !InstrI->isInsideBundle())
+        break;
+      OpI = InstrI->operands_begin();
+      OpE = InstrI->operands_end();
+    }
+  }
+
+protected:
+  /// MachineOperandIteratorBase - Create an iterator that visits all operands
+  /// on MI, or all operands on every instruction in the bundle containing MI.
+  ///
+  /// @param MI The instruction to examine.
+  /// @param WholeBundle When true, visit all operands on the entire bundle.
+  ///
+  explicit MachineOperandIteratorBase(MachineInstr *MI, bool WholeBundle) {
+    if (WholeBundle) {
+      InstrI = getBundleStart(MI);
+      InstrE = MI->getParent()->instr_end();
+    } else {
+      InstrI = InstrE = MI;
+      ++InstrE;
+    }
+    OpI = InstrI->operands_begin();
+    OpE = InstrI->operands_end();
+    if (WholeBundle)
+      advance();
+  }
+
+  MachineOperand &deref() const { return *OpI; }
+
+public:
+  /// isValid - Returns true until all the operands have been visited.
+  bool isValid() const { return OpI != OpE; }
+
+  /// Preincrement.  Move to the next operand.
+  void operator++() {
+    assert(isValid() && "Cannot advance MIOperands beyond the last operand");
+    ++OpI;
+    advance();
+  }
+
+  /// getOperandNo - Returns the number of the current operand relative to its
+  /// instruction.
+  ///
+  unsigned getOperandNo() const {
+    return OpI - InstrI->operands_begin();
+  }
+
+  /// RegInfo - Information about a virtual register used by a set of operands.
+  ///
+  struct RegInfo {
+    /// Reads - One of the operands read the virtual register.  This does not
+    /// include <undef> or <internal> use operands, see MO::readsReg().
+    bool Reads;
+
+    /// Writes - One of the operands writes the virtual register.
+    bool Writes;
+
+    /// Tied - Uses and defs must use the same register. This can be because of
+    /// a two-address constraint, or there may be a partial redefinition of a
+    /// sub-register.
+    bool Tied;
+  };
+
+  /// analyzeVirtReg - Analyze how the current instruction or bundle uses a
+  /// virtual register.  This function should not be called after operator++(),
+  /// it expects a fresh iterator.
+  ///
+  /// @param Reg The virtual register to analyze.
+  /// @param Ops When set, this vector will receive an (MI, OpNum) entry for
+  ///            each operand referring to Reg.
+  /// @returns A filled-in RegInfo struct.
+  RegInfo analyzeVirtReg(unsigned Reg,
+                 SmallVectorImpl<std::pair<MachineInstr*, unsigned> > *Ops = 0);
+};
+
+/// MIOperands - Iterate over operands of a single instruction.
+///
+class MIOperands : public MachineOperandIteratorBase {
+public:
+  MIOperands(MachineInstr *MI) : MachineOperandIteratorBase(MI, false) {}
+  MachineOperand &operator* () const { return deref(); }
+  MachineOperand *operator->() const { return &deref(); }
+};
+
+/// ConstMIOperands - Iterate over operands of a single const instruction.
+///
+class ConstMIOperands : public MachineOperandIteratorBase {
+public:
+  ConstMIOperands(const MachineInstr *MI)
+    : MachineOperandIteratorBase(const_cast<MachineInstr*>(MI), false) {}
+  const MachineOperand &operator* () const { return deref(); }
+  const MachineOperand *operator->() const { return &deref(); }
+};
+
+/// MIBundleOperands - Iterate over all operands in a bundle of machine
+/// instructions.
+///
+class MIBundleOperands : public MachineOperandIteratorBase {
+public:
+  MIBundleOperands(MachineInstr *MI) : MachineOperandIteratorBase(MI, true) {}
+  MachineOperand &operator* () const { return deref(); }
+  MachineOperand *operator->() const { return &deref(); }
+};
+
+/// ConstMIBundleOperands - Iterate over all operands in a const bundle of
+/// machine instructions.
+///
+class ConstMIBundleOperands : public MachineOperandIteratorBase {
+public:
+  ConstMIBundleOperands(const MachineInstr *MI)
+    : MachineOperandIteratorBase(const_cast<MachineInstr*>(MI), true) {}
+  const MachineOperand &operator* () const { return deref(); }
+  const MachineOperand *operator->() const { return &deref(); }
+};
+
+} // End llvm namespace
+
+#endif
diff --git a/include/llvm/CodeGen/MachineJumpTableInfo.h b/include/llvm/CodeGen/MachineJumpTableInfo.h
index 6264349..6bd6682 100644
--- a/include/llvm/CodeGen/MachineJumpTableInfo.h
+++ b/include/llvm/CodeGen/MachineJumpTableInfo.h
@@ -47,7 +47,12 @@ public:
     /// EK_BlockAddress - Each entry is a plain address of block, e.g.:
     ///     .word LBB123
     EK_BlockAddress,
-    
+
+    /// EK_GPRel64BlockAddress - Each entry is an address of block, encoded
+    /// with a relocation as gp-relative, e.g.:
+    ///     .gpdword LBB123
+    EK_GPRel64BlockAddress,
+
     /// EK_GPRel32BlockAddress - Each entry is an address of block, encoded
     /// with a relocation as gp-relative, e.g.:
     ///     .gprel32 LBB123
diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h
index 768ce47..1ac9080 100644
--- a/include/llvm/CodeGen/MachineMemOperand.h
+++ b/include/llvm/CodeGen/MachineMemOperand.h
@@ -22,6 +22,7 @@ namespace llvm {
 
 class Value;
 class FoldingSetNodeID;
+class MDNode;
 class raw_ostream;
 
 /// MachinePointerInfo - This class contains a discriminated union of
@@ -83,6 +84,7 @@ class MachineMemOperand {
   uint64_t Size;
   unsigned Flags;
   const MDNode *TBAAInfo;
+  const MDNode *Ranges;
 
 public:
   /// Flags values. These may be or'd together.
@@ -95,14 +97,17 @@ public:
     MOVolatile = 4,
     /// The memory access is non-temporal.
     MONonTemporal = 8,
+    /// The memory access is invariant.
+    MOInvariant = 16,
     // This is the number of bits we need to represent flags.
-    MOMaxBits = 4
+    MOMaxBits = 5
   };
 
   /// MachineMemOperand - Construct an MachineMemOperand object with the
   /// specified PtrInfo, flags, size, and base alignment.
   MachineMemOperand(MachinePointerInfo PtrInfo, unsigned flags, uint64_t s,
-                    unsigned base_alignment, const MDNode *TBAAInfo = 0);
+                    unsigned base_alignment, const MDNode *TBAAInfo = 0,
+                    const MDNode *Ranges = 0);
 
   const MachinePointerInfo &getPointerInfo() const { return PtrInfo; }
   
@@ -137,10 +142,14 @@ public:
   /// getTBAAInfo - Return the TBAA tag for the memory reference.
   const MDNode *getTBAAInfo() const { return TBAAInfo; }
 
+  /// getRanges - Return the range tag for the memory reference.
+  const MDNode *getRanges() const { return Ranges; }
+
   bool isLoad() const { return Flags & MOLoad; }
   bool isStore() const { return Flags & MOStore; }
   bool isVolatile() const { return Flags & MOVolatile; }
   bool isNonTemporal() const { return Flags & MONonTemporal; }
+  bool isInvariant() const { return Flags & MOInvariant; }
 
   /// refineAlignment - Update this MachineMemOperand to reflect the alignment
   /// of MMO, if it has a greater alignment. This must only be used when the
diff --git a/include/llvm/CodeGen/MachineModuleInfo.h b/include/llvm/CodeGen/MachineModuleInfo.h
index 2bf7f17..6b88d4a 100644
--- a/include/llvm/CodeGen/MachineModuleInfo.h
+++ b/include/llvm/CodeGen/MachineModuleInfo.h
@@ -161,10 +161,10 @@ class MachineModuleInfo : public ImmutablePass {
   /// in this module.
   bool DbgInfoAvailable;
 
-  /// CallsExternalVAFunctionWithFloatingPointArguments - True if this module
-  /// calls VarArg function with floating point arguments.  This is used to emit
-  /// an undefined reference to fltused on Windows targets.
-  bool CallsExternalVAFunctionWithFloatingPointArguments;
+  /// UsesVAFloatArgument - True if this module calls VarArg function with
+  /// floating-point arguments.  This is used to emit an undefined reference
+  /// to _fltused on Windows targets.
+  bool UsesVAFloatArgument;
 
 public:
   static char ID; // Pass identification, replacement for typeid
@@ -223,12 +223,12 @@ public:
   bool callsUnwindInit() const { return CallsUnwindInit; }
   void setCallsUnwindInit(bool b) { CallsUnwindInit = b; }
 
-  bool callsExternalVAFunctionWithFloatingPointArguments() const {
-    return CallsExternalVAFunctionWithFloatingPointArguments;
+  bool usesVAFloatArgument() const {
+    return UsesVAFloatArgument;
   }
 
-  void setCallsExternalVAFunctionWithFloatingPointArguments(bool b) {
-    CallsExternalVAFunctionWithFloatingPointArguments = b;
+  void setUsesVAFloatArgument(bool b) {
+    UsesVAFloatArgument = b;
   }
 
   /// getFrameMoves - Returns a reference to a list of moves done in the current
diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h
index 5440a63..d244dd9 100644
--- a/include/llvm/CodeGen/MachineOperand.h
+++ b/include/llvm/CodeGen/MachineOperand.h
@@ -48,6 +48,7 @@ public:
     MO_ExternalSymbol,         ///< Name of external global symbol
     MO_GlobalAddress,          ///< Address of a global value
     MO_BlockAddress,           ///< Address of a basic block
+    MO_RegisterMask,           ///< Mask of preserved registers.
     MO_Metadata,               ///< Metadata reference (for debug info)
     MO_MCSymbol                ///< MCSymbol reference (for debug/eh info)
   };
@@ -102,6 +103,17 @@ private:
   ///
   bool IsUndef : 1;
 
+  /// IsInternalRead - True if this operand reads a value that was defined
+  /// inside the same instruction or bundle.  This flag can be set on both use
+  /// and def operands.  On a sub-register def operand, it refers to the part
+  /// of the register that isn't written.  On a full-register def operand, it
+  /// is a noop.
+  ///
+  /// When this flag is set, the instruction bundle must contain at least one
+  /// other def of the register.  If multiple instructions in the bundle define
+  /// the register, the meaning is target-defined.
+  bool IsInternalRead : 1;
+
   /// IsEarlyClobber - True if this MO_Register 'def' operand is written to
   /// by the MachineInstr before all input registers are read.  This is used to
   /// model the GCC inline asm '&' constraint modifier.
@@ -130,6 +142,7 @@ private:
     const ConstantFP *CFP;    // For MO_FPImmediate.
     const ConstantInt *CI;    // For MO_CImmediate. Integers > 64bit.
     int64_t ImmVal;           // For MO_Immediate.
+    const uint32_t *RegMask;  // For MO_RegisterMask.
     const MDNode *MD;         // For MO_Metadata.
     MCSymbol *Sym;            // For MO_MCSymbol
 
@@ -209,10 +222,13 @@ public:
   bool isSymbol() const { return OpKind == MO_ExternalSymbol; }
   /// isBlockAddress - Tests if this is a MO_BlockAddress operand.
   bool isBlockAddress() const { return OpKind == MO_BlockAddress; }
+  /// isRegMask - Tests if this is a MO_RegisterMask operand.
+  bool isRegMask() const { return OpKind == MO_RegisterMask; }
   /// isMetadata - Tests if this is a MO_Metadata operand.
   bool isMetadata() const { return OpKind == MO_Metadata; }
   bool isMCSymbol() const { return OpKind == MO_MCSymbol; }
 
+
   //===--------------------------------------------------------------------===//
   // Accessors for Register Operands
   //===--------------------------------------------------------------------===//
@@ -258,6 +274,11 @@ public:
     return IsUndef;
   }
 
+  bool isInternalRead() const {
+    assert(isReg() && "Wrong MachineOperand accessor");
+    return IsInternalRead;
+  }
+
   bool isEarlyClobber() const {
     assert(isReg() && "Wrong MachineOperand accessor");
     return IsEarlyClobber;
@@ -272,9 +293,12 @@ public:
   /// register.  A use operand with the <undef> flag set doesn't read its
   /// register.  A sub-register def implicitly reads the other parts of the
   /// register being redefined unless the <undef> flag is set.
+  ///
+  /// This refers to reading the register value from before the current
+  /// instruction or bundle. Internal bundle reads are not included.
   bool readsReg() const {
     assert(isReg() && "Wrong MachineOperand accessor");
-    return !isUndef() && (isUse() || getSubReg());
+    return !isUndef() && !isInternalRead() && (isUse() || getSubReg());
   }
 
   /// getNextOperandForReg - Return the next MachineOperand in the function that
@@ -343,6 +367,11 @@ public:
     IsUndef = Val;
   }
 
+  void setIsInternalRead(bool Val = true) {
+    assert(isReg() && "Wrong MachineOperand accessor");
+    IsInternalRead = Val;
+  }
+
   void setIsEarlyClobber(bool Val = true) {
     assert(isReg() && IsDef && "Wrong MachineOperand accessor");
     IsEarlyClobber = Val;
@@ -412,6 +441,28 @@ public:
     return Contents.OffsetedInfo.Val.SymbolName;
   }
 
+  /// clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
+  /// It is sometimes necessary to detach the register mask pointer from its
+  /// machine operand. This static method can be used for such detached bit
+  /// mask pointers.
+  static bool clobbersPhysReg(const uint32_t *RegMask, unsigned PhysReg) {
+    // See TargetRegisterInfo.h.
+    assert(PhysReg < (1u << 30) && "Not a physical register");
+    return !(RegMask[PhysReg / 32] & (1u << PhysReg % 32));
+  }
+
+  /// clobbersPhysReg - Returns true if this RegMask operand clobbers PhysReg.
+  bool clobbersPhysReg(unsigned PhysReg) const {
+     return clobbersPhysReg(getRegMask(), PhysReg);
+  }
+
+  /// getRegMask - Returns a bit mask of registers preserved by this RegMask
+  /// operand.
+  const uint32_t *getRegMask() const {
+    assert(isRegMask() && "Wrong MachineOperand accessor");
+    return Contents.RegMask;
+  }
+
   const MDNode *getMetadata() const {
     assert(isMetadata() && "Wrong MachineOperand accessor");
     return Contents.MD;
@@ -498,6 +549,7 @@ public:
     Op.IsKill = isKill;
     Op.IsDead = isDead;
     Op.IsUndef = isUndef;
+    Op.IsInternalRead = false;
     Op.IsEarlyClobber = isEarlyClobber;
     Op.IsDebug = isDebug;
     Op.SmallContents.RegNo = Reg;
@@ -557,6 +609,24 @@ public:
     Op.setTargetFlags(TargetFlags);
     return Op;
   }
+  /// CreateRegMask - Creates a register mask operand referencing Mask.  The
+  /// operand does not take ownership of the memory referenced by Mask, it must
+  /// remain valid for the lifetime of the operand.
+  ///
+  /// A RegMask operand represents a set of non-clobbered physical registers on
+  /// an instruction that clobbers many registers, typically a call.  The bit
+  /// mask has a bit set for each physreg that is preserved by this
+  /// instruction, as described in the documentation for
+  /// TargetRegisterInfo::getCallPreservedMask().
+  ///
+  /// Any physreg with a 0 bit in the mask is clobbered by the instruction.
+  ///
+  static MachineOperand CreateRegMask(const uint32_t *Mask) {
+    assert(Mask && "Missing register mask");
+    MachineOperand Op(MachineOperand::MO_RegisterMask);
+    Op.Contents.RegMask = Mask;
+    return Op;
+  }
   static MachineOperand CreateMetadata(const MDNode *Meta) {
     MachineOperand Op(MachineOperand::MO_Metadata);
     Op.Contents.MD = Meta;
diff --git a/include/llvm/CodeGen/MachinePassRegistry.h b/include/llvm/CodeGen/MachinePassRegistry.h
index 6ee2e90..c41e8e26 100644
--- a/include/llvm/CodeGen/MachinePassRegistry.h
+++ b/include/llvm/CodeGen/MachinePassRegistry.h
@@ -33,6 +33,7 @@ typedef void *(*MachinePassCtor)();
 ///
 //===----------------------------------------------------------------------===//
 class MachinePassRegistryListener {
+  virtual void anchor();
 public:
   MachinePassRegistryListener() {}
   virtual ~MachinePassRegistryListener() {}
diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h
index 3866b26..3272fbd 100644
--- a/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -15,12 +15,13 @@
 #define LLVM_CODEGEN_MACHINEREGISTERINFO_H
 
 #include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/CodeGen/MachineInstrBundle.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/IndexedMap.h"
 #include <vector>
 
 namespace llvm {
-  
+
 /// MachineRegisterInfo - Keep track of information for virtual and physical
 /// registers, including vreg register classes, use/def chains for registers,
 /// etc.
@@ -31,6 +32,11 @@ class MachineRegisterInfo {
   /// registers have a single def.
   bool IsSSA;
 
+  /// TracksLiveness - True while register liveness is being tracked accurately.
+  /// Basic block live-in lists, kill flags, and implicit defs may not be
+  /// accurate when after this flag is cleared.
+  bool TracksLiveness;
+
   /// VRegInfo - Information we keep for each virtual register.
   ///
   /// Each element in this list contains the register class of the vreg and the
@@ -46,18 +52,32 @@ class MachineRegisterInfo {
   /// the allocator should prefer the physical register allocated to the virtual
   /// register of the hint.
   IndexedMap<std::pair<unsigned, unsigned>, VirtReg2IndexFunctor> RegAllocHints;
-  
+
   /// PhysRegUseDefLists - This is an array of the head of the use/def list for
   /// physical registers.
-  MachineOperand **PhysRegUseDefLists; 
-  
+  MachineOperand **PhysRegUseDefLists;
+
   /// UsedPhysRegs - This is a bit vector that is computed and set by the
   /// register allocator, and must be kept up to date by passes that run after
   /// register allocation (though most don't modify this).  This is used
   /// so that the code generator knows which callee save registers to save and
   /// for other target specific uses.
+  /// This vector only has bits set for registers explicitly used, not their
+  /// aliases.
   BitVector UsedPhysRegs;
-  
+
+  /// UsedPhysRegMask - Additional used physregs, but including aliases.
+  BitVector UsedPhysRegMask;
+
+  /// ReservedRegs - This is a bit vector of reserved registers.  The target
+  /// may change its mind about which registers should be reserved.  This
+  /// vector is the frozen set of reserved registers when register allocation
+  /// started.
+  BitVector ReservedRegs;
+
+  /// AllocatableRegs - From TRI->getAllocatableSet.
+  mutable BitVector AllocatableRegs;
+
   /// LiveIns/LiveOuts - Keep track of the physical registers that are
   /// livein/liveout of the function.  Live in values are typically arguments in
   /// registers, live out values are typically return values in registers.
@@ -65,7 +85,7 @@ class MachineRegisterInfo {
   /// stored in the second element.
   std::vector<std::pair<unsigned, unsigned> > LiveIns;
   std::vector<unsigned> LiveOuts;
-  
+
   MachineRegisterInfo(const MachineRegisterInfo&); // DO NOT IMPLEMENT
   void operator=(const MachineRegisterInfo&);      // DO NOT IMPLEMENT
 public:
@@ -88,6 +108,23 @@ public:
   // leaveSSA - Indicates that the machine function is no longer in SSA form.
   void leaveSSA() { IsSSA = false; }
 
+  /// tracksLiveness - Returns true when tracking register liveness accurately.
+  ///
+  /// While this flag is true, register liveness information in basic block
+  /// live-in lists and machine instruction operands is accurate. This means it
+  /// can be used to change the code in ways that affect the values in
+  /// registers, for example by the register scavenger.
+  ///
+  /// When this flag is false, liveness is no longer reliable.
+  bool tracksLiveness() const { return TracksLiveness; }
+
+  /// invalidateLiveness - Indicates that register liveness is no longer being
+  /// tracked accurately.
+  ///
+  /// This should be called by late passes that invalidate the liveness
+  /// information.
+  void invalidateLiveness() { TracksLiveness = false; }
+
   //===--------------------------------------------------------------------===//
   // Register Info
   //===--------------------------------------------------------------------===//
@@ -141,7 +178,7 @@ public:
     return use_iterator(getRegUseDefListHead(RegNo));
   }
   static use_iterator use_end() { return use_iterator(0); }
-  
+
   /// use_empty - Return true if there are no instructions using the specified
   /// register.
   bool use_empty(unsigned RegNo) const { return use_begin(RegNo) == use_end(); }
@@ -157,7 +194,7 @@ public:
     return use_nodbg_iterator(getRegUseDefListHead(RegNo));
   }
   static use_nodbg_iterator use_nodbg_end() { return use_nodbg_iterator(0); }
-  
+
   /// use_nodbg_empty - Return true if there are no non-Debug instructions
   /// using the specified register.
   bool use_nodbg_empty(unsigned RegNo) const {
@@ -171,8 +208,16 @@ public:
   /// replaceRegWith - Replace all instances of FromReg with ToReg in the
   /// machine function.  This is like llvm-level X->replaceAllUsesWith(Y),
   /// except that it also changes any definitions of the register as well.
+  ///
+  /// Note that it is usually necessary to first constrain ToReg's register
+  /// class to match the FromReg constraints using:
+  ///
+  ///   constrainRegClass(ToReg, getRegClass(FromReg))
+  ///
+  /// That function will return NULL if the virtual registers have incompatible
+  /// constraints.
   void replaceRegWith(unsigned FromReg, unsigned ToReg);
-  
+
   /// getRegUseDefListHead - Return the head pointer for the register use/def
   /// list for the specified virtual or physical register.
   MachineOperand *&getRegUseDefListHead(unsigned RegNo) {
@@ -180,7 +225,7 @@ public:
       return VRegInfo[RegNo].second;
     return PhysRegUseDefLists[RegNo];
   }
-  
+
   MachineOperand *getRegUseDefListHead(unsigned RegNo) const {
     if (TargetRegisterInfo::isVirtualRegister(RegNo))
       return VRegInfo[RegNo].second;
@@ -197,15 +242,20 @@ public:
   /// optimization passes which extend register lifetimes and need only
   /// preserve conservative kill flag information.
   void clearKillFlags(unsigned Reg) const;
-  
+
 #ifndef NDEBUG
   void dumpUses(unsigned RegNo) const;
 #endif
-  
+
+  /// isConstantPhysReg - Returns true if PhysReg is unallocatable and constant
+  /// throughout the function.  It is safe to move instructions that read such
+  /// a physreg.
+  bool isConstantPhysReg(unsigned PhysReg, const MachineFunction &MF) const;
+
   //===--------------------------------------------------------------------===//
   // Virtual Register Info
   //===--------------------------------------------------------------------===//
-  
+
   /// getRegClass - Return the register class of the specified virtual register.
   ///
   const TargetRegisterClass *getRegClass(unsigned Reg) const {
@@ -246,6 +296,9 @@ public:
   ///
   unsigned getNumVirtRegs() const { return VRegInfo.size(); }
 
+  /// clearVirtRegs - Remove all virtual registers (after physreg assignment).
+  void clearVirtRegs();
+
   /// setRegAllocationHint - Specify a register allocation hint for the
   /// specified virtual register.
   void setRegAllocationHint(unsigned Reg, unsigned Type, unsigned PrefReg) {
@@ -271,38 +324,87 @@ public:
   //===--------------------------------------------------------------------===//
   // Physical Register Use Info
   //===--------------------------------------------------------------------===//
-  
+
   /// isPhysRegUsed - Return true if the specified register is used in this
   /// function.  This only works after register allocation.
-  bool isPhysRegUsed(unsigned Reg) const { return UsedPhysRegs[Reg]; }
-  
+  bool isPhysRegUsed(unsigned Reg) const {
+    return UsedPhysRegs.test(Reg) || UsedPhysRegMask.test(Reg);
+  }
+
+  /// isPhysRegOrOverlapUsed - Return true if Reg or any overlapping register
+  /// is used in this function.
+  bool isPhysRegOrOverlapUsed(unsigned Reg) const {
+    if (UsedPhysRegMask.test(Reg))
+      return true;
+    for (const uint16_t *AI = TRI->getOverlaps(Reg); *AI; ++AI)
+      if (UsedPhysRegs.test(*AI))
+        return true;
+    return false;
+  }
+
   /// setPhysRegUsed - Mark the specified register used in this function.
   /// This should only be called during and after register allocation.
-  void setPhysRegUsed(unsigned Reg) { UsedPhysRegs[Reg] = true; }
+  void setPhysRegUsed(unsigned Reg) { UsedPhysRegs.set(Reg); }
 
   /// addPhysRegsUsed - Mark the specified registers used in this function.
   /// This should only be called during and after register allocation.
   void addPhysRegsUsed(const BitVector &Regs) { UsedPhysRegs |= Regs; }
 
+  /// addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
+  /// This corresponds to the bit mask attached to register mask operands.
+  void addPhysRegsUsedFromRegMask(const uint32_t *RegMask) {
+    UsedPhysRegMask.setBitsNotInMask(RegMask);
+  }
+
   /// setPhysRegUnused - Mark the specified register unused in this function.
   /// This should only be called during and after register allocation.
-  void setPhysRegUnused(unsigned Reg) { UsedPhysRegs[Reg] = false; }
+  void setPhysRegUnused(unsigned Reg) {
+    UsedPhysRegs.reset(Reg);
+    UsedPhysRegMask.reset(Reg);
+  }
+
+
+  //===--------------------------------------------------------------------===//
+  // Reserved Register Info
+  //===--------------------------------------------------------------------===//
+  //
+  // The set of reserved registers must be invariant during register
+  // allocation.  For example, the target cannot suddenly decide it needs a
+  // frame pointer when the register allocator has already used the frame
+  // pointer register for something else.
+  //
+  // These methods can be used by target hooks like hasFP() to avoid changing
+  // the reserved register set during register allocation.
+
+  /// freezeReservedRegs - Called by the register allocator to freeze the set
+  /// of reserved registers before allocation begins.
+  void freezeReservedRegs(const MachineFunction&);
+
+  /// reservedRegsFrozen - Returns true after freezeReservedRegs() was called
+  /// to ensure the set of reserved registers stays constant.
+  bool reservedRegsFrozen() const {
+    return !ReservedRegs.empty();
+  }
+
+  /// canReserveReg - Returns true if PhysReg can be used as a reserved
+  /// register.  Any register can be reserved before freezeReservedRegs() is
+  /// called.
+  bool canReserveReg(unsigned PhysReg) const {
+    return !reservedRegsFrozen() || ReservedRegs.test(PhysReg);
+  }
 
-  /// closePhysRegsUsed - Expand UsedPhysRegs to its transitive closure over
-  /// subregisters. That means that if R is used, so are all subregisters.
-  void closePhysRegsUsed(const TargetRegisterInfo&);
 
   //===--------------------------------------------------------------------===//
   // LiveIn/LiveOut Management
   //===--------------------------------------------------------------------===//
-  
+
   /// addLiveIn/Out - Add the specified register as a live in/out.  Note that it
   /// is an error to add the same register to the same set more than once.
   void addLiveIn(unsigned Reg, unsigned vreg = 0) {
     LiveIns.push_back(std::make_pair(Reg, vreg));
   }
   void addLiveOut(unsigned Reg) { LiveOuts.push_back(Reg); }
-  
+
   // Iteration support for live in/out sets.  These sets are kept in sorted
   // order by their register number.
   typedef std::vector<std::pair<unsigned,unsigned> >::const_iterator
@@ -334,7 +436,7 @@ public:
 
 private:
   void HandleVRegListReallocation();
-  
+
 public:
   /// defusechain_iterator - This class provides iterator support for machine
   /// operands in the function that use or define a specific register.  If
@@ -362,31 +464,31 @@ public:
                           MachineInstr, ptrdiff_t>::reference reference;
     typedef std::iterator<std::forward_iterator_tag,
                           MachineInstr, ptrdiff_t>::pointer pointer;
-    
+
     defusechain_iterator(const defusechain_iterator &I) : Op(I.Op) {}
     defusechain_iterator() : Op(0) {}
-    
+
     bool operator==(const defusechain_iterator &x) const {
       return Op == x.Op;
     }
     bool operator!=(const defusechain_iterator &x) const {
       return !operator==(x);
     }
-    
+
     /// atEnd - return true if this iterator is equal to reg_end() on the value.
     bool atEnd() const { return Op == 0; }
-    
+
     // Iterator traversal: forward iteration only
     defusechain_iterator &operator++() {          // Preincrement
       assert(Op && "Cannot increment end iterator!");
       Op = Op->getNextOperandForReg();
-      
+
       // If this is an operand we don't care about, skip it.
-      while (Op && ((!ReturnUses && Op->isUse()) || 
+      while (Op && ((!ReturnUses && Op->isUse()) ||
                     (!ReturnDefs && Op->isDef()) ||
                     (SkipDebug && Op->isDebug())))
         Op = Op->getNextOperandForReg();
-      
+
       return *this;
     }
     defusechain_iterator operator++(int) {        // Postincrement
@@ -404,30 +506,38 @@ public:
       return MI;
     }
 
+    MachineInstr *skipBundle() {
+      if (!Op) return 0;
+      MachineInstr *MI = getBundleStart(Op->getParent());
+      do ++*this;
+      while (Op && getBundleStart(Op->getParent()) == MI);
+      return MI;
+    }
+
     MachineOperand &getOperand() const {
       assert(Op && "Cannot dereference end iterator!");
       return *Op;
     }
-    
+
     /// getOperandNo - Return the operand # of this MachineOperand in its
     /// MachineInstr.
     unsigned getOperandNo() const {
       assert(Op && "Cannot dereference end iterator!");
       return Op - &Op->getParent()->getOperand(0);
     }
-    
+
     // Retrieve a reference to the current operand.
     MachineInstr &operator*() const {
       assert(Op && "Cannot dereference end iterator!");
       return *Op->getParent();
     }
-    
+
     MachineInstr *operator->() const {
       assert(Op && "Cannot dereference end iterator!");
       return Op->getParent();
     }
   };
-  
+
 };
 
 } // End llvm namespace
diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h
new file mode 100644
index 0000000..e852009
--- /dev/null
+++ b/include/llvm/CodeGen/MachineScheduler.h
@@ -0,0 +1,91 @@
+//==- MachineScheduler.h - MachineInstr Scheduling Pass ----------*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides a MachineSchedRegistry for registering alternative machine
+// schedulers. A Target may provide an alternative scheduler implementation by
+// implementing the following boilerplate:
+//
+// static ScheduleDAGInstrs *createCustomMachineSched(MachineSchedContext *C) {
+//  return new CustomMachineScheduler(C);
+// }
+// static MachineSchedRegistry
+// SchedCustomRegistry("custom", "Run my target's custom scheduler",
+//                     createCustomMachineSched);
+//
+// Inside <Target>PassConfig:
+//   enablePass(MachineSchedulerID);
+//   MachineSchedRegistry::setDefault(createCustomMachineSched);
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MACHINESCHEDULER_H
+#define MACHINESCHEDULER_H
+
+#include "llvm/CodeGen/MachinePassRegistry.h"
+
+namespace llvm {
+
+class AliasAnalysis;
+class LiveIntervals;
+class MachineDominatorTree;
+class MachineLoopInfo;
+class ScheduleDAGInstrs;
+
+/// MachineSchedContext provides enough context from the MachineScheduler pass
+/// for the target to instantiate a scheduler.
+struct MachineSchedContext {
+  MachineFunction *MF;
+  const MachineLoopInfo *MLI;
+  const MachineDominatorTree *MDT;
+  const TargetPassConfig *PassConfig;
+  AliasAnalysis *AA;
+  LiveIntervals *LIS;
+
+  MachineSchedContext(): MF(0), MLI(0), MDT(0), PassConfig(0), AA(0), LIS(0) {}
+};
+
+/// MachineSchedRegistry provides a selection of available machine instruction
+/// schedulers.
+class MachineSchedRegistry : public MachinePassRegistryNode {
+public:
+  typedef ScheduleDAGInstrs *(*ScheduleDAGCtor)(MachineSchedContext *);
+
+  // RegisterPassParser requires a (misnamed) FunctionPassCtor type.
+  typedef ScheduleDAGCtor FunctionPassCtor;
+
+  static MachinePassRegistry Registry;
+
+  MachineSchedRegistry(const char *N, const char *D, ScheduleDAGCtor C)
+    : MachinePassRegistryNode(N, D, (MachinePassCtor)C) {
+    Registry.Add(this);
+  }
+  ~MachineSchedRegistry() { Registry.Remove(this); }
+
+  // Accessors.
+  //
+  MachineSchedRegistry *getNext() const {
+    return (MachineSchedRegistry *)MachinePassRegistryNode::getNext();
+  }
+  static MachineSchedRegistry *getList() {
+    return (MachineSchedRegistry *)Registry.getList();
+  }
+  static ScheduleDAGCtor getDefault() {
+    return (ScheduleDAGCtor)Registry.getDefault();
+  }
+  static void setDefault(ScheduleDAGCtor C) {
+    Registry.setDefault((MachinePassCtor)C);
+  }
+  static void setListener(MachinePassRegistryListener *L) {
+    Registry.setListener(L);
+  }
+};
+
+} // namespace llvm
+
+#endif
diff --git a/include/llvm/CodeGen/ObjectCodeEmitter.h b/include/llvm/CodeGen/ObjectCodeEmitter.h
deleted file mode 100644
index d46628c..0000000
--- a/include/llvm/CodeGen/ObjectCodeEmitter.h
+++ /dev/null
@@ -1,171 +0,0 @@
-//===-- llvm/CodeGen/ObjectCodeEmitter.h - Object Code Emitter -*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  Generalized Object Code Emitter, works with ObjectModule and BinaryObject.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_OBJECTCODEEMITTER_H
-#define LLVM_CODEGEN_OBJECTCODEEMITTER_H
-
-#include "llvm/CodeGen/MachineCodeEmitter.h"
-
-namespace llvm {
-
-class BinaryObject;
-class MachineBasicBlock;
-class MachineCodeEmitter;
-class MachineFunction;
-class MachineConstantPool;
-class MachineJumpTableInfo;
-class MachineModuleInfo;
-
-class ObjectCodeEmitter : public MachineCodeEmitter {
-protected:
-
-  /// Binary Object (Section or Segment) we are emitting to.
-  BinaryObject *BO;
-
-  /// MBBLocations - This vector is a mapping from MBB ID's to their address.
-  /// It is filled in by the StartMachineBasicBlock callback and queried by
-  /// the getMachineBasicBlockAddress callback.
-  std::vector<uintptr_t> MBBLocations;
-
-  /// LabelLocations - This vector is a mapping from Label ID's to their 
-  /// address.
-  std::vector<uintptr_t> LabelLocations;
-
-  /// CPLocations - This is a map of constant pool indices to offsets from the
-  /// start of the section for that constant pool index.
-  std::vector<uintptr_t> CPLocations;
-
-  /// CPSections - This is a map of constant pool indices to the Section
-  /// containing the constant pool entry for that index.
-  std::vector<uintptr_t> CPSections;
-
-  /// JTLocations - This is a map of jump table indices to offsets from the
-  /// start of the section for that jump table index.
-  std::vector<uintptr_t> JTLocations;
-
-public:
-  ObjectCodeEmitter();
-  ObjectCodeEmitter(BinaryObject *bo);
-  virtual ~ObjectCodeEmitter();
-
-  /// setBinaryObject - set the BinaryObject we are writting to
-  void setBinaryObject(BinaryObject *bo);
-
-  /// emitByte - This callback is invoked when a byte needs to be 
-  /// written to the data stream, without buffer overflow testing.
-  void emitByte(uint8_t B);
-
-  /// emitWordLE - This callback is invoked when a 32-bit word needs to be
-  /// written to the data stream in little-endian format.
-  void emitWordLE(uint32_t W);
-
-  /// emitWordBE - This callback is invoked when a 32-bit word needs to be
-  /// written to the data stream in big-endian format.
-  void emitWordBE(uint32_t W);
-
-  /// emitDWordLE - This callback is invoked when a 64-bit word needs to be
-  /// written to the data stream in little-endian format.
-  void emitDWordLE(uint64_t W);
-
-  /// emitDWordBE - This callback is invoked when a 64-bit word needs to be
-  /// written to the data stream in big-endian format.
-  void emitDWordBE(uint64_t W);
-
-  /// emitAlignment - Move the CurBufferPtr pointer up to the specified
-  /// alignment (saturated to BufferEnd of course).
-  void emitAlignment(unsigned Alignment = 0, uint8_t fill = 0);
-
-  /// emitULEB128Bytes - This callback is invoked when a ULEB128 needs to be
-  /// written to the data stream.
-  void emitULEB128Bytes(uint64_t Value);
-
-  /// emitSLEB128Bytes - This callback is invoked when a SLEB128 needs to be
-  /// written to the data stream.
-  void emitSLEB128Bytes(uint64_t Value);
-
-  /// emitString - This callback is invoked when a String needs to be
-  /// written to the data stream.
-  void emitString(const std::string &String);
-
-  /// getCurrentPCValue - This returns the address that the next emitted byte
-  /// will be output to.
-  uintptr_t getCurrentPCValue() const;
-
-  /// getCurrentPCOffset - Return the offset from the start of the emitted
-  /// buffer that we are currently writing to.
-  uintptr_t getCurrentPCOffset() const;
-
-  /// addRelocation - Whenever a relocatable address is needed, it should be
-  /// noted with this interface.
-  void addRelocation(const MachineRelocation& relocation);
-
-  /// earlyResolveAddresses - True if the code emitter can use symbol addresses 
-  /// during code emission time. The JIT is capable of doing this because it
-  /// creates jump tables or constant pools in memory on the fly while the
-  /// object code emitters rely on a linker to have real addresses and should
-  /// use relocations instead.
-  bool earlyResolveAddresses() const { return false; }
-
-  /// startFunction - This callback is invoked when the specified function is
-  /// about to be code generated.  This initializes the BufferBegin/End/Ptr
-  /// fields.
-  virtual void startFunction(MachineFunction &F) = 0;
-
-  /// finishFunction - This callback is invoked when the specified function has
-  /// finished code generation.  If a buffer overflow has occurred, this method
-  /// returns true (the callee is required to try again), otherwise it returns
-  /// false.
-  virtual bool finishFunction(MachineFunction &F) = 0;
-
-  /// StartMachineBasicBlock - This should be called by the target when a new
-  /// basic block is about to be emitted.  This way the MCE knows where the
-  /// start of the block is, and can implement getMachineBasicBlockAddress.
-  virtual void StartMachineBasicBlock(MachineBasicBlock *MBB);
-
-  /// getMachineBasicBlockAddress - Return the address of the specified
-  /// MachineBasicBlock, only usable after the label for the MBB has been
-  /// emitted.
-  virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const;
-
-  /// emitJumpTables - Emit all the jump tables for a given jump table info
-  /// record to the appropriate section.
-  virtual void emitJumpTables(MachineJumpTableInfo *MJTI) = 0;
-
-  /// getJumpTableEntryAddress - Return the address of the jump table with index
-  /// 'Index' in the function that last called initJumpTableInfo.
-  virtual uintptr_t getJumpTableEntryAddress(unsigned Index) const;
-
-  /// emitConstantPool - For each constant pool entry, figure out which section
-  /// the constant should live in, allocate space for it, and emit it to the 
-  /// Section data buffer.
-  virtual void emitConstantPool(MachineConstantPool *MCP) = 0;
-
-  /// getConstantPoolEntryAddress - Return the address of the 'Index' entry in
-  /// the constant pool that was last emitted with the emitConstantPool method.
-  virtual uintptr_t getConstantPoolEntryAddress(unsigned Index) const;
-
-  /// getConstantPoolEntrySection - Return the section of the 'Index' entry in
-  /// the constant pool that was last emitted with the emitConstantPool method.
-  virtual uintptr_t getConstantPoolEntrySection(unsigned Index) const;
-
-  /// Specifies the MachineModuleInfo object. This is used for exception handling
-  /// purposes.
-  virtual void setModuleInfo(MachineModuleInfo* Info) = 0;
-  // to be implemented or depreciated with MachineModuleInfo
-
-}; // end class ObjectCodeEmitter
-
-} // end namespace llvm
-
-#endif
-
diff --git a/include/llvm/CodeGen/PBQP/Graph.h b/include/llvm/CodeGen/PBQP/Graph.h
index 5240729..a5d8b0d 100644
--- a/include/llvm/CodeGen/PBQP/Graph.h
+++ b/include/llvm/CodeGen/PBQP/Graph.h
@@ -350,6 +350,43 @@ namespace PBQP {
       numNodes = numEdges = 0;
     }
 
+    /// \brief Dump a graph to an output stream.
+    template <typename OStream>
+    void dump(OStream &os) {
+      os << getNumNodes() << " " << getNumEdges() << "\n";
+
+      for (NodeItr nodeItr = nodesBegin(), nodeEnd = nodesEnd();
+           nodeItr != nodeEnd; ++nodeItr) {
+        const Vector& v = getNodeCosts(nodeItr);
+        os << "\n" << v.getLength() << "\n";
+        assert(v.getLength() != 0 && "Empty vector in graph.");
+        os << v[0];
+        for (unsigned i = 1; i < v.getLength(); ++i) {
+          os << " " << v[i];
+        }
+        os << "\n";
+      }
+
+      for (EdgeItr edgeItr = edgesBegin(), edgeEnd = edgesEnd();
+           edgeItr != edgeEnd; ++edgeItr) {
+        unsigned n1 = std::distance(nodesBegin(), getEdgeNode1(edgeItr));
+        unsigned n2 = std::distance(nodesBegin(), getEdgeNode2(edgeItr));
+        assert(n1 != n2 && "PBQP graphs shound not have self-edges.");
+        const Matrix& m = getEdgeCosts(edgeItr);
+        os << "\n" << n1 << " " << n2 << "\n"
+           << m.getRows() << " " << m.getCols() << "\n";
+        assert(m.getRows() != 0 && "No rows in matrix.");
+        assert(m.getCols() != 0 && "No cols in matrix.");
+        for (unsigned i = 0; i < m.getRows(); ++i) {
+          os << m[i][0];
+          for (unsigned j = 1; j < m.getCols(); ++j) {
+            os << " " << m[i][j];
+          }
+          os << "\n";
+        }
+      }
+    }
+
     /// \brief Print a representation of this graph in DOT format.
     /// @param os Output stream to print on.
     template <typename OStream>
diff --git a/include/llvm/CodeGen/PBQP/HeuristicBase.h b/include/llvm/CodeGen/PBQP/HeuristicBase.h
index 791c227..3fee18c 100644
--- a/include/llvm/CodeGen/PBQP/HeuristicBase.h
+++ b/include/llvm/CodeGen/PBQP/HeuristicBase.h
@@ -157,7 +157,7 @@ namespace PBQP {
         case 0: s.applyR0(nItr); break;
         case 1: s.applyR1(nItr); break;
         case 2: s.applyR2(nItr); break;
-        default: assert(false &&
+        default: llvm_unreachable(
                         "Optimal reductions of degree > 2 nodes is invalid.");
       }
 
@@ -186,7 +186,7 @@ namespace PBQP {
     /// \brief Add a node to the heuristic reduce list.
     /// @param nItr Node iterator to add to the heuristic reduce list.
     void addToHeuristicList(Graph::NodeItr nItr) {
-      assert(false && "Must be implemented in derived class.");
+      llvm_unreachable("Must be implemented in derived class.");
     }
 
     /// \brief Heuristically reduce one of the nodes in the heuristic
@@ -194,25 +194,25 @@ namespace PBQP {
     /// @return True if a reduction takes place, false if the heuristic reduce
     ///         list is empty.
     void heuristicReduce() {
-      assert(false && "Must be implemented in derived class.");
+      llvm_unreachable("Must be implemented in derived class.");
     }
 
     /// \brief Prepare a change in the costs on the given edge.
     /// @param eItr Edge iterator.    
     void preUpdateEdgeCosts(Graph::EdgeItr eItr) {
-      assert(false && "Must be implemented in derived class.");
+      llvm_unreachable("Must be implemented in derived class.");
     }
 
     /// \brief Handle the change in the costs on the given edge.
     /// @param eItr Edge iterator.
     void postUpdateEdgeCostts(Graph::EdgeItr eItr) {
-      assert(false && "Must be implemented in derived class.");
+      llvm_unreachable("Must be implemented in derived class.");
     }
 
     /// \brief Handle the addition of a new edge into the PBQP graph.
     /// @param eItr Edge iterator for the added edge.
     void handleAddEdge(Graph::EdgeItr eItr) {
-      assert(false && "Must be implemented in derived class.");
+      llvm_unreachable("Must be implemented in derived class.");
     }
 
     /// \brief Handle disconnection of an edge from a node.
@@ -223,7 +223,7 @@ namespace PBQP {
     /// method allows for the effect to be computed only for the remaining
     /// node in the graph.
     void handleRemoveEdge(Graph::EdgeItr eItr, Graph::NodeItr nItr) {
-      assert(false && "Must be implemented in derived class.");
+      llvm_unreachable("Must be implemented in derived class.");
     }
 
     /// \brief Clean up any structures used by HeuristicBase.
diff --git a/include/llvm/CodeGen/PBQP/Heuristics/Briggs.h b/include/llvm/CodeGen/PBQP/Heuristics/Briggs.h
index e96c4cb..a859e58 100644
--- a/include/llvm/CodeGen/PBQP/Heuristics/Briggs.h
+++ b/include/llvm/CodeGen/PBQP/Heuristics/Briggs.h
@@ -418,6 +418,12 @@ namespace PBQP {
         unsigned numRegs = getGraph().getNodeCosts(nItr).getLength() - 1;
 
         nd.numDenied = 0;
+        const Vector& nCosts = getGraph().getNodeCosts(nItr);
+        for (unsigned i = 1; i < nCosts.getLength(); ++i) {
+          if (nCosts[i] == std::numeric_limits<PBQPNum>::infinity())
+            ++nd.numDenied;
+        }
+
         nd.numSafe = numRegs;
         nd.unsafeDegrees.resize(numRegs, 0);
 
diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h
index 7a03ce9..3b38199 100644
--- a/include/llvm/CodeGen/Passes.h
+++ b/include/llvm/CodeGen/Passes.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_CODEGEN_PASSES_H
 #define LLVM_CODEGEN_PASSES_H
 
+#include "llvm/Pass.h"
 #include "llvm/Target/TargetMachine.h"
 #include <string>
 
@@ -26,7 +27,211 @@ namespace llvm {
   class TargetLowering;
   class TargetRegisterClass;
   class raw_ostream;
+}
 
+namespace llvm {
+
+extern char &NoPassID; // Allow targets to choose not to run a pass.
+
+class PassConfigImpl;
+
+/// Target-Independent Code Generator Pass Configuration Options.
+///
+/// This is an ImmutablePass solely for the purpose of exposing CodeGen options
+/// to the internals of other CodeGen passes.
+class TargetPassConfig : public ImmutablePass {
+public:
+  /// Pseudo Pass IDs. These are defined within TargetPassConfig because they
+  /// are unregistered pass IDs. They are only useful for use with
+  /// TargetPassConfig APIs to identify multiple occurrences of the same pass.
+  ///
+
+  /// EarlyTailDuplicate - A clone of the TailDuplicate pass that runs early
+  /// during codegen, on SSA form.
+  static char EarlyTailDuplicateID;
+
+  /// PostRAMachineLICM - A clone of the LICM pass that runs during late machine
+  /// optimization after regalloc.
+  static char PostRAMachineLICMID;
+
+protected:
+  TargetMachine *TM;
+  PassManagerBase &PM;
+  PassConfigImpl *Impl; // Internal data structures
+  bool Initialized;     // Flagged after all passes are configured.
+
+  // Target Pass Options
+  // Targets provide a default setting, user flags override.
+  //
+  bool DisableVerify;
+
+  /// Default setting for -enable-tail-merge on this target.
+  bool EnableTailMerge;
+
+public:
+  TargetPassConfig(TargetMachine *tm, PassManagerBase &pm);
+  // Dummy constructor.
+  TargetPassConfig();
+
+  virtual ~TargetPassConfig();
+
+  static char ID;
+
+  /// Get the right type of TargetMachine for this target.
+  template<typename TMC> TMC &getTM() const {
+    return *static_cast<TMC*>(TM);
+  }
+
+  const TargetLowering *getTargetLowering() const {
+    return TM->getTargetLowering();
+  }
+
+  //
+  void setInitialized() { Initialized = true; }
+
+  CodeGenOpt::Level getOptLevel() const { return TM->getOptLevel(); }
+
+  void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); }
+
+  bool getEnableTailMerge() const { return EnableTailMerge; }
+  void setEnableTailMerge(bool Enable) { setOpt(EnableTailMerge, Enable); }
+
+  /// Allow the target to override a specific pass without overriding the pass
+  /// pipeline. When passes are added to the standard pipeline at the
+  /// point where StadardID is expected, add TargetID in its place.
+  void substitutePass(char &StandardID, char &TargetID);
+
+  /// Allow the target to enable a specific standard pass by default.
+  void enablePass(char &ID) { substitutePass(ID, ID); }
+
+  /// Allow the target to disable a specific standard pass by default.
+  void disablePass(char &ID) { substitutePass(ID, NoPassID); }
+
+  /// Return the pass ssubtituted for StandardID by the target.
+  /// If no substitution exists, return StandardID.
+  AnalysisID getPassSubstitution(AnalysisID StandardID) const;
+
+  /// Return true if the optimized regalloc pipeline is enabled.
+  bool getOptimizeRegAlloc() const;
+
+  /// Add common target configurable passes that perform LLVM IR to IR
+  /// transforms following machine independent optimization.
+  virtual void addIRPasses();
+
+  /// Add common passes that perform LLVM IR to IR transforms in preparation for
+  /// instruction selection.
+  virtual void addISelPrepare();
+
+  /// addInstSelector - This method should install an instruction selector pass,
+  /// which converts from LLVM code to machine instructions.
+  virtual bool addInstSelector() {
+    return true;
+  }
+
+  /// Add the complete, standard set of LLVM CodeGen passes.
+  /// Fully developed targets will not generally override this.
+  virtual void addMachinePasses();
+
+protected:
+  // Helper to verify the analysis is really immutable.
+  void setOpt(bool &Opt, bool Val);
+
+  /// Methods with trivial inline returns are convenient points in the common
+  /// codegen pass pipeline where targets may insert passes. Methods with
+  /// out-of-line standard implementations are major CodeGen stages called by
+  /// addMachinePasses. Some targets may override major stages when inserting
+  /// passes is insufficient, but maintaining overriden stages is more work.
+  ///
+
+  /// addPreISelPasses - This method should add any "last minute" LLVM->LLVM
+  /// passes (which are run just before instruction selector).
+  virtual bool addPreISel() {
+    return true;
+  }
+
+  /// addMachineSSAOptimization - Add standard passes that optimize machine
+  /// instructions in SSA form.
+  virtual void addMachineSSAOptimization();
+
+  /// addPreRegAlloc - This method may be implemented by targets that want to
+  /// run passes immediately before register allocation. This should return
+  /// true if -print-machineinstrs should print after these passes.
+  virtual bool addPreRegAlloc() {
+    return false;
+  }
+
+  /// createTargetRegisterAllocator - Create the register allocator pass for
+  /// this target at the current optimization level.
+  virtual FunctionPass *createTargetRegisterAllocator(bool Optimized);
+
+  /// addFastRegAlloc - Add the minimum set of target-independent passes that
+  /// are required for fast register allocation.
+  virtual void addFastRegAlloc(FunctionPass *RegAllocPass);
+
+  /// addOptimizedRegAlloc - Add passes related to register allocation.
+  /// LLVMTargetMachine provides standard regalloc passes for most targets.
+  virtual void addOptimizedRegAlloc(FunctionPass *RegAllocPass);
+
+  /// addFinalizeRegAlloc - This method may be implemented by targets that want
+  /// to run passes within the regalloc pipeline, immediately after the register
+  /// allocation pass itself. These passes run as soon as virtual regisiters
+  /// have been rewritten to physical registers but before and other postRA
+  /// optimization happens. Targets that have marked instructions for bundling
+  /// must have finalized those bundles by the time these passes have run,
+  /// because subsequent passes are not guaranteed to be bundle-aware.
+  virtual bool addFinalizeRegAlloc() {
+    return false;
+  }
+
+  /// addPostRegAlloc - This method may be implemented by targets that want to
+  /// run passes after register allocation pass pipeline but before
+  /// prolog-epilog insertion.  This should return true if -print-machineinstrs
+  /// should print after these passes.
+  virtual bool addPostRegAlloc() {
+    return false;
+  }
+
+  /// Add passes that optimize machine instructions after register allocation.
+  virtual void addMachineLateOptimization();
+
+  /// addPreSched2 - This method may be implemented by targets that want to
+  /// run passes after prolog-epilog insertion and before the second instruction
+  /// scheduling pass.  This should return true if -print-machineinstrs should
+  /// print after these passes.
+  virtual bool addPreSched2() {
+    return false;
+  }
+
+  /// Add standard basic block placement passes.
+  virtual void addBlockPlacement();
+
+  /// addPreEmitPass - This pass may be implemented by targets that want to run
+  /// passes immediately before machine code is emitted.  This should return
+  /// true if -print-machineinstrs should print out the code after the passes.
+  virtual bool addPreEmitPass() {
+    return false;
+  }
+
+  /// Utilities for targets to add passes to the pass manager.
+  ///
+
+  /// Add a CodeGen pass at this point in the pipeline after checking overrides.
+  /// Return the pass that was added, or NoPassID.
+  AnalysisID addPass(char &ID);
+
+  /// addMachinePasses helper to create the target-selected or overriden
+  /// regalloc pass.
+  FunctionPass *createRegAllocPass(bool Optimized);
+
+  /// printAndVerify - Add a pass to dump then verify the machine function, if
+  /// those steps are enabled.
+  ///
+  void printAndVerify(const char *Banner) const;
+};
+} // namespace llvm
+
+/// List of target independent CodeGen pass IDs.
+namespace llvm {
   /// createUnreachableBlockEliminationPass - The LLVM code generator does not
   /// work well with unreachable basic blocks (what live ranges make sense for a
   /// block that cannot be reached?).  As such, a code generator should either
@@ -41,31 +246,29 @@ namespace llvm {
   createMachineFunctionPrinterPass(raw_ostream &OS,
                                    const std::string &Banner ="");
 
-  /// MachineLoopInfo pass - This pass is a loop analysis pass.
-  ///
+  /// MachineLoopInfo - This pass is a loop analysis pass.
   extern char &MachineLoopInfoID;
 
-  /// MachineLoopRanges pass - This pass is an on-demand loop coverage
-  /// analysis pass.
-  ///
+  /// MachineLoopRanges - This pass is an on-demand loop coverage analysis.
   extern char &MachineLoopRangesID;
 
-  /// MachineDominators pass - This pass is a machine dominators analysis pass.
-  ///
+  /// MachineDominators - This pass is a machine dominators analysis pass.
   extern char &MachineDominatorsID;
 
   /// EdgeBundles analysis - Bundle machine CFG edges.
-  ///
   extern char &EdgeBundlesID;
 
-  /// PHIElimination pass - This pass eliminates machine instruction PHI nodes
+  /// LiveVariables pass - This pass computes the set of blocks in which each
+  /// variable is life and sets machine operand kill flags.
+  extern char &LiveVariablesID;
+
+  /// PHIElimination - This pass eliminates machine instruction PHI nodes
   /// by inserting copy instructions.  This destroys SSA information, but is the
   /// desired input for some register allocators.  This pass is "required" by
   /// these register allocator like this: AU.addRequiredID(PHIEliminationID);
-  ///
   extern char &PHIEliminationID;
 
-  /// StrongPHIElimination pass - This pass eliminates machine instruction PHI
+  /// StrongPHIElimination - This pass eliminates machine instruction PHI
   /// nodes by inserting copy instructions.  This destroys SSA information, but
   /// is the desired input for some register allocators.  This pass is
   /// "required" by these register allocator like this:
@@ -76,32 +279,30 @@ namespace llvm {
   /// LiveStacks pass. An analysis keeping track of the liveness of stack slots.
   extern char &LiveStacksID;
 
-  /// TwoAddressInstruction pass - This pass reduces two-address instructions to
+  /// TwoAddressInstruction - This pass reduces two-address instructions to
   /// use two operands. This destroys SSA information but it is desired by
   /// register allocators.
   extern char &TwoAddressInstructionPassID;
 
-  /// RegisteCoalescer pass - This pass merges live ranges to eliminate copies.
-  extern char &RegisterCoalescerPassID;
+  /// ProcessImpicitDefs pass - This pass removes IMPLICIT_DEFs.
+  extern char &ProcessImplicitDefsID;
+
+  /// RegisterCoalescer - This pass merges live ranges to eliminate copies.
+  extern char &RegisterCoalescerID;
+
+  /// MachineScheduler - This pass schedules machine instructions.
+  extern char &MachineSchedulerID;
 
   /// SpillPlacement analysis. Suggest optimal placement of spill code between
   /// basic blocks.
-  ///
   extern char &SpillPlacementID;
 
-  /// UnreachableMachineBlockElimination pass - This pass removes unreachable
+  /// UnreachableMachineBlockElimination - This pass removes unreachable
   /// machine basic blocks.
   extern char &UnreachableMachineBlockElimID;
 
-  /// DeadMachineInstructionElim pass - This pass removes dead machine
-  /// instructions.
-  ///
-  FunctionPass *createDeadMachineInstructionElimPass();
-
-  /// Creates a register allocator as the user specified on the command line, or
-  /// picks one that matches OptLevel.
-  ///
-  FunctionPass *createRegisterAllocator(CodeGenOpt::Level OptLevel);
+  /// DeadMachineInstructionElim - This pass removes dead machine instructions.
+  extern char &DeadMachineInstructionElimID;
 
   /// FastRegisterAllocation Pass - This pass register allocates as fast as
   /// possible. It is best suited for debug code where live ranges are short.
@@ -118,56 +319,59 @@ namespace llvm {
   ///
   FunctionPass *createGreedyRegisterAllocator();
 
-  /// LinearScanRegisterAllocation Pass - This pass implements the linear scan
-  /// register allocation algorithm, a global register allocator.
-  ///
-  FunctionPass *createLinearScanRegisterAllocator();
-
   /// PBQPRegisterAllocation Pass - This pass implements the Partitioned Boolean
   /// Quadratic Prograaming (PBQP) based register allocator.
   ///
   FunctionPass *createDefaultPBQPRegisterAllocator();
 
-  /// PrologEpilogCodeInserter Pass - This pass inserts prolog and epilog code,
+  /// PrologEpilogCodeInserter - This pass inserts prolog and epilog code,
   /// and eliminates abstract frame references.
-  ///
-  FunctionPass *createPrologEpilogCodeInserter();
+  extern char &PrologEpilogCodeInserterID;
 
-  /// ExpandPostRAPseudos Pass - This pass expands pseudo instructions after
+  /// ExpandPostRAPseudos - This pass expands pseudo instructions after
   /// register allocation.
-  ///
-  FunctionPass *createExpandPostRAPseudosPass();
+  extern char &ExpandPostRAPseudosID;
 
   /// createPostRAScheduler - This pass performs post register allocation
   /// scheduling.
-  FunctionPass *createPostRAScheduler(CodeGenOpt::Level OptLevel);
+  extern char &PostRASchedulerID;
 
-  /// BranchFolding Pass - This pass performs machine code CFG based
+  /// BranchFolding - This pass performs machine code CFG based
   /// optimizations to delete branches to branches, eliminate branches to
   /// successor blocks (creating fall throughs), and eliminating branches over
   /// branches.
-  FunctionPass *createBranchFoldingPass(bool DefaultEnableTailMerge);
+  extern char &BranchFolderPassID;
 
-  /// TailDuplicate Pass - Duplicate blocks with unconditional branches
+  /// TailDuplicate - Duplicate blocks with unconditional branches
   /// into tails of their predecessors.
-  FunctionPass *createTailDuplicatePass(bool PreRegAlloc = false);
+  extern char &TailDuplicateID;
+
+  /// IfConverter - This pass performs machine code if conversion.
+  extern char &IfConverterID;
 
-  /// IfConverter Pass - This pass performs machine code if conversion.
-  FunctionPass *createIfConverterPass();
+  /// MachineBlockPlacement - This pass places basic blocks based on branch
+  /// probabilities.
+  extern char &MachineBlockPlacementID;
 
-  /// Code Placement Pass - This pass optimize code placement and aligns loop
+  /// MachineBlockPlacementStats - This pass collects statistics about the
+  /// basic block placement using branch probabilities and block frequency
+  /// information.
+  extern char &MachineBlockPlacementStatsID;
+
+  /// Code Placement - This pass optimize code placement and aligns loop
   /// headers to target specific alignment boundary.
-  FunctionPass *createCodePlacementOptPass();
+  extern char &CodePlacementOptID;
 
-  /// IntrinsicLowering Pass - Performs target-independent LLVM IR
-  /// transformations for highly portable strategies.
+  /// GCLowering Pass - Performs target-independent LLVM IR transformations for
+  /// highly portable strategies.
+  ///
   FunctionPass *createGCLoweringPass();
 
-  /// MachineCodeAnalysis Pass - Target-independent pass to mark safe points in
-  /// machine code. Must be added very late during code generation, just prior
-  /// to output, and importantly after all CFG transformations (such as branch
-  /// folding).
-  FunctionPass *createGCMachineCodeAnalysisPass();
+  /// GCMachineCodeAnalysis - Target-independent pass to mark safe points
+  /// in machine code. Must be added very late during code generation, just
+  /// prior to output, and importantly after all CFG transformations (such as
+  /// branch folding).
+  extern char &GCMachineCodeAnalysisID;
 
   /// Deleter Pass - Releases GC metadata.
   ///
@@ -177,54 +381,56 @@ namespace llvm {
   ///
   FunctionPass *createGCInfoPrinter(raw_ostream &OS);
 
-  /// createMachineCSEPass - This pass performs global CSE on machine
-  /// instructions.
-  FunctionPass *createMachineCSEPass();
+  /// MachineCSE - This pass performs global CSE on machine instructions.
+  extern char &MachineCSEID;
 
-  /// createMachineLICMPass - This pass performs LICM on machine instructions.
-  ///
-  FunctionPass *createMachineLICMPass(bool PreRegAlloc = true);
+  /// MachineLICM - This pass performs LICM on machine instructions.
+  extern char &MachineLICMID;
+
+  /// MachineSinking - This pass performs sinking on machine instructions.
+  extern char &MachineSinkingID;
 
-  /// createMachineSinkingPass - This pass performs sinking on machine
-  /// instructions.
-  FunctionPass *createMachineSinkingPass();
+  /// MachineCopyPropagation - This pass performs copy propagation on
+  /// machine instructions.
+  extern char &MachineCopyPropagationID;
 
-  /// createPeepholeOptimizerPass - This pass performs peephole optimizations -
+  /// PeepholeOptimizer - This pass performs peephole optimizations -
   /// like extension and comparison eliminations.
-  FunctionPass *createPeepholeOptimizerPass();
+  extern char &PeepholeOptimizerID;
 
-  /// createOptimizePHIsPass - This pass optimizes machine instruction PHIs
+  /// OptimizePHIs - This pass optimizes machine instruction PHIs
   /// to take advantage of opportunities created during DAG legalization.
-  FunctionPass *createOptimizePHIsPass();
+  extern char &OptimizePHIsID;
 
-  /// createStackSlotColoringPass - This pass performs stack slot coloring.
-  FunctionPass *createStackSlotColoringPass(bool);
+  /// StackSlotColoring - This pass performs stack slot coloring.
+  extern char &StackSlotColoringID;
 
   /// createStackProtectorPass - This pass adds stack protectors to functions.
+  ///
   FunctionPass *createStackProtectorPass(const TargetLowering *tli);
 
   /// createMachineVerifierPass - This pass verifies cenerated machine code
   /// instructions for correctness.
+  ///
   FunctionPass *createMachineVerifierPass(const char *Banner = 0);
 
   /// createDwarfEHPass - This pass mulches exception handling code into a form
   /// adapted to code generation.  Required if using dwarf exception handling.
   FunctionPass *createDwarfEHPass(const TargetMachine *tm);
 
-  /// createSjLjEHPass - This pass adapts exception handling code to use
+  /// createSjLjEHPreparePass - This pass adapts exception handling code to use
   /// the GCC-style builtin setjmp/longjmp (sjlj) to handling EH control flow.
-  FunctionPass *createSjLjEHPass(const TargetLowering *tli);
+  ///
+  FunctionPass *createSjLjEHPreparePass(const TargetLowering *tli);
 
-  /// createLocalStackSlotAllocationPass - This pass assigns local frame
-  /// indices to stack slots relative to one another and allocates
-  /// base registers to access them when it is estimated by the target to
-  /// be out of range of normal frame pointer or stack pointer index
-  /// addressing.
-  FunctionPass *createLocalStackSlotAllocationPass();
+  /// LocalStackSlotAllocation - This pass assigns local frame indices to stack
+  /// slots relative to one another and allocates base registers to access them
+  /// when it is estimated by the target to be out of range of normal frame
+  /// pointer or stack pointer index addressing.
+  extern char &LocalStackSlotAllocationID;
 
-  /// createExpandISelPseudosPass - This pass expands pseudo-instructions.
-  ///
-  FunctionPass *createExpandISelPseudosPass();
+  /// ExpandISelPseudos - This pass expands pseudo-instructions.
+  extern char &ExpandISelPseudosID;
 
   /// createExecutionDependencyFixPass - This pass fixes execution time
   /// problems with dependent instructions, such as switching execution
@@ -234,6 +440,13 @@ namespace llvm {
   ///
   FunctionPass *createExecutionDependencyFixPass(const TargetRegisterClass *RC);
 
+  /// UnpackMachineBundles - This pass unpack machine instruction bundles.
+  extern char &UnpackMachineBundlesID;
+
+  /// FinalizeMachineBundles - This pass finalize machine instruction
+  /// bundles (created earlier, e.g. during pre-RA scheduling).
+  extern char &FinalizeMachineBundlesID;
+
 } // End llvm namespace
 
 #endif
diff --git a/include/llvm/CodeGen/RegisterScavenging.h b/include/llvm/CodeGen/RegisterScavenging.h
index 26b6773..3986a8d 100644
--- a/include/llvm/CodeGen/RegisterScavenging.h
+++ b/include/llvm/CodeGen/RegisterScavenging.h
@@ -68,6 +68,10 @@ class RegScavenger {
   /// available, unset means the register is currently being used.
   BitVector RegsAvailable;
 
+  // These BitVectors are only used internally to forward(). They are members
+  // to avoid frequent reallocations.
+  BitVector KillRegs, DefRegs;
+
 public:
   RegScavenger()
     : MBB(NULL), NumPhysRegs(0), Tracking(false),
@@ -130,8 +134,9 @@ private:
 
   /// isUsed / isUnused - Test if a register is currently being used.
   ///
-  bool isUsed(unsigned Reg) const   { return !RegsAvailable.test(Reg); }
-  bool isUnused(unsigned Reg) const { return RegsAvailable.test(Reg); }
+  bool isUsed(unsigned Reg) const   {
+    return !RegsAvailable.test(Reg) || ReservedRegs.test(Reg);
+  }
 
   /// isAliasUsed - Is Reg or an alias currently in use?
   bool isAliasUsed(unsigned Reg) const;
@@ -139,7 +144,7 @@ private:
   /// setUsed / setUnused - Mark the state of one or a number of registers.
   ///
   void setUsed(BitVector &Regs) {
-    RegsAvailable &= ~Regs;
+    RegsAvailable.reset(Regs);
   }
   void setUnused(BitVector &Regs) {
     RegsAvailable |= Regs;
@@ -148,9 +153,6 @@ private:
   /// Add Reg and all its sub-registers to BV.
   void addRegWithSubRegs(BitVector &BV, unsigned Reg);
 
-  /// Add Reg and its aliases to BV.
-  void addRegWithAliases(BitVector &BV, unsigned Reg);
-
   /// findSurvivorReg - Return the candidate register that is unused for the
   /// longest after StartMI. UseMI is set to the instruction where the search
   /// stopped.
diff --git a/include/llvm/CodeGen/ResourcePriorityQueue.h b/include/llvm/CodeGen/ResourcePriorityQueue.h
new file mode 100644
index 0000000..56b5855
--- /dev/null
+++ b/include/llvm/CodeGen/ResourcePriorityQueue.h
@@ -0,0 +1,142 @@
+//===----- ResourcePriorityQueue.h - A DFA-oriented priority queue -------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the ResourcePriorityQueue class, which is a
+// SchedulingPriorityQueue that schedules using DFA state to
+// reduce the length of the critical path through the basic block
+// on VLIW platforms.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef RESOURCE_PRIORITY_QUEUE_H
+#define RESOURCE_PRIORITY_QUEUE_H
+
+#include "llvm/CodeGen/DFAPacketizer.h"
+#include "llvm/CodeGen/SelectionDAGISel.h"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/MC/MCInstrItineraries.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+
+namespace llvm {
+  class ResourcePriorityQueue;
+
+  /// Sorting functions for the Available queue.
+  struct resource_sort : public std::binary_function<SUnit*, SUnit*, bool> {
+    ResourcePriorityQueue *PQ;
+    explicit resource_sort(ResourcePriorityQueue *pq) : PQ(pq) {}
+
+    bool operator()(const SUnit* left, const SUnit* right) const;
+  };
+
+  class ResourcePriorityQueue : public SchedulingPriorityQueue {
+    /// SUnits - The SUnits for the current graph.
+    std::vector<SUnit> *SUnits;
+
+    /// NumNodesSolelyBlocking - This vector contains, for every node in the
+    /// Queue, the number of nodes that the node is the sole unscheduled
+    /// predecessor for.  This is used as a tie-breaker heuristic for better
+    /// mobility.
+    std::vector<unsigned> NumNodesSolelyBlocking;
+
+    /// Queue - The queue.
+    std::vector<SUnit*> Queue;
+
+    /// RegPressure - Tracking current reg pressure per register class.
+    ///
+    std::vector<unsigned> RegPressure;
+
+    /// RegLimit - Tracking the number of allocatable registers per register
+    /// class.
+    std::vector<unsigned> RegLimit;
+
+    resource_sort Picker;
+    const TargetRegisterInfo *TRI;
+    const TargetLowering *TLI;
+    const TargetInstrInfo *TII;
+    const InstrItineraryData* InstrItins;
+    /// ResourcesModel - Represents VLIW state.
+    /// Not limited to VLIW targets per say, but assumes
+    /// definition of DFA by a target.
+    DFAPacketizer *ResourcesModel;
+
+    /// Resource model - packet/bundle model. Purely
+    /// internal at the time.
+    std::vector<SUnit*> Packet;
+
+    /// Heuristics for estimating register pressure.
+    unsigned ParallelLiveRanges;
+    signed HorizontalVerticalBalance;
+
+  public:
+    ResourcePriorityQueue(SelectionDAGISel *IS);
+
+    ~ResourcePriorityQueue() {
+      delete ResourcesModel;
+    }
+
+    bool isBottomUp() const { return false; }
+
+    void initNodes(std::vector<SUnit> &sunits);
+
+    void addNode(const SUnit *SU) {
+      NumNodesSolelyBlocking.resize(SUnits->size(), 0);
+    }
+
+    void updateNode(const SUnit *SU) {}
+
+    void releaseState() {
+      SUnits = 0;
+    }
+
+    unsigned getLatency(unsigned NodeNum) const {
+      assert(NodeNum < (*SUnits).size());
+      return (*SUnits)[NodeNum].getHeight();
+    }
+
+    unsigned getNumSolelyBlockNodes(unsigned NodeNum) const {
+      assert(NodeNum < NumNodesSolelyBlocking.size());
+      return NumNodesSolelyBlocking[NodeNum];
+    }
+
+    /// Single cost function reflecting benefit of scheduling SU
+    /// in the current cycle.
+    signed SUSchedulingCost (SUnit *SU);
+
+    /// InitNumRegDefsLeft - Determine the # of regs defined by this node.
+    ///
+    void initNumRegDefsLeft(SUnit *SU);
+    void updateNumRegDefsLeft(SUnit *SU);
+    signed regPressureDelta(SUnit *SU, bool RawPressure = false);
+    signed rawRegPressureDelta (SUnit *SU, unsigned RCId);
+
+    bool empty() const { return Queue.empty(); }
+
+    virtual void push(SUnit *U);
+
+    virtual SUnit *pop();
+
+    virtual void remove(SUnit *SU);
+
+    virtual void dump(ScheduleDAG* DAG) const;
+
+    /// scheduledNode - Main resource tracking point.
+    void scheduledNode(SUnit *Node);
+    bool isResourceAvailable(SUnit *SU);
+    void reserveResources(SUnit *SU);
+
+private:
+    void adjustPriorityOfUnscheduledPreds(SUnit *SU);
+    SUnit *getSingleUnscheduledPred(SUnit *SU);
+    unsigned numberRCValPredInSU (SUnit *SU, unsigned RCId);
+    unsigned numberRCValSuccInSU (SUnit *SU, unsigned RCId);
+  };
+}
+
+#endif
diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h
index 1bbc6c5..f4de693 100644
--- a/include/llvm/CodeGen/ScheduleDAG.h
+++ b/include/llvm/CodeGen/ScheduleDAG.h
@@ -8,7 +8,8 @@
 //===----------------------------------------------------------------------===//
 //
 // This file implements the ScheduleDAG class, which is used as the common
-// base class for instruction schedulers.
+// base class for instruction schedulers. This encapsulates the scheduling DAG,
+// which is shared between SelectionDAG and MachineInstr scheduling.
 //
 //===----------------------------------------------------------------------===//
 
@@ -16,7 +17,7 @@
 #define LLVM_CODEGEN_SCHEDULEDAG_H
 
 #include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetLowering.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/GraphTraits.h"
@@ -129,8 +130,7 @@ namespace llvm {
                Contents.Order.isMustAlias == Other.Contents.Order.isMustAlias &&
                Contents.Order.isArtificial == Other.Contents.Order.isArtificial;
       }
-      assert(0 && "Invalid dependency kind!");
-      return false;
+      llvm_unreachable("Invalid dependency kind!");
     }
 
     bool operator!=(const SDep &Other) const {
@@ -232,6 +232,7 @@ namespace llvm {
   public:
     SUnit *OrigNode;                    // If not this, the node from which
                                         // this node was cloned.
+                                        // (SD scheduling only)
 
     // Preds/Succs - The SUnits before/after us in the graph.
     SmallVector<SDep, 4> Preds;  // All sunit predecessors.
@@ -409,6 +410,13 @@ namespace llvm {
       return false;
     }
 
+    bool isTopReady() const {
+      return NumPredsLeft == 0;
+    }
+    bool isBottomReady() const {
+      return NumSuccsLeft == 0;
+    }
+
     void dump(const ScheduleDAG *G) const;
     void dumpAll(const ScheduleDAG *G) const;
     void print(raw_ostream &O, const ScheduleDAG *G) const;
@@ -427,6 +435,7 @@ namespace llvm {
   /// implementation to decide.
   ///
   class SchedulingPriorityQueue {
+    virtual void anchor();
     unsigned CurCycle;
     bool HasReadyFilter;
   public:
@@ -465,13 +474,13 @@ namespace llvm {
 
     virtual void dump(ScheduleDAG *) const {}
 
-    /// ScheduledNode - As each node is scheduled, this method is invoked.  This
+    /// scheduledNode - As each node is scheduled, this method is invoked.  This
     /// allows the priority function to adjust the priority of related
     /// unscheduled nodes, for example.
     ///
-    virtual void ScheduledNode(SUnit *) {}
+    virtual void scheduledNode(SUnit *) {}
 
-    virtual void UnscheduledNode(SUnit *) {}
+    virtual void unscheduledNode(SUnit *) {}
 
     void setCurCycle(unsigned Cycle) {
       CurCycle = Cycle;
@@ -484,15 +493,11 @@ namespace llvm {
 
   class ScheduleDAG {
   public:
-    MachineBasicBlock *BB;          // The block in which to insert instructions
-    MachineBasicBlock::iterator InsertPos;// The position to insert instructions
     const TargetMachine &TM;              // Target processor
     const TargetInstrInfo *TII;           // Target instruction information
     const TargetRegisterInfo *TRI;        // Target processor register info
     MachineFunction &MF;                  // Machine function
     MachineRegisterInfo &MRI;             // Virtual/real register map
-    std::vector<SUnit*> Sequence;         // The schedule. Null SUnit*'s
-                                          // represent noop instructions.
     std::vector<SUnit> SUnits;            // The scheduling units.
     SUnit EntrySU;                        // Special node for the region entry.
     SUnit ExitSU;                         // Special node for the region exit.
@@ -507,6 +512,9 @@ namespace llvm {
 
     virtual ~ScheduleDAG();
 
+    /// clearDAG - clear the DAG state (between regions).
+    void clearDAG();
+
     /// getInstrDesc - Return the MCInstrDesc of this SUnit.
     /// Return NULL for SDNodes without a machine opcode.
     const MCInstrDesc *getInstrDesc(const SUnit *SU) const {
@@ -517,66 +525,43 @@ namespace llvm {
     /// viewGraph - Pop up a GraphViz/gv window with the ScheduleDAG rendered
     /// using 'dot'.
     ///
+    void viewGraph(const Twine &Name, const Twine &Title);
     void viewGraph();
 
-    /// EmitSchedule - Insert MachineInstrs into the MachineBasicBlock
-    /// according to the order specified in Sequence.
-    ///
-    virtual MachineBasicBlock *EmitSchedule() = 0;
-
-    void dumpSchedule() const;
-
     virtual void dumpNode(const SUnit *SU) const = 0;
 
     /// getGraphNodeLabel - Return a label for an SUnit node in a visualization
     /// of the ScheduleDAG.
     virtual std::string getGraphNodeLabel(const SUnit *SU) const = 0;
 
+    /// getDAGLabel - Return a label for the region of code covered by the DAG.
+    virtual std::string getDAGName() const = 0;
+
     /// addCustomGraphFeatures - Add custom features for a visualization of
     /// the ScheduleDAG.
     virtual void addCustomGraphFeatures(GraphWriter<ScheduleDAG*> &) const {}
 
 #ifndef NDEBUG
-    /// VerifySchedule - Verify that all SUnits were scheduled and that
-    /// their state is consistent.
-    void VerifySchedule(bool isBottomUp);
+    /// VerifyScheduledDAG - Verify that all SUnits were scheduled and that
+    /// their state is consistent. Return the number of scheduled SUnits.
+    unsigned VerifyScheduledDAG(bool isBottomUp);
 #endif
 
   protected:
-    /// Run - perform scheduling.
-    ///
-    void Run(MachineBasicBlock *bb, MachineBasicBlock::iterator insertPos);
-
-    /// BuildSchedGraph - Build SUnits and set up their Preds and Succs
-    /// to form the scheduling dependency graph.
-    ///
-    virtual void BuildSchedGraph(AliasAnalysis *AA) = 0;
-
     /// ComputeLatency - Compute node latency.
     ///
-    virtual void ComputeLatency(SUnit *SU) = 0;
+    virtual void computeLatency(SUnit *SU) = 0;
 
     /// ComputeOperandLatency - Override dependence edge latency using
     /// operand use/def information
     ///
-    virtual void ComputeOperandLatency(SUnit *, SUnit *,
+    virtual void computeOperandLatency(SUnit *, SUnit *,
                                        SDep&) const { }
 
-    /// Schedule - Order nodes according to selected style, filling
-    /// in the Sequence member.
-    ///
-    virtual void Schedule() = 0;
-
     /// ForceUnitLatencies - Return true if all scheduling edges should be given
     /// a latency value of one.  The default is to return false; schedulers may
     /// override this as needed.
-    virtual bool ForceUnitLatencies() const { return false; }
-
-    /// EmitNoop - Emit a noop instruction.
-    ///
-    void EmitNoop();
-
-    void EmitPhysRegCopy(SUnit *SU, DenseMap<SUnit*, unsigned> &VRBaseMap);
+    virtual bool forceUnitLatencies() const { return false; }
 
   private:
     // Return the MCInstrDesc of this SDNode or NULL.
diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h
new file mode 100644
index 0000000..c8de7bc
--- /dev/null
+++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h
@@ -0,0 +1,344 @@
+//==- ScheduleDAGInstrs.h - MachineInstr Scheduling --------------*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the ScheduleDAGInstrs class, which implements
+// scheduling for a MachineInstr-based dependency graph.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCHEDULEDAGINSTRS_H
+#define SCHEDULEDAGINSTRS_H
+
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SparseSet.h"
+#include <map>
+
+namespace llvm {
+  class MachineLoopInfo;
+  class MachineDominatorTree;
+  class LiveIntervals;
+
+  /// LoopDependencies - This class analyzes loop-oriented register
+  /// dependencies, which are used to guide scheduling decisions.
+  /// For example, loop induction variable increments should be
+  /// scheduled as soon as possible after the variable's last use.
+  ///
+  class LoopDependencies {
+    const MachineLoopInfo &MLI;
+    const MachineDominatorTree &MDT;
+
+  public:
+    typedef std::map<unsigned, std::pair<const MachineOperand *, unsigned> >
+      LoopDeps;
+    LoopDeps Deps;
+
+    LoopDependencies(const MachineLoopInfo &mli,
+                     const MachineDominatorTree &mdt) :
+      MLI(mli), MDT(mdt) {}
+
+    /// VisitLoop - Clear out any previous state and analyze the given loop.
+    ///
+    void VisitLoop(const MachineLoop *Loop) {
+      assert(Deps.empty() && "stale loop dependencies");
+
+      MachineBasicBlock *Header = Loop->getHeader();
+      SmallSet<unsigned, 8> LoopLiveIns;
+      for (MachineBasicBlock::livein_iterator LI = Header->livein_begin(),
+           LE = Header->livein_end(); LI != LE; ++LI)
+        LoopLiveIns.insert(*LI);
+
+      const MachineDomTreeNode *Node = MDT.getNode(Header);
+      const MachineBasicBlock *MBB = Node->getBlock();
+      assert(Loop->contains(MBB) &&
+             "Loop does not contain header!");
+      VisitRegion(Node, MBB, Loop, LoopLiveIns);
+    }
+
+  private:
+    void VisitRegion(const MachineDomTreeNode *Node,
+                     const MachineBasicBlock *MBB,
+                     const MachineLoop *Loop,
+                     const SmallSet<unsigned, 8> &LoopLiveIns) {
+      unsigned Count = 0;
+      for (MachineBasicBlock::const_iterator I = MBB->begin(), E = MBB->end();
+           I != E; ++I) {
+        const MachineInstr *MI = I;
+        if (MI->isDebugValue())
+          continue;
+        for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+          const MachineOperand &MO = MI->getOperand(i);
+          if (!MO.isReg() || !MO.isUse())
+            continue;
+          unsigned MOReg = MO.getReg();
+          if (LoopLiveIns.count(MOReg))
+            Deps.insert(std::make_pair(MOReg, std::make_pair(&MO, Count)));
+        }
+        ++Count; // Not every iteration due to dbg_value above.
+      }
+
+      const std::vector<MachineDomTreeNode*> &Children = Node->getChildren();
+      for (std::vector<MachineDomTreeNode*>::const_iterator I =
+           Children.begin(), E = Children.end(); I != E; ++I) {
+        const MachineDomTreeNode *ChildNode = *I;
+        MachineBasicBlock *ChildBlock = ChildNode->getBlock();
+        if (Loop->contains(ChildBlock))
+          VisitRegion(ChildNode, ChildBlock, Loop, LoopLiveIns);
+      }
+    }
+  };
+
+  /// An individual mapping from virtual register number to SUnit.
+  struct VReg2SUnit {
+    unsigned VirtReg;
+    SUnit *SU;
+
+    VReg2SUnit(unsigned reg, SUnit *su): VirtReg(reg), SU(su) {}
+
+    unsigned getSparseSetKey() const {
+      return TargetRegisterInfo::virtReg2Index(VirtReg);
+    }
+  };
+
+  /// Combine a SparseSet with a 1x1 vector to track physical registers.
+  /// The SparseSet allows iterating over the (few) live registers for quickly
+  /// comparing against a regmask or clearing the set.
+  ///
+  /// Storage for the map is allocated once for the pass. The map can be
+  /// cleared between scheduling regions without freeing unused entries.
+  class Reg2SUnitsMap {
+    SparseSet<unsigned> PhysRegSet;
+    std::vector<std::vector<SUnit*> > SUnits;
+  public:
+    typedef SparseSet<unsigned>::const_iterator const_iterator;
+
+    // Allow iteration over register numbers (keys) in the map. If needed, we
+    // can provide an iterator over SUnits (values) as well.
+    const_iterator reg_begin() const { return PhysRegSet.begin(); }
+    const_iterator reg_end() const { return PhysRegSet.end(); }
+
+    /// Initialize the map with the number of registers.
+    /// If the map is already large enough, no allocation occurs.
+    /// For simplicity we expect the map to be empty().
+    void setRegLimit(unsigned Limit);
+
+    /// Returns true if the map is empty.
+    bool empty() const { return PhysRegSet.empty(); }
+
+    /// Clear the map without deallocating storage.
+    void clear();
+
+    bool contains(unsigned Reg) const { return PhysRegSet.count(Reg); }
+
+    /// If this register is mapped, return its existing SUnits vector.
+    /// Otherwise map the register and return an empty SUnits vector.
+    std::vector<SUnit *> &operator[](unsigned Reg) {
+      bool New = PhysRegSet.insert(Reg).second;
+      assert((!New || SUnits[Reg].empty()) && "stale SUnits vector");
+      (void)New;
+      return SUnits[Reg];
+    }
+
+    /// Erase an existing element without freeing memory.
+    void erase(unsigned Reg) {
+      PhysRegSet.erase(Reg);
+      SUnits[Reg].clear();
+    }
+  };
+
+  /// Use SparseSet as a SparseMap by relying on the fact that it never
+  /// compares ValueT's, only unsigned keys. This allows the set to be cleared
+  /// between scheduling regions in constant time as long as ValueT does not
+  /// require a destructor.
+  typedef SparseSet<VReg2SUnit> VReg2SUnitMap;
+
+  /// ScheduleDAGInstrs - A ScheduleDAG subclass for scheduling lists of
+  /// MachineInstrs.
+  class ScheduleDAGInstrs : public ScheduleDAG {
+  protected:
+    const MachineLoopInfo &MLI;
+    const MachineDominatorTree &MDT;
+    const MachineFrameInfo *MFI;
+    const InstrItineraryData *InstrItins;
+
+    /// Live Intervals provides reaching defs in preRA scheduling.
+    LiveIntervals *LIS;
+
+    /// isPostRA flag indicates vregs cannot be present.
+    bool IsPostRA;
+
+    /// UnitLatencies (misnamed) flag avoids computing def-use latencies, using
+    /// the def-side latency only.
+    bool UnitLatencies;
+
+    /// State specific to the current scheduling region.
+    /// ------------------------------------------------
+
+    /// The block in which to insert instructions
+    MachineBasicBlock *BB;
+
+    /// The beginning of the range to be scheduled.
+    MachineBasicBlock::iterator RegionBegin;
+
+    /// The end of the range to be scheduled.
+    MachineBasicBlock::iterator RegionEnd;
+
+    /// The index in BB of RegionEnd.
+    unsigned EndIndex;
+
+    /// After calling BuildSchedGraph, each machine instruction in the current
+    /// scheduling region is mapped to an SUnit.
+    DenseMap<MachineInstr*, SUnit*> MISUnitMap;
+
+    /// State internal to DAG building.
+    /// -------------------------------
+
+    /// Defs, Uses - Remember where defs and uses of each register are as we
+    /// iterate upward through the instructions. This is allocated here instead
+    /// of inside BuildSchedGraph to avoid the need for it to be initialized and
+    /// destructed for each block.
+    Reg2SUnitsMap Defs;
+    Reg2SUnitsMap Uses;
+
+    /// Track the last instructon in this region defining each virtual register.
+    VReg2SUnitMap VRegDefs;
+
+    /// PendingLoads - Remember where unknown loads are after the most recent
+    /// unknown store, as we iterate. As with Defs and Uses, this is here
+    /// to minimize construction/destruction.
+    std::vector<SUnit *> PendingLoads;
+
+    /// LoopRegs - Track which registers are used for loop-carried dependencies.
+    ///
+    LoopDependencies LoopRegs;
+
+    /// DbgValues - Remember instruction that preceeds DBG_VALUE.
+    /// These are generated by buildSchedGraph but persist so they can be
+    /// referenced when emitting the final schedule.
+    typedef std::vector<std::pair<MachineInstr *, MachineInstr *> >
+      DbgValueVector;
+    DbgValueVector DbgValues;
+    MachineInstr *FirstDbgValue;
+
+  public:
+    explicit ScheduleDAGInstrs(MachineFunction &mf,
+                               const MachineLoopInfo &mli,
+                               const MachineDominatorTree &mdt,
+                               bool IsPostRAFlag,
+                               LiveIntervals *LIS = 0);
+
+    virtual ~ScheduleDAGInstrs() {}
+
+    /// begin - Return an iterator to the top of the current scheduling region.
+    MachineBasicBlock::iterator begin() const { return RegionBegin; }
+
+    /// end - Return an iterator to the bottom of the current scheduling region.
+    MachineBasicBlock::iterator end() const { return RegionEnd; }
+
+    /// newSUnit - Creates a new SUnit and return a ptr to it.
+    SUnit *newSUnit(MachineInstr *MI);
+
+    /// getSUnit - Return an existing SUnit for this MI, or NULL.
+    SUnit *getSUnit(MachineInstr *MI) const;
+
+    /// startBlock - Prepare to perform scheduling in the given block.
+    virtual void startBlock(MachineBasicBlock *BB);
+
+    /// finishBlock - Clean up after scheduling in the given block.
+    virtual void finishBlock();
+
+    /// Initialize the scheduler state for the next scheduling region.
+    virtual void enterRegion(MachineBasicBlock *bb,
+                             MachineBasicBlock::iterator begin,
+                             MachineBasicBlock::iterator end,
+                             unsigned endcount);
+
+    /// Notify that the scheduler has finished scheduling the current region.
+    virtual void exitRegion();
+
+    /// buildSchedGraph - Build SUnits from the MachineBasicBlock that we are
+    /// input.
+    void buildSchedGraph(AliasAnalysis *AA);
+
+    /// addSchedBarrierDeps - Add dependencies from instructions in the current
+    /// list of instructions being scheduled to scheduling barrier. We want to
+    /// make sure instructions which define registers that are either used by
+    /// the terminator or are live-out are properly scheduled. This is
+    /// especially important when the definition latency of the return value(s)
+    /// are too high to be hidden by the branch or when the liveout registers
+    /// used by instructions in the fallthrough block.
+    void addSchedBarrierDeps();
+
+    /// computeLatency - Compute node latency.
+    ///
+    virtual void computeLatency(SUnit *SU);
+
+    /// computeOperandLatency - Override dependence edge latency using
+    /// operand use/def information
+    ///
+    virtual void computeOperandLatency(SUnit *Def, SUnit *Use,
+                                       SDep& dep) const;
+
+    /// schedule - Order nodes according to selected style, filling
+    /// in the Sequence member.
+    ///
+    /// Typically, a scheduling algorithm will implement schedule() without
+    /// overriding enterRegion() or exitRegion().
+    virtual void schedule() = 0;
+
+    /// finalizeSchedule - Allow targets to perform final scheduling actions at
+    /// the level of the whole MachineFunction. By default does nothing.
+    virtual void finalizeSchedule() {}
+
+    virtual void dumpNode(const SUnit *SU) const;
+
+    /// Return a label for a DAG node that points to an instruction.
+    virtual std::string getGraphNodeLabel(const SUnit *SU) const;
+
+    /// Return a label for the region of code covered by the DAG.
+    virtual std::string getDAGName() const;
+
+  protected:
+    void initSUnits();
+    void addPhysRegDataDeps(SUnit *SU, const MachineOperand &MO);
+    void addPhysRegDeps(SUnit *SU, unsigned OperIdx);
+    void addVRegDefDeps(SUnit *SU, unsigned OperIdx);
+    void addVRegUseDeps(SUnit *SU, unsigned OperIdx);
+
+    VReg2SUnitMap::iterator findVRegDef(unsigned VirtReg) {
+      return VRegDefs.find(TargetRegisterInfo::virtReg2Index(VirtReg));
+    }
+  };
+
+  /// newSUnit - Creates a new SUnit and return a ptr to it.
+  inline SUnit *ScheduleDAGInstrs::newSUnit(MachineInstr *MI) {
+#ifndef NDEBUG
+    const SUnit *Addr = SUnits.empty() ? 0 : &SUnits[0];
+#endif
+    SUnits.push_back(SUnit(MI, (unsigned)SUnits.size()));
+    assert((Addr == 0 || Addr == &SUnits[0]) &&
+           "SUnits std::vector reallocated on the fly!");
+    SUnits.back().OrigNode = &SUnits.back();
+    return &SUnits.back();
+  }
+
+  /// getSUnit - Return an existing SUnit for this MI, or NULL.
+  inline SUnit *ScheduleDAGInstrs::getSUnit(MachineInstr *MI) const {
+    DenseMap<MachineInstr*, SUnit*>::const_iterator I = MISUnitMap.find(MI);
+    if (I == MISUnitMap.end())
+      return 0;
+    return I->second;
+  }
+} // namespace llvm
+
+#endif
diff --git a/include/llvm/CodeGen/SchedulerRegistry.h b/include/llvm/CodeGen/SchedulerRegistry.h
index 96573dd..a582b0c 100644
--- a/include/llvm/CodeGen/SchedulerRegistry.h
+++ b/include/llvm/CodeGen/SchedulerRegistry.h
@@ -42,7 +42,7 @@ public:
   : MachinePassRegistryNode(N, D, (MachinePassCtor)C)
   { Registry.Add(this); }
   ~RegisterScheduler() { Registry.Remove(this); }
-  
+
 
   // Accessors.
   //
@@ -68,11 +68,6 @@ public:
 ScheduleDAGSDNodes *createBURRListDAGScheduler(SelectionDAGISel *IS,
                                                CodeGenOpt::Level OptLevel);
 
-/// createTDRRListDAGScheduler - This creates a top down register usage
-/// reduction list scheduler.
-ScheduleDAGSDNodes *createTDRRListDAGScheduler(SelectionDAGISel *IS,
-                                               CodeGenOpt::Level OptLevel);
-
 /// createBURRListDAGScheduler - This creates a bottom up list scheduler that
 /// schedules nodes in source code order when possible.
 ScheduleDAGSDNodes *createSourceListDAGScheduler(SelectionDAGISel *IS,
@@ -91,16 +86,17 @@ ScheduleDAGSDNodes *createHybridListDAGScheduler(SelectionDAGISel *IS,
 /// to reduce register pressure.
 ScheduleDAGSDNodes *createILPListDAGScheduler(SelectionDAGISel *IS,
                                               CodeGenOpt::Level);
-/// createTDListDAGScheduler - This creates a top-down list scheduler with
-/// a hazard recognizer.
-ScheduleDAGSDNodes *createTDListDAGScheduler(SelectionDAGISel *IS,
-                                             CodeGenOpt::Level OptLevel);
 
 /// createFastDAGScheduler - This creates a "fast" scheduler.
 ///
 ScheduleDAGSDNodes *createFastDAGScheduler(SelectionDAGISel *IS,
                                            CodeGenOpt::Level OptLevel);
 
+/// createVLIWDAGScheduler - Scheduler for VLIW targets. This creates top down
+/// DFA driven list scheduler with clustering heuristic to control
+/// register pressure.
+ScheduleDAGSDNodes *createVLIWDAGScheduler(SelectionDAGISel *IS,
+                                           CodeGenOpt::Level OptLevel);
 /// createDefaultScheduler - This creates an instruction scheduler appropriate
 /// for the target.
 ScheduleDAGSDNodes *createDefaultScheduler(SelectionDAGISel *IS,
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index 132983c..6a7a87e 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -51,7 +51,7 @@ public:
   static void noteHead(SDNode*, SDNode*) {}
 
   static void deleteNode(SDNode *) {
-    assert(0 && "ilist_traits<SDNode> shouldn't see a deleteNode call!");
+    llvm_unreachable("ilist_traits<SDNode> shouldn't see a deleteNode call!");
   }
 private:
   static void createNode(const SDNode &);
@@ -112,9 +112,10 @@ public:
 };
 
 enum CombineLevel {
-  Unrestricted,   // Combine may create illegal operations and illegal types.
-  NoIllegalTypes, // Combine may create illegal operations but no illegal types.
-  NoIllegalOperations // Combine may only create legal operations and types.
+  BeforeLegalizeTypes,
+  AfterLegalizeTypes,
+  AfterLegalizeVectorOps,
+  AfterLegalizeDAG
 };
 
 class SelectionDAG;
@@ -138,6 +139,7 @@ class SelectionDAG {
   const TargetSelectionDAGInfo &TSI;
   MachineFunction *MF;
   LLVMContext *Context;
+  CodeGenOpt::Level OptLevel;
 
   /// EntryNode - The starting token.
   SDNode EntryNode;
@@ -186,7 +188,7 @@ class SelectionDAG {
   SelectionDAG(const SelectionDAG&);   // Do not implement.
 
 public:
-  explicit SelectionDAG(const TargetMachine &TM);
+  explicit SelectionDAG(const TargetMachine &TM, llvm::CodeGenOpt::Level);
   ~SelectionDAG();
 
   /// init - Prepare this SelectionDAG to process code in the given
@@ -392,6 +394,7 @@ public:
                                   unsigned char TargetFlags = 0);
   SDValue getValueType(EVT);
   SDValue getRegister(unsigned Reg, EVT VT);
+  SDValue getRegisterMask(const uint32_t *RegMask);
   SDValue getEHLabel(DebugLoc dl, SDValue Root, MCSymbol *Label);
   SDValue getBlockAddress(const BlockAddress *BA, EVT VT,
                           bool isTarget = false, unsigned char TargetFlags = 0);
@@ -650,8 +653,8 @@ public:
   ///
   SDValue getLoad(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr,
                   MachinePointerInfo PtrInfo, bool isVolatile,
-                  bool isNonTemporal, unsigned Alignment,
-                  const MDNode *TBAAInfo = 0);
+                  bool isNonTemporal, bool isInvariant, unsigned Alignment,
+                  const MDNode *TBAAInfo = 0, const MDNode *Ranges = 0);
   SDValue getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT,
                      SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo,
                      EVT MemVT, bool isVolatile,
@@ -663,8 +666,9 @@ public:
                   EVT VT, DebugLoc dl,
                   SDValue Chain, SDValue Ptr, SDValue Offset,
                   MachinePointerInfo PtrInfo, EVT MemVT,
-                  bool isVolatile, bool isNonTemporal, unsigned Alignment,
-                  const MDNode *TBAAInfo = 0);
+                  bool isVolatile, bool isNonTemporal, bool isInvariant,
+                  unsigned Alignment, const MDNode *TBAAInfo = 0,
+                  const MDNode *Ranges = 0);
   SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
                   EVT VT, DebugLoc dl,
                   SDValue Chain, SDValue Ptr, SDValue Offset,
@@ -976,8 +980,8 @@ public:
   /// bitsets.  This code only analyzes bits in Mask, in order to short-circuit
   /// processing.  Targets can implement the computeMaskedBitsForTargetNode
   /// method in the TargetLowering class to allow target nodes to be understood.
-  void ComputeMaskedBits(SDValue Op, const APInt &Mask, APInt &KnownZero,
-                         APInt &KnownOne, unsigned Depth = 0) const;
+  void ComputeMaskedBits(SDValue Op, APInt &KnownZero, APInt &KnownOne,
+                         unsigned Depth = 0) const;
 
   /// ComputeNumSignBits - Return the number of times the sign bit of the
   /// register is replicated into the other bits.  We know that at least 1 bit
@@ -1033,6 +1037,7 @@ private:
                                void *&InsertPos);
   SDNode *FindModifiedNodeSlot(SDNode *N, const SDValue *Ops, unsigned NumOps,
                                void *&InsertPos);
+  SDNode *UpdadeDebugLocOnMergedSDNode(SDNode *N, DebugLoc loc);
 
   void DeleteNodeNotInCSEMaps(SDNode *N);
   void DeallocateNode(SDNode *N);
diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h
index ecf3947..ee3f231 100644
--- a/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/include/llvm/CodeGen/SelectionDAGISel.h
@@ -29,6 +29,7 @@ namespace llvm {
   class MachineFunction;
   class MachineInstr;
   class TargetLowering;
+  class TargetLibraryInfo;
   class TargetInstrInfo;
   class FunctionLoweringInfo;
   class ScheduleHazardRecognizer;
@@ -42,6 +43,7 @@ class SelectionDAGISel : public MachineFunctionPass {
 public:
   const TargetMachine &TM;
   const TargetLowering &TLI;
+  const TargetLibraryInfo *LibInfo;
   FunctionLoweringInfo *FuncInfo;
   MachineFunction *MF;
   MachineRegisterInfo *RegInfo;
@@ -92,7 +94,7 @@ public:
 
   /// IsLegalToFold - Returns true if the specific operand node N of
   /// U can be folded during instruction selection that starts at Root.
-  /// FIXME: This is a static member function because the MSP430/SystemZ/X86
+  /// FIXME: This is a static member function because the MSP430/X86
   /// targets, which uses it during isel.  This could become a proper member.
   static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root,
                             CodeGenOpt::Level OptLevel,
@@ -179,6 +181,7 @@ protected:
   /// ISelUpdater - helper class to handle updates of the
   /// instruction selection graph.
   class ISelUpdater : public SelectionDAG::DAGUpdateListener {
+    virtual void anchor();
     SelectionDAG::allnodes_iterator &ISelPosition;
   public:
     explicit ISelUpdater(SelectionDAG::allnodes_iterator &isp)
@@ -237,8 +240,7 @@ public:
   /// succeeds or false if it fails.  The number is a private implementation
   /// detail to the code tblgen produces.
   virtual bool CheckPatternPredicate(unsigned PredNo) const {
-    assert(0 && "Tblgen should generate the implementation of this!");
-    return 0;
+    llvm_unreachable("Tblgen should generate the implementation of this!");
   }
 
   /// CheckNodePredicate - This function is generated by tblgen in the target.
@@ -246,20 +248,17 @@ public:
   /// false if it fails.  The number is a private implementation
   /// detail to the code tblgen produces.
   virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const {
-    assert(0 && "Tblgen should generate the implementation of this!");
-    return 0;
+    llvm_unreachable("Tblgen should generate the implementation of this!");
   }
 
   virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N,
                                    unsigned PatternNo,
                         SmallVectorImpl<std::pair<SDValue, SDNode*> > &Result) {
-    assert(0 && "Tblgen should generate the implementation of this!");
-    return false;
+    llvm_unreachable("Tblgen should generate the implementation of this!");
   }
 
   virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo) {
-    assert(0 && "Tblgen should generate this!");
-    return SDValue();
+    llvm_unreachable("Tblgen should generate this!");
   }
 
   SDNode *SelectCodeCommon(SDNode *NodeToMatch,
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index 6c7be69..f8248b8 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -917,12 +917,13 @@ public:
   // with MachineMemOperand information.
   bool isVolatile() const { return (SubclassData >> 5) & 1; }
   bool isNonTemporal() const { return (SubclassData >> 6) & 1; }
+  bool isInvariant() const { return (SubclassData >> 7) & 1; }
 
   AtomicOrdering getOrdering() const {
-    return AtomicOrdering((SubclassData >> 7) & 15);
+    return AtomicOrdering((SubclassData >> 8) & 15);
   }
   SynchronizationScope getSynchScope() const {
-    return SynchronizationScope((SubclassData >> 11) & 1);
+    return SynchronizationScope((SubclassData >> 12) & 1);
   }
 
   /// Returns the SrcValue and offset that describes the location of the access
@@ -932,6 +933,9 @@ public:
   /// Returns the TBAAInfo that describes the dereference.
   const MDNode *getTBAAInfo() const { return MMO->getTBAAInfo(); }
 
+  /// Returns the Ranges that describes the dereference.
+  const MDNode *getRanges() const { return MMO->getRanges(); }
+
   /// getMemoryVT - Return the type of the in-memory value.
   EVT getMemoryVT() const { return MemoryVT; }
 
@@ -993,8 +997,8 @@ class AtomicSDNode : public MemSDNode {
            "Ordering may not require more than 4 bits!");
     assert((SynchScope & 1) == SynchScope &&
            "SynchScope may not require more than 1 bit!");
-    SubclassData |= Ordering << 7;
-    SubclassData |= SynchScope << 11;
+    SubclassData |= Ordering << 8;
+    SubclassData |= SynchScope << 12;
     assert(getOrdering() == Ordering && "Ordering encoding error!");
     assert(getSynchScope() == SynchScope && "Synch-scope encoding error!");
 
@@ -1113,11 +1117,9 @@ protected:
   }
 public:
 
-  void getMask(SmallVectorImpl<int> &M) const {
+  ArrayRef<int> getMask() const {
     EVT VT = getValueType(0);
-    M.clear();
-    for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
-      M.push_back(Mask[i]);
+    return makeArrayRef(Mask, VT.getVectorNumElements());
   }
   int getMaskElt(unsigned Idx) const {
     assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!");
@@ -1434,6 +1436,23 @@ public:
   }
 };
 
+class RegisterMaskSDNode : public SDNode {
+  // The memory for RegMask is not owned by the node.
+  const uint32_t *RegMask;
+  friend class SelectionDAG;
+  RegisterMaskSDNode(const uint32_t *mask)
+    : SDNode(ISD::RegisterMask, DebugLoc(), getSDVTList(MVT::Untyped)),
+      RegMask(mask) {}
+public:
+
+  const uint32_t *getRegMask() const { return RegMask; }
+
+  static bool classof(const RegisterMaskSDNode *) { return true; }
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::RegisterMask;
+  }
+};
+
 class BlockAddressSDNode : public SDNode {
   const BlockAddress *BA;
   unsigned char TargetFlags;
@@ -1684,6 +1703,8 @@ public:
   /// setMemRefs - Assign this MachineSDNodes's memory reference descriptor
   /// list. This does not transfer ownership.
   void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) {
+    for (mmo_iterator MMI = NewMemRefs, MME = NewMemRefsEnd; MMI != MME; ++MMI)
+      assert(*MMI && "Null mem ref detected!");
     MemRefs = NewMemRefs;
     MemRefsEnd = NewMemRefsEnd;
   }
diff --git a/include/llvm/CodeGen/SlotIndexes.h b/include/llvm/CodeGen/SlotIndexes.h
index 2d98864..d868cb8 100644
--- a/include/llvm/CodeGen/SlotIndexes.h
+++ b/include/llvm/CodeGen/SlotIndexes.h
@@ -19,7 +19,7 @@
 #ifndef LLVM_CODEGEN_SLOTINDEXES_H
 #define LLVM_CODEGEN_SLOTINDEXES_H
 
-#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineInstrBundle.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/ADT/PointerIntPair.h"
@@ -83,7 +83,29 @@ namespace llvm {
     friend class SlotIndexes;
     friend struct DenseMapInfo<SlotIndex>;
 
-    enum Slot { LOAD, USE, DEF, STORE, NUM };
+    enum Slot {
+      /// Basic block boundary.  Used for live ranges entering and leaving a
+      /// block without being live in the layout neighbor.  Also used as the
+      /// def slot of PHI-defs.
+      Slot_Block,
+
+      /// Early-clobber register use/def slot.  A live range defined at
+      /// Slot_EarlyCLobber interferes with normal live ranges killed at
+      /// Slot_Register.  Also used as the kill slot for live ranges tied to an
+      /// early-clobber def.
+      Slot_EarlyClobber,
+
+      /// Normal register use/def slot.  Normal instructions kill and define
+      /// register live ranges at this slot.
+      Slot_Register,
+
+      /// Dead def kill point.  Kill slot for a live range that is defined by
+      /// the same instruction (Slot_Register or Slot_EarlyClobber), but isn't
+      /// used anywhere.
+      Slot_Dead,
+
+      Slot_Count
+    };
 
     PointerIntPair<IndexListEntry*, 2, unsigned> lie;
 
@@ -113,7 +135,7 @@ namespace llvm {
     enum {
       /// The default distance between instructions as returned by distance().
       /// This may vary as instructions are inserted and removed.
-      InstrDist = 4*NUM
+      InstrDist = 4 * Slot_Count
     };
 
     static inline SlotIndex getEmptyKey() {
@@ -186,69 +208,55 @@ namespace llvm {
       return A.lie.getPointer() == B.lie.getPointer();
     }
 
+    /// isEarlierInstr - Return true if A refers to an instruction earlier than
+    /// B. This is equivalent to A < B && !isSameInstr(A, B).
+    static bool isEarlierInstr(SlotIndex A, SlotIndex B) {
+      return A.entry().getIndex() < B.entry().getIndex();
+    }
+
     /// Return the distance from this index to the given one.
     int distance(SlotIndex other) const {
       return other.getIndex() - getIndex();
     }
 
-    /// isLoad - Return true if this is a LOAD slot.
-    bool isLoad() const {
-      return getSlot() == LOAD;
-    }
+    /// isBlock - Returns true if this is a block boundary slot.
+    bool isBlock() const { return getSlot() == Slot_Block; }
 
-    /// isDef - Return true if this is a DEF slot.
-    bool isDef() const {
-      return getSlot() == DEF;
-    }
+    /// isEarlyClobber - Returns true if this is an early-clobber slot.
+    bool isEarlyClobber() const { return getSlot() == Slot_EarlyClobber; }
 
-    /// isUse - Return true if this is a USE slot.
-    bool isUse() const {
-      return getSlot() == USE;
-    }
+    /// isRegister - Returns true if this is a normal register use/def slot.
+    /// Note that early-clobber slots may also be used for uses and defs.
+    bool isRegister() const { return getSlot() == Slot_Register; }
 
-    /// isStore - Return true if this is a STORE slot.
-    bool isStore() const {
-      return getSlot() == STORE;
-    }
+    /// isDead - Returns true if this is a dead def kill slot.
+    bool isDead() const { return getSlot() == Slot_Dead; }
 
     /// Returns the base index for associated with this index. The base index
-    /// is the one associated with the LOAD slot for the instruction pointed to
-    /// by this index.
+    /// is the one associated with the Slot_Block slot for the instruction
+    /// pointed to by this index.
     SlotIndex getBaseIndex() const {
-      return getLoadIndex();
+      return SlotIndex(&entry(), Slot_Block);
     }
 
     /// Returns the boundary index for associated with this index. The boundary
-    /// index is the one associated with the LOAD slot for the instruction
+    /// index is the one associated with the Slot_Block slot for the instruction
     /// pointed to by this index.
     SlotIndex getBoundaryIndex() const {
-      return getStoreIndex();
+      return SlotIndex(&entry(), Slot_Dead);
     }
 
-    /// Returns the index of the LOAD slot for the instruction pointed to by
-    /// this index.
-    SlotIndex getLoadIndex() const {
-      return SlotIndex(&entry(), SlotIndex::LOAD);
-    }    
-
-    /// Returns the index of the USE slot for the instruction pointed to by
-    /// this index.
-    SlotIndex getUseIndex() const {
-      return SlotIndex(&entry(), SlotIndex::USE);
+    /// Returns the register use/def slot in the current instruction for a
+    /// normal or early-clobber def.
+    SlotIndex getRegSlot(bool EC = false) const {
+      return SlotIndex(&entry(), EC ? Slot_EarlyClobber : Slot_Register);
     }
 
-    /// Returns the index of the DEF slot for the instruction pointed to by
-    /// this index.
-    SlotIndex getDefIndex() const {
-      return SlotIndex(&entry(), SlotIndex::DEF);
+    /// Returns the dead def kill slot for the current instruction.
+    SlotIndex getDeadSlot() const {
+      return SlotIndex(&entry(), Slot_Dead);
     }
 
-    /// Returns the index of the STORE slot for the instruction pointed to by
-    /// this index.
-    SlotIndex getStoreIndex() const {
-      return SlotIndex(&entry(), SlotIndex::STORE);
-    }    
-
     /// Returns the next slot in the index list. This could be either the
     /// next slot for the instruction pointed to by this index or, if this
     /// index is a STORE, the first slot for the next instruction.
@@ -257,8 +265,8 @@ namespace llvm {
     /// use one of those methods.
     SlotIndex getNextSlot() const {
       Slot s = getSlot();
-      if (s == SlotIndex::STORE) {
-        return SlotIndex(entry().getNext(), SlotIndex::LOAD);
+      if (s == Slot_Dead) {
+        return SlotIndex(entry().getNext(), Slot_Block);
       }
       return SlotIndex(&entry(), s + 1);
     }
@@ -271,14 +279,14 @@ namespace llvm {
 
     /// Returns the previous slot in the index list. This could be either the
     /// previous slot for the instruction pointed to by this index or, if this
-    /// index is a LOAD, the last slot for the previous instruction.
+    /// index is a Slot_Block, the last slot for the previous instruction.
     /// WARNING: This method is considerably more expensive than the methods
     /// that return specific slots (getUseIndex(), etc). If you can - please
     /// use one of those methods.
     SlotIndex getPrevSlot() const {
       Slot s = getSlot();
-      if (s == SlotIndex::LOAD) {
-        return SlotIndex(entry().getPrev(), SlotIndex::STORE);
+      if (s == Slot_Block) {
+        return SlotIndex(entry().getPrev(), Slot_Dead);
       }
       return SlotIndex(&entry(), s - 1);
     }
@@ -464,11 +472,6 @@ namespace llvm {
       return SlotIndex(back(), 0);
     }
 
-    /// Returns the invalid index marker for this analysis.
-    SlotIndex getInvalidIndex() {
-      return getZeroIndex();
-    }
-
     /// Returns the distance between the highest and lowest indexes allocated
     /// so far.
     unsigned getIndexesLength() const {
@@ -486,12 +489,13 @@ namespace llvm {
     /// Returns true if the given machine instr is mapped to an index,
     /// otherwise returns false.
     bool hasIndex(const MachineInstr *instr) const {
-      return (mi2iMap.find(instr) != mi2iMap.end());
+      return mi2iMap.count(instr);
     }
 
     /// Returns the base index for the given instruction.
-    SlotIndex getInstructionIndex(const MachineInstr *instr) const {
-      Mi2IndexMap::const_iterator itr = mi2iMap.find(instr);
+    SlotIndex getInstructionIndex(const MachineInstr *MI) const {
+      // Instructions inside a bundle have the same number as the bundle itself.
+      Mi2IndexMap::const_iterator itr = mi2iMap.find(getBundleStart(MI));
       assert(itr != mi2iMap.end() && "Instruction not found in maps.");
       return itr->second;
     }
@@ -645,6 +649,8 @@ namespace llvm {
     /// instructions, create the new index after the null indexes instead of
     /// before them.
     SlotIndex insertMachineInstrInMaps(MachineInstr *mi, bool Late = false) {
+      assert(!mi->isInsideBundle() &&
+             "Instructions inside bundles should use bundle start's slot.");
       assert(mi2iMap.find(mi) == mi2iMap.end() && "Instr already indexed.");
       // Numbering DBG_VALUE instructions could cause code generation to be
       // affected by debug information.
@@ -677,7 +683,7 @@ namespace llvm {
       if (dist == 0)
         renumberIndexes(newEntry);
 
-      SlotIndex newIndex(newEntry, SlotIndex::LOAD);
+      SlotIndex newIndex(newEntry, SlotIndex::Slot_Block);
       mi2iMap.insert(std::make_pair(mi, newIndex));
       return newIndex;
     }
@@ -728,8 +734,8 @@ namespace llvm {
       insert(nextEntry, startEntry);
       insert(nextEntry, stopEntry);
 
-      SlotIndex startIdx(startEntry, SlotIndex::LOAD);
-      SlotIndex endIdx(nextEntry, SlotIndex::LOAD);
+      SlotIndex startIdx(startEntry, SlotIndex::Slot_Block);
+      SlotIndex endIdx(nextEntry, SlotIndex::Slot_Block);
 
       assert(unsigned(mbb->getNumber()) == MBBRanges.size() &&
              "Blocks must be added in order");
diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index ca40ccf..5a42136 100644
--- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -15,9 +15,9 @@
 #ifndef LLVM_CODEGEN_TARGETLOWERINGOBJECTFILEIMPL_H
 #define LLVM_CODEGEN_TARGETLOWERINGOBJECTFILEIMPL_H
 
-#include "llvm/ADT/StringRef.h"
 #include "llvm/MC/SectionKind.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
+#include "llvm/ADT/StringRef.h"
 
 namespace llvm {
   class MachineModuleInfo;
@@ -65,6 +65,11 @@ public:
   virtual MCSymbol *
   getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang,
                           MachineModuleInfo *MMI) const;
+
+  virtual const MCSection *
+  getStaticCtorSection(unsigned Priority = 65535) const;
+  virtual const MCSection *
+  getStaticDtorSection(unsigned Priority = 65535) const;
 };
 
 
@@ -73,6 +78,12 @@ class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile {
 public:
   virtual ~TargetLoweringObjectFileMachO() {}
 
+  /// emitModuleFlags - Emit the module flags that specify the garbage
+  /// collection information.
+  virtual void emitModuleFlags(MCStreamer &Streamer,
+                               ArrayRef<Module::ModuleFlagEntry> ModuleFlags,
+                               Mangler *Mang, const TargetMachine &TM) const;
+
   virtual const MCSection *
   SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
                          Mangler *Mang, const TargetMachine &TM) const;
diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h
index cae0bcb..76c2357 100644
--- a/include/llvm/CodeGen/ValueTypes.h
+++ b/include/llvm/CodeGen/ValueTypes.h
@@ -16,10 +16,11 @@
 #ifndef LLVM_CODEGEN_VALUETYPES_H
 #define LLVM_CODEGEN_VALUETYPES_H
 
-#include <cassert>
-#include <string>
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
+#include <cassert>
+#include <string>
 
 namespace llvm {
   class Type;
@@ -45,49 +46,56 @@ namespace llvm {
       FIRST_INTEGER_VALUETYPE = i1,
       LAST_INTEGER_VALUETYPE  = i128,
 
-      f32            =   7,   // This is a 32 bit floating point value
-      f64            =   8,   // This is a 64 bit floating point value
-      f80            =   9,   // This is a 80 bit floating point value
-      f128           =  10,   // This is a 128 bit floating point value
-      ppcf128        =  11,   // This is a PPC 128-bit floating point value
-
-      v2i8           =  12,   //  2 x i8
-      v4i8           =  13,   //  4 x i8
-      v8i8           =  14,   //  8 x i8
-      v16i8          =  15,   // 16 x i8
-      v32i8          =  16,   // 32 x i8
-      v2i16          =  17,   //  2 x i16
-      v4i16          =  18,   //  4 x i16
-      v8i16          =  19,   //  8 x i16
-      v16i16         =  20,   // 16 x i16
-      v2i32          =  21,   //  2 x i32
-      v4i32          =  22,   //  4 x i32
-      v8i32          =  23,   //  8 x i32
-      v1i64          =  24,   //  1 x i64
-      v2i64          =  25,   //  2 x i64
-      v4i64          =  26,   //  4 x i64
-      v8i64          =  27,   //  8 x i64
-
-      v2f32          =  28,   //  2 x f32
-      v4f32          =  29,   //  4 x f32
-      v8f32          =  30,   //  8 x f32
-      v2f64          =  31,   //  2 x f64
-      v4f64          =  32,   //  4 x f64
+      f16            =   7,   // This is a 16 bit floating point value
+      f32            =   8,   // This is a 32 bit floating point value
+      f64            =   9,   // This is a 64 bit floating point value
+      f80            =  10,   // This is a 80 bit floating point value
+      f128           =  11,   // This is a 128 bit floating point value
+      ppcf128        =  12,   // This is a PPC 128-bit floating point value
+
+      FIRST_FP_VALUETYPE = f16,
+      LAST_FP_VALUETYPE  = ppcf128,
+
+      v2i8           =  13,   //  2 x i8
+      v4i8           =  14,   //  4 x i8
+      v8i8           =  15,   //  8 x i8
+      v16i8          =  16,   // 16 x i8
+      v32i8          =  17,   // 32 x i8
+      v2i16          =  18,   //  2 x i16
+      v4i16          =  19,   //  4 x i16
+      v8i16          =  20,   //  8 x i16
+      v16i16         =  21,   // 16 x i16
+      v2i32          =  22,   //  2 x i32
+      v4i32          =  23,   //  4 x i32
+      v8i32          =  24,   //  8 x i32
+      v1i64          =  25,   //  1 x i64
+      v2i64          =  26,   //  2 x i64
+      v4i64          =  27,   //  4 x i64
+      v8i64          =  28,   //  8 x i64
+
+      v2f16          =  29,   //  2 x f16
+      v2f32          =  30,   //  2 x f32
+      v4f32          =  31,   //  4 x f32
+      v8f32          =  32,   //  8 x f32
+      v2f64          =  33,   //  2 x f64
+      v4f64          =  34,   //  4 x f64
 
       FIRST_VECTOR_VALUETYPE = v2i8,
       LAST_VECTOR_VALUETYPE  = v4f64,
+      FIRST_FP_VECTOR_VALUETYPE = v2f16,
+      LAST_FP_VECTOR_VALUETYPE = v4f64,
 
-      x86mmx         =  33,   // This is an X86 MMX value
+      x86mmx         =  35,   // This is an X86 MMX value
 
-      Glue           =  34,   // This glues nodes together during pre-RA sched
+      Glue           =  36,   // This glues nodes together during pre-RA sched
 
-      isVoid         =  35,   // This has no value
+      isVoid         =  37,   // This has no value
 
-      untyped        =  36,   // This value takes a register, but has
+      Untyped        =  38,   // This value takes a register, but has
                               // unspecified type.  The register class
                               // will be determined by the opcode.
 
-      LAST_VALUETYPE =  37,   // This always remains at the end of the list.
+      LAST_VALUETYPE =  39,   // This always remains at the end of the list.
 
       // This is the current maximum for LAST_VALUETYPE.
       // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors
@@ -143,8 +151,10 @@ namespace llvm {
 
     /// isFloatingPoint - Return true if this is a FP, or a vector FP type.
     bool isFloatingPoint() const {
-      return ((SimpleTy >= MVT::f32 && SimpleTy <= MVT::ppcf128) ||
-	      (SimpleTy >= MVT::v2f32 && SimpleTy <= MVT::v4f64));
+      return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE &&
+               SimpleTy <= MVT::LAST_FP_VALUETYPE) ||
+	      (SimpleTy >= MVT::FIRST_FP_VECTOR_VALUETYPE &&
+         SimpleTy <= MVT::LAST_FP_VECTOR_VALUETYPE));
     }
 
     /// isInteger - Return true if this is an integer, or a vector integer type.
@@ -203,6 +213,7 @@ namespace llvm {
       case v2i64:
       case v4i64:
       case v8i64: return i64;
+      case v2f16: return f16;
       case v2f32:
       case v4f32:
       case v8f32: return f32;
@@ -233,6 +244,7 @@ namespace llvm {
       case v2i16:
       case v2i32:
       case v2i64:
+      case v2f16:
       case v2f32:
       case v2f64: return 2;
       case v1i64: return 1;
@@ -242,21 +254,23 @@ namespace llvm {
     unsigned getSizeInBits() const {
       switch (SimpleTy) {
       case iPTR:
-        assert(0 && "Value type size is target-dependent. Ask TLI.");
+        llvm_unreachable("Value type size is target-dependent. Ask TLI.");
       case iPTRAny:
       case iAny:
       case fAny:
-        assert(0 && "Value type is overloaded.");
+        llvm_unreachable("Value type is overloaded.");
       default:
-        assert(0 && "getSizeInBits called on extended MVT.");
+        llvm_unreachable("getSizeInBits called on extended MVT.");
       case i1  :  return 1;
       case i8  :  return 8;
       case i16 :
+      case f16:
       case v2i8:  return 16;
       case f32 :
       case i32 :
       case v4i8:
-      case v2i16: return 32;
+      case v2i16:
+      case v2f16: return 32;
       case x86mmx:
       case f64 :
       case i64 :
@@ -300,7 +314,9 @@ namespace llvm {
     static MVT getFloatingPointVT(unsigned BitWidth) {
       switch (BitWidth) {
       default:
-        assert(false && "Bad bit width!");
+        llvm_unreachable("Bad bit width!");
+      case 16:
+        return MVT::f16;
       case 32:
         return MVT::f32;
       case 64:
@@ -359,6 +375,9 @@ namespace llvm {
         if (NumElements == 4)  return MVT::v4i64;
         if (NumElements == 8)  return MVT::v8i64;
         break;
+      case MVT::f16:
+        if (NumElements == 2)  return MVT::v2f16;
+        break;
       case MVT::f32:
         if (NumElements == 2)  return MVT::v2f32;
         if (NumElements == 4)  return MVT::v4f32;
@@ -424,20 +443,6 @@ namespace llvm {
       return getExtendedVectorVT(Context, VT, NumElements);
     }
 
-    /// getIntVectorWithNumElements - Return any integer vector type that has
-    /// the specified number of elements.
-    static EVT getIntVectorWithNumElements(LLVMContext &C, unsigned NumElts) {
-      switch (NumElts) {
-      default: return getVectorVT(C, MVT::i8, NumElts);
-      case  1: return MVT::v1i64;
-      case  2: return MVT::v2i32;
-      case  4: return MVT::v4i16;
-      case  8: return MVT::v8i8;
-      case 16: return MVT::v16i8;
-      }
-      return MVT::INVALID_SIMPLE_VALUE_TYPE;
-    }
-
     /// changeVectorElementTypeToInteger - Return a vector with the same number
     /// of elements as this vector, but with the element type converted to an
     /// integer type with the same bitwidth.
diff --git a/include/llvm/CodeGen/ValueTypes.td b/include/llvm/CodeGen/ValueTypes.td
index 0cfb634..6c22690 100644
--- a/include/llvm/CodeGen/ValueTypes.td
+++ b/include/llvm/CodeGen/ValueTypes.td
@@ -26,39 +26,41 @@ def i16    : ValueType<16 ,  3>;   // 16-bit integer value
 def i32    : ValueType<32 ,  4>;   // 32-bit integer value
 def i64    : ValueType<64 ,  5>;   // 64-bit integer value
 def i128   : ValueType<128,  6>;   // 128-bit integer value
-def f32    : ValueType<32 ,  7>;   // 32-bit floating point value
-def f64    : ValueType<64 ,  8>;   // 64-bit floating point value
-def f80    : ValueType<80 ,  9>;   // 80-bit floating point value
-def f128   : ValueType<128, 10>;   // 128-bit floating point value
-def ppcf128: ValueType<128, 11>;   // PPC 128-bit floating point value
+def f16    : ValueType<16 ,  7>;   // 32-bit floating point value
+def f32    : ValueType<32 ,  8>;   // 32-bit floating point value
+def f64    : ValueType<64 ,  9>;   // 64-bit floating point value
+def f80    : ValueType<80 , 10>;   // 80-bit floating point value
+def f128   : ValueType<128, 11>;   // 128-bit floating point value
+def ppcf128: ValueType<128, 12>;   // PPC 128-bit floating point value
 
-def v2i8   : ValueType<16 , 12>;   //  2 x i8  vector value
-def v4i8   : ValueType<32 , 13>;   //  4 x i8  vector value
-def v8i8   : ValueType<64 , 14>;   //  8 x i8  vector value
-def v16i8  : ValueType<128, 15>;   // 16 x i8  vector value
-def v32i8  : ValueType<256, 16>;   // 32 x i8 vector value
-def v2i16  : ValueType<32 , 17>;   //  2 x i16 vector value
-def v4i16  : ValueType<64 , 18>;   //  4 x i16 vector value
-def v8i16  : ValueType<128, 19>;   //  8 x i16 vector value
-def v16i16 : ValueType<256, 20>;   // 16 x i16 vector value
-def v2i32  : ValueType<64 , 21>;   //  2 x i32 vector value
-def v4i32  : ValueType<128, 22>;   //  4 x i32 vector value
-def v8i32  : ValueType<256, 23>;   //  8 x i32 vector value
-def v1i64  : ValueType<64 , 24>;   //  1 x i64 vector value
-def v2i64  : ValueType<128, 25>;   //  2 x i64 vector value
-def v4i64  : ValueType<256, 26>;   //  4 x i64 vector value
-def v8i64  : ValueType<512, 27>;   //  8 x i64 vector value
+def v2i8   : ValueType<16 , 13>;   //  2 x i8  vector value
+def v4i8   : ValueType<32 , 14>;   //  4 x i8  vector value
+def v8i8   : ValueType<64 , 15>;   //  8 x i8  vector value
+def v16i8  : ValueType<128, 16>;   // 16 x i8  vector value
+def v32i8  : ValueType<256, 17>;   // 32 x i8 vector value
+def v2i16  : ValueType<32 , 18>;   //  2 x i16 vector value
+def v4i16  : ValueType<64 , 19>;   //  4 x i16 vector value
+def v8i16  : ValueType<128, 20>;   //  8 x i16 vector value
+def v16i16 : ValueType<256, 21>;   // 16 x i16 vector value
+def v2i32  : ValueType<64 , 22>;   //  2 x i32 vector value
+def v4i32  : ValueType<128, 23>;   //  4 x i32 vector value
+def v8i32  : ValueType<256, 24>;   //  8 x i32 vector value
+def v1i64  : ValueType<64 , 25>;   //  1 x i64 vector value
+def v2i64  : ValueType<128, 26>;   //  2 x i64 vector value
+def v4i64  : ValueType<256, 27>;   //  4 x i64 vector value
+def v8i64  : ValueType<512, 28>;   //  8 x i64 vector value
 
-def v2f32  : ValueType<64 , 28>;   //  2 x f32 vector value
-def v4f32  : ValueType<128, 29>;   //  4 x f32 vector value
-def v8f32  : ValueType<256, 30>;   //  8 x f32 vector value
-def v2f64  : ValueType<128, 31>;   //  2 x f64 vector value
-def v4f64  : ValueType<256, 32>;   //  4 x f64 vector value
+def v2f16  : ValueType<32 , 29>;   //  2 x f16 vector value
+def v2f32  : ValueType<64 , 30>;   //  2 x f32 vector value
+def v4f32  : ValueType<128, 31>;   //  4 x f32 vector value
+def v8f32  : ValueType<256, 32>;   //  8 x f32 vector value
+def v2f64  : ValueType<128, 33>;   //  2 x f64 vector value
+def v4f64  : ValueType<256, 34>;   //  4 x f64 vector value
 
-def x86mmx : ValueType<64 , 33>;   // X86 MMX value
-def FlagVT : ValueType<0  , 34>;   // Pre-RA sched glue
-def isVoid : ValueType<0  , 35>;   // Produces no value
-def untyped: ValueType<8  , 36>;   // Produces an untyped value
+def x86mmx : ValueType<64 , 35>;   // X86 MMX value
+def FlagVT : ValueType<0  , 36>;   // Pre-RA sched glue
+def isVoid : ValueType<0  , 37>;   // Produces no value
+def untyped: ValueType<8  , 38>;   // Produces an untyped value
 
 def MetadataVT: ValueType<0, 250>; // Metadata