Update LLVM to r84949.

30 files changed, 612 insertions, 365 deletions

diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 2914851..d50e5e3 100644
--- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -923,7 +923,7 @@ void DwarfDebug::ConstructTypeDIE(CompileUnit *DW_Unit, DIE &Buffer,
   AddType(DW_Unit, &Buffer, FromTy);
 
   // Add name if not anonymous or intermediate type.
-  if (Name)
+  if (Name && Tag != dwarf::DW_TAG_pointer_type)
     AddString(&Buffer, dwarf::DW_AT_name, dwarf::DW_FORM_string, Name);
 
   // Add size if non-zero (derived types might be zero-sized.)
@@ -1811,10 +1811,18 @@ void DwarfDebug::CollectVariableInfo() {
     DIVariable DV (Var);
     if (DV.isNull()) continue;
     unsigned VSlot = VI->second;
+    DbgScope *Scope = NULL;
     DenseMap<MDNode *, DbgScope *>::iterator DSI = 
       DbgScopeMap.find(DV.getContext().getNode());
-    assert (DSI != DbgScopeMap.end() && "Unable to find variable scope!");
-    DbgScope *Scope = DSI->second;
+    if (DSI != DbgScopeMap.end()) 
+      Scope = DSI->second;
+    else 
+      // There is not any instruction assocated with this scope, so get
+      // a new scope.
+      Scope = getDbgScope(DV.getContext().getNode(), 
+                          NULL /* Not an instruction */,
+                          NULL /* Not inlined */);
+    assert (Scope && "Unable to find variable scope!");
     Scope->AddVariable(new DbgVariable(DV, VSlot, false));
   }
 }
diff --git a/lib/CodeGen/AsmPrinter/DwarfException.cpp b/lib/CodeGen/AsmPrinter/DwarfException.cpp
index 626523b..6c03b55 100644
--- a/lib/CodeGen/AsmPrinter/DwarfException.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfException.cpp
@@ -871,28 +871,7 @@ void DwarfException::EmitExceptionTable() {
     Asm->EOL("Next action");
   }
 
-  // Emit the Catch Clauses. The code for the catch clauses following the same
-  // try is similar to a switch statement. The catch clause action record
-  // informs the runtime about the type of a catch clause and about the
-  // associated switch value.
-  //
-  //  Action Record Fields:
-  //
-  //   * Filter Value
-  //     Positive value, starting at 1. Index in the types table of the
-  //     __typeinfo for the catch-clause type. 1 is the first word preceding
-  //     TTBase, 2 is the second word, and so on. Used by the runtime to check
-  //     if the thrown exception type matches the catch-clause type. Back-end
-  //     generated switch statements check against this value.
-  //
-  //   * Next
-  //     Signed offset, in bytes from the start of this field, to the next
-  //     chained action record, or zero if none.
-  //
-  // The order of the action records determined by the next field is the order
-  // of the catch clauses as they appear in the source code, and must be kept in
-  // the same order. As a result, changing the order of the catch clause would
-  // change the semantics of the program.
+  // Emit the Catch TypeInfos.
   for (std::vector<GlobalVariable *>::const_reverse_iterator
          I = TypeInfos.rbegin(), E = TypeInfos.rend(); I != E; ++I) {
     const GlobalVariable *GV = *I;
@@ -907,12 +886,15 @@ void DwarfException::EmitExceptionTable() {
     Asm->EOL("TypeInfo");
   }
 
-  // Emit the Type Table.
+  // Emit the Exception Specifications.
   for (std::vector<unsigned>::const_iterator
          I = FilterIds.begin(), E = FilterIds.end(); I < E; ++I) {
     unsigned TypeID = *I;
     Asm->EmitULEB128Bytes(TypeID);
-    Asm->EOL("Filter TypeInfo index");
+    if (TypeID != 0)
+      Asm->EOL("Exception specification");
+    else
+      Asm->EOL();
   }
 
   Asm->EmitAlignment(2, 0, 0, false);
diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp
index f9abeac..66c5aa5 100644
--- a/lib/CodeGen/BranchFolding.cpp
+++ b/lib/CodeGen/BranchFolding.cpp
@@ -945,15 +945,15 @@ bool BranchFolder::OptimizeBlock(MachineBasicBlock *MBB) {
       }
     }
     
-    // If this block doesn't fall through (e.g. it ends with an uncond branch or
-    // has no successors) and if the pred falls through into this block, and if
-    // it would otherwise fall through into the block after this, move this
-    // block to the end of the function.
+    // If this block has no successors (e.g. it is a return block or ends with
+    // a call to a no-return function like abort or __cxa_throw) and if the pred
+    // falls through into this block, and if it would otherwise fall through
+    // into the block after this, move this block to the end of the function.
     //
     // We consider it more likely that execution will stay in the function (e.g.
     // due to loops) than it is to exit it.  This asserts in loops etc, moving
     // the assert condition out of the loop body.
-    if (!PriorCond.empty() && PriorFBB == 0 &&
+    if (MBB->succ_empty() && !PriorCond.empty() && PriorFBB == 0 &&
         MachineFunction::iterator(PriorTBB) == FallThrough &&
         !CanFallThrough(MBB)) {
       bool DoTransform = true;
diff --git a/lib/CodeGen/CodePlacementOpt.cpp b/lib/CodeGen/CodePlacementOpt.cpp
index 42b24a6..6fff12c 100644
--- a/lib/CodeGen/CodePlacementOpt.cpp
+++ b/lib/CodeGen/CodePlacementOpt.cpp
@@ -34,14 +34,6 @@ namespace {
     const TargetInstrInfo *TII;
     const TargetLowering  *TLI;
 
-    /// ChangedMBBs - BBs which are modified by OptimizeIntraLoopEdges.
-    SmallPtrSet<MachineBasicBlock*, 8> ChangedMBBs;
-
-    /// UncondJmpMBBs - A list of BBs which are in loops and end with
-    /// unconditional branches.
-    SmallVector<std::pair<MachineBasicBlock*,MachineBasicBlock*>, 4>
-    UncondJmpMBBs;
-
   public:
     static char ID;
     CodePlacementOpt() : MachineFunctionPass(&ID) {}
@@ -58,7 +50,19 @@ namespace {
     }
 
   private:
-    bool OptimizeIntraLoopEdges();
+    bool HasFallthrough(MachineBasicBlock *MBB);
+    bool HasAnalyzableTerminator(MachineBasicBlock *MBB);
+    void Splice(MachineFunction &MF,
+                MachineFunction::iterator InsertPt,
+                MachineFunction::iterator Begin,
+                MachineFunction::iterator End);
+    void UpdateTerminator(MachineBasicBlock *MBB);
+    bool EliminateUnconditionalJumpsToTop(MachineFunction &MF,
+                                          MachineLoop *L);
+    bool MoveDiscontiguousLoopBlocks(MachineFunction &MF,
+                                     MachineLoop *L);
+    bool OptimizeIntraLoopEdgesInLoopNest(MachineFunction &MF, MachineLoop *L);
+    bool OptimizeIntraLoopEdges(MachineFunction &MF);
     bool AlignLoops(MachineFunction &MF);
     bool AlignLoop(MachineFunction &MF, MachineLoop *L, unsigned Align);
   };
@@ -70,168 +74,354 @@ FunctionPass *llvm::createCodePlacementOptPass() {
   return new CodePlacementOpt();
 }
 
-/// OptimizeBackEdges - Place loop back edges to move unconditional branches
-/// out of the loop.
-///
-///       A:
-///       ...
-///       <fallthrough to B>
+/// HasFallthrough - Test whether the given branch has a fallthrough, either as
+/// a plain fallthrough or as a fallthrough case of a conditional branch.
 ///
-///       B:  --> loop header
-///       ...
-///       jcc <cond> C, [exit]
-///
-///       C:
-///       ...
-///       jmp B
+bool CodePlacementOpt::HasFallthrough(MachineBasicBlock *MBB) {
+  MachineBasicBlock *TBB = 0, *FBB = 0;
+  SmallVector<MachineOperand, 4> Cond;
+  if (TII->AnalyzeBranch(*MBB, TBB, FBB, Cond))
+    return false;
+  // This conditional branch has no fallthrough.
+  if (FBB)
+    return false;
+  // An unconditional branch has no fallthrough.
+  if (Cond.empty() && TBB)
+    return false;
+  // It has a fallthrough.
+  return true;
+}
+
+/// HasAnalyzableTerminator - Test whether AnalyzeBranch will succeed on MBB.
+/// This is called before major changes are begun to test whether it will be
+/// possible to complete the changes.
 ///
-/// ==>
+/// Target-specific code is hereby encouraged to make AnalyzeBranch succeed
+/// whenever possible.
 ///
-///       A:
-///       ...
-///       jmp B
+bool CodePlacementOpt::HasAnalyzableTerminator(MachineBasicBlock *MBB) {
+  // Conservatively ignore EH landing pads.
+  if (MBB->isLandingPad()) return false;
+
+  // Ignore blocks which look like they might have EH-related control flow.
+  // At the time of this writing, there are blocks which AnalyzeBranch
+  // thinks end in single uncoditional branches, yet which have two CFG
+  // successors. Code in this file is not prepared to reason about such things.
+  if (!MBB->empty() && MBB->back().getOpcode() == TargetInstrInfo::EH_LABEL)
+    return false;
+
+  // Aggressively handle return blocks and similar constructs.
+  if (MBB->succ_empty()) return true;
+
+  // Ask the target's AnalyzeBranch if it can handle this block.
+  MachineBasicBlock *TBB = 0, *FBB = 0;
+  SmallVector<MachineOperand, 4> Cond;
+  // Make the the terminator is understood.
+  if (TII->AnalyzeBranch(*MBB, TBB, FBB, Cond))
+    return false;
+  // Make sure we have the option of reversing the condition.
+  if (!Cond.empty() && TII->ReverseBranchCondition(Cond))
+    return false;
+  return true;
+}
+
+/// Splice - Move the sequence of instructions [Begin,End) to just before
+/// InsertPt. Update branch instructions as needed to account for broken
+/// fallthrough edges and to take advantage of newly exposed fallthrough
+/// opportunities.
 ///
-///       C:
-///       ...
-///       <fallthough to B>
-///       
-///       B:  --> loop header
-///       ...
-///       jcc <cond> C, [exit]
+void CodePlacementOpt::Splice(MachineFunction &MF,
+                              MachineFunction::iterator InsertPt,
+                              MachineFunction::iterator Begin,
+                              MachineFunction::iterator End) {
+  assert(Begin != MF.begin() && End != MF.begin() && InsertPt != MF.begin() &&
+         "Splice can't change the entry block!");
+  MachineFunction::iterator OldBeginPrior = prior(Begin);
+  MachineFunction::iterator OldEndPrior = prior(End);
+
+  MF.splice(InsertPt, Begin, End);
+
+  UpdateTerminator(prior(Begin));
+  UpdateTerminator(OldBeginPrior);
+  UpdateTerminator(OldEndPrior);
+}
+
+/// UpdateTerminator - Update the terminator instructions in MBB to account
+/// for changes to the layout. If the block previously used a fallthrough,
+/// it may now need a branch, and if it previously used branching it may now
+/// be able to use a fallthrough.
 ///
-bool CodePlacementOpt::OptimizeIntraLoopEdges() {
-  if (!TLI->shouldOptimizeCodePlacement())
-    return false;
+void CodePlacementOpt::UpdateTerminator(MachineBasicBlock *MBB) {
+  // A block with no successors has no concerns with fall-through edges.
+  if (MBB->succ_empty()) return;
+
+  MachineBasicBlock *TBB = 0, *FBB = 0;
+  SmallVector<MachineOperand, 4> Cond;
+  bool B = TII->AnalyzeBranch(*MBB, TBB, FBB, Cond);
+  (void) B;
+  assert(!B && "UpdateTerminators requires analyzable predecessors!");
+  if (Cond.empty()) {
+    if (TBB) {
+      // The block has an unconditional branch. If its successor is now
+      // its layout successor, delete the branch.
+      if (MBB->isLayoutSuccessor(TBB))
+        TII->RemoveBranch(*MBB);
+    } else {
+      // The block has an unconditional fallthrough. If its successor is not
+      // its layout successor, insert a branch.
+      TBB = *MBB->succ_begin();
+      if (!MBB->isLayoutSuccessor(TBB))
+        TII->InsertBranch(*MBB, TBB, 0, Cond);
+    }
+  } else {
+    if (FBB) {
+      // The block has a non-fallthrough conditional branch. If one of its
+      // successors is its layout successor, rewrite it to a fallthrough
+      // conditional branch.
+      if (MBB->isLayoutSuccessor(TBB)) {
+        TII->RemoveBranch(*MBB);
+        TII->ReverseBranchCondition(Cond);
+        TII->InsertBranch(*MBB, FBB, 0, Cond);
+      } else if (MBB->isLayoutSuccessor(FBB)) {
+        TII->RemoveBranch(*MBB);
+        TII->InsertBranch(*MBB, TBB, 0, Cond);
+      }
+    } else {
+      // The block has a fallthrough conditional branch.
+      MachineBasicBlock *MBBA = *MBB->succ_begin();
+      MachineBasicBlock *MBBB = *next(MBB->succ_begin());
+      if (MBBA == TBB) std::swap(MBBB, MBBA);
+      if (MBB->isLayoutSuccessor(TBB)) {
+        TII->RemoveBranch(*MBB);
+        TII->ReverseBranchCondition(Cond);
+        TII->InsertBranch(*MBB, MBBA, 0, Cond);
+      } else if (!MBB->isLayoutSuccessor(MBBA)) {
+        TII->RemoveBranch(*MBB);
+        TII->InsertBranch(*MBB, TBB, MBBA, Cond);
+      }
+    }
+  }
+}
 
+/// EliminateUnconditionalJumpsToTop - Move blocks which unconditionally jump
+/// to the loop top to the top of the loop so that they have a fall through.
+/// This can introduce a branch on entry to the loop, but it can eliminate a
+/// branch within the loop. See the @simple case in
+/// test/CodeGen/X86/loop_blocks.ll for an example of this.
+bool CodePlacementOpt::EliminateUnconditionalJumpsToTop(MachineFunction &MF,
+                                                        MachineLoop *L) {
   bool Changed = false;
-  for (unsigned i = 0, e = UncondJmpMBBs.size(); i != e; ++i) {
-    MachineBasicBlock *MBB = UncondJmpMBBs[i].first;
-    MachineBasicBlock *SuccMBB = UncondJmpMBBs[i].second;
-    MachineLoop *L = MLI->getLoopFor(MBB);
-    assert(L && "BB is expected to be in a loop!");
-
-    if (ChangedMBBs.count(MBB)) {
-      // BB has been modified, re-analyze.
-      MachineBasicBlock *TBB = 0, *FBB = 0;
-      SmallVector<MachineOperand, 4> Cond;
-      if (TII->AnalyzeBranch(*MBB, TBB, FBB, Cond) || !Cond.empty())
+  MachineBasicBlock *TopMBB = L->getTopBlock();
+
+  bool BotHasFallthrough = HasFallthrough(L->getBottomBlock());
+
+  if (TopMBB == MF.begin() ||
+      HasAnalyzableTerminator(prior(MachineFunction::iterator(TopMBB)))) {
+  new_top:
+    for (MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin(),
+         PE = TopMBB->pred_end(); PI != PE; ++PI) {
+      MachineBasicBlock *Pred = *PI;
+      if (Pred == TopMBB) continue;
+      if (HasFallthrough(Pred)) continue;
+      if (!L->contains(Pred)) continue;
+
+      // Verify that we can analyze all the loop entry edges before beginning
+      // any changes which will require us to be able to analyze them.
+      if (Pred == MF.begin())
         continue;
-      if (MLI->getLoopFor(TBB) != L || TBB->isLandingPad())
+      if (!HasAnalyzableTerminator(Pred))
+        continue;
+      if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(Pred))))
         continue;
-      SuccMBB = TBB;
-    } else {
-      assert(MLI->getLoopFor(SuccMBB) == L &&
-             "Successor is not in the same loop!");
-    }
 
-    if (MBB->isLayoutSuccessor(SuccMBB)) {
-      // Successor is right after MBB, just eliminate the unconditional jmp.
-      // Can this happen?
-      TII->RemoveBranch(*MBB);
-      ChangedMBBs.insert(MBB);
-      ++NumIntraElim;
+      // Move the block.
       Changed = true;
-      continue;
-    }
 
-    // Now check if the predecessor is fallthrough from any BB. If there is,
-    // that BB should be from outside the loop since edge will become a jmp.
-    bool OkToMove = true;
-    MachineBasicBlock *FtMBB = 0, *FtTBB = 0, *FtFBB = 0;
-    SmallVector<MachineOperand, 4> FtCond;    
-    for (MachineBasicBlock::pred_iterator PI = SuccMBB->pred_begin(),
-           PE = SuccMBB->pred_end(); PI != PE; ++PI) {
-      MachineBasicBlock *PredMBB = *PI;
-      if (PredMBB->isLayoutSuccessor(SuccMBB)) {
-        if (TII->AnalyzeBranch(*PredMBB, FtTBB, FtFBB, FtCond)) {
-          OkToMove = false;
+      // Move it and all the blocks that can reach it via fallthrough edges
+      // exclusively, to keep existing fallthrough edges intact.
+      MachineFunction::iterator Begin = Pred;
+      MachineFunction::iterator End = next(Begin);
+      while (Begin != MF.begin()) {
+        MachineFunction::iterator Prior = prior(Begin);
+        if (Prior == MF.begin())
+          break;
+        // Stop when a non-fallthrough edge is found.
+        if (!HasFallthrough(Prior))
+          break;
+        // Stop if a block which could fall-through out of the loop is found.
+        if (Prior->isSuccessor(End))
+          break;
+        // If we've reached the top, stop scanning.
+        if (Prior == MachineFunction::iterator(TopMBB)) {
+          // We know top currently has a fall through (because we just checked
+          // it) which would be lost if we do the transformation, so it isn't
+          // worthwhile to do the transformation unless it would expose a new
+          // fallthrough edge.
+          if (!Prior->isSuccessor(End))
+            goto next_pred;
+          // Otherwise we can stop scanning and procede to move the blocks.
           break;
         }
-        if (!FtTBB)
-          FtTBB = SuccMBB;
-        else if (!FtFBB) {
-          assert(FtFBB != SuccMBB && "Unexpected control flow!");
-          FtFBB = SuccMBB;
-        }
-        
-        // A fallthrough.
-        FtMBB = PredMBB;
-        MachineLoop *PL = MLI->getLoopFor(PredMBB);
-        if (PL && (PL == L || PL->getLoopDepth() >= L->getLoopDepth()))
-          OkToMove = false;
-
-        break;
+        // If we hit a switch or something complicated, don't move anything
+        // for this predecessor.
+        if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(Prior))))
+          break;
+        // Ok, the block prior to Begin will be moved along with the rest.
+        // Extend the range to include it.
+        Begin = Prior;
+        ++NumIntraMoved;
       }
+
+      // Move the blocks.
+      Splice(MF, TopMBB, Begin, End);
+
+      // Update TopMBB.
+      TopMBB = L->getTopBlock();
+
+      // We have a new loop top. Iterate on it. We shouldn't have to do this
+      // too many times if BranchFolding has done a reasonable job.
+      goto new_top;
+    next_pred:;
     }
+  }
+
+  // If the loop previously didn't exit with a fall-through and it now does,
+  // we eliminated a branch.
+  if (Changed &&
+      !BotHasFallthrough &&
+      HasFallthrough(L->getBottomBlock())) {
+    ++NumIntraElim;
+    BotHasFallthrough = true;
+  }
+
+  return Changed;
+}
+
+/// MoveDiscontiguousLoopBlocks - Move any loop blocks that are not in the
+/// portion of the loop contiguous with the header. This usually makes the loop
+/// contiguous, provided that AnalyzeBranch can handle all the relevant
+/// branching. See the @cfg_islands case in test/CodeGen/X86/loop_blocks.ll
+/// for an example of this.
+bool CodePlacementOpt::MoveDiscontiguousLoopBlocks(MachineFunction &MF,
+                                                   MachineLoop *L) {
+  bool Changed = false;
+  MachineBasicBlock *TopMBB = L->getTopBlock();
+  MachineBasicBlock *BotMBB = L->getBottomBlock();
+
+  // Determine a position to move orphaned loop blocks to. If TopMBB is not
+  // entered via fallthrough and BotMBB is exited via fallthrough, prepend them
+  // to the top of the loop to avoid loosing that fallthrough. Otherwise append
+  // them to the bottom, even if it previously had a fallthrough, on the theory
+  // that it's worth an extra branch to keep the loop contiguous.
+  MachineFunction::iterator InsertPt = next(MachineFunction::iterator(BotMBB));
+  bool InsertAtTop = false;
+  if (TopMBB != MF.begin() &&
+      !HasFallthrough(prior(MachineFunction::iterator(TopMBB))) &&
+      HasFallthrough(BotMBB)) {
+    InsertPt = TopMBB;
+    InsertAtTop = true;
+  }
+
+  // Keep a record of which blocks are in the portion of the loop contiguous
+  // with the loop header.
+  SmallPtrSet<MachineBasicBlock *, 8> ContiguousBlocks;
+  for (MachineFunction::iterator I = TopMBB,
+       E = next(MachineFunction::iterator(BotMBB)); I != E; ++I)
+    ContiguousBlocks.insert(I);
+
+  // Find non-contigous blocks and fix them.
+  if (InsertPt != MF.begin() && HasAnalyzableTerminator(prior(InsertPt)))
+    for (MachineLoop::block_iterator BI = L->block_begin(), BE = L->block_end();
+         BI != BE; ++BI) {
+      MachineBasicBlock *BB = *BI;
+
+      // Verify that we can analyze all the loop entry edges before beginning
+      // any changes which will require us to be able to analyze them.
+      if (!HasAnalyzableTerminator(BB))
+        continue;
+      if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(BB))))
+        continue;
+
+      // If the layout predecessor is part of the loop, this block will be
+      // processed along with it. This keeps them in their relative order.
+      if (BB != MF.begin() &&
+          L->contains(prior(MachineFunction::iterator(BB))))
+        continue;
 
-    if (!OkToMove)
-      continue;
-
-    // Is it profitable? If SuccMBB can fallthrough itself, that can be changed
-    // into a jmp.
-    MachineBasicBlock *TBB = 0, *FBB = 0;
-    SmallVector<MachineOperand, 4> Cond;
-    if (TII->AnalyzeBranch(*SuccMBB, TBB, FBB, Cond))
-      continue;
-    if (!TBB && Cond.empty())
-      TBB = next(MachineFunction::iterator(SuccMBB));
-    else if (!FBB && !Cond.empty())
-      FBB = next(MachineFunction::iterator(SuccMBB));
-
-    // This calculate the cost of the transformation. Also, it finds the *only*
-    // intra-loop edge if there is one.
-    int Cost = 0;
-    bool HasOneIntraSucc = true;
-    MachineBasicBlock *IntraSucc = 0;
-    for (MachineBasicBlock::succ_iterator SI = SuccMBB->succ_begin(),
-           SE = SuccMBB->succ_end(); SI != SE; ++SI) {
-      MachineBasicBlock *SSMBB = *SI;
-      if (MLI->getLoopFor(SSMBB) == L) {
-        if (!IntraSucc)
-          IntraSucc = SSMBB;
-        else
-          HasOneIntraSucc = false;
+      // Check to see if this block is already contiguous with the main
+      // portion of the loop.
+      if (!ContiguousBlocks.insert(BB))
+        continue;
+
+      // Move the block.
+      Changed = true;
+
+      // Process this block and all loop blocks contiguous with it, to keep
+      // them in their relative order.
+      MachineFunction::iterator Begin = BB;
+      MachineFunction::iterator End = next(MachineFunction::iterator(BB));
+      for (; End != MF.end(); ++End) {
+        if (!L->contains(End)) break;
+        if (!HasAnalyzableTerminator(End)) break;
+        ContiguousBlocks.insert(End);
+        ++NumIntraMoved;
       }
 
-      if (SuccMBB->isLayoutSuccessor(SSMBB))
-        // This will become a jmp.
-        ++Cost;
-      else if (MBB->isLayoutSuccessor(SSMBB)) {
-        // One of the successor will become the new fallthrough.
-        if (SSMBB == FBB) {
-          FBB = 0;
-          --Cost;
-        } else if (!FBB && SSMBB == TBB && Cond.empty()) {
-          TBB = 0;
-          --Cost;
-        } else if (!Cond.empty() && !TII->ReverseBranchCondition(Cond)) {
-          assert(SSMBB == TBB);
-          TBB = FBB;
-          FBB = 0;
-          --Cost;
+      // If we're inserting at the bottom of the loop, and the code we're
+      // moving originally had fall-through successors, bring the sucessors
+      // up with the loop blocks to preserve the fall-through edges.
+      if (!InsertAtTop)
+        for (; End != MF.end(); ++End) {
+          if (L->contains(End)) break;
+          if (!HasAnalyzableTerminator(End)) break;
+          if (!HasFallthrough(prior(End))) break;
         }
-      }
-    }
-    if (Cost)
-      continue;
-
-    // Now, let's move the successor to below the BB to eliminate the jmp.
-    SuccMBB->moveAfter(MBB);
-    TII->RemoveBranch(*MBB);
-    TII->RemoveBranch(*SuccMBB);
-    if (TBB)
-      TII->InsertBranch(*SuccMBB, TBB, FBB, Cond);
-    ChangedMBBs.insert(MBB);
-    ChangedMBBs.insert(SuccMBB);
-    if (FtMBB) {
-      TII->RemoveBranch(*FtMBB);
-      TII->InsertBranch(*FtMBB, FtTBB, FtFBB, FtCond);
-      ChangedMBBs.insert(FtMBB);
+
+      // Move the blocks. This may invalidate TopMBB and/or BotMBB, but
+      // we don't need them anymore at this point.
+      Splice(MF, InsertPt, Begin, End);
     }
-    Changed = true;
-  }
 
-  ++NumIntraMoved;
+  return Changed;
+}
+
+/// OptimizeIntraLoopEdgesInLoopNest - Reposition loop blocks to minimize
+/// intra-loop branching and to form contiguous loops.
+///
+/// This code takes the approach of making minor changes to the existing
+/// layout to fix specific loop-oriented problems. Also, it depends on
+/// AnalyzeBranch, which can't understand complex control instructions.
+///
+bool CodePlacementOpt::OptimizeIntraLoopEdgesInLoopNest(MachineFunction &MF,
+                                                        MachineLoop *L) {
+  bool Changed = false;
+
+  // Do optimization for nested loops.
+  for (MachineLoop::iterator I = L->begin(), E = L->end(); I != E; ++I)
+    Changed |= OptimizeIntraLoopEdgesInLoopNest(MF, *I);
+
+  // Do optimization for this loop.
+  Changed |= EliminateUnconditionalJumpsToTop(MF, L);
+  Changed |= MoveDiscontiguousLoopBlocks(MF, L);
+
+  return Changed;
+}
+
+/// OptimizeIntraLoopEdges - Reposition loop blocks to minimize
+/// intra-loop branching and to form contiguous loops.
+///
+bool CodePlacementOpt::OptimizeIntraLoopEdges(MachineFunction &MF) {
+  bool Changed = false;
+
+  if (!TLI->shouldOptimizeCodePlacement())
+    return Changed;
+
+  // Do optimization for each loop in the function.
+  for (MachineLoopInfo::iterator I = MLI->begin(), E = MLI->end();
+       I != E; ++I)
+    if (!(*I)->getParentLoop())
+      Changed |= OptimizeIntraLoopEdgesInLoopNest(MF, *I);
+
   return Changed;
 }
 
@@ -255,6 +445,8 @@ bool CodePlacementOpt::AlignLoops(MachineFunction &MF) {
   return Changed;
 }
 
+/// AlignLoop - Align loop headers to target preferred alignments.
+///
 bool CodePlacementOpt::AlignLoop(MachineFunction &MF, MachineLoop *L,
                                  unsigned Align) {
   bool Changed = false;
@@ -263,17 +455,7 @@ bool CodePlacementOpt::AlignLoop(MachineFunction &MF, MachineLoop *L,
   for (MachineLoop::iterator I = L->begin(), E = L->end(); I != E; ++I)
     Changed |= AlignLoop(MF, *I, Align);
 
-  MachineBasicBlock *TopMBB = L->getHeader();
-  if (TopMBB == MF.begin()) return Changed;
-
-  MachineBasicBlock *PredMBB = prior(MachineFunction::iterator(TopMBB));
-  while (MLI->getLoopFor(PredMBB) == L) {
-    TopMBB = PredMBB;
-    if (TopMBB == MF.begin()) return Changed;
-    PredMBB = prior(MachineFunction::iterator(TopMBB));
-  }
-
-  TopMBB->setAlignment(Align);
+  L->getTopBlock()->setAlignment(Align);
   Changed = true;
   ++NumLoopsAligned;
 
@@ -288,30 +470,9 @@ bool CodePlacementOpt::runOnMachineFunction(MachineFunction &MF) {
   TLI = MF.getTarget().getTargetLowering();
   TII = MF.getTarget().getInstrInfo();
 
-  // Analyze the BBs first and keep track of BBs that
-  // end with an unconditional jmp to another block in the same loop.
-  for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) {
-    MachineBasicBlock *MBB = I;
-    if (MBB->isLandingPad())
-      continue;
-    MachineLoop *L = MLI->getLoopFor(MBB);
-    if (!L)
-      continue;
-
-    MachineBasicBlock *TBB = 0, *FBB = 0;
-    SmallVector<MachineOperand, 4> Cond;
-    if (TII->AnalyzeBranch(*MBB, TBB, FBB, Cond) || !Cond.empty())
-      continue;
-    if (MLI->getLoopFor(TBB) == L && !TBB->isLandingPad())
-      UncondJmpMBBs.push_back(std::make_pair(MBB, TBB));
-  }
-
-  bool Changed = OptimizeIntraLoopEdges();
+  bool Changed = OptimizeIntraLoopEdges(MF);
 
   Changed |= AlignLoops(MF);
 
-  ChangedMBBs.clear();
-  UncondJmpMBBs.clear();
-
   return Changed;
 }
diff --git a/lib/CodeGen/ExactHazardRecognizer.cpp b/lib/CodeGen/ExactHazardRecognizer.cpp
index 4f32c2b..f35d196 100644
--- a/lib/CodeGen/ExactHazardRecognizer.cpp
+++ b/lib/CodeGen/ExactHazardRecognizer.cpp
@@ -22,7 +22,8 @@
 
 using namespace llvm;
 
-ExactHazardRecognizer::ExactHazardRecognizer(const InstrItineraryData &LItinData) :
+ExactHazardRecognizer::
+ExactHazardRecognizer(const InstrItineraryData &LItinData) :
   ScheduleHazardRecognizer(), ItinData(LItinData) 
 {
   // Determine the maximum depth of any itinerary. This determines the
diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp
index 4e713a6..e58a9ca 100644
--- a/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/lib/CodeGen/LLVMTargetMachine.cpp
@@ -323,7 +323,7 @@ bool LLVMTargetMachine::addCommonCodeGenPasses(PassManagerBase &PM,
 
   // Second pass scheduler.
   if (OptLevel != CodeGenOpt::None) {
-    PM.add(createPostRAScheduler());
+    PM.add(createPostRAScheduler(OptLevel));
     printAndVerify(PM);
   }
 
diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp
index 93d3d4c..79f46f3 100644
--- a/lib/CodeGen/LiveIntervalAnalysis.cpp
+++ b/lib/CodeGen/LiveIntervalAnalysis.cpp
@@ -28,7 +28,6 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/Passes.h"
-#include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetMachine.h"
@@ -2603,7 +2602,19 @@ bool LiveIntervals::spillPhysRegAroundRegDefsUses(const LiveInterval &li,
            tri_->isSuperRegister(*AS, SpillReg));
 
   bool Cut = false;
-  LiveInterval &pli = getInterval(SpillReg);
+  SmallVector<unsigned, 4> PRegs;
+  if (hasInterval(SpillReg))
+    PRegs.push_back(SpillReg);
+  else {
+    SmallSet<unsigned, 4> Added;
+    for (const unsigned* AS = tri_->getSubRegisters(SpillReg); *AS; ++AS)
+      if (Added.insert(*AS) && hasInterval(*AS)) {
+        PRegs.push_back(*AS);
+        for (const unsigned* ASS = tri_->getSubRegisters(*AS); *ASS; ++ASS)
+          Added.insert(*ASS);
+      }
+  }
+
   SmallPtrSet<MachineInstr*, 8> SeenMIs;
   for (MachineRegisterInfo::reg_iterator I = mri_->reg_begin(li.reg),
          E = mri_->reg_end(); I != E; ++I) {
@@ -2613,8 +2624,12 @@ bool LiveIntervals::spillPhysRegAroundRegDefsUses(const LiveInterval &li,
       continue;
     SeenMIs.insert(MI);
     LiveIndex Index = getInstructionIndex(MI);
-    if (pli.liveAt(Index)) {
-      vrm.addEmergencySpill(SpillReg, MI);
+    for (unsigned i = 0, e = PRegs.size(); i != e; ++i) {
+      unsigned PReg = PRegs[i];
+      LiveInterval &pli = getInterval(PReg);
+      if (!pli.liveAt(Index))
+        continue;
+      vrm.addEmergencySpill(PReg, MI);
       LiveIndex StartIdx = getLoadIndex(Index);
       LiveIndex EndIdx = getNextSlot(getStoreIndex(Index));
       if (pli.isInOneLiveRange(StartIdx, EndIdx)) {
@@ -2626,12 +2641,12 @@ bool LiveIntervals::spillPhysRegAroundRegDefsUses(const LiveInterval &li,
         Msg << "Ran out of registers during register allocation!";
         if (MI->getOpcode() == TargetInstrInfo::INLINEASM) {
           Msg << "\nPlease check your inline asm statement for invalid "
-               << "constraints:\n";
+              << "constraints:\n";
           MI->print(Msg, tm_);
         }
         llvm_report_error(Msg.str());
       }
-      for (const unsigned* AS = tri_->getSubRegisters(SpillReg); *AS; ++AS) {
+      for (const unsigned* AS = tri_->getSubRegisters(PReg); *AS; ++AS) {
         if (!hasInterval(*AS))
           continue;
         LiveInterval &spli = getInterval(*AS);
diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp
index b9d3ba7..1f85e92 100644
--- a/lib/CodeGen/MachineInstr.cpp
+++ b/lib/CodeGen/MachineInstr.cpp
@@ -220,8 +220,10 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const {
           OS << "imp-";
         OS << "def";
         NeedComma = true;
-      } else if (isImplicit())
+      } else if (isImplicit()) {
           OS << "imp-use";
+          NeedComma = true;
+      }
 
       if (isKill() || isDead() || isUndef()) {
         if (NeedComma) OS << ',';
diff --git a/lib/CodeGen/MachineLoopInfo.cpp b/lib/CodeGen/MachineLoopInfo.cpp
index 2da8e37..db77d19 100644
--- a/lib/CodeGen/MachineLoopInfo.cpp
+++ b/lib/CodeGen/MachineLoopInfo.cpp
@@ -43,3 +43,31 @@ void MachineLoopInfo::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<MachineDominatorTree>();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
+
+MachineBasicBlock *MachineLoop::getTopBlock() {
+  MachineBasicBlock *TopMBB = getHeader();
+  MachineFunction::iterator Begin = TopMBB->getParent()->begin();
+  if (TopMBB != Begin) {
+    MachineBasicBlock *PriorMBB = prior(MachineFunction::iterator(TopMBB));
+    while (contains(PriorMBB)) {
+      TopMBB = PriorMBB;
+      if (TopMBB == Begin) break;
+      PriorMBB = prior(MachineFunction::iterator(TopMBB));
+    }
+  }
+  return TopMBB;
+}
+
+MachineBasicBlock *MachineLoop::getBottomBlock() {
+  MachineBasicBlock *BotMBB = getHeader();
+  MachineFunction::iterator End = BotMBB->getParent()->end();
+  if (BotMBB != prior(End)) {
+    MachineBasicBlock *NextMBB = next(MachineFunction::iterator(BotMBB));
+    while (contains(NextMBB)) {
+      BotMBB = NextMBB;
+      if (BotMBB == next(MachineFunction::iterator(BotMBB))) break;
+      NextMBB = next(MachineFunction::iterator(BotMBB));
+    }
+  }
+  return BotMBB;
+}
diff --git a/lib/CodeGen/MachineSink.cpp b/lib/CodeGen/MachineSink.cpp
index 0f3b33f..a00bebb 100644
--- a/lib/CodeGen/MachineSink.cpp
+++ b/lib/CodeGen/MachineSink.cpp
@@ -34,10 +34,8 @@ STATISTIC(NumSunk, "Number of machine instructions sunk");
 
 namespace {
   class VISIBILITY_HIDDEN MachineSinking : public MachineFunctionPass {
-    const TargetMachine   *TM;
     const TargetInstrInfo *TII;
     const TargetRegisterInfo *TRI;
-    MachineFunction       *CurMF; // Current MachineFunction
     MachineRegisterInfo  *RegInfo; // Machine register information
     MachineDominatorTree *DT;   // Machine dominator tree
     AliasAnalysis *AA;
@@ -92,19 +90,16 @@ bool MachineSinking::AllUsesDominatedByBlock(unsigned Reg,
   return true;
 }
 
-
-
 bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
   DEBUG(errs() << "******** Machine Sinking ********\n");
   
-  CurMF = &MF;
-  TM = &CurMF->getTarget();
-  TII = TM->getInstrInfo();
-  TRI = TM->getRegisterInfo();
-  RegInfo = &CurMF->getRegInfo();
+  const TargetMachine &TM = MF.getTarget();
+  TII = TM.getInstrInfo();
+  TRI = TM.getRegisterInfo();
+  RegInfo = &MF.getRegInfo();
   DT = &getAnalysis<MachineDominatorTree>();
   AA = &getAnalysis<AliasAnalysis>();
-  AllocatableSet = TRI->getAllocatableSet(*CurMF);
+  AllocatableSet = TRI->getAllocatableSet(MF);
 
   bool EverMadeChange = false;
   
@@ -112,7 +107,7 @@ bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
     bool MadeChange = false;
 
     // Process all basic blocks.
-    for (MachineFunction::iterator I = CurMF->begin(), E = CurMF->end(); 
+    for (MachineFunction::iterator I = MF.begin(), E = MF.end(); 
          I != E; ++I)
       MadeChange |= ProcessBlock(*I);
     
@@ -256,7 +251,7 @@ bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) {
   if (SuccToSinkTo->isLandingPad())
     return false;
   
-  // If is not possible to sink an instruction into its own block.  This can
+  // It is not possible to sink an instruction into its own block.  This can
   // happen with loops.
   if (MI->getParent() == SuccToSinkTo)
     return false;
diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp
index e52158c..8fdbe9b 100644
--- a/lib/CodeGen/PostRASchedulerList.cpp
+++ b/lib/CodeGen/PostRASchedulerList.cpp
@@ -78,10 +78,12 @@ DebugMod("postra-sched-debugmod",
 namespace {
   class VISIBILITY_HIDDEN PostRAScheduler : public MachineFunctionPass {
     AliasAnalysis *AA;
+    CodeGenOpt::Level OptLevel;
 
   public:
     static char ID;
-    PostRAScheduler() : MachineFunctionPass(&ID) {}
+    PostRAScheduler(CodeGenOpt::Level ol) :
+      MachineFunctionPass(&ID), OptLevel(ol) {}
 
     void getAnalysisUsage(AnalysisUsage &AU) const {
       AU.setPreservesCFG();
@@ -126,6 +128,9 @@ namespace {
     /// AA - AliasAnalysis for making memory reference queries.
     AliasAnalysis *AA;
 
+    /// AntiDepMode - Anti-dependence breaking mode
+    TargetSubtarget::AntiDepBreakMode AntiDepMode;
+
     /// Classes - For live regs that are only used in one register class in a
     /// live range, the register class. If the register is not live, the
     /// corresponding value is null. If the register is live but used in
@@ -154,10 +159,11 @@ namespace {
                          const MachineLoopInfo &MLI,
                          const MachineDominatorTree &MDT,
                          ScheduleHazardRecognizer *HR,
-                         AliasAnalysis *aa)
+                         AliasAnalysis *aa,
+                         TargetSubtarget::AntiDepBreakMode adm)
       : ScheduleDAGInstrs(MF, MLI, MDT), Topo(SUnits),
         AllocatableSet(TRI->getAllocatableSet(MF)),
-        HazardRec(HR), AA(aa) {}
+      HazardRec(HR), AA(aa), AntiDepMode(adm) {}
 
     ~SchedulePostRATDList() {
       delete HazardRec;
@@ -232,14 +238,21 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) {
   AA = &getAnalysis<AliasAnalysis>();
 
   // Check for explicit enable/disable of post-ra scheduling.
+  TargetSubtarget::AntiDepBreakMode AntiDepMode = TargetSubtarget::ANTIDEP_NONE;
   if (EnablePostRAScheduler.getPosition() > 0) {
     if (!EnablePostRAScheduler)
-      return true;
+      return false;
   } else {
-    // Check that post-RA scheduling is enabled for this function
+    // Check that post-RA scheduling is enabled for this target.
     const TargetSubtarget &ST = Fn.getTarget().getSubtarget<TargetSubtarget>();
-    if (!ST.enablePostRAScheduler())
-      return true;
+    if (!ST.enablePostRAScheduler(OptLevel, AntiDepMode))
+      return false;
+  }
+
+  // Check for antidep breaking override...
+  if (EnableAntiDepBreaking.getPosition() > 0) {
+    AntiDepMode = (EnableAntiDepBreaking) ? 
+      TargetSubtarget::ANTIDEP_CRITICAL : TargetSubtarget::ANTIDEP_NONE;
   }
 
   DEBUG(errs() << "PostRAScheduler\n");
@@ -251,7 +264,7 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) {
     (ScheduleHazardRecognizer *)new ExactHazardRecognizer(InstrItins) :
     (ScheduleHazardRecognizer *)new SimpleHazardRecognizer();
 
-  SchedulePostRATDList Scheduler(Fn, MLI, MDT, HR, AA);
+  SchedulePostRATDList Scheduler(Fn, MLI, MDT, HR, AA, AntiDepMode);
 
   // Loop over all of the basic blocks
   for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
@@ -391,7 +404,7 @@ void SchedulePostRATDList::Schedule() {
   // Build the scheduling graph.
   BuildSchedGraph(AA);
 
-  if (EnableAntiDepBreaking) {
+  if (AntiDepMode != TargetSubtarget::ANTIDEP_NONE) {
     if (BreakAntiDependencies()) {
       // We made changes. Update the dependency graph.
       // Theoretically we could update the graph in place:
@@ -1195,6 +1208,6 @@ void SchedulePostRATDList::ListScheduleTopDown() {
 //                         Public Constructor Functions
 //===----------------------------------------------------------------------===//
 
-FunctionPass *llvm::createPostRAScheduler() {
-  return new PostRAScheduler();
+FunctionPass *llvm::createPostRAScheduler(CodeGenOpt::Level OptLevel) {
+  return new PostRAScheduler(OptLevel);
 }
diff --git a/lib/CodeGen/PreAllocSplitting.cpp b/lib/CodeGen/PreAllocSplitting.cpp
index 8fa07d4..726869a 100644
--- a/lib/CodeGen/PreAllocSplitting.cpp
+++ b/lib/CodeGen/PreAllocSplitting.cpp
@@ -952,7 +952,7 @@ MachineInstr* PreAllocSplitting::FoldSpill(unsigned vreg,
   if (I != IntervalSSMap.end()) {
     SS = I->second;
   } else {
-    SS = MFI->CreateStackObject(RC->getSize(), RC->getAlignment());    
+    SS = MFI->CreateStackObject(RC->getSize(), RC->getAlignment());
   }
   
   MachineInstr* FMI = TII->foldMemoryOperand(*MBB->getParent(),
diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp
index 7af0bba..a0860a1 100644
--- a/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/lib/CodeGen/PrologEpilogInserter.cpp
@@ -259,7 +259,7 @@ void PEI::calculateCalleeSavedRegisters(MachineFunction &Fn) {
       // the TargetRegisterClass if the stack alignment is smaller. Use the
       // min.
       Align = std::min(Align, StackAlign);
-      FrameIdx = FFI->CreateStackObject(RC->getSize(), Align);
+      FrameIdx = FFI->CreateStackObject(RC->getSize(), Align, true);
       if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
       if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
     } else {
@@ -729,10 +729,12 @@ void PEI::replaceFrameIndices(MachineFunction &Fn) {
 /// the instruciton range. Return the operand number of the kill in Operand.
 static MachineBasicBlock::iterator
 findLastUseReg(MachineBasicBlock::iterator I, MachineBasicBlock::iterator ME,
-               unsigned Reg, unsigned *Operand) {
+               unsigned Reg) {
   // Scan forward to find the last use of this virtual register
   for (++I; I != ME; ++I) {
     MachineInstr *MI = I;
+    bool isDefInsn = false;
+    bool isKillInsn = false;
     for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i)
       if (MI->getOperand(i).isReg()) {
         unsigned OpReg = MI->getOperand(i).getReg();
@@ -740,13 +742,14 @@ findLastUseReg(MachineBasicBlock::iterator I, MachineBasicBlock::iterator ME,
           continue;
         assert (OpReg == Reg
                 && "overlapping use of scavenged index register!");
-        // If this is the killing use, we're done
-        if (MI->getOperand(i).isKill()) {
-          if (Operand)
-            *Operand = i;
-          return I;
-        }
+        // If this is the killing use, we have a candidate.
+        if (MI->getOperand(i).isKill())
+          isKillInsn = true;
+        else if (MI->getOperand(i).isDef())
+          isDefInsn = true;
       }
+    if (isKillInsn && !isDefInsn)
+      return I;
   }
   // If we hit the end of the basic block, there was no kill of
   // the virtual register, which is wrong.
@@ -763,10 +766,13 @@ void PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) {
        E = Fn.end(); BB != E; ++BB) {
     RS->enterBasicBlock(BB);
 
+    // FIXME: The logic flow in this function is still too convoluted.
+    // It needs a cleanup refactoring. Do that in preparation for tracking
+    // more than one scratch register value and using ranges to find
+    // available scratch registers.
     unsigned CurrentVirtReg = 0;
     unsigned CurrentScratchReg = 0;
     bool havePrevValue = false;
-    unsigned PrevScratchReg = 0;
     int PrevValue = 0;
     MachineInstr *PrevLastUseMI = NULL;
     unsigned PrevLastUseOp = 0;
@@ -776,37 +782,39 @@ void PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) {
 
     // The instruction stream may change in the loop, so check BB->end()
     // directly.
-    for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ++I) {
+    for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ) {
       MachineInstr *MI = I;
-      for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i)
+      bool isDefInsn = false;
+      bool isKillInsn = false;
+      bool clobbersScratchReg = false;
+      bool DoIncr = true;
+      for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
         if (MI->getOperand(i).isReg()) {
           MachineOperand &MO = MI->getOperand(i);
           unsigned Reg = MO.getReg();
           if (Reg == 0)
             continue;
           if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
-            // If we have an active scavenged register, we shouldn't be
-            // seeing any references to it.
-            assert (Reg != CurrentScratchReg
-                    && "overlapping use of scavenged frame index register!");
-
             // If we have a previous scratch reg, check and see if anything
             // here kills whatever value is in there.
-            if (Reg == PrevScratchReg) {
+            if (Reg == CurrentScratchReg) {
               if (MO.isUse()) {
                 // Two-address operands implicitly kill
-                if (MO.isKill() || MI->isRegTiedToDefOperand(i)) {
-                  havePrevValue = false;
-                  PrevScratchReg = 0;
-                }
+                if (MO.isKill() || MI->isRegTiedToDefOperand(i))
+                  clobbersScratchReg = true;
               } else {
                 assert (MO.isDef());
-                havePrevValue = false;
-                PrevScratchReg = 0;
+                clobbersScratchReg = true;
               }
             }
             continue;
           }
+          // If this is a def, remember that this insn defines the value.
+          // This lets us properly consider insns which re-use the scratch
+          // register, such as r2 = sub r2, #imm, in the middle of the
+          // scratch range.
+          if (MO.isDef())
+            isDefInsn = true;
 
           // Have we already allocated a scratch register for this virtual?
           if (Reg != CurrentVirtReg) {
@@ -837,50 +845,75 @@ void PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) {
               // for the virtual register are exclusively for the purpose
               // of populating the value in the register. That's reasonable
               // for these frame index registers, but it's still a very, very
-              // strong assumption. Perhaps this implies that the frame index
-              // elimination should be before register allocation, with
-              // conservative heuristics since we'll know less then, and
-              // the reuse calculations done directly when doing the code-gen?
+              // strong assumption. rdar://7322732. Better would be to
+              // explicitly check each instruction in the range for references
+              // to the virtual register. Only delete those insns that
+              // touch the virtual register.
 
               // Find the last use of the new virtual register. Remove all
               // instruction between here and there, and update the current
               // instruction to reference the last use insn instead.
               MachineBasicBlock::iterator LastUseMI =
-                findLastUseReg(I, BB->end(), Reg, &i);
+                findLastUseReg(I, BB->end(), Reg);
+
               // Remove all instructions up 'til the last use, since they're
               // just calculating the value we already have.
               BB->erase(I, LastUseMI);
               MI = I = LastUseMI;
-              e = MI->getNumOperands();
 
-              CurrentScratchReg = PrevScratchReg;
-              // Extend the live range of the register
+              // Extend the live range of the scratch register
               PrevLastUseMI->getOperand(PrevLastUseOp).setIsKill(false);
               RS->setUsed(CurrentScratchReg);
-            } else {
               CurrentVirtReg = Reg;
-              const TargetRegisterClass *RC = Fn.getRegInfo().getRegClass(Reg);
-              CurrentScratchReg = RS->FindUnusedReg(RC);
-              if (CurrentScratchReg == 0)
-                // No register is "free". Scavenge a register.
-                CurrentScratchReg = RS->scavengeRegister(RC, I, SPAdj);
 
-              PrevValue = Value;
+              // We deleted the instruction we were scanning the operands of.
+              // Jump back to the instruction iterator loop. Don't increment
+              // past this instruction since we updated the iterator already.
+              DoIncr = false;
+              break;
             }
+
+            // Scavenge a new scratch register
+            CurrentVirtReg = Reg;
+            const TargetRegisterClass *RC = Fn.getRegInfo().getRegClass(Reg);
+            CurrentScratchReg = RS->FindUnusedReg(RC);
+            if (CurrentScratchReg == 0)
+              // No register is "free". Scavenge a register.
+              CurrentScratchReg = RS->scavengeRegister(RC, I, SPAdj);
+
+            PrevValue = Value;
           }
+          // replace this reference to the virtual register with the
+          // scratch register.
           assert (CurrentScratchReg && "Missing scratch register!");
           MI->getOperand(i).setReg(CurrentScratchReg);
 
-          // If this is the last use of the register, stop tracking it.
           if (MI->getOperand(i).isKill()) {
-            PrevScratchReg = CurrentScratchReg;
-            PrevLastUseMI = MI;
+            isKillInsn = true;
             PrevLastUseOp = i;
-            CurrentScratchReg = CurrentVirtReg = 0;
-            havePrevValue = trackingCurrentValue;
+            PrevLastUseMI = MI;
           }
         }
-      RS->forward(MI);
+      }
+      // If this is the last use of the scratch, stop tracking it. The
+      // last use will be a kill operand in an instruction that does
+      // not also define the scratch register.
+      if (isKillInsn && !isDefInsn) {
+        CurrentVirtReg = 0;
+        havePrevValue = trackingCurrentValue;
+      }
+      // Similarly, notice if instruction clobbered the value in the
+      // register we're tracking for possible later reuse. This is noted
+      // above, but enforced here since the value is still live while we
+      // process the rest of the operands of the instruction.
+      if (clobbersScratchReg) {
+        havePrevValue = false;
+        CurrentScratchReg = 0;
+      }
+      if (DoIncr) {
+        RS->forward(I);
+        ++I;
+      }
     }
   }
 }
diff --git a/lib/CodeGen/PseudoSourceValue.cpp b/lib/CodeGen/PseudoSourceValue.cpp
index 00c5d46..70e8640 100644
--- a/lib/CodeGen/PseudoSourceValue.cpp
+++ b/lib/CodeGen/PseudoSourceValue.cpp
@@ -63,6 +63,8 @@ namespace {
 
     virtual bool isConstant(const MachineFrameInfo *MFI) const;
 
+    virtual bool isAliased(const MachineFrameInfo *MFI) const;
+
     virtual void printCustom(raw_ostream &OS) const {
       OS << "FixedStack" << FI;
     }
@@ -89,6 +91,26 @@ bool PseudoSourceValue::isConstant(const MachineFrameInfo *) const {
   return false;
 }
 
+bool PseudoSourceValue::isAliased(const MachineFrameInfo *MFI) const {
+  if (this == getStack() ||
+      this == getGOT() ||
+      this == getConstantPool() ||
+      this == getJumpTable())
+    return false;
+  llvm_unreachable("Unknown PseudoSourceValue!");
+  return true;
+}
+
 bool FixedStackPseudoSourceValue::isConstant(const MachineFrameInfo *MFI) const{
   return MFI && MFI->isImmutableObjectIndex(FI);
 }
+
+bool FixedStackPseudoSourceValue::isAliased(const MachineFrameInfo *MFI) const {
+  // Negative frame indices are used for special things that don't
+  // appear in LLVM IR. Non-negative indices may be used for things
+  // like static allocas.
+  if (!MFI)
+    return FI >= 0;
+  // Spill slots should not alias others.
+  return !MFI->isFixedObjectIndex(FI) && !MFI->isSpillSlotObjectIndex(FI);
+}
diff --git a/lib/CodeGen/RegAllocLocal.cpp b/lib/CodeGen/RegAllocLocal.cpp
index 6caa2d3..28ede55 100644
--- a/lib/CodeGen/RegAllocLocal.cpp
+++ b/lib/CodeGen/RegAllocLocal.cpp
@@ -263,7 +263,7 @@ int RALocal::getStackSpaceFor(unsigned VirtReg, const TargetRegisterClass *RC) {
 
   // Allocate a new stack object for this spill location...
   int FrameIdx = MF->getFrameInfo()->CreateStackObject(RC->getSize(),
-                                                       RC->getAlignment());
+                                                       RC->getAlignment(),true);
 
   // Assign the slot...
   StackSlotForVirtReg[VirtReg] = FrameIdx;
diff --git a/lib/CodeGen/RegisterScavenging.cpp b/lib/CodeGen/RegisterScavenging.cpp
index 5f1c4e2..2518ce1 100644
--- a/lib/CodeGen/RegisterScavenging.cpp
+++ b/lib/CodeGen/RegisterScavenging.cpp
@@ -300,7 +300,7 @@ unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC,
 
   // If the target knows how to save/restore the register, let it do so;
   // otherwise, use the emergency stack spill slot.
-  if (!TRI->saveScavengerRegister(*MBB, I, RC, SReg)) {
+  if (!TRI->saveScavengerRegister(*MBB, I, UseMI, RC, SReg)) {
     // Spill the scavenged register before I.
     assert(ScavengingFrameIndex >= 0 &&
            "Cannot scavenge register without an emergency spill slot!");
@@ -310,8 +310,9 @@ unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC,
 
     // Restore the scavenged register before its use (or first terminator).
     TII->loadRegFromStackSlot(*MBB, UseMI, SReg, ScavengingFrameIndex, RC);
-  } else
-    TRI->restoreScavengerRegister(*MBB, UseMI, RC, SReg);
+    II = prior(UseMI);
+    TRI->eliminateFrameIndex(II, SPAdj, NULL, this);
+  }
 
   ScavengeRestore = prior(UseMI);
 
diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp
index 44e9296..43454dd 100644
--- a/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -32,7 +32,9 @@ using namespace llvm;
 ScheduleDAGInstrs::ScheduleDAGInstrs(MachineFunction &mf,
                                      const MachineLoopInfo &mli,
                                      const MachineDominatorTree &mdt)
-  : ScheduleDAG(mf), MLI(mli), MDT(mdt), LoopRegs(MLI, MDT) {}
+  : ScheduleDAG(mf), MLI(mli), MDT(mdt), LoopRegs(MLI, MDT) {
+  MFI = mf.getFrameInfo();
+}
 
 /// Run - perform scheduling.
 ///
@@ -95,7 +97,8 @@ static const Value *getUnderlyingObject(const Value *V) {
 /// getUnderlyingObjectForInstr - If this machine instr has memory reference
 /// information and it can be tracked to a normal reference to a known
 /// object, return the Value for that object. Otherwise return null.
-static const Value *getUnderlyingObjectForInstr(const MachineInstr *MI) {
+static const Value *getUnderlyingObjectForInstr(const MachineInstr *MI,
+                                                const MachineFrameInfo *MFI) {
   if (!MI->hasOneMemOperand() ||
       !(*MI->memoperands_begin())->getValue() ||
       (*MI->memoperands_begin())->isVolatile())
@@ -106,10 +109,19 @@ static const Value *getUnderlyingObjectForInstr(const MachineInstr *MI) {
     return 0;
 
   V = getUnderlyingObject(V);
-  if (!isa<PseudoSourceValue>(V) && !isIdentifiedObject(V))
-    return 0;
+  if (const PseudoSourceValue *PSV = dyn_cast<PseudoSourceValue>(V)) {
+    // For now, ignore PseudoSourceValues which may alias LLVM IR values
+    // because the code that uses this function has no way to cope with
+    // such aliases.
+    if (PSV->isAliased(MFI))
+      return 0;
+    return V;
+  }
 
-  return V;
+  if (isIdentifiedObject(V))
+    return V;
+
+  return 0;
 }
 
 void ScheduleDAGInstrs::StartBlock(MachineBasicBlock *BB) {
@@ -344,7 +356,7 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) {
         // Unknown memory accesses. Assume the worst.
         ChainMMO = 0;
     } else if (TID.mayStore()) {
-      if (const Value *V = getUnderlyingObjectForInstr(MI)) {
+      if (const Value *V = getUnderlyingObjectForInstr(MI, MFI)) {
         // A store to a specific PseudoSourceValue. Add precise dependencies.
         // Handle the def in MemDefs, if there is one.
         std::map<const Value *, SUnit *>::iterator I = MemDefs.find(V);
@@ -377,7 +389,7 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) {
     } else if (TID.mayLoad()) {
       if (MI->isInvariantLoad(AA)) {
         // Invariant load, no chain dependencies needed!
-      } else if (const Value *V = getUnderlyingObjectForInstr(MI)) {
+      } else if (const Value *V = getUnderlyingObjectForInstr(MI, MFI)) {
         // A load from a specific PseudoSourceValue. Add precise dependencies.
         std::map<const Value *, SUnit *>::iterator I = MemDefs.find(V);
         if (I != MemDefs.end())
diff --git a/lib/CodeGen/ScheduleDAGInstrs.h b/lib/CodeGen/ScheduleDAGInstrs.h
index 29e1c98..366c3a8 100644
--- a/lib/CodeGen/ScheduleDAGInstrs.h
+++ b/lib/CodeGen/ScheduleDAGInstrs.h
@@ -98,6 +98,7 @@ namespace llvm {
   class VISIBILITY_HIDDEN ScheduleDAGInstrs : public ScheduleDAG {
     const MachineLoopInfo &MLI;
     const MachineDominatorTree &MDT;
+    const MachineFrameInfo *MFI;
 
     /// Defs, Uses - Remember where defs and uses of each physical register
     /// are as we iterate upward through the instructions. This is allocated
diff --git a/lib/CodeGen/ScheduleDAGPrinter.cpp b/lib/CodeGen/ScheduleDAGPrinter.cpp
index 95ad05e..4851d49 100644
--- a/lib/CodeGen/ScheduleDAGPrinter.cpp
+++ b/lib/CodeGen/ScheduleDAGPrinter.cpp
@@ -18,7 +18,6 @@
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
-#include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Support/Debug.h"
diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 1ed3082..e3f8f0f 100644
--- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -4381,15 +4381,17 @@ SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
 
 SDValue DAGCombiner::visitFNEG(SDNode *N) {
   SDValue N0 = N->getOperand(0);
+  EVT VT = N->getValueType(0);
 
   if (isNegatibleForFree(N0, LegalOperations))
     return GetNegatedExpression(N0, DAG, LegalOperations);
 
   // Transform fneg(bitconvert(x)) -> bitconvert(x^sign) to avoid loading
   // constant pool values.
-  if (N0.getOpcode() == ISD::BIT_CONVERT && N0.getNode()->hasOneUse() &&
-      N0.getOperand(0).getValueType().isInteger() &&
-      !N0.getOperand(0).getValueType().isVector()) {
+  if (N0.getOpcode() == ISD::BIT_CONVERT && 
+      !VT.isVector() &&
+      N0.getNode()->hasOneUse() &&
+      N0.getOperand(0).getValueType().isInteger()) {
     SDValue Int = N0.getOperand(0);
     EVT IntVT = Int.getValueType();
     if (IntVT.isInteger() && !IntVT.isVector()) {
@@ -4397,7 +4399,7 @@ SDValue DAGCombiner::visitFNEG(SDNode *N) {
               DAG.getConstant(APInt::getSignBit(IntVT.getSizeInBits()), IntVT));
       AddToWorkList(Int.getNode());
       return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(),
-                         N->getValueType(0), Int);
+                         VT, Int);
     }
   }
 
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index fc01b07..7138dd2 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1655,8 +1655,7 @@ void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
 }
 
 /// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and
-/// condition code CC on the current target. This routine assumes LHS and rHS
-/// have already been legalized by LegalizeSetCCOperands. It expands SETCC with
+/// condition code CC on the current target. This routine expands SETCC with
 /// illegal condition code into AND / OR of multiple SETCC values.
 void SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT,
                                                  SDValue &LHS, SDValue &RHS,
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 859c656..e1b7022 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -617,6 +617,7 @@ private:
   SDValue WidenVecOp_BIT_CONVERT(SDNode *N);
   SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
   SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
+  SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);
   SDValue WidenVecOp_STORE(SDNode* N);
 
   SDValue WidenVecOp_Convert(SDNode *N);
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index 0eafe62..dbd3e39 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -115,7 +115,8 @@ void DAGTypeLegalizer::ExpandRes_BIT_CONVERT(SDNode *N, SDValue &Lo,
   // Create the stack frame object.  Make sure it is aligned for both
   // the source and expanded destination types.
   unsigned Alignment =
-    TLI.getTargetData()->getPrefTypeAlignment(NOutVT.getTypeForEVT(*DAG.getContext()));
+    TLI.getTargetData()->getPrefTypeAlignment(NOutVT.
+                                              getTypeForEVT(*DAG.getContext()));
   SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
   const Value *SV = PseudoSourceValue::getFixedStack(SPFI);
diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index a03f825..75e1239 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1789,6 +1789,7 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) {
 
   case ISD::BIT_CONVERT:        Res = WidenVecOp_BIT_CONVERT(N); break;
   case ISD::CONCAT_VECTORS:     Res = WidenVecOp_CONCAT_VECTORS(N); break;
+  case ISD::EXTRACT_SUBVECTOR:  Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
   case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
   case ISD::STORE:              Res = WidenVecOp_STORE(N); break;
 
@@ -1893,6 +1894,12 @@ SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
   return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts);
 }
 
+SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
+  SDValue InOp = GetWidenedVector(N->getOperand(0));
+  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(),
+                     N->getValueType(0), InOp, N->getOperand(1));
+}
+
 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
   SDValue InOp = GetWidenedVector(N->getOperand(0));
   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(),
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 542bf64..37736c0 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -30,6 +30,7 @@
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetIntrinsicInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -4600,7 +4601,7 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
       N->InitOperands(new SDUse[NumOps], Ops, NumOps);
       N->OperandsNeedDelete = true;
     } else
-      MN->InitOperands(MN->OperandList, Ops, NumOps);
+      N->InitOperands(N->OperandList, Ops, NumOps);
   }
 
   // Delete any nodes that are still dead after adding the uses for the
@@ -5404,14 +5405,16 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::EH_RETURN: return "EH_RETURN";
   case ISD::ConstantPool:  return "ConstantPool";
   case ISD::ExternalSymbol: return "ExternalSymbol";
-  case ISD::INTRINSIC_WO_CHAIN: {
-    unsigned IID = cast<ConstantSDNode>(getOperand(0))->getZExtValue();
-    return Intrinsic::getName((Intrinsic::ID)IID);
-  }
+  case ISD::INTRINSIC_WO_CHAIN:
   case ISD::INTRINSIC_VOID:
   case ISD::INTRINSIC_W_CHAIN: {
-    unsigned IID = cast<ConstantSDNode>(getOperand(1))->getZExtValue();
-    return Intrinsic::getName((Intrinsic::ID)IID);
+    unsigned OpNo = getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1;
+    unsigned IID = cast<ConstantSDNode>(getOperand(OpNo))->getZExtValue();
+    if (IID < Intrinsic::num_intrinsics)
+      return Intrinsic::getName((Intrinsic::ID)IID);
+    else if (const TargetIntrinsicInfo *TII = G->getTarget().getIntrinsicInfo())
+      return TII->getName(IID);
+    llvm_unreachable("Invalid intrinsic ID");
   }
 
   case ISD::BUILD_VECTOR:   return "BUILD_VECTOR";
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp
index 9017e43..adcc532 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp
@@ -5485,48 +5485,6 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) {
   DAG.setRoot(Chain);
 }
 
-
-void SelectionDAGLowering::visitMalloc(MallocInst &I) {
-  SDValue Src = getValue(I.getOperand(0));
-
-  // Scale up by the type size in the original i32 type width.  Various
-  // mid-level optimizers may make assumptions about demanded bits etc from the
-  // i32-ness of the optimizer: we do not want to promote to i64 and then
-  // multiply on 64-bit targets.
-  // FIXME: Malloc inst should go away: PR715.
-  uint64_t ElementSize = TD->getTypeAllocSize(I.getType()->getElementType());
-  if (ElementSize != 1) {
-    // Src is always 32-bits, make sure the constant fits.
-    assert(Src.getValueType() == MVT::i32);
-    ElementSize = (uint32_t)ElementSize;
-    Src = DAG.getNode(ISD::MUL, getCurDebugLoc(), Src.getValueType(),
-                      Src, DAG.getConstant(ElementSize, Src.getValueType()));
-  }
-  
-  EVT IntPtr = TLI.getPointerTy();
-
-  Src = DAG.getZExtOrTrunc(Src, getCurDebugLoc(), IntPtr);
-
-  TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Node = Src;
-  Entry.Ty = TLI.getTargetData()->getIntPtrType(*DAG.getContext());
-  Args.push_back(Entry);
-
-  bool isTailCall = PerformTailCallOpt &&
-                    isInTailCallPosition(&I, Attribute::None, TLI);
-  std::pair<SDValue,SDValue> Result =
-    TLI.LowerCallTo(getRoot(), I.getType(), false, false, false, false,
-                    0, CallingConv::C, isTailCall,
-                    /*isReturnValueUsed=*/true,
-                    DAG.getExternalSymbol("malloc", IntPtr),
-                    Args, DAG, getCurDebugLoc());
-  if (Result.first.getNode())
-    setValue(&I, Result.first);  // Pointers always fit in registers
-  if (Result.second.getNode())
-    DAG.setRoot(Result.second);
-}
-
 void SelectionDAGLowering::visitFree(FreeInst &I) {
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.h
index 06acc8a..722b1d8 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.h
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.h
@@ -60,7 +60,6 @@ class MachineFunction;
 class MachineInstr;
 class MachineModuleInfo;
 class MachineRegisterInfo;
-class MallocInst;
 class PHINode;
 class PtrToIntInst;
 class ReturnInst;
@@ -529,7 +528,6 @@ private:
   void visitGetElementPtr(User &I);
   void visitSelect(User &I);
 
-  void visitMalloc(MallocInst &I);
   void visitFree(FreeInst &I);
   void visitAlloca(AllocaInst &I);
   void visitLoad(LoadInst &I);
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index ae98da5..72e7f58 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -226,7 +226,7 @@ static void EmitLiveInCopy(MachineBasicBlock *MBB,
   assert(Emitted && "Unable to issue a live-in copy instruction!\n");
   (void) Emitted;
 
-CopyRegMap.insert(std::make_pair(prior(Pos), VirtReg));
+  CopyRegMap.insert(std::make_pair(prior(Pos), VirtReg));
   if (Coalesced) {
     if (&*InsertPos == UseMI) ++InsertPos;
     MBB->erase(UseMI);
diff --git a/lib/CodeGen/StackProtector.cpp b/lib/CodeGen/StackProtector.cpp
index 350bc6e..0204969 100644
--- a/lib/CodeGen/StackProtector.cpp
+++ b/lib/CodeGen/StackProtector.cpp
@@ -111,11 +111,16 @@ bool StackProtector::RequiresStackProtector() const {
           // protectors.
           return true;
 
-        if (const ArrayType *AT = dyn_cast<ArrayType>(AI->getAllocatedType()))
+        if (const ArrayType *AT = dyn_cast<ArrayType>(AI->getAllocatedType())) {
+          // We apparently only care about character arrays.
+          if (AT->getElementType() != Type::getInt8Ty(AT->getContext()))
+            continue;
+
           // If an array has more than SSPBufferSize bytes of allocated space,
           // then we emit stack protectors.
           if (SSPBufferSize <= TD->getTypeAllocSize(AT))
             return true;
+        }
       }
   }
 
diff --git a/lib/CodeGen/VirtRegMap.cpp b/lib/CodeGen/VirtRegMap.cpp
index c78f35b..cac098b 100644
--- a/lib/CodeGen/VirtRegMap.cpp
+++ b/lib/CodeGen/VirtRegMap.cpp
@@ -118,7 +118,7 @@ int VirtRegMap::assignVirt2StackSlot(unsigned virtReg) {
          "attempt to assign stack slot to already spilled register");
   const TargetRegisterClass* RC = MF->getRegInfo().getRegClass(virtReg);
   int SS = MF->getFrameInfo()->CreateStackObject(RC->getSize(),
-                                                RC->getAlignment());
+                                              RC->getAlignment(), /*isSS*/true);
   if (LowSpillSlot == NO_STACK_SLOT)
     LowSpillSlot = SS;
   if (HighSpillSlot == NO_STACK_SLOT || SS > HighSpillSlot)
@@ -162,7 +162,7 @@ int VirtRegMap::getEmergencySpillSlot(const TargetRegisterClass *RC) {
   if (I != EmergencySpillSlots.end())
     return I->second;
   int SS = MF->getFrameInfo()->CreateStackObject(RC->getSize(),
-                                                RC->getAlignment());
+                                              RC->getAlignment(), /*isSS*/true);
   if (LowSpillSlot == NO_STACK_SLOT)
     LowSpillSlot = SS;
   if (HighSpillSlot == NO_STACK_SLOT || SS > HighSpillSlot)