Update clang to r108428.

50 files changed, 1298 insertions, 390 deletions

diff --git a/lib/AST/DeclTemplate.cpp b/lib/AST/DeclTemplate.cpp
index f00eb04..9e1d79d 100644
--- a/lib/AST/DeclTemplate.cpp
+++ b/lib/AST/DeclTemplate.cpp
@@ -223,7 +223,7 @@ ClassTemplateDecl::getInjectedClassNameSpecialization() {
     } else if (NonTypeTemplateParmDecl *NTTP =
                  dyn_cast<NonTypeTemplateParmDecl>(*Param)) {
       Expr *E = new (Context) DeclRefExpr(NTTP,
-                                          NTTP->getType().getNonReferenceType(),
+                                  NTTP->getType().getNonLValueExprType(Context),
                                           NTTP->getLocation());
       TemplateArgs.push_back(TemplateArgument(E));
     } else {
diff --git a/lib/AST/Expr.cpp b/lib/AST/Expr.cpp
index bd97b88..6524a31 100644
--- a/lib/AST/Expr.cpp
+++ b/lib/AST/Expr.cpp
@@ -684,6 +684,8 @@ const char *CastExpr::getCastKindName() const {
     return "Unknown";
   case CastExpr::CK_BitCast:
     return "BitCast";
+  case CastExpr::CK_LValueBitCast:
+    return "LValueBitCast";
   case CastExpr::CK_NoOp:
     return "NoOp";
   case CastExpr::CK_BaseToDerived:
diff --git a/lib/AST/ExprConstant.cpp b/lib/AST/ExprConstant.cpp
index a963182..3c97420 100644
--- a/lib/AST/ExprConstant.cpp
+++ b/lib/AST/ExprConstant.cpp
@@ -563,6 +563,7 @@ bool PointerExprEvaluator::VisitCastExpr(CastExpr* E) {
 
   case CastExpr::CK_NoOp:
   case CastExpr::CK_BitCast:
+  case CastExpr::CK_LValueBitCast:
   case CastExpr::CK_AnyPointerToObjCPointerCast:
   case CastExpr::CK_AnyPointerToBlockPointerCast:
     return Visit(SubExpr);
diff --git a/lib/AST/Type.cpp b/lib/AST/Type.cpp
index ab64eaf..d7929304 100644
--- a/lib/AST/Type.cpp
+++ b/lib/AST/Type.cpp
@@ -992,7 +992,7 @@ const char *BuiltinType::getName(const LangOptions &LO) const {
 
 void FunctionType::ANCHOR() {} // Key function for FunctionType.
 
-QualType QualType::getCallResultType(ASTContext &Context) const {
+QualType QualType::getNonLValueExprType(ASTContext &Context) const {
   if (const ReferenceType *RefType = getTypePtr()->getAs<ReferenceType>())
     return RefType->getPointeeType();
   
@@ -1002,7 +1002,7 @@ QualType QualType::getCallResultType(ASTContext &Context) const {
   //
   // See also C99 6.3.2.1p2.
   if (!Context.getLangOptions().CPlusPlus ||
-      !getTypePtr()->isDependentType() && !getTypePtr()->isRecordType())
+      (!getTypePtr()->isDependentType() && !getTypePtr()->isRecordType()))
     return getUnqualifiedType();
   
   return *this;
diff --git a/lib/Basic/FileManager.cpp b/lib/Basic/FileManager.cpp
index c4296c3..3c91a0f 100644
--- a/lib/Basic/FileManager.cpp
+++ b/lib/Basic/FileManager.cpp
@@ -331,8 +331,8 @@ const FileEntry *FileManager::getFile(const char *NameStart,
 }
 
 const FileEntry *
-FileManager::getVirtualFile(const llvm::StringRef &Filename,
-                            off_t Size, time_t ModificationTime) {
+FileManager::getVirtualFile(llvm::StringRef Filename, off_t Size,
+                            time_t ModificationTime) {
   const char *NameStart = Filename.begin(), *NameEnd = Filename.end();
 
   ++NumFileLookups;
diff --git a/lib/Basic/TargetInfo.cpp b/lib/Basic/TargetInfo.cpp
index e4eaf63..7fcf372 100644
--- a/lib/Basic/TargetInfo.cpp
+++ b/lib/Basic/TargetInfo.cpp
@@ -55,6 +55,9 @@ TargetInfo::TargetInfo(const std::string &T) : Triple(T) {
                       "i64:64:64-f32:32:32-f64:64:64-n32";
   UserLabelPrefix = "_";
   HasAlignMac68kSupport = false;
+
+  // Default to no types using fpret.
+  RealTypeUsesObjCFPRet = 0;
 }
 
 // Out of line virtual dtor for TargetInfo.
diff --git a/lib/Basic/Targets.cpp b/lib/Basic/Targets.cpp
index 8d79316..fdf63e7 100644
--- a/lib/Basic/Targets.cpp
+++ b/lib/Basic/Targets.cpp
@@ -1245,6 +1245,11 @@ public:
     PtrDiffType = SignedInt;
     IntPtrType = SignedInt;
     RegParmMax = 3;
+
+    // Use fpret for all types.
+    RealTypeUsesObjCFPRet = ((1 << TargetInfo::Float) |
+                             (1 << TargetInfo::Double) |
+                             (1 << TargetInfo::LongDouble));
   }
   virtual const char *getVAListDeclaration() const {
     return "typedef char* __builtin_va_list;";
@@ -1411,6 +1416,9 @@ public:
     DescriptionString = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
                         "i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-"
                         "a0:0:64-s0:64:64-f80:128:128-n8:16:32:64";
+
+    // Use fpret only for long double.
+    RealTypeUsesObjCFPRet = (1 << TargetInfo::LongDouble);
   }
   virtual const char *getVAListDeclaration() const {
     return "typedef struct __va_list_tag {"
diff --git a/lib/Checker/GRExprEngine.cpp b/lib/Checker/GRExprEngine.cpp
index 4652a4c..07fee9d 100644
--- a/lib/Checker/GRExprEngine.cpp
+++ b/lib/Checker/GRExprEngine.cpp
@@ -2438,6 +2438,7 @@ void GRExprEngine::VisitCast(CastExpr *CastE, Expr *Ex, ExplodedNode *Pred,
   case CastExpr::CK_Unknown:
   case CastExpr::CK_ArrayToPointerDecay:
   case CastExpr::CK_BitCast:
+  case CastExpr::CK_LValueBitCast:
   case CastExpr::CK_IntegralCast:
   case CastExpr::CK_IntegralToPointer:
   case CastExpr::CK_PointerToIntegral:
diff --git a/lib/Checker/LLVMConventionsChecker.cpp b/lib/Checker/LLVMConventionsChecker.cpp
index 0576f08..c121257 100644
--- a/lib/Checker/LLVMConventionsChecker.cpp
+++ b/lib/Checker/LLVMConventionsChecker.cpp
@@ -36,7 +36,7 @@ static bool IsLLVMStringRef(QualType T) {
 
 /// Check whether the declaration is semantically inside the top-level
 /// namespace named by ns.
-static bool InNamespace(const Decl *D, const llvm::StringRef &NS) {
+static bool InNamespace(const Decl *D, llvm::StringRef NS) {
   const DeclContext *DC = D->getDeclContext();
   const NamespaceDecl *ND = dyn_cast<NamespaceDecl>(D->getDeclContext());
   if (!ND)
diff --git a/lib/CodeGen/CGBlocks.cpp b/lib/CodeGen/CGBlocks.cpp
index 0d05a62..cb9e636 100644
--- a/lib/CodeGen/CGBlocks.cpp
+++ b/lib/CodeGen/CGBlocks.cpp
@@ -253,7 +253,7 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
       CodeGenTypes &Types = CGM.getTypes();
       const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, Args,
                                                        FunctionType::ExtInfo());
-      if (CGM.ReturnTypeUsesSret(FnInfo))
+      if (CGM.ReturnTypeUsesSRet(FnInfo))
         flags |= BLOCK_USE_STRET;
     }
     const llvm::IntegerType *IntTy = cast<llvm::IntegerType>(
diff --git a/lib/CodeGen/CGCall.cpp b/lib/CodeGen/CGCall.cpp
index 1632cb3..3d1e143 100644
--- a/lib/CodeGen/CGCall.cpp
+++ b/lib/CodeGen/CGCall.cpp
@@ -564,10 +564,28 @@ static void CreateCoercedStore(llvm::Value *Src,
 
 /***/
 
-bool CodeGenModule::ReturnTypeUsesSret(const CGFunctionInfo &FI) {
+bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
   return FI.getReturnInfo().isIndirect();
 }
 
+bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
+  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
+    switch (BT->getKind()) {
+    default:
+      return false;
+    case BuiltinType::Float:
+      return getContext().Target.useObjCFPRetForRealType(TargetInfo::Float);
+    case BuiltinType::Double:
+      return getContext().Target.useObjCFPRetForRealType(TargetInfo::Double);
+    case BuiltinType::LongDouble:
+      return getContext().Target.useObjCFPRetForRealType(
+        TargetInfo::LongDouble);
+    }
+  }
+
+  return false;
+}
+
 const llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
   const CGFunctionInfo &FI = getFunctionInfo(GD);
   
@@ -841,7 +859,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
   llvm::Function::arg_iterator AI = Fn->arg_begin();
 
   // Name the struct return argument.
-  if (CGM.ReturnTypeUsesSret(FI)) {
+  if (CGM.ReturnTypeUsesSRet(FI)) {
     AI->setName("agg.result");
     ++AI;
   }
@@ -1116,7 +1134,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
   // If the call returns a temporary with struct return, create a temporary
   // alloca to hold the result, unless one is given to us.
-  if (CGM.ReturnTypeUsesSret(CallInfo)) {
+  if (CGM.ReturnTypeUsesSRet(CallInfo)) {
     llvm::Value *Value = ReturnValue.getValue();
     if (!Value)
       Value = CreateMemTemp(RetTy);
diff --git a/lib/CodeGen/CGClass.cpp b/lib/CodeGen/CGClass.cpp
index df5ea18..c50fe90 100644
--- a/lib/CodeGen/CGClass.cpp
+++ b/lib/CodeGen/CGClass.cpp
@@ -340,7 +340,7 @@ static void EmitBaseInitializer(CodeGenFunction &CGF,
   
   if (CGF.Exceptions && !BaseClassDecl->hasTrivialDestructor()) {
     // FIXME: Is this OK for C++0x delegating constructors?
-    CodeGenFunction::CleanupBlock Cleanup(CGF, CodeGenFunction::EHCleanup);
+    CodeGenFunction::CleanupBlock Cleanup(CGF, EHCleanup);
 
     CXXDestructorDecl *DD = BaseClassDecl->getDestructor();
     CGF.EmitCXXDestructorCall(DD, Dtor_Base, isBaseVirtual, V);
@@ -534,7 +534,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
     CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
     if (!RD->hasTrivialDestructor()) {
       // FIXME: Is this OK for C++0x delegating constructors?
-      CodeGenFunction::CleanupBlock Cleanup(CGF, CodeGenFunction::EHCleanup);
+      CodeGenFunction::CleanupBlock Cleanup(CGF, EHCleanup);
       
       llvm::Value *ThisPtr = CGF.LoadCXXThis();
       LValue LHS = CGF.EmitLValueForField(ThisPtr, Field, 0);
diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp
index a3c4003..4e15895 100644
--- a/lib/CodeGen/CGDebugInfo.cpp
+++ b/lib/CodeGen/CGDebugInfo.cpp
@@ -537,11 +537,17 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
   llvm::DIType ThisPtrType = 
     DebugFactory.CreateArtificialType(getOrCreateType(ThisPtr, Unit));
 
-  if (Method->getTypeQualifiers() && Qualifiers::Const)
+  unsigned Quals = Method->getTypeQualifiers();
+  if (Quals & Qualifiers::Const)
     ThisPtrType = 
       DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_const_type, 
                                      Unit, "", Unit,
                                      0, 0, 0, 0, 0, ThisPtrType);
+  if (Quals & Qualifiers::Volatile)
+    ThisPtrType = 
+      DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_volatile_type, 
+                                     Unit, "", Unit,
+                                     0, 0, 0, 0, 0, ThisPtrType);
 
   TypeCache[ThisPtr.getAsOpaquePtr()] = ThisPtrType;  
   Elts.push_back(ThisPtrType);
diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp
index 959a9ae..1a62ea9 100644
--- a/lib/CodeGen/CGDecl.cpp
+++ b/lib/CodeGen/CGDecl.cpp
@@ -388,6 +388,58 @@ const llvm::Type *CodeGenFunction::BuildByRefType(const ValueDecl *D) {
   return Info.first;
 }
 
+namespace {
+  struct CallArrayDtor : EHScopeStack::LazyCleanup {
+    CallArrayDtor(const CXXDestructorDecl *Dtor, 
+                  const ConstantArrayType *Type,
+                  llvm::Value *Loc)
+      : Dtor(Dtor), Type(Type), Loc(Loc) {}
+
+    const CXXDestructorDecl *Dtor;
+    const ConstantArrayType *Type;
+    llvm::Value *Loc;
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      QualType BaseElementTy = CGF.getContext().getBaseElementType(Type);
+      const llvm::Type *BasePtr = CGF.ConvertType(BaseElementTy);
+      BasePtr = llvm::PointerType::getUnqual(BasePtr);
+      llvm::Value *BaseAddrPtr = CGF.Builder.CreateBitCast(Loc, BasePtr);
+      CGF.EmitCXXAggrDestructorCall(Dtor, Type, BaseAddrPtr);
+    }
+  };
+
+  struct CallVarDtor : EHScopeStack::LazyCleanup {
+    CallVarDtor(const CXXDestructorDecl *Dtor,
+                llvm::Value *NRVOFlag,
+                llvm::Value *Loc)
+      : Dtor(Dtor), NRVOFlag(NRVOFlag), Loc(Loc) {}
+
+    const CXXDestructorDecl *Dtor;
+    llvm::Value *NRVOFlag;
+    llvm::Value *Loc;
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      // Along the exceptions path we always execute the dtor.
+      bool NRVO = !IsForEH && NRVOFlag;
+
+      llvm::BasicBlock *SkipDtorBB = 0;
+      if (NRVO) {
+        // If we exited via NRVO, we skip the destructor call.
+        llvm::BasicBlock *RunDtorBB = CGF.createBasicBlock("nrvo.unused");
+        SkipDtorBB = CGF.createBasicBlock("nrvo.skipdtor");
+        llvm::Value *DidNRVO = CGF.Builder.CreateLoad(NRVOFlag, "nrvo.val");
+        CGF.Builder.CreateCondBr(DidNRVO, SkipDtorBB, RunDtorBB);
+        CGF.EmitBlock(RunDtorBB);
+      }
+          
+      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
+                                /*ForVirtualBase=*/false, Loc);
+
+      if (NRVO) CGF.EmitBlock(SkipDtorBB);
+    }
+  };
+}
+
 /// EmitLocalBlockVarDecl - Emit code and set up an entry in LocalDeclMap for a
 /// variable declaration with auto, register, or no storage class specifier.
 /// These turn into simple stack objects, or GlobalValues depending on target.
@@ -686,53 +738,11 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
         
         if (const ConstantArrayType *Array = 
               getContext().getAsConstantArrayType(Ty)) {
-          CleanupBlock Scope(*this, NormalCleanup);
-
-          QualType BaseElementTy = getContext().getBaseElementType(Array);
-          const llvm::Type *BasePtr = ConvertType(BaseElementTy);
-          BasePtr = llvm::PointerType::getUnqual(BasePtr);
-          llvm::Value *BaseAddrPtr =
-            Builder.CreateBitCast(Loc, BasePtr);
-          EmitCXXAggrDestructorCall(D, Array, BaseAddrPtr);
-
-          if (Exceptions) {
-            Scope.beginEHCleanup();
-
-            QualType BaseElementTy = getContext().getBaseElementType(Array);
-            const llvm::Type *BasePtr = ConvertType(BaseElementTy);
-            BasePtr = llvm::PointerType::getUnqual(BasePtr);
-            llvm::Value *BaseAddrPtr =
-              Builder.CreateBitCast(Loc, BasePtr);
-            EmitCXXAggrDestructorCall(D, Array, BaseAddrPtr);
-          }
+          EHStack.pushLazyCleanup<CallArrayDtor>(NormalAndEHCleanup,
+                                                 D, Array, Loc);
         } else {
-          // Normal destruction. 
-          CleanupBlock Scope(*this, NormalCleanup);
-
-          llvm::BasicBlock *SkipDtor = 0;
-          if (NRVO) {
-            // If we exited via NRVO, we skip the destructor call.
-            llvm::BasicBlock *NoNRVO = createBasicBlock("nrvo.unused");
-            SkipDtor = createBasicBlock("nrvo.skipdtor");
-            Builder.CreateCondBr(Builder.CreateLoad(NRVOFlag, "nrvo.val"),
-                                 SkipDtor,
-                                 NoNRVO);
-            EmitBlock(NoNRVO);
-          }
-          
-          // We don't call the destructor along the normal edge if we're
-          // applying the NRVO.
-          EmitCXXDestructorCall(D, Dtor_Complete, /*ForVirtualBase=*/false,
-                                Loc);
-
-          if (NRVO) EmitBlock(SkipDtor);
-
-          // Along the exceptions path we always execute the dtor.
-          if (Exceptions) {
-            Scope.beginEHCleanup();
-            EmitCXXDestructorCall(D, Dtor_Complete, /*ForVirtualBase=*/false,
-                                  Loc);
-          }
+          EHStack.pushLazyCleanup<CallVarDtor>(NormalAndEHCleanup,
+                                               D, NRVOFlag, Loc);
         }
       }
   }
diff --git a/lib/CodeGen/CGException.cpp b/lib/CodeGen/CGException.cpp
index 085dddd..4980aad 100644
--- a/lib/CodeGen/CGException.cpp
+++ b/lib/CodeGen/CGException.cpp
@@ -62,12 +62,37 @@ EHScopeStack::getEnclosingEHCleanup(iterator it) const {
         return stabilize(it);
       return cast<EHCleanupScope>(*it).getEnclosingEHCleanup();
     }
+    if (isa<EHLazyCleanupScope>(*it)) {
+      if (cast<EHLazyCleanupScope>(*it).isEHCleanup())
+        return stabilize(it);
+      return cast<EHLazyCleanupScope>(*it).getEnclosingEHCleanup();
+    }
     ++it;
   } while (it != end());
   return stable_end();
 }
 
 
+void *EHScopeStack::pushLazyCleanup(CleanupKind Kind, size_t Size) {
+  assert(((Size % sizeof(void*)) == 0) && "cleanup type is misaligned");
+  char *Buffer = allocate(EHLazyCleanupScope::getSizeForCleanupSize(Size));
+  bool IsNormalCleanup = Kind != EHCleanup;
+  bool IsEHCleanup = Kind != NormalCleanup;
+  EHLazyCleanupScope *Scope =
+    new (Buffer) EHLazyCleanupScope(IsNormalCleanup,
+                                    IsEHCleanup,
+                                    Size,
+                                    BranchFixups.size(),
+                                    InnermostNormalCleanup,
+                                    InnermostEHCleanup);
+  if (IsNormalCleanup)
+    InnermostNormalCleanup = stable_begin();
+  if (IsEHCleanup)
+    InnermostEHCleanup = stable_begin();
+
+  return Scope->getCleanupBuffer();
+}
+
 void EHScopeStack::pushCleanup(llvm::BasicBlock *NormalEntry,
                                llvm::BasicBlock *NormalExit,
                                llvm::BasicBlock *EHEntry,
@@ -86,11 +111,18 @@ void EHScopeStack::pushCleanup(llvm::BasicBlock *NormalEntry,
 void EHScopeStack::popCleanup() {
   assert(!empty() && "popping exception stack when not empty");
 
-  assert(isa<EHCleanupScope>(*begin()));
-  EHCleanupScope &Cleanup = cast<EHCleanupScope>(*begin());
-  InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
-  InnermostEHCleanup = Cleanup.getEnclosingEHCleanup();
-  StartOfData += EHCleanupScope::getSize();
+  if (isa<EHLazyCleanupScope>(*begin())) {
+    EHLazyCleanupScope &Cleanup = cast<EHLazyCleanupScope>(*begin());
+    InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
+    InnermostEHCleanup = Cleanup.getEnclosingEHCleanup();
+    StartOfData += Cleanup.getAllocatedSize();
+  } else {
+    assert(isa<EHCleanupScope>(*begin()));
+    EHCleanupScope &Cleanup = cast<EHCleanupScope>(*begin());
+    InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
+    InnermostEHCleanup = Cleanup.getEnclosingEHCleanup();
+    StartOfData += EHCleanupScope::getSize();
+  }
 
   // Check whether we can shrink the branch-fixups stack.
   if (!BranchFixups.empty()) {
@@ -144,7 +176,11 @@ void EHScopeStack::popNullFixups() {
   assert(hasNormalCleanups());
 
   EHScopeStack::iterator it = find(InnermostNormalCleanup);
-  unsigned MinSize = cast<EHCleanupScope>(*it).getFixupDepth();
+  unsigned MinSize;
+  if (isa<EHCleanupScope>(*it))
+    MinSize = cast<EHCleanupScope>(*it).getFixupDepth();
+  else
+    MinSize = cast<EHLazyCleanupScope>(*it).getFixupDepth();
   assert(BranchFixups.size() >= MinSize && "fixup stack out of order");
 
   while (BranchFixups.size() > MinSize &&
@@ -364,6 +400,33 @@ static llvm::Constant *getCleanupValue(CodeGenFunction &CGF) {
   return llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0);
 }
 
+namespace {
+  /// A cleanup to free the exception object if its initialization
+  /// throws.
+  struct FreeExceptionCleanup : EHScopeStack::LazyCleanup {
+    FreeExceptionCleanup(llvm::Value *ShouldFreeVar,
+                         llvm::Value *ExnLocVar)
+      : ShouldFreeVar(ShouldFreeVar), ExnLocVar(ExnLocVar) {}
+
+    llvm::Value *ShouldFreeVar;
+    llvm::Value *ExnLocVar;
+    
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      llvm::BasicBlock *FreeBB = CGF.createBasicBlock("free-exnobj");
+      llvm::BasicBlock *DoneBB = CGF.createBasicBlock("free-exnobj.done");
+
+      llvm::Value *ShouldFree = CGF.Builder.CreateLoad(ShouldFreeVar,
+                                                       "should-free-exnobj");
+      CGF.Builder.CreateCondBr(ShouldFree, FreeBB, DoneBB);
+      CGF.EmitBlock(FreeBB);
+      llvm::Value *ExnLocLocal = CGF.Builder.CreateLoad(ExnLocVar, "exnobj");
+      CGF.Builder.CreateCall(getFreeExceptionFn(CGF), ExnLocLocal)
+        ->setDoesNotThrow();
+      CGF.EmitBlock(DoneBB);
+    }
+  };
+}
+
 // Emits an exception expression into the given location.  This
 // differs from EmitAnyExprToMem only in that, if a final copy-ctor
 // call is required, an exception within that copy ctor causes
@@ -388,22 +451,11 @@ static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *E,
 
   // Make sure the exception object is cleaned up if there's an
   // exception during initialization.
-  // FIXME: StmtExprs probably force this to include a non-EH
-  // handler.
-  {
-    CodeGenFunction::CleanupBlock Cleanup(CGF, CodeGenFunction::EHCleanup);
-    llvm::BasicBlock *FreeBB = CGF.createBasicBlock("free-exnobj");
-    llvm::BasicBlock *DoneBB = CGF.createBasicBlock("free-exnobj.done");
-
-    llvm::Value *ShouldFree = CGF.Builder.CreateLoad(ShouldFreeVar,
-                                                     "should-free-exnobj");
-    CGF.Builder.CreateCondBr(ShouldFree, FreeBB, DoneBB);
-    CGF.EmitBlock(FreeBB);
-    llvm::Value *ExnLocLocal = CGF.Builder.CreateLoad(ExnLocVar, "exnobj");
-    CGF.Builder.CreateCall(getFreeExceptionFn(CGF), ExnLocLocal)
-      ->setDoesNotThrow();
-    CGF.EmitBlock(DoneBB);
-  }
+  // FIXME: stmt expressions might require this to be a normal
+  // cleanup, too.
+  CGF.EHStack.pushLazyCleanup<FreeExceptionCleanup>(EHCleanup,
+                                                    ShouldFreeVar,
+                                                    ExnLocVar);
   EHScopeStack::stable_iterator Cleanup = CGF.EHStack.stable_begin();
 
   CGF.Builder.CreateStore(ExnLoc, ExnLocVar);
@@ -598,13 +650,28 @@ void CodeGenFunction::EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
 /// affect exception handling.  Currently, the only non-EH scopes are
 /// normal-only cleanup scopes.
 static bool isNonEHScope(const EHScope &S) {
-  return isa<EHCleanupScope>(S) && !cast<EHCleanupScope>(S).isEHCleanup();
+  switch (S.getKind()) {
+  case EHScope::Cleanup:
+    return !cast<EHCleanupScope>(S).isEHCleanup();
+  case EHScope::LazyCleanup:
+    return !cast<EHLazyCleanupScope>(S).isEHCleanup();
+  case EHScope::Filter:
+  case EHScope::Catch:
+  case EHScope::Terminate:
+    return false;
+  }
+
+  // Suppress warning.
+  return false;
 }
 
 llvm::BasicBlock *CodeGenFunction::getInvokeDestImpl() {
   assert(EHStack.requiresLandingPad());
   assert(!EHStack.empty());
 
+  if (!Exceptions)
+    return 0;
+
   // Check the innermost scope for a cached landing pad.  If this is
   // a non-EH cleanup, we'll check enclosing scopes in EmitLandingPad.
   llvm::BasicBlock *LP = EHStack.begin()->getCachedLandingPad();
@@ -713,6 +780,12 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
          I != E; ++I) {
 
     switch (I->getKind()) {
+    case EHScope::LazyCleanup:
+      if (!HasEHCleanup)
+        HasEHCleanup = cast<EHLazyCleanupScope>(*I).isEHCleanup();
+      // We otherwise don't care about cleanups.
+      continue;
+
     case EHScope::Cleanup:
       if (!HasEHCleanup)
         HasEHCleanup = cast<EHCleanupScope>(*I).isEHCleanup();
@@ -947,19 +1020,45 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
   return LP;
 }
 
+namespace {
+  /// A cleanup to call __cxa_end_catch.  In many cases, the caught
+  /// exception type lets us state definitively that the thrown exception
+  /// type does not have a destructor.  In particular:
+  ///   - Catch-alls tell us nothing, so we have to conservatively
+  ///     assume that the thrown exception might have a destructor.
+  ///   - Catches by reference behave according to their base types.
+  ///   - Catches of non-record types will only trigger for exceptions
+  ///     of non-record types, which never have destructors.
+  ///   - Catches of record types can trigger for arbitrary subclasses
+  ///     of the caught type, so we have to assume the actual thrown
+  ///     exception type might have a throwing destructor, even if the
+  ///     caught type's destructor is trivial or nothrow.
+  struct CallEndCatch : EHScopeStack::LazyCleanup {
+    CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {}
+    bool MightThrow;
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      if (!MightThrow) {
+        CGF.Builder.CreateCall(getEndCatchFn(CGF))->setDoesNotThrow();
+        return;
+      }
+
+      CGF.EmitCallOrInvoke(getEndCatchFn(CGF), 0, 0);
+    }
+  };
+}
+
 /// Emits a call to __cxa_begin_catch and enters a cleanup to call
 /// __cxa_end_catch.
-static llvm::Value *CallBeginCatch(CodeGenFunction &CGF, llvm::Value *Exn) {
+///
+/// \param EndMightThrow - true if __cxa_end_catch might throw
+static llvm::Value *CallBeginCatch(CodeGenFunction &CGF,
+                                   llvm::Value *Exn,
+                                   bool EndMightThrow) {
   llvm::CallInst *Call = CGF.Builder.CreateCall(getBeginCatchFn(CGF), Exn);
   Call->setDoesNotThrow();
 
-  {
-    CodeGenFunction::CleanupBlock EndCatchCleanup(CGF,
-                                  CodeGenFunction::NormalAndEHCleanup);
-
-    // __cxa_end_catch never throws, so this can just be a call.
-    CGF.Builder.CreateCall(getEndCatchFn(CGF))->setDoesNotThrow();
-  }
+  CGF.EHStack.pushLazyCleanup<CallEndCatch>(NormalAndEHCleanup, EndMightThrow);
 
   return Call;
 }
@@ -979,8 +1078,11 @@ static void InitCatchParam(CodeGenFunction &CGF,
   // If we're catching by reference, we can just cast the object
   // pointer to the appropriate pointer.
   if (isa<ReferenceType>(CatchType)) {
+    bool EndCatchMightThrow = cast<ReferenceType>(CatchType)->getPointeeType()
+      ->isRecordType();
+
     // __cxa_begin_catch returns the adjusted object pointer.
-    llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn);
+    llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndCatchMightThrow);
     llvm::Value *ExnCast =
       CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.byref");
     CGF.Builder.CreateStore(ExnCast, ParamAddr);
@@ -991,7 +1093,7 @@ static void InitCatchParam(CodeGenFunction &CGF,
   bool IsComplex = false;
   if (!CGF.hasAggregateLLVMType(CatchType) ||
       (IsComplex = CatchType->isAnyComplexType())) {
-    llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn);
+    llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, false);
     
     // If the catch type is a pointer type, __cxa_begin_catch returns
     // the pointer by value.
@@ -1026,7 +1128,7 @@ static void InitCatchParam(CodeGenFunction &CGF,
   const llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
 
   if (RD->hasTrivialCopyConstructor()) {
-    llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn);
+    llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, true);
     llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
     CGF.EmitAggregateCopy(ParamAddr, Cast, CatchType);
     return;
@@ -1059,7 +1161,7 @@ static void InitCatchParam(CodeGenFunction &CGF,
   CGF.EHStack.popTerminate();
 
   // Finally we can call __cxa_begin_catch.
-  CallBeginCatch(CGF, Exn);
+  CallBeginCatch(CGF, Exn, true);
 }
 
 /// Begins a catch statement by initializing the catch variable and
@@ -1092,7 +1194,7 @@ static void BeginCatch(CodeGenFunction &CGF,
   VarDecl *CatchParam = S->getExceptionDecl();
   if (!CatchParam) {
     llvm::Value *Exn = CGF.Builder.CreateLoad(CGF.getExceptionSlot(), "exn");
-    CallBeginCatch(CGF, Exn);
+    CallBeginCatch(CGF, Exn, true);
     return;
   }
 
@@ -1100,6 +1202,14 @@ static void BeginCatch(CodeGenFunction &CGF,
   CGF.EmitLocalBlockVarDecl(*CatchParam, &InitCatchParam);
 }
 
+namespace {
+  struct CallRethrow : EHScopeStack::LazyCleanup {
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      CGF.EmitCallOrInvoke(getReThrowFn(CGF), 0, 0);
+    }
+  };
+}
+
 void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
   unsigned NumHandlers = S.getNumHandlers();
   EHCatchScope &CatchScope = cast<EHCatchScope>(*EHStack.begin());
@@ -1140,12 +1250,10 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
     BeginCatch(*this, C);
 
     // If there's an implicit rethrow, push a normal "cleanup" to call
-    // _cxa_rethrow.  This needs to happen before _cxa_end_catch is
-    // called.
-    if (ImplicitRethrow) {
-      CleanupBlock Rethrow(*this, NormalCleanup);
-      EmitCallOrInvoke(getReThrowFn(*this), 0, 0);
-    }
+    // _cxa_rethrow.  This needs to happen before __cxa_end_catch is
+    // called, and so it is pushed after BeginCatch.
+    if (ImplicitRethrow)
+      EHStack.pushLazyCleanup<CallRethrow>(NormalCleanup);
 
     // Perform the body of the catch.
     EmitStmt(C->getHandlerBlock());
@@ -1213,13 +1321,11 @@ CodeGenFunction::EnterFinallyBlock(const Stmt *Body,
 
   // Enter a normal cleanup which will perform the @finally block.
   {
-    CodeGenFunction::CleanupBlock
-      NormalCleanup(*this, CodeGenFunction::NormalCleanup);
+    CodeGenFunction::CleanupBlock Cleanup(*this, NormalCleanup);
 
     // Enter a cleanup to call the end-catch function if one was provided.
     if (EndCatchFn) {
-      CodeGenFunction::CleanupBlock
-        FinallyExitCleanup(CGF, CodeGenFunction::NormalAndEHCleanup);
+      CodeGenFunction::CleanupBlock FinallyExitCleanup(CGF, NormalAndEHCleanup);
 
       llvm::BasicBlock *EndCatchBB = createBasicBlock("finally.endcatch");
       llvm::BasicBlock *CleanupContBB = createBasicBlock("finally.cleanup.cont");
@@ -1228,7 +1334,7 @@ CodeGenFunction::EnterFinallyBlock(const Stmt *Body,
         Builder.CreateLoad(ForEHVar, "finally.endcatch");
       Builder.CreateCondBr(ShouldEndCatch, EndCatchBB, CleanupContBB);
       EmitBlock(EndCatchBB);
-      Builder.CreateCall(EndCatchFn)->setDoesNotThrow();
+      EmitCallOrInvoke(EndCatchFn, 0, 0); // catch-all, so might throw
       EmitBlock(CleanupContBB);
     }
 
@@ -1435,3 +1541,6 @@ CodeGenFunction::CleanupBlock::~CleanupBlock() {
   CGF.Builder.restoreIP(SavedIP);
 }
 
+EHScopeStack::LazyCleanup::~LazyCleanup() {
+  llvm_unreachable("LazyCleanup is indestructable");
+}
diff --git a/lib/CodeGen/CGException.h b/lib/CodeGen/CGException.h
index 8755dca..80739cd 100644
--- a/lib/CodeGen/CGException.h
+++ b/lib/CodeGen/CGException.h
@@ -31,13 +31,13 @@ namespace CodeGen {
 class EHScope {
   llvm::BasicBlock *CachedLandingPad;
 
-  unsigned K : 2;
+  unsigned K : 3;
 
 protected:
-  enum { BitsRemaining = 30 };
+  enum { BitsRemaining = 29 };
 
 public:
-  enum Kind { Cleanup, Catch, Terminate, Filter };
+  enum Kind { Cleanup, LazyCleanup, Catch, Terminate, Filter };
 
   EHScope(Kind K) : CachedLandingPad(0), K(K) {}
 
@@ -127,6 +127,87 @@ public:
   }
 };
 
+/// A cleanup scope which generates the cleanup blocks lazily.
+class EHLazyCleanupScope : public EHScope {
+  /// Whether this cleanup needs to be run along normal edges.
+  bool IsNormalCleanup : 1;
+
+  /// Whether this cleanup needs to be run along exception edges.
+  bool IsEHCleanup : 1;
+
+  /// The amount of extra storage needed by the LazyCleanup.
+  /// Always a multiple of the scope-stack alignment.
+  unsigned CleanupSize : 12;
+
+  /// The number of fixups required by enclosing scopes (not including
+  /// this one).  If this is the top cleanup scope, all the fixups
+  /// from this index onwards belong to this scope.
+  unsigned FixupDepth : BitsRemaining - 14;
+
+  /// The nearest normal cleanup scope enclosing this one.
+  EHScopeStack::stable_iterator EnclosingNormal;
+
+  /// The nearest EH cleanup scope enclosing this one.
+  EHScopeStack::stable_iterator EnclosingEH;
+
+  /// The dual entry/exit block along the normal edge.  This is lazily
+  /// created if needed before the cleanup is popped.
+  llvm::BasicBlock *NormalBlock;
+
+  /// The dual entry/exit block along the EH edge.  This is lazily
+  /// created if needed before the cleanup is popped.
+  llvm::BasicBlock *EHBlock;
+
+public:
+  /// Gets the size required for a lazy cleanup scope with the given
+  /// cleanup-data requirements.
+  static size_t getSizeForCleanupSize(size_t Size) {
+    return sizeof(EHLazyCleanupScope) + Size;
+  }
+
+  size_t getAllocatedSize() const {
+    return sizeof(EHLazyCleanupScope) + CleanupSize;
+  }
+
+  EHLazyCleanupScope(bool IsNormal, bool IsEH, unsigned CleanupSize,
+                     unsigned FixupDepth,
+                     EHScopeStack::stable_iterator EnclosingNormal,
+                     EHScopeStack::stable_iterator EnclosingEH)
+    : EHScope(EHScope::LazyCleanup),
+      IsNormalCleanup(IsNormal), IsEHCleanup(IsEH),
+      CleanupSize(CleanupSize), FixupDepth(FixupDepth),
+      EnclosingNormal(EnclosingNormal), EnclosingEH(EnclosingEH),
+      NormalBlock(0), EHBlock(0)
+  {}
+
+  bool isNormalCleanup() const { return IsNormalCleanup; }
+  llvm::BasicBlock *getNormalBlock() const { return NormalBlock; }
+  void setNormalBlock(llvm::BasicBlock *BB) { NormalBlock = BB; }
+
+  bool isEHCleanup() const { return IsEHCleanup; }
+  llvm::BasicBlock *getEHBlock() const { return EHBlock; }
+  void setEHBlock(llvm::BasicBlock *BB) { EHBlock = BB; }
+
+  unsigned getFixupDepth() const { return FixupDepth; }
+  EHScopeStack::stable_iterator getEnclosingNormalCleanup() const {
+    return EnclosingNormal;
+  }
+  EHScopeStack::stable_iterator getEnclosingEHCleanup() const {
+    return EnclosingEH;
+  }
+
+  size_t getCleanupSize() const { return CleanupSize; }
+  void *getCleanupBuffer() { return this + 1; }
+
+  EHScopeStack::LazyCleanup *getCleanup() {
+    return reinterpret_cast<EHScopeStack::LazyCleanup*>(getCleanupBuffer());
+  }
+
+  static bool classof(const EHScope *Scope) {
+    return (Scope->getKind() == LazyCleanup);
+  }
+};
+
 /// A scope which needs to execute some code if we try to unwind ---
 /// either normally, via the EH mechanism, or both --- through it.
 class EHCleanupScope : public EHScope {
@@ -267,6 +348,11 @@ public:
           static_cast<const EHFilterScope*>(get())->getNumFilters());
       break;
 
+    case EHScope::LazyCleanup:
+      Ptr += static_cast<const EHLazyCleanupScope*>(get())
+        ->getAllocatedSize();
+      break;
+
     case EHScope::Cleanup:
       Ptr += EHCleanupScope::getSize();
       break;
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
index 0426a60..43bab9f 100644
--- a/lib/CodeGen/CGExpr.cpp
+++ b/lib/CodeGen/CGExpr.cpp
@@ -1816,7 +1816,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
     
     return LValue::MakeAddr(Derived, MakeQualifiers(E->getType()));
   }
-  case CastExpr::CK_BitCast: {
+  case CastExpr::CK_LValueBitCast: {
     // This must be a reinterpret_cast (or c-style equivalent).
     const ExplicitCastExpr *CE = cast<ExplicitCastExpr>(E);
     
diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp
index 20722f7..219a5f9 100644
--- a/lib/CodeGen/CGExprAgg.cpp
+++ b/lib/CodeGen/CGExprAgg.cpp
@@ -307,6 +307,10 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
     break;
   }
       
+  case CastExpr::CK_LValueBitCast:
+    llvm_unreachable("there are no lvalue bit-casts on aggregates");
+    break;
+      
   case CastExpr::CK_BitCast: {
     // This must be a member function pointer cast.
     Visit(E->getSubExpr());
diff --git a/lib/CodeGen/CGExprComplex.cpp b/lib/CodeGen/CGExprComplex.cpp
index 90b6446..0927319 100644
--- a/lib/CodeGen/CGExprComplex.cpp
+++ b/lib/CodeGen/CGExprComplex.cpp
@@ -131,14 +131,14 @@ public:
 
   // FIXME: CompoundLiteralExpr
 
-  ComplexPairTy EmitCast(Expr *Op, QualType DestTy);
+  ComplexPairTy EmitCast(CastExpr::CastKind CK, Expr *Op, QualType DestTy);
   ComplexPairTy VisitImplicitCastExpr(ImplicitCastExpr *E) {
     // Unlike for scalars, we don't have to worry about function->ptr demotion
     // here.
-    return EmitCast(E->getSubExpr(), E->getType());
+    return EmitCast(E->getCastKind(), E->getSubExpr(), E->getType());
   }
   ComplexPairTy VisitCastExpr(CastExpr *E) {
-    return EmitCast(E->getSubExpr(), E->getType());
+    return EmitCast(E->getCastKind(), E->getSubExpr(), E->getType());
   }
   ComplexPairTy VisitCallExpr(const CallExpr *E);
   ComplexPairTy VisitStmtExpr(const StmtExpr *E);
@@ -339,11 +339,22 @@ ComplexPairTy ComplexExprEmitter::EmitComplexToComplexCast(ComplexPairTy Val,
   return Val;
 }
 
-ComplexPairTy ComplexExprEmitter::EmitCast(Expr *Op, QualType DestTy) {
+ComplexPairTy ComplexExprEmitter::EmitCast(CastExpr::CastKind CK, Expr *Op, 
+                                           QualType DestTy) {
   // Two cases here: cast from (complex to complex) and (scalar to complex).
   if (Op->getType()->isAnyComplexType())
     return EmitComplexToComplexCast(Visit(Op), Op->getType(), DestTy);
 
+  // FIXME: We should be looking at all of the cast kinds here, not
+  // cherry-picking the ones we have test cases for.
+  if (CK == CastExpr::CK_LValueBitCast) {
+    llvm::Value *V = CGF.EmitLValue(Op).getAddress();
+    V = Builder.CreateBitCast(V, 
+                      CGF.ConvertType(CGF.getContext().getPointerType(DestTy)));
+    // FIXME: Are the qualifiers correct here?
+    return EmitLoadOfComplex(V, DestTy.isVolatileQualified());
+  }
+  
   // C99 6.3.1.7: When a value of real type is converted to a complex type, the
   // real part of the complex result value is determined by the rules of
   // conversion to the corresponding real type and the imaginary part of the
@@ -521,7 +532,7 @@ EmitCompoundAssign(const CompoundAssignOperator *E,
   // improve codegen a little.  It is possible for the RHS to be complex or
   // scalar.
   OpInfo.Ty = E->getComputationResultType();
-  OpInfo.RHS = EmitCast(E->getRHS(), OpInfo.Ty);
+  OpInfo.RHS = EmitCast(CastExpr::CK_Unknown, E->getRHS(), OpInfo.Ty);
   
   LValue LHS = CGF.EmitLValue(E->getLHS());
   // We know the LHS is a complex lvalue.
@@ -572,8 +583,8 @@ ComplexPairTy ComplexExprEmitter::VisitBinAssign(const BinaryOperator *E) {
   TestAndClearIgnoreImag();
   bool ignreal = TestAndClearIgnoreRealAssign();
   bool ignimag = TestAndClearIgnoreImagAssign();
-  assert(CGF.getContext().getCanonicalType(E->getLHS()->getType()) ==
-         CGF.getContext().getCanonicalType(E->getRHS()->getType()) &&
+  assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(), 
+                                                 E->getRHS()->getType()) &&
          "Invalid assignment");
   // Emit the RHS.
   ComplexPairTy Val = Visit(E->getRHS());
diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp
index 1ebc2c5..ef38209 100644
--- a/lib/CodeGen/CGExprScalar.cpp
+++ b/lib/CodeGen/CGExprScalar.cpp
@@ -925,6 +925,15 @@ Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
     //assert(0 && "Unknown cast kind!");
     break;
 
+  case CastExpr::CK_LValueBitCast: {
+    Value *V = EmitLValue(E).getAddress();
+    V = Builder.CreateBitCast(V, 
+                          ConvertType(CGF.getContext().getPointerType(DestTy)));
+    // FIXME: Are the qualifiers correct here?
+    return EmitLoadOfLValue(LValue::MakeAddr(V, CGF.MakeQualifiers(DestTy)), 
+                            DestTy);
+  }
+      
   case CastExpr::CK_AnyPointerToObjCPointerCast:
   case CastExpr::CK_AnyPointerToBlockPointerCast:
   case CastExpr::CK_BitCast: {
diff --git a/lib/CodeGen/CGObjCGNU.cpp b/lib/CodeGen/CGObjCGNU.cpp
index c9da348..f3c80bc 100644
--- a/lib/CodeGen/CGObjCGNU.cpp
+++ b/lib/CodeGen/CGObjCGNU.cpp
@@ -1871,8 +1871,7 @@ void CGObjCGNU::EmitSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
 
   // Register an all-paths cleanup to release the lock.
   {
-    CodeGenFunction::CleanupBlock
-      ReleaseScope(CGF, CodeGenFunction::NormalAndEHCleanup);
+    CodeGenFunction::CleanupBlock ReleaseScope(CGF, NormalAndEHCleanup);
 
     llvm::Value *SyncExit = CGM.CreateRuntimeFunction(FTy, "objc_sync_exit");
     SyncArg = CGF.Builder.CreateBitCast(SyncArg, IdTy);
diff --git a/lib/CodeGen/CGObjCMac.cpp b/lib/CodeGen/CGObjCMac.cpp
index 0a766d5..01ead9e 100644
--- a/lib/CodeGen/CGObjCMac.cpp
+++ b/lib/CodeGen/CGObjCMac.cpp
@@ -32,6 +32,7 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Support/CallSite.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetData.h"
 #include <cstdio>
@@ -1649,29 +1650,17 @@ CGObjCCommonMac::EmitLegacyMessageSend(CodeGen::CodeGenFunction &CGF,
            "Result type mismatch!");
 
   llvm::Constant *Fn = NULL;
-  if (CGM.ReturnTypeUsesSret(FnInfo)) {
+  if (CGM.ReturnTypeUsesSRet(FnInfo)) {
     Fn = (ObjCABI == 2) ?  ObjCTypes.getSendStretFn2(IsSuper)
       : ObjCTypes.getSendStretFn(IsSuper);
-  } else if (ResultType->isRealFloatingType()) {
-    if (ObjCABI == 2) {
-      if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
-        BuiltinType::Kind k = BT->getKind();
-        Fn = (k == BuiltinType::LongDouble) ? ObjCTypes.getSendFpretFn2(IsSuper)
-          : ObjCTypes.getSendFn2(IsSuper);
-      } else {
-        Fn = ObjCTypes.getSendFn2(IsSuper);
-      }
-    } else
-      // FIXME. This currently matches gcc's API for x86-32. May need to change
-      // for others if we have their API.
-      Fn = ObjCTypes.getSendFpretFn(IsSuper);
+  } else if (CGM.ReturnTypeUsesFPRet(ResultType)) {
+    Fn = (ObjCABI == 2) ? ObjCTypes.getSendFpretFn2(IsSuper)
+      : ObjCTypes.getSendFpretFn(IsSuper);
   } else {
     Fn = (ObjCABI == 2) ? ObjCTypes.getSendFn2(IsSuper)
       : ObjCTypes.getSendFn(IsSuper);
   }
-  assert(Fn && "EmitLegacyMessageSend - unknown API");
-  Fn = llvm::ConstantExpr::getBitCast(Fn,
-                                      llvm::PointerType::getUnqual(FTy));
+  Fn = llvm::ConstantExpr::getBitCast(Fn, llvm::PointerType::getUnqual(FTy));
   return CGF.EmitCall(FnInfo, Fn, Return, ActualArgs);
 }
 
@@ -2697,8 +2686,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
 
   // Push a normal cleanup to leave the try scope.
   {
-    CodeGenFunction::CleanupBlock
-      FinallyScope(CGF, CodeGenFunction::NormalCleanup);
+    CodeGenFunction::CleanupBlock FinallyScope(CGF, NormalCleanup);
 
     // Check whether we need to call objc_exception_try_exit.
     // In optimized code, this branch will always be folded.
@@ -5295,7 +5283,7 @@ CodeGen::RValue CGObjCNonFragileABIMac::EmitMessageSend(
                               FunctionType::ExtInfo());
   llvm::Constant *Fn = 0;
   std::string Name("\01l_");
-  if (CGM.ReturnTypeUsesSret(FnInfo)) {
+  if (CGM.ReturnTypeUsesSRet(FnInfo)) {
 #if 0
     // unlike what is documented. gcc never generates this API!!
     if (Receiver->getType() == ObjCTypes.ObjectPtrTy) {
@@ -5312,14 +5300,9 @@ CodeGen::RValue CGObjCNonFragileABIMac::EmitMessageSend(
         Fn = ObjCTypes.getMessageSendStretFixupFn();
         Name += "objc_msgSend_stret_fixup";
       }
-  } else if (!IsSuper && ResultType->isRealFloatingType()) {
-    if (ResultType->isSpecificBuiltinType(BuiltinType::LongDouble)) {
-      Fn = ObjCTypes.getMessageSendFpretFixupFn();
-      Name += "objc_msgSend_fpret_fixup";
-    } else {
-      Fn = ObjCTypes.getMessageSendFixupFn();
-      Name += "objc_msgSend_fixup";
-    }
+  } else if (!IsSuper && CGM.ReturnTypeUsesFPRet(ResultType)) {
+    Fn = ObjCTypes.getMessageSendFpretFixupFn();
+    Name += "objc_msgSend_fpret_fixup";
   } else {
 #if 0
 // unlike what is documented. gcc never generates this API!!
@@ -5693,8 +5676,7 @@ CGObjCNonFragileABIMac::EmitSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
 
   // Register an all-paths cleanup to release the lock.
   {
-    CodeGenFunction::CleanupBlock
-      ReleaseScope(CGF, CodeGenFunction::NormalAndEHCleanup);
+    CodeGenFunction::CleanupBlock ReleaseScope(CGF, NormalAndEHCleanup);
 
     CGF.Builder.CreateCall(ObjCTypes.getSyncExitFn(), SyncArg)
       ->setDoesNotThrow();
@@ -5714,6 +5696,22 @@ namespace {
     llvm::BasicBlock *Block;
     llvm::Value *TypeInfo;
   };
+
+  struct CallObjCEndCatch : EHScopeStack::LazyCleanup {
+    CallObjCEndCatch(bool MightThrow, llvm::Value *Fn) :
+      MightThrow(MightThrow), Fn(Fn) {}
+    bool MightThrow;
+    llvm::Value *Fn;
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      if (!MightThrow) {
+        CGF.Builder.CreateCall(Fn)->setDoesNotThrow();
+        return;
+      }
+
+      CGF.EmitCallOrInvoke(Fn, 0, 0);
+    }
+  };
 }
 
 void CGObjCNonFragileABIMac::EmitTryStmt(CodeGen::CodeGenFunction &CGF,
@@ -5803,14 +5801,10 @@ void CGObjCNonFragileABIMac::EmitTryStmt(CodeGen::CodeGenFunction &CGF,
     Exn->setDoesNotThrow();
 
     // Add a cleanup to leave the catch.
-    {
-      CodeGenFunction::CleanupBlock
-        EndCatchBlock(CGF, CodeGenFunction::NormalAndEHCleanup);
-
-      // __objc_end_catch never throws.
-      CGF.Builder.CreateCall(ObjCTypes.getObjCEndCatchFn())
-        ->setDoesNotThrow();
-    }
+    bool EndCatchMightThrow = (Handler.Variable == 0);
+    CGF.EHStack.pushLazyCleanup<CallObjCEndCatch>(NormalAndEHCleanup,
+                                                  EndCatchMightThrow,
+                                                  ObjCTypes.getObjCEndCatchFn());
 
     // Bind the catch parameter if it exists.
     if (const VarDecl *CatchParam = Handler.Variable) {
diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp
index 5e505c2..eb6c436 100644
--- a/lib/CodeGen/CodeGenFunction.cpp
+++ b/lib/CodeGen/CodeGenFunction.cpp
@@ -825,11 +825,168 @@ static void SimplifyCleanupEdges(CodeGenFunction &CGF,
   SimplifyCleanupEntry(CGF, Entry);  
 }
 
+static void EmitLazyCleanup(CodeGenFunction &CGF,
+                            EHScopeStack::LazyCleanup *Fn,
+                            bool ForEH) {
+  if (ForEH) CGF.EHStack.pushTerminate();
+  Fn->Emit(CGF, ForEH);
+  if (ForEH) CGF.EHStack.popTerminate();
+  assert(CGF.HaveInsertPoint() && "cleanup ended with no insertion point?");
+}
+
+static void SplitAndEmitLazyCleanup(CodeGenFunction &CGF,
+                                    EHScopeStack::LazyCleanup *Fn,
+                                    bool ForEH,
+                                    llvm::BasicBlock *Entry) {
+  assert(Entry && "no entry block for cleanup");
+
+  // Remove the switch and load from the end of the entry block.
+  llvm::Instruction *Switch = &Entry->getInstList().back();
+  Entry->getInstList().remove(Switch);
+  assert(isa<llvm::SwitchInst>(Switch));
+  llvm::Instruction *Load = &Entry->getInstList().back();
+  Entry->getInstList().remove(Load);
+  assert(isa<llvm::LoadInst>(Load));
+
+  assert(Entry->getInstList().empty() &&
+         "lazy cleanup block not empty after removing load/switch pair?");
+
+  // Emit the actual cleanup at the end of the entry block.
+  CGF.Builder.SetInsertPoint(Entry);
+  EmitLazyCleanup(CGF, Fn, ForEH);
+
+  // Put the load and switch at the end of the exit block.
+  llvm::BasicBlock *Exit = CGF.Builder.GetInsertBlock();
+  Exit->getInstList().push_back(Load);
+  Exit->getInstList().push_back(Switch);
+
+  // Clean up the edges if possible.
+  SimplifyCleanupEdges(CGF, Entry, Exit);
+
+  CGF.Builder.ClearInsertionPoint();
+}
+
+static void PopLazyCleanupBlock(CodeGenFunction &CGF) {
+  assert(isa<EHLazyCleanupScope>(*CGF.EHStack.begin()) && "top not a cleanup!");
+  EHLazyCleanupScope &Scope = cast<EHLazyCleanupScope>(*CGF.EHStack.begin());
+  assert(Scope.getFixupDepth() <= CGF.EHStack.getNumBranchFixups());
+
+  // Check whether we need an EH cleanup.  This is only true if we've
+  // generated a lazy EH cleanup block.
+  llvm::BasicBlock *EHEntry = Scope.getEHBlock();
+  bool RequiresEHCleanup = (EHEntry != 0);
+
+  // Check the three conditions which might require a normal cleanup:
+
+  // - whether there are branch fix-ups through this cleanup
+  unsigned FixupDepth = Scope.getFixupDepth();
+  bool HasFixups = CGF.EHStack.getNumBranchFixups() != FixupDepth;
+
+  // - whether control has already been threaded through this cleanup
+  llvm::BasicBlock *NormalEntry = Scope.getNormalBlock();
+  bool HasExistingBranches = (NormalEntry != 0);
+
+  // - whether there's a fallthrough
+  llvm::BasicBlock *FallthroughSource = CGF.Builder.GetInsertBlock();
+  bool HasFallthrough = (FallthroughSource != 0);
+
+  bool RequiresNormalCleanup = false;
+  if (Scope.isNormalCleanup() &&
+      (HasFixups || HasExistingBranches || HasFallthrough)) {
+    RequiresNormalCleanup = true;
+  }
+
+  // If we don't need the cleanup at all, we're done.
+  if (!RequiresNormalCleanup && !RequiresEHCleanup) {
+    CGF.EHStack.popCleanup();
+    assert(CGF.EHStack.getNumBranchFixups() == 0 ||
+           CGF.EHStack.hasNormalCleanups());
+    return;
+  }
+
+  // Copy the cleanup emission data out.  Note that SmallVector
+  // guarantees maximal alignment for its buffer regardless of its
+  // type parameter.
+  llvm::SmallVector<char, 8*sizeof(void*)> CleanupBuffer;
+  CleanupBuffer.reserve(Scope.getCleanupSize());
+  memcpy(CleanupBuffer.data(),
+         Scope.getCleanupBuffer(), Scope.getCleanupSize());
+  CleanupBuffer.set_size(Scope.getCleanupSize());
+  EHScopeStack::LazyCleanup *Fn =
+    reinterpret_cast<EHScopeStack::LazyCleanup*>(CleanupBuffer.data());
+
+  // We're done with the scope; pop it off so we can emit the cleanups.
+  CGF.EHStack.popCleanup();
+
+  if (RequiresNormalCleanup) {
+    // If we have a fallthrough and no other need for the cleanup,
+    // emit it directly.
+    if (HasFallthrough && !HasFixups && !HasExistingBranches) {
+      EmitLazyCleanup(CGF, Fn, /*ForEH*/ false);
+
+    // Otherwise, the best approach is to thread everything through
+    // the cleanup block and then try to clean up after ourselves.
+    } else {
+      // Force the entry block to exist.
+      if (!HasExistingBranches) {
+        NormalEntry = CGF.createBasicBlock("cleanup");
+        CreateCleanupSwitch(CGF, NormalEntry);
+      }
+
+      CGF.EmitBlock(NormalEntry);
+
+      // Thread the fallthrough edge through the (momentarily trivial)
+      // cleanup.
+      llvm::BasicBlock *FallthroughDestination = 0;
+      if (HasFallthrough) {
+        assert(isa<llvm::BranchInst>(FallthroughSource->getTerminator()));
+        FallthroughDestination = CGF.createBasicBlock("cleanup.cont");
+
+        BranchFixup Fix;
+        Fix.Destination = FallthroughDestination;
+        Fix.LatestBranch = FallthroughSource->getTerminator();
+        Fix.LatestBranchIndex = 0;
+        Fix.Origin = Fix.LatestBranch;
+
+        // Restore fixup invariant.  EmitBlock added a branch to the
+        // cleanup which we need to redirect to the destination.
+        cast<llvm::BranchInst>(Fix.LatestBranch)
+          ->setSuccessor(0, Fix.Destination);
+
+        ThreadFixupThroughCleanup(CGF, Fix, NormalEntry, NormalEntry);
+      }
+
+      // Thread any "real" fixups we need to thread.
+      for (unsigned I = FixupDepth, E = CGF.EHStack.getNumBranchFixups();
+           I != E; ++I)
+        if (CGF.EHStack.getBranchFixup(I).Destination)
+          ThreadFixupThroughCleanup(CGF, CGF.EHStack.getBranchFixup(I),
+                                    NormalEntry, NormalEntry);
+
+      SplitAndEmitLazyCleanup(CGF, Fn, /*ForEH*/ false, NormalEntry);
+
+      if (HasFallthrough)
+        CGF.EmitBlock(FallthroughDestination);
+    }
+  }
+
+  // Emit the EH cleanup if required.
+  if (RequiresEHCleanup) {
+    CGBuilderTy::InsertPoint SavedIP = CGF.Builder.saveAndClearIP();
+    CGF.EmitBlock(EHEntry);
+    SplitAndEmitLazyCleanup(CGF, Fn, /*ForEH*/ true, EHEntry);
+    CGF.Builder.restoreIP(SavedIP);
+  }
+}
+
 /// Pops a cleanup block.  If the block includes a normal cleanup, the
 /// current insertion point is threaded through the cleanup, as are
 /// any branch fixups on the cleanup.
 void CodeGenFunction::PopCleanupBlock() {
   assert(!EHStack.empty() && "cleanup stack is empty!");
+  if (isa<EHLazyCleanupScope>(*EHStack.begin()))
+    return PopLazyCleanupBlock(*this);
+
   assert(isa<EHCleanupScope>(*EHStack.begin()) && "top not a cleanup!");
   EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
   assert(Scope.getFixupDepth() <= EHStack.getNumBranchFixups());
@@ -1007,6 +1164,16 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) {
       if (Scope.isNormalCleanup())
         ThreadFixupThroughCleanup(*this, Fixup, Scope.getNormalEntry(),
                                   Scope.getNormalExit());
+    } else if (isa<EHLazyCleanupScope>(*I)) {
+      EHLazyCleanupScope &Scope = cast<EHLazyCleanupScope>(*I);
+      if (Scope.isNormalCleanup()) {
+        llvm::BasicBlock *Block = Scope.getNormalBlock();
+        if (!Block) {
+          Block = createBasicBlock("cleanup");
+          Scope.setNormalBlock(Block);
+        }
+        ThreadFixupThroughCleanup(*this, Fixup, Block, Block);
+      }
     }
   }
   
@@ -1046,6 +1213,16 @@ void CodeGenFunction::EmitBranchThroughEHCleanup(JumpDest Dest) {
       if (Scope.isEHCleanup())
         ThreadFixupThroughCleanup(*this, Fixup, Scope.getEHEntry(),
                                   Scope.getEHExit());
+    } else if (isa<EHLazyCleanupScope>(*I)) {
+      EHLazyCleanupScope &Scope = cast<EHLazyCleanupScope>(*I);
+      if (Scope.isEHCleanup()) {
+        llvm::BasicBlock *Block = Scope.getEHBlock();
+        if (!Block) {
+          Block = createBasicBlock("eh.cleanup");
+          Scope.setEHBlock(Block);
+        }
+        ThreadFixupThroughCleanup(*this, Fixup, Block, Block);
+      }
     }
   }
   
diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h
index 26fb882..5ee3db0 100644
--- a/lib/CodeGen/CodeGenFunction.h
+++ b/lib/CodeGen/CodeGenFunction.h
@@ -95,6 +95,8 @@ struct BranchFixup {
   unsigned LatestBranchIndex;
 };
 
+enum CleanupKind { NormalAndEHCleanup, EHCleanup, NormalCleanup };
+
 /// A stack of scopes which respond to exceptions, including cleanups
 /// and catch blocks.
 class EHScopeStack {
@@ -123,6 +125,33 @@ public:
     }
   };
 
+  /// A lazy cleanup.  Subclasses must be POD-like:  cleanups will
+  /// not be destructed, and they will be allocated on the cleanup
+  /// stack and freely copied and moved around.
+  ///
+  /// LazyCleanup implementations should generally be declared in an
+  /// anonymous namespace.
+  class LazyCleanup {
+  public:
+    // Anchor the construction vtable.  We use the destructor because
+    // gcc gives an obnoxious warning if there are virtual methods
+    // with an accessible non-virtual destructor.  Unfortunately,
+    // declaring this destructor makes it non-trivial, but there
+    // doesn't seem to be any other way around this warning.
+    //
+    // This destructor will never be called.
+    virtual ~LazyCleanup();
+
+    /// Emit the cleanup.  For normal cleanups, this is run in the
+    /// same EH context as when the cleanup was pushed, i.e. the
+    /// immediately-enclosing context of the cleanup scope.  For
+    /// EH cleanups, this is run in a terminate context.
+    ///
+    // \param IsForEHCleanup true if this is for an EH cleanup, false
+    ///  if for a normal cleanup.
+    virtual void Emit(CodeGenFunction &CGF, bool IsForEHCleanup) = 0;
+  };
+
 private:
   // The implementation for this class is in CGException.h and
   // CGException.cpp; the definition is here because it's used as a
@@ -171,6 +200,8 @@ private:
 
   void popNullFixups();
 
+  void *pushLazyCleanup(CleanupKind K, size_t DataSize);
+
 public:
   EHScopeStack() : StartOfBuffer(0), EndOfBuffer(0), StartOfData(0),
                    InnermostNormalCleanup(stable_end()),
@@ -178,6 +209,48 @@ public:
                    CatchDepth(0) {}
   ~EHScopeStack() { delete[] StartOfBuffer; }
 
+  // Variadic templates would make this not terrible.
+
+  /// Push a lazily-created cleanup on the stack.
+  template <class T>
+  void pushLazyCleanup(CleanupKind Kind) {
+    void *Buffer = pushLazyCleanup(Kind, sizeof(T));
+    LazyCleanup *Obj = new(Buffer) T();
+    (void) Obj;
+  }
+
+  /// Push a lazily-created cleanup on the stack.
+  template <class T, class A0>
+  void pushLazyCleanup(CleanupKind Kind, A0 a0) {
+    void *Buffer = pushLazyCleanup(Kind, sizeof(T));
+    LazyCleanup *Obj = new(Buffer) T(a0);
+    (void) Obj;
+  }
+
+  /// Push a lazily-created cleanup on the stack.
+  template <class T, class A0, class A1>
+  void pushLazyCleanup(CleanupKind Kind, A0 a0, A1 a1) {
+    void *Buffer = pushLazyCleanup(Kind, sizeof(T));
+    LazyCleanup *Obj = new(Buffer) T(a0, a1);
+    (void) Obj;
+  }
+
+  /// Push a lazily-created cleanup on the stack.
+  template <class T, class A0, class A1, class A2>
+  void pushLazyCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2) {
+    void *Buffer = pushLazyCleanup(Kind, sizeof(T));
+    LazyCleanup *Obj = new(Buffer) T(a0, a1, a2);
+    (void) Obj;
+  }
+
+  /// Push a lazily-created cleanup on the stack.
+  template <class T, class A0, class A1, class A2, class A3>
+  void pushLazyCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2, A3 a3) {
+    void *Buffer = pushLazyCleanup(Kind, sizeof(T));
+    LazyCleanup *Obj = new(Buffer) T(a0, a1, a2, a3);
+    (void) Obj;
+  }
+
   /// Push a cleanup on the stack.
   void pushCleanup(llvm::BasicBlock *NormalEntry,
                    llvm::BasicBlock *NormalExit,
@@ -375,8 +448,6 @@ public:
                                 llvm::Constant *RethrowFn);
   void ExitFinallyBlock(FinallyInfo &FinallyInfo);
 
-  enum CleanupKind { NormalAndEHCleanup, EHCleanup, NormalCleanup };
-
   /// PushDestructorCleanup - Push a cleanup to call the
   /// complete-object destructor of an object of the given type at the
   /// given address.  Does nothing if T is not a C++ class type with a
diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h
index a1ea0ec..27f15fc 100644
--- a/lib/CodeGen/CodeGenModule.h
+++ b/lib/CodeGen/CodeGenModule.h
@@ -444,9 +444,13 @@ public:
   /// which only apply to a function definintion.
   void SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F);
 
-  /// ReturnTypeUsesSret - Return true iff the given type uses 'sret' when used
+  /// ReturnTypeUsesSRet - Return true iff the given type uses 'sret' when used
   /// as a return type.
-  bool ReturnTypeUsesSret(const CGFunctionInfo &FI);
+  bool ReturnTypeUsesSRet(const CGFunctionInfo &FI);
+
+  /// ReturnTypeUsesSret - Return true iff the given type uses 'fpret' when used
+  /// as a return type.
+  bool ReturnTypeUsesFPRet(QualType ResultType);
 
   /// ConstructAttributeList - Get the LLVM attributes and calling convention to
   /// use for a particular function type.
diff --git a/lib/CodeGen/Mangle.cpp b/lib/CodeGen/Mangle.cpp
index 2ae1919..30ee541 100644
--- a/lib/CodeGen/Mangle.cpp
+++ b/lib/CodeGen/Mangle.cpp
@@ -169,7 +169,9 @@ public:
 
   void mangle(const NamedDecl *D, llvm::StringRef Prefix = "_Z");
   void mangleCallOffset(int64_t NonVirtual, int64_t Virtual);
+  void mangleNumber(const llvm::APSInt &I);
   void mangleNumber(int64_t Number);
+  void mangleFloat(const llvm::APFloat &F);
   void mangleFunctionEncoding(const FunctionDecl *FD);
   void mangleName(const NamedDecl *ND);
   void mangleType(QualType T);
@@ -230,6 +232,7 @@ private:
 #include "clang/AST/TypeNodes.def"
 
   void mangleType(const TagType*);
+  void mangleType(TemplateName);
   void mangleBareFunctionType(const FunctionType *T,
                               bool MangleReturnType);
 
@@ -526,6 +529,21 @@ void CXXNameMangler::mangleUnscopedTemplateName(TemplateName Template) {
   addSubstitution(Template);
 }
 
+void CXXNameMangler::mangleFloat(const llvm::APFloat &F) {
+  // TODO: avoid this copy with careful stream management.
+  llvm::SmallString<20> Buffer;
+  F.bitcastToAPInt().toString(Buffer, 16, false);
+  Out.write(Buffer.data(), Buffer.size());
+}
+
+void CXXNameMangler::mangleNumber(const llvm::APSInt &Value) {
+  if (Value.isSigned() && Value.isNegative()) {
+    Out << 'n';
+    Value.abs().print(Out, true);
+  } else
+    Value.print(Out, Value.isSigned());
+}
+
 void CXXNameMangler::mangleNumber(int64_t Number) {
   //  <number> ::= [n] <non-negative decimal integer>
   if (Number < 0) {
@@ -939,6 +957,53 @@ void CXXNameMangler::mangleTemplatePrefix(const TemplateDecl *ND) {
   addSubstitution(ND);
 }
 
+/// Mangles a template name under the production <type>.  Required for
+/// template template arguments.
+///   <type> ::= <class-enum-type>
+///          ::= <template-param>
+///          ::= <substitution>
+void CXXNameMangler::mangleType(TemplateName TN) {
+  if (mangleSubstitution(TN))
+    return;
+      
+  TemplateDecl *TD = 0;
+
+  switch (TN.getKind()) {
+  case TemplateName::QualifiedTemplate:
+    TD = TN.getAsQualifiedTemplateName()->getTemplateDecl();
+    goto HaveDecl;
+
+  case TemplateName::Template:
+    TD = TN.getAsTemplateDecl();
+    goto HaveDecl;
+
+  HaveDecl:
+    if (isa<TemplateTemplateParmDecl>(TD))
+      mangleTemplateParameter(cast<TemplateTemplateParmDecl>(TD)->getIndex());
+    else
+      mangleName(TD);
+    break;
+
+  case TemplateName::OverloadedTemplate:
+    llvm_unreachable("can't mangle an overloaded template name as a <type>");
+    break;
+
+  case TemplateName::DependentTemplate: {
+    const DependentTemplateName *Dependent = TN.getAsDependentTemplateName();
+    assert(Dependent->isIdentifier());
+
+    // <class-enum-type> ::= <name>
+    // <name> ::= <nested-name>
+    mangleUnresolvedScope(Dependent->getQualifier());
+    mangleSourceName(Dependent->getIdentifier());
+    break;
+  }
+
+  }
+
+  addSubstitution(TN);
+}
+
 void
 CXXNameMangler::mangleOperatorName(OverloadedOperatorKind OO, unsigned Arity) {
   switch (OO) {
@@ -1301,8 +1366,6 @@ void CXXNameMangler::mangleType(const TemplateTypeParmType *T) {
   mangleTemplateParameter(T->getIndex());
 }
 
-// FIXME: <type> ::= <template-template-param> <template-args>
-
 // <type> ::= P <type>   # pointer-to
 void CXXNameMangler::mangleType(const PointerType *T) {
   Out << 'P';
@@ -1467,11 +1530,7 @@ void CXXNameMangler::mangleIntegerLiteral(QualType T,
     // Boolean values are encoded as 0/1.
     Out << (Value.getBoolValue() ? '1' : '0');
   } else {
-    if (Value.isSigned() && Value.isNegative()) {
-      Out << 'n';
-      Value.abs().print(Out, true);
-    } else
-      Value.print(Out, Value.isSigned());
+    mangleNumber(Value);
   }
   Out << 'E';
 
@@ -1535,10 +1594,44 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
 #define STMT(Type, Base) \
   case Expr::Type##Class:
 #include "clang/AST/StmtNodes.inc"
+    // fallthrough
+
+  // These all can only appear in local or variable-initialization
+  // contexts and so should never appear in a mangling.
+  case Expr::AddrLabelExprClass:
+  case Expr::BlockDeclRefExprClass:
+  case Expr::CXXThisExprClass:
+  case Expr::DesignatedInitExprClass:
+  case Expr::ImplicitValueInitExprClass:
+  case Expr::InitListExprClass:
+  case Expr::ParenListExprClass:
+  case Expr::CXXScalarValueInitExprClass:
     llvm_unreachable("unexpected statement kind");
     break;
 
-  default: {
+  // FIXME: invent manglings for all these.
+  case Expr::BlockExprClass:
+  case Expr::CXXPseudoDestructorExprClass:
+  case Expr::ChooseExprClass:
+  case Expr::CompoundLiteralExprClass:
+  case Expr::ExtVectorElementExprClass:
+  case Expr::ObjCEncodeExprClass:
+  case Expr::ObjCImplicitSetterGetterRefExprClass:
+  case Expr::ObjCIsaExprClass:
+  case Expr::ObjCIvarRefExprClass:
+  case Expr::ObjCMessageExprClass:
+  case Expr::ObjCPropertyRefExprClass:
+  case Expr::ObjCProtocolExprClass:
+  case Expr::ObjCSelectorExprClass:
+  case Expr::ObjCStringLiteralClass:
+  case Expr::ObjCSuperExprClass:
+  case Expr::OffsetOfExprClass:
+  case Expr::PredefinedExprClass:
+  case Expr::ShuffleVectorExprClass:
+  case Expr::StmtExprClass:
+  case Expr::TypesCompatibleExprClass:
+  case Expr::UnaryTypeTraitExprClass:
+  case Expr::VAArgExprClass: {
     // As bad as this diagnostic is, it's better than crashing.
     Diagnostic &Diags = Context.getDiags();
     unsigned DiagID = Diags.getCustomDiagID(Diagnostic::Error,
@@ -1550,6 +1643,11 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     break;
   }
 
+  case Expr::CXXDefaultArgExprClass:
+    mangleExpression(cast<CXXDefaultArgExpr>(E)->getExpr());
+    break;
+
+  case Expr::CXXMemberCallExprClass: // fallthrough
   case Expr::CallExprClass: {
     const CallExpr *CE = cast<CallExpr>(E);
     Out << "cl";
@@ -1560,6 +1658,26 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     break;
   }
 
+  case Expr::CXXNewExprClass: {
+    // Proposal from David Vandervoorde, 2010.06.30
+    const CXXNewExpr *New = cast<CXXNewExpr>(E);
+    if (New->isGlobalNew()) Out << "gs";
+    Out << (New->isArray() ? "na" : "nw");
+    for (CXXNewExpr::const_arg_iterator I = New->placement_arg_begin(),
+           E = New->placement_arg_end(); I != E; ++I)
+      mangleExpression(*I);
+    Out << '_';
+    mangleType(New->getAllocatedType());
+    if (New->hasInitializer()) {
+      Out << "pi";
+      for (CXXNewExpr::const_arg_iterator I = New->constructor_arg_begin(),
+             E = New->constructor_arg_end(); I != E; ++I)
+        mangleExpression(*I);
+    }
+    Out << 'E';
+    break;
+  }
+
   case Expr::MemberExprClass: {
     const MemberExpr *ME = cast<MemberExpr>(E);
     mangleMemberExpr(ME->getBase(), ME->isArrow(),
@@ -1633,6 +1751,43 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     break;
   }
 
+  case Expr::CXXThrowExprClass: {
+    const CXXThrowExpr *TE = cast<CXXThrowExpr>(E);
+
+    // Proposal from David Vandervoorde, 2010.06.30
+    if (TE->getSubExpr()) {
+      Out << "tw";
+      mangleExpression(TE->getSubExpr());
+    } else {
+      Out << "tr";
+    }
+    break;
+  }
+
+  case Expr::CXXTypeidExprClass: {
+    const CXXTypeidExpr *TIE = cast<CXXTypeidExpr>(E);
+
+    // Proposal from David Vandervoorde, 2010.06.30
+    if (TIE->isTypeOperand()) {
+      Out << "ti";
+      mangleType(TIE->getTypeOperand());
+    } else {
+      Out << "te";
+      mangleExpression(TIE->getExprOperand());
+    }
+    break;
+  }
+
+  case Expr::CXXDeleteExprClass: {
+    const CXXDeleteExpr *DE = cast<CXXDeleteExpr>(E);
+
+    // Proposal from David Vandervoorde, 2010.06.30
+    if (DE->isGlobalDelete()) Out << "gs";
+    Out << (DE->isArrayForm() ? "da" : "dl");
+    mangleExpression(DE->getArgument());
+    break;
+  }
+
   case Expr::UnaryOperatorClass: {
     const UnaryOperator *UO = cast<UnaryOperator>(E);
     mangleOperatorName(UnaryOperator::getOverloadedOperator(UO->getOpcode()),
@@ -1641,6 +1796,18 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     break;
   }
 
+  case Expr::ArraySubscriptExprClass: {
+    const ArraySubscriptExpr *AE = cast<ArraySubscriptExpr>(E);
+
+    // Array subscript is treated as a syntactically wierd form of
+    // binary operator.
+    Out << "ix";
+    mangleExpression(AE->getLHS());
+    mangleExpression(AE->getRHS());
+    break;
+  }
+
+  case Expr::CompoundAssignOperatorClass: // fallthrough
   case Expr::BinaryOperatorClass: {
     const BinaryOperator *BO = cast<BinaryOperator>(E);
     mangleOperatorName(BinaryOperator::getOverloadedOperator(BO->getOpcode()),
@@ -1757,12 +1924,7 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     const FloatingLiteral *FL = cast<FloatingLiteral>(E);
     Out << 'L';
     mangleType(FL->getType());
-
-    // TODO: avoid this copy with careful stream management.
-    llvm::SmallString<20> Buffer;
-    FL->getValue().bitcastToAPInt().toString(Buffer, 16, false);
-    Out.write(Buffer.data(), Buffer.size());
-
+    mangleFloat(FL->getValue());
     Out << 'E';
     break;
   }
@@ -1780,16 +1942,62 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     Out << 'E';
     break;
 
-  case Expr::IntegerLiteralClass:
-    mangleIntegerLiteral(E->getType(),
-                         llvm::APSInt(cast<IntegerLiteral>(E)->getValue()));
+  case Expr::IntegerLiteralClass: {
+    llvm::APSInt Value(cast<IntegerLiteral>(E)->getValue());
+    if (E->getType()->isSignedIntegerType())
+      Value.setIsSigned(true);
+    mangleIntegerLiteral(E->getType(), Value);
     break;
+  }
 
+  case Expr::ImaginaryLiteralClass: {
+    const ImaginaryLiteral *IE = cast<ImaginaryLiteral>(E);
+    // Mangle as if a complex literal.
+    // Proposal from David Vandervoorde, 2010.06.30.
+    Out << 'L';
+    mangleType(E->getType());
+    if (const FloatingLiteral *Imag =
+          dyn_cast<FloatingLiteral>(IE->getSubExpr())) {
+      // Mangle a floating-point zero of the appropriate type.
+      mangleFloat(llvm::APFloat(Imag->getValue().getSemantics()));
+      Out << '_';
+      mangleFloat(Imag->getValue());
+    } else {
+      Out << '0' << '_';
+      llvm::APSInt Value(cast<IntegerLiteral>(IE->getSubExpr())->getValue());
+      if (IE->getSubExpr()->getType()->isSignedIntegerType())
+        Value.setIsSigned(true);
+      mangleNumber(Value);
+    }
+    Out << 'E';
+    break;
+  }
+
+  case Expr::StringLiteralClass: {
+    // Proposal from David Vandervoorde, 2010.06.30.
+    // I've sent a comment off asking whether this needs to also
+    // represent the length of the string.
+    Out << 'L';
+    const ConstantArrayType *T = cast<ConstantArrayType>(E->getType());
+    QualType CharTy = T->getElementType().getUnqualifiedType();
+    mangleType(CharTy);
+    Out << 'E';
+    break;
   }
-}
 
-// FIXME: <type> ::= G <type>   # imaginary (C 2000)
-// FIXME: <type> ::= U <source-name> <type>     # vendor extended type qualifier
+  case Expr::GNUNullExprClass:
+    // FIXME: should this really be mangled the same as nullptr?
+    // fallthrough
+
+  case Expr::CXXNullPtrLiteralExprClass: {
+    // Proposal from David Vandervoorde, 2010.06.30, as
+    // modified by ABI list discussion.
+    Out << "LDnE";
+    break;
+  }
+
+  }
+}
 
 void CXXNameMangler::mangleCXXCtorType(CXXCtorType T) {
   // <ctor-dtor-name> ::= C1  # complete object constructor
@@ -1874,9 +2082,8 @@ void CXXNameMangler::mangleTemplateArg(const NamedDecl *P,
     mangleType(A.getAsType());
     break;
   case TemplateArgument::Template:
-    assert(A.getAsTemplate().getAsTemplateDecl() &&
-           "Can't get dependent template names here");
-    mangleName(A.getAsTemplate().getAsTemplateDecl());
+    // This is mangled as <type>.
+    mangleType(A.getAsTemplate());
     break;
   case TemplateArgument::Expression:
     Out << 'X';
diff --git a/lib/CodeGen/TargetInfo.cpp b/lib/CodeGen/TargetInfo.cpp
index 73530d4..c65f203 100644
--- a/lib/CodeGen/TargetInfo.cpp
+++ b/lib/CodeGen/TargetInfo.cpp
@@ -2365,7 +2365,6 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() const {
     case llvm::Triple::DragonFly:
     case llvm::Triple::FreeBSD:
     case llvm::Triple::OpenBSD:
-    case llvm::Triple::Minix:
       return *(TheTargetCodeGenInfo =
                new X86_32TargetCodeGenInfo(Context, false, true));
 
diff --git a/lib/Driver/ArgList.cpp b/lib/Driver/ArgList.cpp
index 95fef89..9101523 100644
--- a/lib/Driver/ArgList.cpp
+++ b/lib/Driver/ArgList.cpp
@@ -95,6 +95,25 @@ Arg *ArgList::getLastArg(OptSpecifier Id0, OptSpecifier Id1,
   return Res;
 }
 
+Arg *ArgList::getLastArg(OptSpecifier Id0, OptSpecifier Id1,
+                         OptSpecifier Id2, OptSpecifier Id3) const {
+  Arg *Res = 0;
+  for (const_reverse_iterator it = rbegin(), ie = rend(); it != ie; ++it) {
+    if ((*it)->getOption().matches(Id0) ||
+        (*it)->getOption().matches(Id1) ||
+        (*it)->getOption().matches(Id2) ||
+        (*it)->getOption().matches(Id3)) {
+      Res = *it;
+      break;
+    }
+  }
+
+  if (Res)
+    Res->claim();
+
+  return Res;
+}
+
 bool ArgList::hasFlag(OptSpecifier Pos, OptSpecifier Neg, bool Default) const {
   if (Arg *A = getLastArg(Pos, Neg))
     return A->getOption().matches(Pos);
diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp
index 5da7908..2fc0a53 100644
--- a/lib/Driver/Driver.cpp
+++ b/lib/Driver/Driver.cpp
@@ -75,6 +75,11 @@ Driver::Driver(llvm::StringRef _Name, llvm::StringRef _Dir,
   P.appendComponent("clang");
   P.appendComponent(CLANG_VERSION_STRING);
   ResourceDir = P.str();
+
+  // Save the original clang executable path.
+  P = Dir;
+  P.appendComponent(Name);
+  ClangExecutable = P.str();
 }
 
 Driver::~Driver() {
diff --git a/lib/Driver/ToolChain.cpp b/lib/Driver/ToolChain.cpp
index 9b6264a..9fae67d 100644
--- a/lib/Driver/ToolChain.cpp
+++ b/lib/Driver/ToolChain.cpp
@@ -26,14 +26,11 @@ const Driver &ToolChain::getDriver() const {
  return Host.getDriver();
 }
 
-std::string ToolChain::GetFilePath(const Compilation &C,
-                                   const char *Name) const {
+std::string ToolChain::GetFilePath(const char *Name) const {
   return Host.getDriver().GetFilePath(Name, *this);
 
 }
 
-std::string ToolChain::GetProgramPath(const Compilation &C,
-                                      const char *Name,
-                                      bool WantFile) const {
+std::string ToolChain::GetProgramPath(const char *Name, bool WantFile) const {
   return Host.getDriver().GetProgramPath(Name, *this, WantFile);
 }
diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp
index 7876339..a78d153 100644
--- a/lib/Driver/ToolChains.cpp
+++ b/lib/Driver/ToolChains.cpp
@@ -469,19 +469,10 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
       iPhoneVersion = DAL->MakeJoinedArg(0, O, iPhoneOSTarget);
       DAL->append(iPhoneVersion);
     } else {
-      // Otherwise, choose a default platform based on the tool chain.
-      //
-      // FIXME: Don't hardcode default here.
-      if (getTriple().getArch() == llvm::Triple::arm ||
-          getTriple().getArch() == llvm::Triple::thumb) {
-        const Option *O = Opts.getOption(options::OPT_miphoneos_version_min_EQ);
-        iPhoneVersion = DAL->MakeJoinedArg(0, O, "3.0");
-        DAL->append(iPhoneVersion);
-      } else {
-        const Option *O = Opts.getOption(options::OPT_mmacosx_version_min_EQ);
-        OSXVersion = DAL->MakeJoinedArg(0, O, MacosxVersionMin);
-        DAL->append(OSXVersion);
-      }
+      // Otherwise, assume we are targeting OS X.
+      const Option *O = Opts.getOption(options::OPT_mmacosx_version_min_EQ);
+      OSXVersion = DAL->MakeJoinedArg(0, O, MacosxVersionMin);
+      DAL->append(OSXVersion);
     }
   }
 
diff --git a/lib/Driver/Tools.cpp b/lib/Driver/Tools.cpp
index 9e03a18..f423d4e 100644
--- a/lib/Driver/Tools.cpp
+++ b/lib/Driver/Tools.cpp
@@ -1489,8 +1489,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddAllArgs(CmdArgs, options::OPT_undef);
 
-  const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "clang"));
+  std::string Exec = getToolChain().getDriver().getClangProgramPath();
 
   // Optionally embed the -cc1 level arguments into the debug info, for build
   // analysis.
@@ -1510,7 +1509,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(Flags.str()));
   }
 
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  Dest.addCommand(new Command(JA, *this, Exec.c_str(), CmdArgs));
 
   // Explicitly warn that these options are unsupported, even though
   // we are allowing compilation to continue.
@@ -1589,9 +1588,8 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Input.getFilename());
   }
 
-  const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "clang"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  std::string Exec = getToolChain().getDriver().getClangProgramPath();
+  Dest.addCommand(new Command(JA, *this, Exec.c_str(), CmdArgs));
 }
 
 void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
@@ -1691,7 +1689,7 @@ void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *GCCName = getToolChain().getDriver().CCCGenericGCCName.c_str();
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, GCCName));
+    Args.MakeArgString(getToolChain().GetProgramPath(GCCName));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2098,7 +2096,7 @@ void darwin::Preprocess::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *CC1Name = getCC1Name(Inputs[0].getType());
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, CC1Name));
+    Args.MakeArgString(getToolChain().GetProgramPath(CC1Name));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2198,7 +2196,7 @@ void darwin::Compile::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *CC1Name = getCC1Name(Inputs[0].getType());
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, CC1Name));
+    Args.MakeArgString(getToolChain().GetProgramPath(CC1Name));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2253,7 +2251,7 @@ void darwin::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   // asm_final spec is empty.
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "as"));
+    Args.MakeArgString(getToolChain().GetProgramPath("as"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2356,8 +2354,15 @@ void darwin::Link::AddLinkArgs(const ArgList &Args,
   Args.AddAllArgs(CmdArgs, options::OPT_multiply__defined);
   Args.AddAllArgs(CmdArgs, options::OPT_multiply__defined__unused);
 
-  if (Args.hasArg(options::OPT_fpie))
-    CmdArgs.push_back("-pie");
+  if (const Arg *A = Args.getLastArg(options::OPT_fpie, options::OPT_fPIE,
+                                     options::OPT_fno_pie,
+                                     options::OPT_fno_PIE)) {
+    if (A->getOption().matches(options::OPT_fpie) ||
+        A->getOption().matches(options::OPT_fPIE))
+      CmdArgs.push_back("-pie");
+    else
+      CmdArgs.push_back("-no_pie");
+  }
 
   Args.AddLastArg(CmdArgs, options::OPT_prebind);
   Args.AddLastArg(CmdArgs, options::OPT_noprebind);
@@ -2505,7 +2510,7 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
         Args.hasArg(options::OPT_shared_libgcc) &&
         getDarwinToolChain().isMacosxVersionLT(10, 5)) {
       const char *Str =
-        Args.MakeArgString(getToolChain().GetFilePath(C, "crt3.o"));
+        Args.MakeArgString(getToolChain().GetFilePath("crt3.o"));
       CmdArgs.push_back(Str);
     }
   }
@@ -2565,7 +2570,7 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddAllArgs(CmdArgs, options::OPT_F);
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "ld"));
+    Args.MakeArgString(getToolChain().GetProgramPath("ld"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2589,7 +2594,7 @@ void darwin::Lipo::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(II.getFilename());
   }
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "lipo"));
+    Args.MakeArgString(getToolChain().GetProgramPath("lipo"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2609,7 +2614,7 @@ void darwin::Dsymutil::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back(Output.getFilename());
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "dsymutil"));
+    Args.MakeArgString(getToolChain().GetProgramPath("dsymutil"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2639,7 +2644,7 @@ void auroraux::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "gas"));
+    Args.MakeArgString(getToolChain().GetProgramPath("gas"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2685,17 +2690,17 @@ void auroraux::Link::ConstructJob(Compilation &C, const JobAction &JA,
       !Args.hasArg(options::OPT_nostartfiles)) {
     if (!Args.hasArg(options::OPT_shared)) {
       CmdArgs.push_back(Args.MakeArgString(
-                                getToolChain().GetFilePath(C, "crt1.o")));
+                                getToolChain().GetFilePath("crt1.o")));
       CmdArgs.push_back(Args.MakeArgString(
-                                getToolChain().GetFilePath(C, "crti.o")));
+                                getToolChain().GetFilePath("crti.o")));
       CmdArgs.push_back(Args.MakeArgString(
-                                getToolChain().GetFilePath(C, "crtbegin.o")));
+                                getToolChain().GetFilePath("crtbegin.o")));
     } else {
       CmdArgs.push_back(Args.MakeArgString(
-                                getToolChain().GetFilePath(C, "crti.o")));
+                                getToolChain().GetFilePath("crti.o")));
     }
     CmdArgs.push_back(Args.MakeArgString(
-                                getToolChain().GetFilePath(C, "crtn.o")));
+                                getToolChain().GetFilePath("crtn.o")));
   }
 
   CmdArgs.push_back(Args.MakeArgString("-L/opt/gcc4/lib/gcc/"
@@ -2741,11 +2746,11 @@ void auroraux::Link::ConstructJob(Compilation &C, const JobAction &JA,
       !Args.hasArg(options::OPT_nostartfiles)) {
     if (!Args.hasArg(options::OPT_shared))
       CmdArgs.push_back(Args.MakeArgString(
-                                getToolChain().GetFilePath(C, "crtend.o")));
+                                getToolChain().GetFilePath("crtend.o")));
   }
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "ld"));
+    Args.MakeArgString(getToolChain().GetProgramPath("ld"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2775,7 +2780,7 @@ void openbsd::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "as"));
+    Args.MakeArgString(getToolChain().GetProgramPath("as"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2820,12 +2825,12 @@ void openbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
       !Args.hasArg(options::OPT_nostartfiles)) {
     if (!Args.hasArg(options::OPT_shared)) {
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crt0.o")));
+                              getToolChain().GetFilePath("crt0.o")));
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crtbegin.o")));
+                              getToolChain().GetFilePath("crtbegin.o")));
     } else {
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crtbeginS.o")));
+                              getToolChain().GetFilePath("crtbeginS.o")));
     }
   }
 
@@ -2874,14 +2879,14 @@ void openbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
       !Args.hasArg(options::OPT_nostartfiles)) {
     if (!Args.hasArg(options::OPT_shared))
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crtend.o")));
+                              getToolChain().GetFilePath("crtend.o")));
     else
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crtendS.o")));
+                              getToolChain().GetFilePath("crtendS.o")));
   }
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "ld"));
+    Args.MakeArgString(getToolChain().GetProgramPath("ld"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2923,7 +2928,7 @@ void freebsd::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "as"));
+    Args.MakeArgString(getToolChain().GetProgramPath("as"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -2968,16 +2973,16 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
       !Args.hasArg(options::OPT_nostartfiles)) {
     if (!Args.hasArg(options::OPT_shared)) {
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crt1.o")));
+                              getToolChain().GetFilePath("crt1.o")));
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crti.o")));
+                              getToolChain().GetFilePath("crti.o")));
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crtbegin.o")));
+                              getToolChain().GetFilePath("crtbegin.o")));
     } else {
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crti.o")));
+                              getToolChain().GetFilePath("crti.o")));
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crtbeginS.o")));
+                              getToolChain().GetFilePath("crtbeginS.o")));
     }
   }
 
@@ -3037,17 +3042,17 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nostartfiles)) {
     if (!Args.hasArg(options::OPT_shared))
-      CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(C,
+      CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(
                                                                   "crtend.o")));
     else
-      CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(C,
+      CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(
                                                                  "crtendS.o")));
-    CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(C,
+    CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(
                                                                     "crtn.o")));
   }
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "ld"));
+    Args.MakeArgString(getToolChain().GetProgramPath("ld"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -3077,7 +3082,7 @@ void minix::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "gas"));
+    Args.MakeArgString(getToolChain().GetProgramPath("gas"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -3101,7 +3106,7 @@ void minix::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nostartfiles))
-    CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(C,
+    CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(
                                                       "/usr/gnu/lib/crtso.o")));
 
   Args.AddAllArgs(CmdArgs, options::OPT_L);
@@ -3145,12 +3150,12 @@ void minix::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nostartfiles)) {
-    CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(C,
+    CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(
                                               "/usr/gnu/lib/libend.a")));
   }
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "/usr/gnu/bin/gld"));
+    Args.MakeArgString(getToolChain().GetProgramPath("/usr/gnu/bin/gld"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -3189,7 +3194,7 @@ void dragonfly::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "as"));
+    Args.MakeArgString(getToolChain().GetProgramPath("as"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -3233,16 +3238,16 @@ void dragonfly::Link::ConstructJob(Compilation &C, const JobAction &JA,
       !Args.hasArg(options::OPT_nostartfiles)) {
     if (!Args.hasArg(options::OPT_shared)) {
       CmdArgs.push_back(
-            Args.MakeArgString(getToolChain().GetFilePath(C, "crt1.o")));
+            Args.MakeArgString(getToolChain().GetFilePath("crt1.o")));
       CmdArgs.push_back(
-            Args.MakeArgString(getToolChain().GetFilePath(C, "crti.o")));
+            Args.MakeArgString(getToolChain().GetFilePath("crti.o")));
       CmdArgs.push_back(
-            Args.MakeArgString(getToolChain().GetFilePath(C, "crtbegin.o")));
+            Args.MakeArgString(getToolChain().GetFilePath("crtbegin.o")));
     } else {
       CmdArgs.push_back(
-            Args.MakeArgString(getToolChain().GetFilePath(C, "crti.o")));
+            Args.MakeArgString(getToolChain().GetFilePath("crti.o")));
       CmdArgs.push_back(
-            Args.MakeArgString(getToolChain().GetFilePath(C, "crtbeginS.o")));
+            Args.MakeArgString(getToolChain().GetFilePath("crtbeginS.o")));
     }
   }
 
@@ -3313,15 +3318,15 @@ void dragonfly::Link::ConstructJob(Compilation &C, const JobAction &JA,
       !Args.hasArg(options::OPT_nostartfiles)) {
     if (!Args.hasArg(options::OPT_shared))
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crtend.o")));
+                              getToolChain().GetFilePath("crtend.o")));
     else
       CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crtendS.o")));
+                              getToolChain().GetFilePath("crtendS.o")));
     CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath(C, "crtn.o")));
+                              getToolChain().GetFilePath("crtn.o")));
   }
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath(C, "ld"));
+    Args.MakeArgString(getToolChain().GetProgramPath("ld"));
   Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp
index 314e253..88f0037 100644
--- a/lib/Frontend/ASTUnit.cpp
+++ b/lib/Frontend/ASTUnit.cpp
@@ -74,11 +74,13 @@ public:
     return false;
   }
 
-  virtual bool ReadPredefinesBuffer(llvm::StringRef PCHPredef,
-                                    FileID PCHBufferID,
+  virtual bool ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
                                     llvm::StringRef OriginalFileName,
                                     std::string &SuggestedPredefines) {
-    Predefines = PCHPredef;
+    Predefines = Buffers[0].Data;
+    for (unsigned I = 1, N = Buffers.size(); I != N; ++I) {
+      Predefines += Buffers[I].Data;
+    }
     return false;
   }
 
diff --git a/lib/Frontend/FrontendActions.cpp b/lib/Frontend/FrontendActions.cpp
index 670b6b8..3a53dee 100644
--- a/lib/Frontend/FrontendActions.cpp
+++ b/lib/Frontend/FrontendActions.cpp
@@ -80,7 +80,7 @@ ASTConsumer *GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI,
   if (!OS)
     return 0;
 
-  const PCHReader *Chain = CI.getInvocation().getFrontendOpts().ChainedPCH ?
+  PCHReader *Chain = CI.getInvocation().getFrontendOpts().ChainedPCH ?
                                CI.getPCHReader() : 0;
   const char *isysroot = CI.getFrontendOpts().RelocatablePCH ?
                              Sysroot.c_str() : 0;
diff --git a/lib/Frontend/GeneratePCH.cpp b/lib/Frontend/GeneratePCH.cpp
index 9be103e..2f3df94 100644
--- a/lib/Frontend/GeneratePCH.cpp
+++ b/lib/Frontend/GeneratePCH.cpp
@@ -28,28 +28,28 @@ using namespace clang;
 namespace {
   class PCHGenerator : public SemaConsumer {
     const Preprocessor &PP;
-    const PCHReader *Chain;
     const char *isysroot;
     llvm::raw_ostream *Out;
     Sema *SemaPtr;
     MemorizeStatCalls *StatCalls; // owned by the FileManager
+    std::vector<unsigned char> Buffer;
+    llvm::BitstreamWriter Stream;
+    PCHWriter Writer;
 
   public:
-    explicit PCHGenerator(const Preprocessor &PP,
-                          const PCHReader *Chain,
-                          const char *isysroot,
-                          llvm::raw_ostream *Out);
+    PCHGenerator(const Preprocessor &PP, PCHReader *Chain,
+                 const char *isysroot, llvm::raw_ostream *Out);
     virtual void InitializeSema(Sema &S) { SemaPtr = &S; }
     virtual void HandleTranslationUnit(ASTContext &Ctx);
   };
 }
 
 PCHGenerator::PCHGenerator(const Preprocessor &PP,
-                           const PCHReader *Chain,
+                           PCHReader *Chain,
                            const char *isysroot,
                            llvm::raw_ostream *OS)
-  : PP(PP), Chain(Chain), isysroot(isysroot), Out(OS), SemaPtr(0),
-    StatCalls(0) {
+  : PP(PP), isysroot(isysroot), Out(OS), SemaPtr(0), StatCalls(0),
+    Stream(Buffer), Writer(Stream, Chain) {
 
   // Install a stat() listener to keep track of all of the stat()
   // calls.
@@ -61,25 +61,23 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
   if (PP.getDiagnostics().hasErrorOccurred())
     return;
 
-  // Write the PCH contents into a buffer
-  std::vector<unsigned char> Buffer;
-  llvm::BitstreamWriter Stream(Buffer);
-  PCHWriter Writer(Stream);
-
   // Emit the PCH file
   assert(SemaPtr && "No Sema?");
-  Writer.WritePCH(*SemaPtr, StatCalls, Chain, isysroot);
+  Writer.WritePCH(*SemaPtr, StatCalls, isysroot);
 
   // Write the generated bitstream to "Out".
   Out->write((char *)&Buffer.front(), Buffer.size());
 
   // Make sure it hits disk now.
   Out->flush();
+
+  // Free up some memory, in case the process is kept alive.
+  Buffer.clear();
 }
 
 ASTConsumer *clang::CreatePCHGenerator(const Preprocessor &PP,
                                        llvm::raw_ostream *OS,
-                                       const PCHReader *Chain,
+                                       PCHReader *Chain,
                                        const char *isysroot) {
   return new PCHGenerator(PP, Chain, isysroot, OS);
 }
diff --git a/lib/Frontend/PCHReader.cpp b/lib/Frontend/PCHReader.cpp
index b452a0d..00aee49 100644
--- a/lib/Frontend/PCHReader.cpp
+++ b/lib/Frontend/PCHReader.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/Frontend/PCHReader.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Frontend/PCHDeserializationListener.h"
 #include "clang/Frontend/Utils.h"
 #include "../Sema/Sema.h" // FIXME: move Sema headers elsewhere
 #include "clang/AST/ASTConsumer.h"
@@ -140,8 +141,86 @@ bool PCHValidator::ReadTargetTriple(llvm::StringRef Triple) {
   return true;
 }
 
-bool PCHValidator::ReadPredefinesBuffer(llvm::StringRef PCHPredef,
-                                        FileID PCHBufferID,
+struct EmptyStringRef {
+  bool operator ()(llvm::StringRef r) const { return r.empty(); }
+};
+struct EmptyBlock {
+  bool operator ()(const PCHPredefinesBlock &r) const { return r.Data.empty(); }
+};
+
+static bool EqualConcatenations(llvm::SmallVector<llvm::StringRef, 2> L,
+                                PCHPredefinesBlocks R) {
+  // First, sum up the lengths.
+  unsigned LL = 0, RL = 0;
+  for (unsigned I = 0, N = L.size(); I != N; ++I) {
+    LL += L[I].size();
+  }
+  for (unsigned I = 0, N = R.size(); I != N; ++I) {
+    RL += R[I].Data.size();
+  }
+  if (LL != RL)
+    return false;
+  if (LL == 0 && RL == 0)
+    return true;
+
+  // Kick out empty parts, they confuse the algorithm below.
+  L.erase(std::remove_if(L.begin(), L.end(), EmptyStringRef()), L.end());
+  R.erase(std::remove_if(R.begin(), R.end(), EmptyBlock()), R.end());
+
+  // Do it the hard way. At this point, both vectors must be non-empty.
+  llvm::StringRef LR = L[0], RR = R[0].Data;
+  unsigned LI = 0, RI = 0, LN = L.size(), RN = R.size();
+  for (;;) {
+    // Compare the current pieces.
+    if (LR.size() == RR.size()) {
+      // If they're the same length, it's pretty easy.
+      if (LR != RR)
+        return false;
+      // Both pieces are done, advance.
+      ++LI;
+      ++RI;
+      // If either string is done, they're both done, since they're the same
+      // length.
+      if (LI == LN) {
+        assert(RI == RN && "Strings not the same length after all?");
+        return true;
+      }
+      LR = L[LI];
+      RR = R[RI].Data;
+    } else if (LR.size() < RR.size()) {
+      // Right piece is longer.
+      if (!RR.startswith(LR))
+        return false;
+      ++LI;
+      assert(LI != LN && "Strings not the same length after all?");
+      RR = RR.substr(LR.size());
+      LR = L[LI];
+    } else {
+      // Left piece is longer.
+      if (!LR.startswith(RR))
+        return false;
+      ++RI;
+      assert(RI != RN && "Strings not the same length after all?");
+      LR = LR.substr(RR.size());
+      RR = R[RI].Data;
+    }
+  }
+}
+
+static std::pair<FileID, llvm::StringRef::size_type>
+FindMacro(const PCHPredefinesBlocks &Buffers, llvm::StringRef MacroDef) {
+  std::pair<FileID, llvm::StringRef::size_type> Res;
+  for (unsigned I = 0, N = Buffers.size(); I != N; ++I) {
+    Res.second = Buffers[I].Data.find(MacroDef);
+    if (Res.second != llvm::StringRef::npos) {
+      Res.first = Buffers[I].BufferID;
+      break;
+    }
+  }
+  return Res;
+}
+
+bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
                                         llvm::StringRef OriginalFileName,
                                         std::string &SuggestedPredefines) {
   // We are in the context of an implicit include, so the predefines buffer will
@@ -160,9 +239,15 @@ bool PCHValidator::ReadPredefinesBuffer(llvm::StringRef PCHPredef,
     return true;
   }
 
-  // If the predefines is equal to the joined left and right halves, we're done!
-  if (Left.size() + Right.size() == PCHPredef.size() &&
-      PCHPredef.startswith(Left) && PCHPredef.endswith(Right))
+  // If the concatenation of all the PCH buffers is equal to the adjusted
+  // command line, we're done.
+  // We build a SmallVector of the command line here, because we'll eventually
+  // need to support an arbitrary amount of pieces anyway (when we have chained
+  // PCH reading).
+  llvm::SmallVector<llvm::StringRef, 2> CommandLine;
+  CommandLine.push_back(Left);
+  CommandLine.push_back(Right);
+  if (EqualConcatenations(CommandLine, Buffers))
     return false;
 
   SourceManager &SourceMgr = PP.getSourceManager();
@@ -170,7 +255,8 @@ bool PCHValidator::ReadPredefinesBuffer(llvm::StringRef PCHPredef,
   // The predefines buffers are different. Determine what the differences are,
   // and whether they require us to reject the PCH file.
   llvm::SmallVector<llvm::StringRef, 8> PCHLines;
-  PCHPredef.split(PCHLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+  for (unsigned I = 0, N = Buffers.size(); I != N; ++I)
+    Buffers[I].Data.split(PCHLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
 
   llvm::SmallVector<llvm::StringRef, 8> CmdLineLines;
   Left.split(CmdLineLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
@@ -235,10 +321,12 @@ bool PCHValidator::ReadPredefinesBuffer(llvm::StringRef PCHPredef,
           << MacroName;
 
       // Show the definition of this macro within the PCH file.
-      llvm::StringRef::size_type Offset = PCHPredef.find(Missing);
-      assert(Offset != llvm::StringRef::npos && "Unable to find macro!");
-      SourceLocation PCHMissingLoc = SourceMgr.getLocForStartOfFile(PCHBufferID)
-        .getFileLocWithOffset(Offset);
+      std::pair<FileID, llvm::StringRef::size_type> MacroLoc =
+          FindMacro(Buffers, Missing);
+      assert(MacroLoc.second!=llvm::StringRef::npos && "Unable to find macro!");
+      SourceLocation PCHMissingLoc =
+          SourceMgr.getLocForStartOfFile(MacroLoc.first)
+            .getFileLocWithOffset(MacroLoc.second);
       Reader.Diag(PCHMissingLoc, diag::note_pch_macro_defined_as) << MacroName;
 
       ConflictingDefines = true;
@@ -256,10 +344,12 @@ bool PCHValidator::ReadPredefinesBuffer(llvm::StringRef PCHPredef,
     }
 
     // Show the definition of this macro within the PCH file.
-    llvm::StringRef::size_type Offset = PCHPredef.find(Missing);
-    assert(Offset != llvm::StringRef::npos && "Unable to find macro!");
-    SourceLocation PCHMissingLoc = SourceMgr.getLocForStartOfFile(PCHBufferID)
-      .getFileLocWithOffset(Offset);
+    std::pair<FileID, llvm::StringRef::size_type> MacroLoc =
+        FindMacro(Buffers, Missing);
+    assert(MacroLoc.second!=llvm::StringRef::npos && "Unable to find macro!");
+    SourceLocation PCHMissingLoc =
+        SourceMgr.getLocForStartOfFile(MacroLoc.first)
+          .getFileLocWithOffset(MacroLoc.second);
     Reader.Diag(PCHMissingLoc, diag::note_using_macro_def_from_pch);
   }
 
@@ -324,10 +414,10 @@ void PCHValidator::ReadCounter(unsigned Value) {
 
 PCHReader::PCHReader(Preprocessor &PP, ASTContext *Context,
                      const char *isysroot)
-  : Listener(new PCHValidator(PP, *this)), SourceMgr(PP.getSourceManager()),
-    FileMgr(PP.getFileManager()), Diags(PP.getDiagnostics()),
-    SemaObj(0), PP(&PP), Context(Context), StatCache(0), Consumer(0),
-    IdentifierTableData(0), IdentifierLookupTable(0),
+  : Listener(new PCHValidator(PP, *this)), DeserializationListener(0),
+    SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()),
+    Diags(PP.getDiagnostics()), SemaObj(0), PP(&PP), Context(Context),
+    StatCache(0), Consumer(0), IdentifierTableData(0), IdentifierLookupTable(0),
     IdentifierOffsets(0),
     MethodPoolLookupTable(0), MethodPoolLookupTableData(0),
     TotalSelectorsInMethodPool(0), SelectorOffsets(0),
@@ -343,8 +433,8 @@ PCHReader::PCHReader(Preprocessor &PP, ASTContext *Context,
 
 PCHReader::PCHReader(SourceManager &SourceMgr, FileManager &FileMgr,
                      Diagnostic &Diags, const char *isysroot)
-  : SourceMgr(SourceMgr), FileMgr(FileMgr), Diags(Diags),
-    SemaObj(0), PP(0), Context(0), StatCache(0), Consumer(0),
+  : DeserializationListener(0), SourceMgr(SourceMgr), FileMgr(FileMgr),
+    Diags(Diags), SemaObj(0), PP(0), Context(0), StatCache(0), Consumer(0),
     IdentifierTableData(0), IdentifierLookupTable(0),
     IdentifierOffsets(0),
     MethodPoolLookupTable(0), MethodPoolLookupTableData(0),
@@ -609,27 +699,18 @@ void PCHReader::Error(const char *Msg) {
   Diag(diag::err_fe_pch_malformed) << Msg;
 }
 
-/// \brief Check the contents of the predefines buffer against the
-/// contents of the predefines buffer used to build the PCH file.
-///
-/// The contents of the two predefines buffers should be the same. If
-/// not, then some command-line option changed the preprocessor state
-/// and we must reject the PCH file.
+/// \brief Check the contents of the concatenation of all predefines buffers in
+/// the PCH chain against the contents of the predefines buffer of the current
+/// compiler invocation.
 ///
-/// \param PCHPredef The start of the predefines buffer in the PCH
-/// file.
-///
-/// \param PCHPredefLen The length of the predefines buffer in the PCH
-/// file.
-///
-/// \param PCHBufferID The FileID for the PCH predefines buffer.
+/// The contents should be the same. If not, then some command-line option
+/// changed the preprocessor state and we must probably reject the PCH file.
 ///
 /// \returns true if there was a mismatch (in which case the PCH file
 /// should be ignored), or false otherwise.
-bool PCHReader::CheckPredefinesBuffer(llvm::StringRef PCHPredef,
-                                      FileID PCHBufferID) {
+bool PCHReader::CheckPredefinesBuffers() {
   if (Listener)
-    return Listener->ReadPredefinesBuffer(PCHPredef, PCHBufferID,
+    return Listener->ReadPredefinesBuffer(PCHPredefinesBuffers,
                                           ActualOriginalFileName,
                                           SuggestedPredefines);
   return false;
@@ -958,9 +1039,11 @@ PCHReader::PCHReadResult PCHReader::ReadSLocEntryRecord(unsigned ID) {
     FileID BufferID = SourceMgr.createFileIDForMemBuffer(Buffer, ID, Offset);
 
     if (strcmp(Name, "<built-in>") == 0) {
-      PCHPredefinesBufferID = BufferID;
-      PCHPredefines = BlobStart;
-      PCHPredefinesLen = BlobLen - 1;
+      PCHPredefinesBlock Block = {
+        BufferID,
+        llvm::StringRef(BlobStart, BlobLen - 1)
+      };
+      PCHPredefinesBuffers.push_back(Block);
     }
 
     break;
@@ -1636,8 +1719,7 @@ PCHReader::PCHReadResult PCHReader::ReadPCH(const std::string &FileName) {
   }
 
   // Check the predefines buffer.
-  if (CheckPredefinesBuffer(llvm::StringRef(PCHPredefines, PCHPredefinesLen),
-                            PCHPredefinesBufferID))
+  if (CheckPredefinesBuffers())
     return IgnorePCH;
 
   if (PP) {
@@ -2545,8 +2627,12 @@ QualType PCHReader::GetType(pch::TypeID ID) {
 
   Index -= pch::NUM_PREDEF_TYPE_IDS;
   //assert(Index < TypesLoaded.size() && "Type index out-of-range");
-  if (TypesLoaded[Index].isNull())
+  if (TypesLoaded[Index].isNull()) {
     TypesLoaded[Index] = ReadTypeRecord(TypeOffsets[Index]);
+    TypesLoaded[Index]->setFromPCH();
+    if (DeserializationListener)
+      DeserializationListener->TypeRead(ID, TypesLoaded[Index]);
+  }
 
   return TypesLoaded[Index].withFastQualifiers(FastQuals);
 }
@@ -2592,8 +2678,11 @@ Decl *PCHReader::GetExternalDecl(uint32_t ID) {
 }
 
 TranslationUnitDecl *PCHReader::GetTranslationUnitDecl() {
-  if (!DeclsLoaded[0])
+  if (!DeclsLoaded[0]) {
     ReadDeclRecord(DeclOffsets[0], 0);
+    if (DeserializationListener)
+      DeserializationListener->DeclRead(0, DeclsLoaded[0]);
+  }
 
   return cast<TranslationUnitDecl>(DeclsLoaded[0]);
 }
@@ -2608,8 +2697,11 @@ Decl *PCHReader::GetDecl(pch::DeclID ID) {
   }
 
   unsigned Index = ID - 1;
-  if (!DeclsLoaded[Index])
+  if (!DeclsLoaded[Index]) {
     ReadDeclRecord(DeclOffsets[Index], Index);
+    if (DeserializationListener)
+      DeserializationListener->DeclRead(ID, DeclsLoaded[Index]);
+  }
 
   return DeclsLoaded[Index];
 }
diff --git a/lib/Frontend/PCHWriter.cpp b/lib/Frontend/PCHWriter.cpp
index e863998..093c1e3 100644
--- a/lib/Frontend/PCHWriter.cpp
+++ b/lib/Frontend/PCHWriter.cpp
@@ -743,8 +743,7 @@ adjustFilenameForRelocatablePCH(const char *Filename, const char *isysroot) {
 }
 
 /// \brief Write the PCH metadata (e.g., i686-apple-darwin9).
-void PCHWriter::WriteMetadata(ASTContext &Context, const PCHReader *Chain,
-                              const char *isysroot) {
+void PCHWriter::WriteMetadata(ASTContext &Context, const char *isysroot) {
   using namespace llvm;
 
   // Metadata
@@ -2074,13 +2073,16 @@ void PCHWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) {
   SelectorOffsets[ID - 1] = Offset;
 }
 
-PCHWriter::PCHWriter(llvm::BitstreamWriter &Stream)
-  : Stream(Stream), NextTypeID(pch::NUM_PREDEF_TYPE_IDS),
+PCHWriter::PCHWriter(llvm::BitstreamWriter &Stream, PCHReader *Chain)
+  : Stream(Stream), Chain(Chain), NextTypeID(pch::NUM_PREDEF_TYPE_IDS),
     CollectedStmts(&StmtsToEmit), NumStatements(0), NumMacros(0),
-    NumLexicalDeclContexts(0), NumVisibleDeclContexts(0) { }
+    NumLexicalDeclContexts(0), NumVisibleDeclContexts(0) {
+  if (Chain)
+    Chain->setDeserializationListener(this);
+}
 
 void PCHWriter::WritePCH(Sema &SemaRef, MemorizeStatCalls *StatCalls,
-                         const PCHReader *Chain, const char *isysroot) {
+                         const char *isysroot) {
   // Emit the file header.
   Stream.Emit((unsigned)'C', 8);
   Stream.Emit((unsigned)'P', 8);
@@ -2090,7 +2092,7 @@ void PCHWriter::WritePCH(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   WriteBlockInfoBlock();
 
   if (Chain)
-    WritePCHChain(SemaRef, StatCalls, Chain, isysroot);
+    WritePCHChain(SemaRef, StatCalls, isysroot);
   else
     WritePCHCore(SemaRef, StatCalls, isysroot);
 }
@@ -2164,7 +2166,7 @@ void PCHWriter::WritePCHCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   // Write the remaining PCH contents.
   RecordData Record;
   Stream.EnterSubblock(pch::PCH_BLOCK_ID, 5);
-  WriteMetadata(Context, 0, isysroot);
+  WriteMetadata(Context, isysroot);
   WriteLanguageOptions(Context.getLangOptions());
   if (StatCalls && !isysroot)
     WriteStatCache(*StatCalls);
@@ -2275,7 +2277,7 @@ void PCHWriter::WritePCHCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 }
 
 void PCHWriter::WritePCHChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
-                              const PCHReader *Chain, const char *isysroot) {
+                              const char *isysroot) {
   using namespace llvm;
 
   ASTContext &Context = SemaRef.Context;
@@ -2284,7 +2286,7 @@ void PCHWriter::WritePCHChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   
   RecordData Record;
   Stream.EnterSubblock(pch::PCH_BLOCK_ID, 5);
-  WriteMetadata(Context, Chain, isysroot);
+  WriteMetadata(Context, isysroot);
   // FIXME: StatCache
   // FIXME: Source manager block
 
@@ -2737,3 +2739,10 @@ void PCHWriter::AddCXXBaseSpecifier(const CXXBaseSpecifier &Base,
   AddTypeRef(Base.getType(), Record);
   AddSourceRange(Base.getSourceRange(), Record);
 }
+
+void PCHWriter::TypeRead(pch::TypeID ID, QualType T) {
+}
+
+void PCHWriter::DeclRead(pch::DeclID ID, const Decl *D) {
+}
+
diff --git a/lib/Rewrite/Rewriter.cpp b/lib/Rewrite/Rewriter.cpp
index 8b0bf91..92e2b03 100644
--- a/lib/Rewrite/Rewriter.cpp
+++ b/lib/Rewrite/Rewriter.cpp
@@ -40,7 +40,7 @@ void RewriteBuffer::RemoveText(unsigned OrigOffset, unsigned Size) {
   AddReplaceDelta(OrigOffset, -Size);
 }
 
-void RewriteBuffer::InsertText(unsigned OrigOffset, const llvm::StringRef &Str,
+void RewriteBuffer::InsertText(unsigned OrigOffset, llvm::StringRef Str,
                                bool InsertAfter) {
 
   // Nothing to insert, exit early.
@@ -57,7 +57,7 @@ void RewriteBuffer::InsertText(unsigned OrigOffset, const llvm::StringRef &Str,
 /// buffer with a new string.  This is effectively a combined "remove+insert"
 /// operation.
 void RewriteBuffer::ReplaceText(unsigned OrigOffset, unsigned OrigLength,
-                                const llvm::StringRef &NewStr) {
+                                llvm::StringRef NewStr) {
   unsigned RealOffset = getMappedOffset(OrigOffset, true);
   Buffer.erase(RealOffset, OrigLength);
   Buffer.insert(RealOffset, NewStr.begin(), NewStr.end());
@@ -185,7 +185,7 @@ RewriteBuffer &Rewriter::getEditBuffer(FileID FID) {
 
 /// InsertText - Insert the specified string at the specified location in the
 /// original buffer.
-bool Rewriter::InsertText(SourceLocation Loc, const llvm::StringRef &Str,
+bool Rewriter::InsertText(SourceLocation Loc, llvm::StringRef Str,
                           bool InsertAfter) {
   if (!isRewritable(Loc)) return true;
   FileID FID;
@@ -207,7 +207,7 @@ bool Rewriter::RemoveText(SourceLocation Start, unsigned Length) {
 /// buffer with a new string.  This is effectively a combined "remove/insert"
 /// operation.
 bool Rewriter::ReplaceText(SourceLocation Start, unsigned OrigLength,
-                           const llvm::StringRef &NewStr) {
+                           llvm::StringRef NewStr) {
   if (!isRewritable(Start)) return true;
   FileID StartFileID;
   unsigned StartOffs = getLocationOffsetAndFileID(Start, StartFileID);
diff --git a/lib/Sema/Sema.h b/lib/Sema/Sema.h
index ddbe7e0..8336918 100644
--- a/lib/Sema/Sema.h
+++ b/lib/Sema/Sema.h
@@ -118,7 +118,8 @@ struct FunctionScopeInfo {
 
   /// \brief Set true when a function, method contains a VLA or ObjC try block,
   /// which introduce scopes that need to be checked for goto conditions.  If a
-  /// function does not contain this, then it need not have the jump checker run on it.
+  /// function does not contain this, then it need not have the jump checker run
+  /// on it.
   bool NeedsScopeChecking;
 
   /// \brief The number of errors that had occurred before starting this
@@ -1540,7 +1541,7 @@ public:
 
   // Decl attributes - this routine is the top level dispatcher.
   void ProcessDeclAttributes(Scope *S, Decl *D, const Declarator &PD);
-  void ProcessDeclAttributeList(Scope *S, Decl *D, const AttributeList *AttrList);
+  void ProcessDeclAttributeList(Scope *S, Decl *D, const AttributeList *AL);
 
   void WarnUndefinedMethod(SourceLocation ImpLoc, ObjCMethodDecl *method,
                            bool &IncompleteImpl, unsigned DiagID);
@@ -2955,7 +2956,8 @@ public:
                                           TemplateParameterList **ParamLists,
                                           unsigned NumParamLists,
                                           bool IsFriend,
-                                          bool &IsExplicitSpecialization);
+                                          bool &IsExplicitSpecialization,
+                                          bool &Invalid);
 
   DeclResult CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
                                 SourceLocation KWLoc, CXXScopeSpec &SS,
@@ -3021,7 +3023,7 @@ public:
                                             Declarator &D);
 
   virtual DeclPtrTy ActOnStartOfFunctionTemplateDef(Scope *FnBodyScope,
-                                                    MultiTemplateParamsArg TemplateParameterLists,
+                                  MultiTemplateParamsArg TemplateParameterLists,
                                                     Declarator &D);
 
   bool
@@ -3029,7 +3031,7 @@ public:
                                          TemplateSpecializationKind NewTSK,
                                          NamedDecl *PrevDecl,
                                          TemplateSpecializationKind PrevTSK,
-                                         SourceLocation PrevPointOfInstantiation,
+                                         SourceLocation PrevPtOfInstantiation,
                                          bool &SuppressNew);
 
   bool CheckDependentFunctionTemplateSpecialization(FunctionDecl *FD,
@@ -3969,7 +3971,7 @@ public:
                                                  SourceLocation *IdentLocs,
                                                  unsigned NumElts);
 
-  virtual DeclPtrTy ActOnForwardProtocolDeclaration(SourceLocation AtProtocolLoc,
+  virtual DeclPtrTy ActOnForwardProtocolDeclaration(SourceLocation AtProtoclLoc,
                                             const IdentifierLocPair *IdentList,
                                                   unsigned NumElts,
                                                   AttributeList *attrList);
@@ -4326,7 +4328,7 @@ public:
                                  bool IgnoreBaseAccess = false);
   bool PerformImplicitConversion(Expr *&From, QualType ToType,
                                  const StandardConversionSequence& SCS,
-                                 AssignmentAction Action, bool IgnoreBaseAccess);
+                                 AssignmentAction Action,bool IgnoreBaseAccess);
 
   /// the following "Check" methods will return a valid/converted QualType
   /// or a null QualType (indicating an error diagnostic was issued).
@@ -4347,11 +4349,12 @@ public:
   QualType CheckShiftOperands( // C99 6.5.7
     Expr *&lex, Expr *&rex, SourceLocation OpLoc, bool isCompAssign = false);
   QualType CheckCompareOperands( // C99 6.5.8/9
-    Expr *&lex, Expr *&rex, SourceLocation OpLoc, unsigned Opc, bool isRelational);
+    Expr *&lex, Expr *&rex, SourceLocation OpLoc, unsigned Opc,
+                                bool isRelational);
   QualType CheckBitwiseOperands( // C99 6.5.[10...12]
     Expr *&lex, Expr *&rex, SourceLocation OpLoc, bool isCompAssign = false);
   QualType CheckLogicalOperands( // C99 6.5.[13,14]
-    Expr *&lex, Expr *&rex, SourceLocation OpLoc);
+    Expr *&lex, Expr *&rex, SourceLocation OpLoc, unsigned Opc);
   // CheckAssignmentOperands is used for both simple and compound assignment.
   // For simple assignment, pass both expressions and a null converted type.
   // For compound assignment, pass both expressions and the converted type.
diff --git a/lib/Sema/SemaCXXCast.cpp b/lib/Sema/SemaCXXCast.cpp
index b7e855f..b8e27e7 100644
--- a/lib/Sema/SemaCXXCast.cpp
+++ b/lib/Sema/SemaCXXCast.cpp
@@ -153,7 +153,8 @@ Sema::BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
   case tok::kw_const_cast:
     if (!TypeDependent)
       CheckConstCast(*this, Ex, DestType, OpRange, DestRange);
-    return Owned(new (Context) CXXConstCastExpr(DestType.getNonReferenceType(),
+    return Owned(new (Context) CXXConstCastExpr(
+                                        DestType.getNonLValueExprType(Context),
                                                 Ex, DestTInfo, OpLoc));
 
   case tok::kw_dynamic_cast: {
@@ -161,7 +162,8 @@ Sema::BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
     CXXBaseSpecifierArray BasePath;
     if (!TypeDependent)
       CheckDynamicCast(*this, Ex, DestType, OpRange, DestRange, Kind, BasePath);
-    return Owned(new (Context)CXXDynamicCastExpr(DestType.getNonReferenceType(),
+    return Owned(new (Context)CXXDynamicCastExpr(
+                                          DestType.getNonLValueExprType(Context),
                                                  Kind, Ex, BasePath, DestTInfo,
                                                  OpLoc));
   }
@@ -170,7 +172,7 @@ Sema::BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
     if (!TypeDependent)
       CheckReinterpretCast(*this, Ex, DestType, OpRange, DestRange, Kind);
     return Owned(new (Context) CXXReinterpretCastExpr(
-                                  DestType.getNonReferenceType(),
+                                  DestType.getNonLValueExprType(Context),
                                   Kind, Ex, CXXBaseSpecifierArray(), 
                                   DestTInfo, OpLoc));
   }
@@ -180,7 +182,8 @@ Sema::BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
     if (!TypeDependent)
       CheckStaticCast(*this, Ex, DestType, OpRange, Kind, BasePath);
     
-    return Owned(new (Context) CXXStaticCastExpr(DestType.getNonReferenceType(),
+    return Owned(new (Context) CXXStaticCastExpr(
+                                         DestType.getNonLValueExprType(Context),
                                                  Kind, Ex, BasePath,
                                                  DestTInfo, OpLoc));
   }
@@ -1049,6 +1052,8 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
                                         const SourceRange &OpRange,
                                         unsigned &msg,
                                         CastExpr::CastKind &Kind) {
+  bool IsLValueCast = false;
+  
   DestType = Self.Context.getCanonicalType(DestType);
   QualType SrcType = SrcExpr->getType();
   if (const ReferenceType *DestTypeTmp = DestType->getAs<ReferenceType>()) {
@@ -1066,6 +1071,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
     // This code does this transformation for the checked types.
     DestType = Self.Context.getPointerType(DestTypeTmp->getPointeeType());
     SrcType = Self.Context.getPointerType(SrcType);
+    IsLValueCast = true;
   }
 
   // Canonicalize source for comparison.
@@ -1092,7 +1098,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
     }
 
     // A valid member pointer cast.
-    Kind = CastExpr::CK_BitCast;
+    Kind = IsLValueCast? CastExpr::CK_LValueBitCast : CastExpr::CK_BitCast;
     return TC_Success;
   }
 
@@ -1209,7 +1215,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
     
   // Not casting away constness, so the only remaining check is for compatible
   // pointer categories.
-  Kind = CastExpr::CK_BitCast;
+  Kind = IsLValueCast? CastExpr::CK_LValueBitCast : CastExpr::CK_BitCast;
 
   if (SrcType->isFunctionPointerType()) {
     if (DestType->isFunctionPointerType()) {
diff --git a/lib/Sema/SemaCodeComplete.cpp b/lib/Sema/SemaCodeComplete.cpp
index 8286110..5528875 100644
--- a/lib/Sema/SemaCodeComplete.cpp
+++ b/lib/Sema/SemaCodeComplete.cpp
@@ -407,13 +407,16 @@ bool ResultBuilder::isInterestingDecl(NamedDecl *ND,
       return false;
     
     // Filter out names reserved for the implementation (C99 7.1.3, 
-    // C++ [lib.global.names]). Users don't need to see those.
+    // C++ [lib.global.names]) if they come from a system header.
     //
     // FIXME: Add predicate for this.
     if (Id->getLength() >= 2) {
       const char *Name = Id->getNameStart();
       if (Name[0] == '_' &&
-          (Name[1] == '_' || (Name[1] >= 'A' && Name[1] <= 'Z')))
+          (Name[1] == '_' || (Name[1] >= 'A' && Name[1] <= 'Z')) &&
+          (ND->getLocation().isInvalid() ||
+           SemaRef.SourceMgr.isInSystemHeader(
+                          SemaRef.SourceMgr.getSpellingLoc(ND->getLocation()))))
         return false;
     }
   }
diff --git a/lib/Sema/SemaDecl.cpp b/lib/Sema/SemaDecl.cpp
index 9c683f7..c1c898f 100644
--- a/lib/Sema/SemaDecl.cpp
+++ b/lib/Sema/SemaDecl.cpp
@@ -2572,6 +2572,7 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   // determine whether we have a template or a template specialization.
   bool isExplicitSpecialization = false;
   unsigned NumMatchedTemplateParamLists = TemplateParamLists.size();
+  bool Invalid = false;
   if (TemplateParameterList *TemplateParams
         = MatchTemplateParametersToScopeSpecifier(
                                   D.getDeclSpec().getSourceRange().getBegin(),
@@ -2579,7 +2580,8 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
                         (TemplateParameterList**)TemplateParamLists.get(),
                                                    TemplateParamLists.size(),
                                                   /*never a friend*/ false,
-                                                  isExplicitSpecialization)) {
+                                                  isExplicitSpecialization,
+                                                  Invalid)) {
     // All but one template parameter lists have been matching.
     --NumMatchedTemplateParamLists;
 
@@ -2606,7 +2608,7 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   VarDecl *NewVD = VarDecl::Create(Context, DC, D.getIdentifierLoc(),
                                    II, R, TInfo, SC, SCAsWritten);
 
-  if (D.isInvalidType())
+  if (D.isInvalidType() || Invalid)
     NewVD->setInvalidDecl();
 
   SetNestedNameSpecifier(NewVD, D);
@@ -3156,6 +3158,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   bool isExplicitSpecialization = false;
   bool isFunctionTemplateSpecialization = false;
   unsigned NumMatchedTemplateParamLists = TemplateParamLists.size();
+  bool Invalid = false;
   if (TemplateParameterList *TemplateParams
         = MatchTemplateParametersToScopeSpecifier(
                                   D.getDeclSpec().getSourceRange().getBegin(),
@@ -3163,7 +3166,8 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
                            (TemplateParameterList**)TemplateParamLists.get(),
                                                   TemplateParamLists.size(),
                                                   isFriend,
-                                                  isExplicitSpecialization)) {
+                                                  isExplicitSpecialization,
+                                                  Invalid)) {
     // All but one template parameter lists have been matching.
     --NumMatchedTemplateParamLists;
 
@@ -3214,6 +3218,12 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
                         (TemplateParameterList**)TemplateParamLists.release());
   }
 
+  if (Invalid) {
+    NewFD->setInvalidDecl();
+    if (FunctionTemplate)
+      FunctionTemplate->setInvalidDecl();
+  }
+  
   // C++ [dcl.fct.spec]p5:
   //   The virtual specifier shall only be used in declarations of
   //   nonstatic class member functions that appear within a
@@ -5037,19 +5047,24 @@ Sema::DeclPtrTy Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
   // FIXME: Check explicit specializations more carefully.
   bool isExplicitSpecialization = false;
   unsigned NumMatchedTemplateParamLists = TemplateParameterLists.size();
+  bool Invalid = false;
   if (TUK != TUK_Reference) {
     if (TemplateParameterList *TemplateParams
           = MatchTemplateParametersToScopeSpecifier(KWLoc, SS,
                         (TemplateParameterList**)TemplateParameterLists.get(),
                                               TemplateParameterLists.size(),
                                                     TUK == TUK_Friend,
-                                                    isExplicitSpecialization)) {
+                                                    isExplicitSpecialization,
+                                                    Invalid)) {
       // All but one template parameter lists have been matching.
       --NumMatchedTemplateParamLists;
 
       if (TemplateParams->size() > 0) {
         // This is a declaration or definition of a class template (which may
         // be a member of another template).
+        if (Invalid)
+          return DeclPtrTy();
+        
         OwnedDecl = false;
         DeclResult Result = CheckClassTemplate(S, TagSpec, TUK, KWLoc,
                                                SS, Name, NameLoc, Attr,
@@ -5069,7 +5084,6 @@ Sema::DeclPtrTy Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
   DeclContext *SearchDC = CurContext;
   DeclContext *DC = CurContext;
   bool isStdBadAlloc = false;
-  bool Invalid = false;
 
   RedeclarationKind Redecl = ForRedeclaration;
   if (TUK == TUK_Friend || TUK == TUK_Reference)
diff --git a/lib/Sema/SemaExpr.cpp b/lib/Sema/SemaExpr.cpp
index ce3bf11..5f46a97 100644
--- a/lib/Sema/SemaExpr.cpp
+++ b/lib/Sema/SemaExpr.cpp
@@ -164,7 +164,7 @@ void Sema::DiagnoseSentinelCalls(NamedDecl *D, SourceLocation Loc,
   if (!sentinelExpr) return;
   if (sentinelExpr->isTypeDependent()) return;
   if (sentinelExpr->isValueDependent()) return;
-  if (sentinelExpr->getType()->isPointerType() &&
+  if (sentinelExpr->getType()->isAnyPointerType() &&
       sentinelExpr->IgnoreParenCasts()->isNullPointerConstant(Context,
                                             Expr::NPC_ValueDependentIsNull))
     return;
@@ -475,6 +475,7 @@ Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, SourceLocation Loc,
     if (isa<NonTypeTemplateParmDecl>(VD)) {
       // Non-type template parameters can be referenced anywhere they are
       // visible.
+      Ty = Ty.getNonLValueExprType(Context);
     } else if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(CurContext)) {
       if (const FunctionDecl *FD = MD->getParent()->isLocalClass()) {
         if (VD->hasLocalStorage() && VD->getDeclContext() != CurContext) {
@@ -4008,7 +4009,8 @@ Sema::BuildCStyleCastExpr(SourceLocation LParenLoc, TypeSourceInfo *Ty,
     return ExprError();
 
   Op.release();
-  return Owned(new (Context) CStyleCastExpr(Ty->getType().getNonReferenceType(),
+  return Owned(new (Context) CStyleCastExpr(
+                                    Ty->getType().getNonLValueExprType(Context),
                                             Kind, castExpr, BasePath, Ty,
                                             LParenLoc, RParenLoc));
 }
@@ -4904,7 +4906,7 @@ Sema::CheckSingleAssignmentConstraints(QualType lhsType, Expr *&rExpr) {
   // The getNonReferenceType() call makes sure that the resulting expression
   // does not have reference type.
   if (result != Incompatible && rExpr->getType() != lhsType)
-    ImpCastExprToType(rExpr, lhsType.getNonReferenceType(),
+    ImpCastExprToType(rExpr, lhsType.getNonLValueExprType(Context),
                       CastExpr::CK_Unknown);
   return result;
 }
@@ -5733,7 +5735,25 @@ inline QualType Sema::CheckBitwiseOperands(
 }
 
 inline QualType Sema::CheckLogicalOperands( // C99 6.5.[13,14]
-  Expr *&lex, Expr *&rex, SourceLocation Loc) {
+  Expr *&lex, Expr *&rex, SourceLocation Loc, unsigned Opc) {
+  
+  // Diagnose cases where the user write a logical and/or but probably meant a
+  // bitwise one.  We do this when the LHS is a non-bool integer and the RHS
+  // is a constant.
+  if (lex->getType()->isIntegerType() && !lex->getType()->isBooleanType() &&
+      rex->getType()->isIntegerType() && rex->isEvaluatable(Context) &&
+      // Don't warn if the RHS is a (constant folded) boolean expression like
+      // "sizeof(int) == 4".
+      !rex->isKnownToHaveBooleanValue() &&
+      // Don't warn in macros.
+      !Loc.isMacroID())
+    Diag(Loc, diag::warn_logical_instead_of_bitwise)
+     << rex->getSourceRange()
+      << (Opc == BinaryOperator::LAnd ? "&&" : "||")
+      << (Opc == BinaryOperator::LAnd ? "&" : "|");
+  
+  
+  
   if (!Context.getLangOptions().CPlusPlus) {
     UsualUnaryConversions(lex);
     UsualUnaryConversions(rex);
@@ -6361,7 +6381,7 @@ Action::OwningExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
     break;
   case BinaryOperator::LAnd:
   case BinaryOperator::LOr:
-    ResultTy = CheckLogicalOperands(lhs, rhs, OpLoc);
+    ResultTy = CheckLogicalOperands(lhs, rhs, OpLoc, Opc);
     break;
   case BinaryOperator::MulAssign:
   case BinaryOperator::DivAssign:
@@ -7348,7 +7368,8 @@ Sema::OwningExprResult Sema::ActOnVAArg(SourceLocation BuiltinLoc,
   // FIXME: Warn if a non-POD type is passed in.
 
   expr.release();
-  return Owned(new (Context) VAArgExpr(BuiltinLoc, E, T.getNonReferenceType(),
+  return Owned(new (Context) VAArgExpr(BuiltinLoc, E, 
+                                       T.getNonLValueExprType(Context),
                                        RPLoc));
 }
 
diff --git a/lib/Sema/SemaExprCXX.cpp b/lib/Sema/SemaExprCXX.cpp
index a5abfe8..090400f 100644
--- a/lib/Sema/SemaExprCXX.cpp
+++ b/lib/Sema/SemaExprCXX.cpp
@@ -540,7 +540,8 @@ Sema::ActOnCXXTypeConstructExpr(SourceRange TypeRange, TypeTy *TypeRep,
 
     exprs.release();
 
-    return Owned(new (Context) CXXFunctionalCastExpr(Ty.getNonReferenceType(),
+    return Owned(new (Context) CXXFunctionalCastExpr(
+                                              Ty.getNonLValueExprType(Context),
                                                      TInfo, TyBeginLoc, Kind,
                                                      Exprs[0], BasePath,
                                                      RParenLoc));
@@ -1879,7 +1880,7 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
   case ICK_Qualification:
     // FIXME: Not sure about lvalue vs rvalue here in the presence of rvalue
     // references.
-    ImpCastExprToType(From, ToType.getNonReferenceType(),
+    ImpCastExprToType(From, ToType.getNonLValueExprType(Context),
                       CastExpr::CK_NoOp, ToType->isLValueReferenceType());
 
     if (SCS.DeprecatedStringLiteralToCharPtr)
diff --git a/lib/Sema/SemaInit.cpp b/lib/Sema/SemaInit.cpp
index 7536289..7ad1775 100644
--- a/lib/Sema/SemaInit.cpp
+++ b/lib/Sema/SemaInit.cpp
@@ -523,8 +523,9 @@ void InitListChecker::CheckExplicitInitList(const InitializedEntity &Entity,
   StructuredList->setSyntacticForm(IList);
   CheckListElementTypes(Entity, IList, T, /*SubobjectIsDesignatorContext=*/true, 
                         Index, StructuredList, StructuredIndex, TopLevelObject);
-  IList->setType(T.getNonReferenceType());
-  StructuredList->setType(T.getNonReferenceType());
+  QualType ExprTy = T.getNonLValueExprType(SemaRef.Context);
+  IList->setType(ExprTy);
+  StructuredList->setType(ExprTy);
   if (hadError)
     return;
 
@@ -1716,7 +1717,7 @@ InitListChecker::getStructuredSubobjectInit(InitListExpr *IList, unsigned Index,
                                          InitRange.getBegin(), 0, 0,
                                          InitRange.getEnd());
 
-  Result->setType(CurrentObjectType.getNonReferenceType());
+  Result->setType(CurrentObjectType.getNonLValueExprType(SemaRef.Context));
 
   // Pre-allocate storage for the structured initializer list.
   unsigned NumElements = 0;
@@ -2370,13 +2371,14 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
 
   // Add the user-defined conversion step.
   Sequence.AddUserConversionStep(Function, Best->FoundDecl,
-                                 T2.getNonReferenceType());
+                                 T2.getNonLValueExprType(S.Context));
 
   // Determine whether we need to perform derived-to-base or 
   // cv-qualification adjustments.
   bool NewDerivedToBase = false;
   Sema::ReferenceCompareResult NewRefRelationship
-    = S.CompareReferenceRelationship(DeclLoc, T1, T2.getNonReferenceType(),
+    = S.CompareReferenceRelationship(DeclLoc, T1, 
+                                     T2.getNonLValueExprType(S.Context),
                                      NewDerivedToBase);
   if (NewRefRelationship == Sema::Ref_Incompatible) {
     // If the type we've converted to is not reference-related to the
diff --git a/lib/Sema/SemaObjCProperty.cpp b/lib/Sema/SemaObjCProperty.cpp
index 44cd271..ff60599 100644
--- a/lib/Sema/SemaObjCProperty.cpp
+++ b/lib/Sema/SemaObjCProperty.cpp
@@ -52,18 +52,22 @@ Sema::DeclPtrTy Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
     cast<ObjCContainerDecl>(ClassCategory.getAs<Decl>());
 
   if (ObjCCategoryDecl *CDecl = dyn_cast<ObjCCategoryDecl>(ClassDecl))
-    if (CDecl->IsClassExtension())
-      return HandlePropertyInClassExtension(S, CDecl, AtLoc,
-                                            FD, GetterSel, SetterSel,
-                                            isAssign, isReadWrite,
-                                            Attributes,
-                                            isOverridingProperty, TSI,
-                                            MethodImplKind);
-
+    if (CDecl->IsClassExtension()) {
+      DeclPtrTy Res = HandlePropertyInClassExtension(S, CDecl, AtLoc,
+                                           FD, GetterSel, SetterSel,
+                                           isAssign, isReadWrite,
+                                           Attributes,
+                                           isOverridingProperty, TSI,
+                                           MethodImplKind);
+      if (Res)
+        CheckObjCPropertyAttributes(Res, AtLoc, Attributes);
+      return Res;
+    }
+  
   DeclPtrTy Res =  DeclPtrTy::make(CreatePropertyDecl(S, ClassDecl, AtLoc, FD,
-                                            GetterSel, SetterSel,
-                                            isAssign, isReadWrite,
-                                            Attributes, TSI, MethodImplKind));
+                                              GetterSel, SetterSel,
+                                              isAssign, isReadWrite,
+                                              Attributes, TSI, MethodImplKind));
   // Validate the attributes on the @property.
   CheckObjCPropertyAttributes(Res, AtLoc, Attributes);
   return Res;
@@ -926,6 +930,10 @@ void Sema::DefaultSynthesizeProperties (Scope *S, ObjCImplDecl* IMPDecl,
         Prop->getPropertyImplementation() == ObjCPropertyDecl::Optional ||
         IMPDecl->FindPropertyImplIvarDecl(Prop->getIdentifier()))
       continue;
+    // Property may have been synthesized by user.
+    if (IMPDecl->FindPropertyImplDecl(Prop->getIdentifier()))
+      continue;
+
     ActOnPropertyImplDecl(S, IMPDecl->getLocation(), IMPDecl->getLocation(),
                           true, DeclPtrTy::make(IMPDecl),
                           Prop->getIdentifier(), Prop->getIdentifier());
diff --git a/lib/Sema/SemaOverload.cpp b/lib/Sema/SemaOverload.cpp
index ee4c479..c4ab906 100644
--- a/lib/Sema/SemaOverload.cpp
+++ b/lib/Sema/SemaOverload.cpp
@@ -3727,7 +3727,7 @@ Sema::AddConversionCandidate(CXXConversionDecl *Conversion,
   // there are 0 arguments (i.e., nothing is allocated using ASTContext's
   // allocator).
   CallExpr Call(Context, &ConversionFn, 0, 0,
-                Conversion->getConversionType().getNonReferenceType(),
+                Conversion->getConversionType().getNonLValueExprType(Context),
                 From->getLocStart());
   ImplicitConversionSequence ICS =
     TryCopyInitialization(*this, &Call, ToType,
@@ -6287,7 +6287,8 @@ FunctionDecl *Sema::ResolveSingleFunctionTemplateSpecialization(Expr *From) {
     //   specified and it, along with any default template arguments, 
     //   identifies a single function template specialization, then the 
     //   template-id is an lvalue for the function template specialization.
-    FunctionTemplateDecl *FunctionTemplate = cast<FunctionTemplateDecl>(*I);
+    FunctionTemplateDecl *FunctionTemplate
+      = cast<FunctionTemplateDecl>((*I)->getUnderlyingDecl());
     
     // C++ [over.over]p2:
     //   If the name is a function template, template argument deduction is
diff --git a/lib/Sema/SemaTemplate.cpp b/lib/Sema/SemaTemplate.cpp
index 7cc4317..f121954 100644
--- a/lib/Sema/SemaTemplate.cpp
+++ b/lib/Sema/SemaTemplate.cpp
@@ -1233,7 +1233,8 @@ Sema::MatchTemplateParametersToScopeSpecifier(SourceLocation DeclStartLoc,
                                           TemplateParameterList **ParamLists,
                                               unsigned NumParamLists,
                                               bool IsFriend,
-                                              bool &IsExplicitSpecialization) {
+                                              bool &IsExplicitSpecialization,
+                                              bool &Invalid) {
   IsExplicitSpecialization = false;
   
   // Find the template-ids that occur within the nested-name-specifier. These
@@ -1311,6 +1312,7 @@ Sema::MatchTemplateParametersToScopeSpecifier(SourceLocation DeclStartLoc,
              diag::err_template_spec_needs_template_parameters)
           << TemplateId
           << SS.getRange();
+        Invalid = true;
       } else {
         Diag(SS.getRange().getBegin(), diag::err_template_spec_needs_header)
           << SS.getRange()
@@ -1373,7 +1375,13 @@ Sema::MatchTemplateParametersToScopeSpecifier(SourceLocation DeclStartLoc,
           << ExplicitSpecializationsInSpecifier.back();
         ExplicitSpecializationsInSpecifier.pop_back();
       }
-        
+
+      // We have a template parameter list with no corresponding scope, which 
+      // means that the resulting template declaration can't be instantiated
+      // properly (we'll end up with dependent nodes when we shouldn't).
+      if (!isExplicitSpecHeader)
+        Invalid = true;
+      
       ++Idx;
     }
   }
@@ -3622,12 +3630,17 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
   // template.
   // FIXME: We probably shouldn't complain about these headers for
   // friend declarations.
+  bool Invalid = false;
   TemplateParameterList *TemplateParams
     = MatchTemplateParametersToScopeSpecifier(TemplateNameLoc, SS,
                         (TemplateParameterList**)TemplateParameterLists.get(),
                                               TemplateParameterLists.size(),
                                               TUK == TUK_Friend,
-                                              isExplicitSpecialization);
+                                              isExplicitSpecialization,
+                                              Invalid);
+  if (Invalid)
+    return true;
+    
   unsigned NumMatchedTemplateParamLists = TemplateParameterLists.size();
   if (TemplateParams)
     --NumMatchedTemplateParamLists;
diff --git a/lib/Sema/SemaTemplateInstantiate.cpp b/lib/Sema/SemaTemplateInstantiate.cpp
index 6db0916..0cdc8a1 100644
--- a/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/lib/Sema/SemaTemplateInstantiate.cpp
@@ -1067,6 +1067,9 @@ ParmVarDecl *Sema::SubstParmVarDecl(ParmVarDecl *OldParm,
   NewParm->setHasInheritedDefaultArg(OldParm->hasInheritedDefaultArg());
 
   CurrentInstantiationScope->InstantiatedLocal(OldParm, NewParm);
+  // Set DeclContext if inside a Block.
+  NewParm->setDeclContext(CurContext);
+  
   return NewParm;  
 }
 
diff --git a/lib/Sema/TreeTransform.h b/lib/Sema/TreeTransform.h
index 132d049..17103c5 100644
--- a/lib/Sema/TreeTransform.h
+++ b/lib/Sema/TreeTransform.h
@@ -4198,10 +4198,6 @@ TreeTransform<Derived>::TransformDeclRefExpr(DeclRefExpr *E) {
   if (!ND)
     return SemaRef.ExprError();
 
-  // Set DeclContext if inside a Block.
-  if (BlockScopeInfo *CurBlock = SemaRef.getCurBlock())
-    ND->setDeclContext(CurBlock->TheDecl);
-  
   if (!getDerived().AlwaysRebuild() && 
       Qualifier == E->getQualifier() &&
       ND == E->getDecl() &&