Update clang to r108243.

1 files changed, 338 insertions, 120 deletions

diff --git a/lib/CodeGen/CGCall.cpp b/lib/CodeGen/CGCall.cpp
index 73cee3c..1632cb3 100644
--- a/lib/CodeGen/CGCall.cpp
+++ b/lib/CodeGen/CGCall.cpp
@@ -13,26 +13,22 @@
 //===----------------------------------------------------------------------===//
 
 #include "CGCall.h"
+#include "ABIInfo.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
-#include "clang/CodeGen/CodeGenOptions.h"
+#include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/Attributes.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Target/TargetData.h"
-
-#include "ABIInfo.h"
-
 using namespace clang;
 using namespace CodeGen;
 
 /***/
 
-// FIXME: Use iterator and sidestep silly type array creation.
-
 static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) {
   switch (CC) {
   default: return llvm::CallingConv::C;
@@ -65,29 +61,31 @@ static CanQualType GetReturnType(QualType RetTy) {
 }
 
 const CGFunctionInfo &
-CodeGenTypes::getFunctionInfo(CanQual<FunctionNoProtoType> FTNP) {
+CodeGenTypes::getFunctionInfo(CanQual<FunctionNoProtoType> FTNP,
+                              bool IsRecursive) {
   return getFunctionInfo(FTNP->getResultType().getUnqualifiedType(),
                          llvm::SmallVector<CanQualType, 16>(),
-                         FTNP->getExtInfo());
+                         FTNP->getExtInfo(), IsRecursive);
 }
 
 /// \param Args - contains any initial parameters besides those
 ///   in the formal type
 static const CGFunctionInfo &getFunctionInfo(CodeGenTypes &CGT,
                                   llvm::SmallVectorImpl<CanQualType> &ArgTys,
-                                             CanQual<FunctionProtoType> FTP) {
+                                             CanQual<FunctionProtoType> FTP,
+                                             bool IsRecursive = false) {
   // FIXME: Kill copy.
   for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
     ArgTys.push_back(FTP->getArgType(i));
   CanQualType ResTy = FTP->getResultType().getUnqualifiedType();
-  return CGT.getFunctionInfo(ResTy, ArgTys,
-                             FTP->getExtInfo());
+  return CGT.getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo(), IsRecursive);
 }
 
 const CGFunctionInfo &
-CodeGenTypes::getFunctionInfo(CanQual<FunctionProtoType> FTP) {
+CodeGenTypes::getFunctionInfo(CanQual<FunctionProtoType> FTP,
+                              bool IsRecursive) {
   llvm::SmallVector<CanQualType, 16> ArgTys;
-  return ::getFunctionInfo(*this, ArgTys, FTP);
+  return ::getFunctionInfo(*this, ArgTys, FTP, IsRecursive);
 }
 
 static CallingConv getCallingConventionForDecl(const Decl *D) {
@@ -220,7 +218,8 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(CanQualType ResTy,
                            const llvm::SmallVectorImpl<CanQualType> &ArgTys,
-                                            const FunctionType::ExtInfo &Info) {
+                                            const FunctionType::ExtInfo &Info,
+                                                    bool IsRecursive) {
 #ifndef NDEBUG
   for (llvm::SmallVectorImpl<CanQualType>::const_iterator
          I = ArgTys.begin(), E = ArgTys.end(); I != E; ++I)
@@ -240,35 +239,65 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(CanQualType ResTy,
     return *FI;
 
   // Construct the function info.
-  FI = new CGFunctionInfo(CC, Info.getNoReturn(), Info.getRegParm(), ResTy, ArgTys);
+  FI = new CGFunctionInfo(CC, Info.getNoReturn(), Info.getRegParm(), ResTy,
+                          ArgTys.data(), ArgTys.size());
   FunctionInfos.InsertNode(FI, InsertPos);
 
+  // ABI lowering wants to know what our preferred type for the argument is in
+  // various situations, pass it in.
+  llvm::SmallVector<const llvm::Type *, 8> PreferredArgTypes;
+  for (llvm::SmallVectorImpl<CanQualType>::const_iterator
+       I = ArgTys.begin(), E = ArgTys.end(); I != E; ++I) {
+    // If this is being called from the guts of the ConvertType loop, make sure
+    // to call ConvertTypeRecursive so we don't get into issues with cyclic
+    // pointer type structures.
+    PreferredArgTypes.push_back(ConvertTypeRecursive(*I));
+  }
+  
   // Compute ABI information.
-  getABIInfo().computeInfo(*FI, getContext(), TheModule.getContext());
-
+  getABIInfo().computeInfo(*FI, getContext(), TheModule.getContext(),
+                           PreferredArgTypes.data(), PreferredArgTypes.size());
+
+  // If this is a top-level call and ConvertTypeRecursive hit unresolved pointer
+  // types, resolve them now.  These pointers may point to this function, which
+  // we *just* filled in the FunctionInfo for.
+  if (!IsRecursive && !PointersToResolve.empty()) {
+    // Use PATypeHolder's so that our preferred types don't dangle under
+    // refinement.
+    llvm::SmallVector<llvm::PATypeHolder, 8> Handles(PreferredArgTypes.begin(),
+                                                     PreferredArgTypes.end());
+    HandleLateResolvedPointers();
+    PreferredArgTypes.clear();
+    PreferredArgTypes.append(Handles.begin(), Handles.end());
+  }
+  
+  
   return *FI;
 }
 
 CGFunctionInfo::CGFunctionInfo(unsigned _CallingConvention,
-                               bool _NoReturn,
-                               unsigned _RegParm,
+                               bool _NoReturn, unsigned _RegParm,
                                CanQualType ResTy,
-                               const llvm::SmallVectorImpl<CanQualType> &ArgTys)
+                               const CanQualType *ArgTys,
+                               unsigned NumArgTys)
   : CallingConvention(_CallingConvention),
     EffectiveCallingConvention(_CallingConvention),
     NoReturn(_NoReturn), RegParm(_RegParm)
 {
-  NumArgs = ArgTys.size();
-  Args = new ArgInfo[1 + NumArgs];
+  NumArgs = NumArgTys;
+  
+  // FIXME: Coallocate with the CGFunctionInfo object.
+  Args = new ArgInfo[1 + NumArgTys];
   Args[0].type = ResTy;
-  for (unsigned i = 0; i < NumArgs; ++i)
+  for (unsigned i = 0; i != NumArgTys; ++i)
     Args[1 + i].type = ArgTys[i];
 }
 
 /***/
 
 void CodeGenTypes::GetExpandedTypes(QualType Ty,
-                                    std::vector<const llvm::Type*> &ArgTys) {
+                                    std::vector<const llvm::Type*> &ArgTys,
+                                    bool IsRecursive) {
   const RecordType *RT = Ty->getAsStructureType();
   assert(RT && "Can only expand structure types.");
   const RecordDecl *RD = RT->getDecl();
@@ -283,9 +312,9 @@ void CodeGenTypes::GetExpandedTypes(QualType Ty,
 
     QualType FT = FD->getType();
     if (CodeGenFunction::hasAggregateLLVMType(FT)) {
-      GetExpandedTypes(FT, ArgTys);
+      GetExpandedTypes(FT, ArgTys, IsRecursive);
     } else {
-      ArgTys.push_back(ConvertType(FT));
+      ArgTys.push_back(ConvertType(FT, IsRecursive));
     }
   }
 }
@@ -345,6 +374,71 @@ CodeGenFunction::ExpandTypeToArgs(QualType Ty, RValue RV,
   }
 }
 
+/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
+/// accessing some number of bytes out of it, try to gep into the struct to get
+/// at its inner goodness.  Dive as deep as possible without entering an element
+/// with an in-memory size smaller than DstSize.
+static llvm::Value *
+EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
+                                   const llvm::StructType *SrcSTy,
+                                   uint64_t DstSize, CodeGenFunction &CGF) {
+  // We can't dive into a zero-element struct.
+  if (SrcSTy->getNumElements() == 0) return SrcPtr;
+  
+  const llvm::Type *FirstElt = SrcSTy->getElementType(0);
+  
+  // If the first elt is at least as large as what we're looking for, or if the
+  // first element is the same size as the whole struct, we can enter it.
+  uint64_t FirstEltSize = 
+    CGF.CGM.getTargetData().getTypeAllocSize(FirstElt);
+  if (FirstEltSize < DstSize && 
+      FirstEltSize < CGF.CGM.getTargetData().getTypeAllocSize(SrcSTy))
+    return SrcPtr;
+  
+  // GEP into the first element.
+  SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive");
+  
+  // If the first element is a struct, recurse.
+  const llvm::Type *SrcTy =
+    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
+  if (const llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
+    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
+
+  return SrcPtr;
+}
+
+/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
+/// are either integers or pointers.  This does a truncation of the value if it
+/// is too large or a zero extension if it is too small.
+static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
+                                             const llvm::Type *Ty,
+                                             CodeGenFunction &CGF) {
+  if (Val->getType() == Ty)
+    return Val;
+  
+  if (isa<llvm::PointerType>(Val->getType())) {
+    // If this is Pointer->Pointer avoid conversion to and from int.
+    if (isa<llvm::PointerType>(Ty))
+      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
+  
+    // Convert the pointer to an integer so we can play with its width.
+    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
+  }
+  
+  const llvm::Type *DestIntTy = Ty;
+  if (isa<llvm::PointerType>(DestIntTy))
+    DestIntTy = CGF.IntPtrTy;
+  
+  if (Val->getType() != DestIntTy)
+    Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
+  
+  if (isa<llvm::PointerType>(Ty))
+    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
+  return Val;
+}
+
+
+
 /// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
 /// a pointer to an object of type \arg Ty.
 ///
@@ -356,9 +450,28 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
                                       CodeGenFunction &CGF) {
   const llvm::Type *SrcTy =
     cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
-  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
+  
+  // If SrcTy and Ty are the same, just do a load.
+  if (SrcTy == Ty)
+    return CGF.Builder.CreateLoad(SrcPtr);
+  
   uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(Ty);
+  
+  if (const llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
+    SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
+    SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
+  }
+  
+  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
 
+  // If the source and destination are integer or pointer types, just do an
+  // extension or truncation to the desired type.
+  if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
+      (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
+    llvm::LoadInst *Load = CGF.Builder.CreateLoad(SrcPtr);
+    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
+  }
+  
   // If load is legal, just bitcast the src pointer.
   if (SrcSize >= DstSize) {
     // Generally SrcSize is never greater than DstSize, since this means we are
@@ -373,18 +486,18 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
     // FIXME: Use better alignment / avoid requiring aligned load.
     Load->setAlignment(1);
     return Load;
-  } else {
-    // Otherwise do coercion through memory. This is stupid, but
-    // simple.
-    llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
-    llvm::Value *Casted =
-      CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(SrcTy));
-    llvm::StoreInst *Store =
-      CGF.Builder.CreateStore(CGF.Builder.CreateLoad(SrcPtr), Casted);
-    // FIXME: Use better alignment / avoid requiring aligned store.
-    Store->setAlignment(1);
-    return CGF.Builder.CreateLoad(Tmp);
   }
+  
+  // Otherwise do coercion through memory. This is stupid, but
+  // simple.
+  llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
+  llvm::Value *Casted =
+    CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(SrcTy));
+  llvm::StoreInst *Store =
+    CGF.Builder.CreateStore(CGF.Builder.CreateLoad(SrcPtr), Casted);
+  // FIXME: Use better alignment / avoid requiring aligned store.
+  Store->setAlignment(1);
+  return CGF.Builder.CreateLoad(Tmp);
 }
 
 /// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
@@ -399,8 +512,27 @@ static void CreateCoercedStore(llvm::Value *Src,
   const llvm::Type *SrcTy = Src->getType();
   const llvm::Type *DstTy =
     cast<llvm::PointerType>(DstPtr->getType())->getElementType();
-
+  if (SrcTy == DstTy) {
+    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
+    return;
+  }
+  
   uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
+  
+  if (const llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
+    DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
+    DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType();
+  }
+  
+  // If the source and destination are integer or pointer types, just do an
+  // extension or truncation to the desired type.
+  if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
+      (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
+    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
+    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
+    return;
+  }
+  
   uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(DstTy);
 
   // If store is legal, just bitcast the src pointer.
@@ -445,11 +577,12 @@ const llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
         cast<FunctionDecl>(GD.getDecl())->getType()->getAs<FunctionProtoType>())
     Variadic = FPT->isVariadic();
 
-  return GetFunctionType(FI, Variadic);
+  return GetFunctionType(FI, Variadic, false);
 }
 
 const llvm::FunctionType *
-CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic) {
+CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic,
+                              bool IsRecursive) {
   std::vector<const llvm::Type*> ArgTys;
 
   const llvm::Type *ResultType = 0;
@@ -462,13 +595,13 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic) {
 
   case ABIArgInfo::Extend:
   case ABIArgInfo::Direct:
-    ResultType = ConvertType(RetTy);
+    ResultType = ConvertType(RetTy, IsRecursive);
     break;
 
   case ABIArgInfo::Indirect: {
     assert(!RetAI.getIndirectAlign() && "Align unused on indirect return.");
     ResultType = llvm::Type::getVoidTy(getLLVMContext());
-    const llvm::Type *STy = ConvertType(RetTy);
+    const llvm::Type *STy = ConvertType(RetTy, IsRecursive);
     ArgTys.push_back(llvm::PointerType::get(STy, RetTy.getAddressSpace()));
     break;
   }
@@ -490,24 +623,34 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic) {
     case ABIArgInfo::Ignore:
       break;
 
-    case ABIArgInfo::Coerce:
-      ArgTys.push_back(AI.getCoerceToType());
+    case ABIArgInfo::Coerce: {
+      // If the coerce-to type is a first class aggregate, flatten it.  Either
+      // way is semantically identical, but fast-isel and the optimizer
+      // generally likes scalar values better than FCAs.
+      const llvm::Type *ArgTy = AI.getCoerceToType();
+      if (const llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgTy)) {
+        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
+          ArgTys.push_back(STy->getElementType(i));
+      } else {
+        ArgTys.push_back(ArgTy);
+      }
       break;
+    }
 
     case ABIArgInfo::Indirect: {
       // indirect arguments are always on the stack, which is addr space #0.
-      const llvm::Type *LTy = ConvertTypeForMem(it->type);
+      const llvm::Type *LTy = ConvertTypeForMem(it->type, IsRecursive);
       ArgTys.push_back(llvm::PointerType::getUnqual(LTy));
       break;
     }
 
     case ABIArgInfo::Extend:
     case ABIArgInfo::Direct:
-      ArgTys.push_back(ConvertType(it->type));
+      ArgTys.push_back(ConvertType(it->type, IsRecursive));
       break;
 
     case ABIArgInfo::Expand:
-      GetExpandedTypes(it->type, ArgTys);
+      GetExpandedTypes(it->type, ArgTys, IsRecursive);
       break;
     }
   }
@@ -515,28 +658,12 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic) {
   return llvm::FunctionType::get(ResultType, ArgTys, IsVariadic);
 }
 
-static bool HasIncompleteReturnTypeOrArgumentTypes(const FunctionProtoType *T) {
-  if (const TagType *TT = T->getResultType()->getAs<TagType>()) {
-    if (!TT->getDecl()->isDefinition())
-      return true;
-  }
-
-  for (unsigned i = 0, e = T->getNumArgs(); i != e; ++i) {
-    if (const TagType *TT = T->getArgType(i)->getAs<TagType>()) {
-      if (!TT->getDecl()->isDefinition())
-        return true;
-    }
-  }
-
-  return false;
-}
-
 const llvm::Type *
 CodeGenTypes::GetFunctionTypeForVTable(const CXXMethodDecl *MD) {
   const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
   
-  if (!HasIncompleteReturnTypeOrArgumentTypes(FPT))
-    return GetFunctionType(getFunctionInfo(MD), FPT->isVariadic());
+  if (!VerifyFuncTypeComplete(FPT))
+    return GetFunctionType(getFunctionInfo(MD), FPT->isVariadic(), false);
 
   return llvm::OpaqueType::get(getLLVMContext());
 }
@@ -557,6 +684,12 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
   if (TargetDecl) {
     if (TargetDecl->hasAttr<NoThrowAttr>())
       FuncAttrs |= llvm::Attribute::NoUnwind;
+    else if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
+      const FunctionProtoType *FPT = Fn->getType()->getAs<FunctionProtoType>();
+      if (FPT && FPT->hasEmptyExceptionSpec())
+        FuncAttrs |= llvm::Attribute::NoUnwind;
+    }
+
     if (TargetDecl->hasAttr<NoReturnAttr>())
       FuncAttrs |= llvm::Attribute::NoReturn;
     if (TargetDecl->hasAttr<ConstAttr>())
@@ -626,7 +759,12 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
 
     switch (AI.getKind()) {
     case ABIArgInfo::Coerce:
-      break;
+      if (const llvm::StructType *STy =
+          dyn_cast<llvm::StructType>(AI.getCoerceToType()))
+        Index += STy->getNumElements();
+      else
+        ++Index;
+      continue;  // Skip index increment.
 
     case ABIArgInfo::Indirect:
       if (AI.getIndirectByVal())
@@ -666,7 +804,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
       // FIXME: This is rather inefficient. Do we ever actually need to do
       // anything here? The result should be just reconstructed on the other
       // side, so extension should be a non-issue.
-      getTypes().GetExpandedTypes(ParamType, Tys);
+      getTypes().GetExpandedTypes(ParamType, Tys, false);
       Index += Tys.size();
       continue;
     }
@@ -687,7 +825,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
   // initialize the return value.  TODO: it might be nice to have
   // a more general mechanism for this that didn't require synthesized
   // return statements.
-  if (const FunctionDecl* FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
+  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
     if (FD->hasImplicitReturnZero()) {
       QualType RetTy = FD->getResultType().getUnqualifiedType();
       const llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
@@ -719,7 +857,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
 
     switch (ArgI.getKind()) {
     case ABIArgInfo::Indirect: {
-      llvm::Value* V = AI;
+      llvm::Value *V = AI;
       if (hasAggregateLLVMType(Ty)) {
         // Do nothing, aggregates and complex variables are accessed by
         // reference.
@@ -739,7 +877,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
     case ABIArgInfo::Extend:
     case ABIArgInfo::Direct: {
       assert(AI != Fn->arg_end() && "Argument mismatch!");
-      llvm::Value* V = AI;
+      llvm::Value *V = AI;
       if (hasAggregateLLVMType(Ty)) {
         // Create a temporary alloca to hold the argument; the rest of
         // codegen expects to access aggregates & complex values by
@@ -789,12 +927,35 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       continue;
 
     case ABIArgInfo::Coerce: {
-      assert(AI != Fn->arg_end() && "Argument mismatch!");
       // FIXME: This is very wasteful; EmitParmDecl is just going to drop the
       // result in a new alloca anyway, so we could just store into that
       // directly if we broke the abstraction down more.
-      llvm::Value *V = CreateMemTemp(Ty, "coerce");
-      CreateCoercedStore(AI, V, /*DestIsVolatile=*/false, *this);
+      llvm::AllocaInst *Alloca = CreateMemTemp(Ty, "coerce");
+      Alloca->setAlignment(getContext().getDeclAlign(Arg).getQuantity());
+      llvm::Value *V = Alloca;
+      
+      // If the coerce-to type is a first class aggregate, we flatten it and
+      // pass the elements. Either way is semantically identical, but fast-isel
+      // and the optimizer generally likes scalar values better than FCAs.
+      if (const llvm::StructType *STy =
+            dyn_cast<llvm::StructType>(ArgI.getCoerceToType())) {
+        llvm::Value *Ptr = V;
+        Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
+        
+        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
+          assert(AI != Fn->arg_end() && "Argument mismatch!");
+          AI->setName(Arg->getName() + ".coerce" + llvm::Twine(i));
+          llvm::Value *EltPtr = Builder.CreateConstGEP2_32(Ptr, 0, i);
+          Builder.CreateStore(AI++, EltPtr);
+        }
+      } else {
+        // Simple case, just do a coerced store of the argument into the alloca.
+        assert(AI != Fn->arg_end() && "Argument mismatch!");
+        AI->setName(Arg->getName() + ".coerce");
+        CreateCoercedStore(AI++, V, /*DestIsVolatile=*/false, *this);
+      }
+      
+      
       // Match to what EmitParmDecl is expecting for this type.
       if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
         V = EmitLoadOfScalar(V, false, Ty);
@@ -805,7 +966,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
         }
       }
       EmitParmDecl(*Arg, V);
-      break;
+      continue;  // Skip ++AI increment, already done.
     }
     }
 
@@ -814,52 +975,73 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
   assert(AI == Fn->arg_end() && "Argument mismatch!");
 }
 
-void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
-                                         llvm::Value *ReturnValue) {
-  llvm::Value *RV = 0;
-
+void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
   // Functions with no result always return void.
-  if (ReturnValue) {
-    QualType RetTy = FI.getReturnType();
-    const ABIArgInfo &RetAI = FI.getReturnInfo();
-
-    switch (RetAI.getKind()) {
-    case ABIArgInfo::Indirect:
-      if (RetTy->isAnyComplexType()) {
-        ComplexPairTy RT = LoadComplexFromAddr(ReturnValue, false);
-        StoreComplexToAddr(RT, CurFn->arg_begin(), false);
-      } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
-        // Do nothing; aggregrates get evaluated directly into the destination.
-      } else {
-        EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), CurFn->arg_begin(),
-                          false, RetTy);
-      }
-      break;
-
-    case ABIArgInfo::Extend:
-    case ABIArgInfo::Direct:
-      // The internal return value temp always will have
-      // pointer-to-return-type type.
-      RV = Builder.CreateLoad(ReturnValue);
-      break;
+  if (ReturnValue == 0) {
+    Builder.CreateRetVoid();
+    return;
+  }
 
-    case ABIArgInfo::Ignore:
-      break;
+  llvm::MDNode *RetDbgInfo = 0;
+  llvm::Value *RV = 0;
+  QualType RetTy = FI.getReturnType();
+  const ABIArgInfo &RetAI = FI.getReturnInfo();
 
-    case ABIArgInfo::Coerce:
-      RV = CreateCoercedLoad(ReturnValue, RetAI.getCoerceToType(), *this);
-      break;
+  switch (RetAI.getKind()) {
+  case ABIArgInfo::Indirect:
+    if (RetTy->isAnyComplexType()) {
+      ComplexPairTy RT = LoadComplexFromAddr(ReturnValue, false);
+      StoreComplexToAddr(RT, CurFn->arg_begin(), false);
+    } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
+      // Do nothing; aggregrates get evaluated directly into the destination.
+    } else {
+      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), CurFn->arg_begin(),
+                        false, RetTy);
+    }
+    break;
 
-    case ABIArgInfo::Expand:
-      assert(0 && "Invalid ABI kind for return argument");
+  case ABIArgInfo::Extend:
+  case ABIArgInfo::Direct: {
+    // The internal return value temp always will have pointer-to-return-type
+    // type, just do a load.
+      
+    // If the instruction right before the insertion point is a store to the
+    // return value, we can elide the load, zap the store, and usually zap the
+    // alloca.
+    llvm::BasicBlock *InsertBB = Builder.GetInsertBlock();
+    llvm::StoreInst *SI = 0;
+    if (InsertBB->empty() || 
+        !(SI = dyn_cast<llvm::StoreInst>(&InsertBB->back())) ||
+        SI->getPointerOperand() != ReturnValue || SI->isVolatile()) {
+      RV = Builder.CreateLoad(ReturnValue);
+    } else {
+      // Get the stored value and nuke the now-dead store.
+      RetDbgInfo = SI->getDbgMetadata();
+      RV = SI->getValueOperand();
+      SI->eraseFromParent();
+      
+      // If that was the only use of the return value, nuke it as well now.
+      if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
+        cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
+        ReturnValue = 0;
+      }
     }
+    break;
   }
+  case ABIArgInfo::Ignore:
+    break;
 
-  if (RV) {
-    Builder.CreateRet(RV);
-  } else {
-    Builder.CreateRetVoid();
+  case ABIArgInfo::Coerce:
+    RV = CreateCoercedLoad(ReturnValue, RetAI.getCoerceToType(), *this);
+    break;
+
+  case ABIArgInfo::Expand:
+    assert(0 && "Invalid ABI kind for return argument");
   }
+
+  llvm::Instruction *Ret = RV ? Builder.CreateRet(RV) : Builder.CreateRetVoid();
+  if (RetDbgInfo)
+    Ret->setDbgMetadata(RetDbgInfo);
 }
 
 RValue CodeGenFunction::EmitDelegateCallArg(const VarDecl *Param) {
@@ -894,11 +1076,29 @@ RValue CodeGenFunction::EmitDelegateCallArg(const VarDecl *Param) {
 
 RValue CodeGenFunction::EmitCallArg(const Expr *E, QualType ArgType) {
   if (ArgType->isReferenceType())
-    return EmitReferenceBindingToExpr(E);
+    return EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0);
 
   return EmitAnyExprToTemp(E);
 }
 
+/// Emits a call or invoke instruction to the given function, depending
+/// on the current state of the EH stack.
+llvm::CallSite
+CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
+                                  llvm::Value * const *ArgBegin,
+                                  llvm::Value * const *ArgEnd,
+                                  const llvm::Twine &Name) {
+  llvm::BasicBlock *InvokeDest = getInvokeDest();
+  if (!InvokeDest)
+    return Builder.CreateCall(Callee, ArgBegin, ArgEnd, Name);
+
+  llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
+  llvm::InvokeInst *Invoke = Builder.CreateInvoke(Callee, ContBB, InvokeDest,
+                                                  ArgBegin, ArgEnd, Name);
+  EmitBlock(ContBB);
+  return Invoke;
+}
+
 RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
                                  llvm::Value *Callee,
                                  ReturnValueSlot ReturnValue,
@@ -973,8 +1173,24 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         StoreComplexToAddr(RV.getComplexVal(), SrcPtr, false);
       } else
         SrcPtr = RV.getAggregateAddr();
-      Args.push_back(CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(),
-                                       *this));
+      
+      // If the coerce-to type is a first class aggregate, we flatten it and
+      // pass the elements. Either way is semantically identical, but fast-isel
+      // and the optimizer generally likes scalar values better than FCAs.
+      if (const llvm::StructType *STy =
+            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType())) {
+        SrcPtr = Builder.CreateBitCast(SrcPtr,
+                                       llvm::PointerType::getUnqual(STy));
+        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
+          llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i);
+          Args.push_back(Builder.CreateLoad(EltPtr));
+        }
+      } else {
+        // In the simple case, just pass the coerced loaded value.
+        Args.push_back(CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(),
+                                         *this));
+      }
+      
       break;
     }
 
@@ -1014,15 +1230,18 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     }
 
 
-  llvm::BasicBlock *InvokeDest = getInvokeDest();
   unsigned CallingConv;
   CodeGen::AttributeListType AttributeList;
   CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList, CallingConv);
   llvm::AttrListPtr Attrs = llvm::AttrListPtr::get(AttributeList.begin(),
                                                    AttributeList.end());
 
+  llvm::BasicBlock *InvokeDest = 0;
+  if (!(Attrs.getFnAttributes() & llvm::Attribute::NoUnwind))
+    InvokeDest = getInvokeDest();
+
   llvm::CallSite CS;
-  if (!InvokeDest || (Attrs.getFnAttributes() & llvm::Attribute::NoUnwind)) {
+  if (!InvokeDest) {
     CS = Builder.CreateCall(Callee, Args.data(), Args.data()+Args.size());
   } else {
     llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
@@ -1030,9 +1249,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
                               Args.data(), Args.data()+Args.size());
     EmitBlock(Cont);
   }
-  if (callOrInvoke) {
+  if (callOrInvoke)
     *callOrInvoke = CS.getInstruction();
-  }
 
   CS.setAttributes(Attrs);
   CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));