Update clang to r96341.

36 files changed, 3761 insertions, 1430 deletions

diff --git a/lib/CodeGen/CGBlocks.cpp b/lib/CodeGen/CGBlocks.cpp
index ca5b6fa..46b6244 100644
--- a/lib/CodeGen/CGBlocks.cpp
+++ b/lib/CodeGen/CGBlocks.cpp
@@ -13,6 +13,7 @@
 
 #include "CGDebugInfo.h"
 #include "CodeGenFunction.h"
+#include "CGObjCRuntime.h"
 #include "CodeGenModule.h"
 #include "clang/AST/DeclObjC.h"
 #include "llvm/Module.h"
@@ -176,7 +177,7 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
     // block literal.
     // __invoke
     CharUnits subBlockSize; 
-    uint64_t subBlockAlign;
+    CharUnits subBlockAlign;
     llvm::SmallVector<const Expr *, 8> subBlockDeclRefDecls;
     bool subBlockHasCopyDispose = false;
     llvm::Function *Fn
@@ -249,7 +250,7 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
     llvm::StructType *Ty = llvm::StructType::get(VMContext, Types, true);
 
     llvm::AllocaInst *A = CreateTempAlloca(Ty);
-    A->setAlignment(subBlockAlign);
+    A->setAlignment(subBlockAlign.getQuantity());
     V = A;
 
     std::vector<HelperInfo> NoteForHelper(subBlockDeclRefDecls.size());
@@ -355,7 +356,21 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
   }
 
   QualType BPT = BE->getType();
-  return Builder.CreateBitCast(V, ConvertType(BPT));
+  V = Builder.CreateBitCast(V, ConvertType(BPT));
+  // See if this is a __weak block variable and the must call objc_read_weak
+  // on it.
+  const FunctionType *ftype = BPT->getPointeeType()->getAs<FunctionType>();
+  QualType RES = ftype->getResultType();
+  if (RES.isObjCGCWeak()) {
+    // Must cast argument to id*
+    const llvm::Type *ObjectPtrTy = 
+      ConvertType(CGM.getContext().getObjCIdType());
+    const llvm::Type *PtrObjectPtrTy = 
+      llvm::PointerType::getUnqual(ObjectPtrTy);
+    V = Builder.CreateBitCast(V, PtrObjectPtrTy);
+    V =  CGM.getObjCRuntime().EmitObjCWeakRead(*this, V);
+  }
+  return V;
 }
 
 
@@ -496,10 +511,12 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr* E,
   // Load the function.
   llvm::Value *Func = Builder.CreateLoad(FuncPtr, "tmp");
 
-  QualType ResultType = FnType->getAs<FunctionType>()->getResultType();
+  const FunctionType *FuncTy = FnType->getAs<FunctionType>();
+  QualType ResultType = FuncTy->getResultType();
 
   const CGFunctionInfo &FnInfo =
-    CGM.getTypes().getFunctionInfo(ResultType, Args);
+    CGM.getTypes().getFunctionInfo(ResultType, Args, FuncTy->getCallConv(),
+                                   FuncTy->getNoReturnAttr());
 
   // Cast the function pointer to the right type.
   const llvm::Type *BlockFTy =
@@ -593,8 +610,8 @@ BlockModule::GetAddrOfGlobalBlock(const BlockExpr *BE, const char * n) {
 
   // Block literal size. For global blocks we just use the size of the generic
   // block literal struct.
-  CharUnits BlockLiteralSize = CharUnits::fromQuantity(
-    TheTargetData.getTypeStoreSizeInBits(getGenericBlockLiteralType()) / 8);
+  CharUnits BlockLiteralSize = 
+    CGM.GetTargetTypeStoreSize(getGenericBlockLiteralType());
   DescriptorFields[1] =
     llvm::ConstantInt::get(UnsignedLongTy,BlockLiteralSize.getQuantity());
 
@@ -615,7 +632,7 @@ BlockModule::GetAddrOfGlobalBlock(const BlockExpr *BE, const char * n) {
 
   CodeGenFunction::BlockInfo Info(0, n);
   CharUnits subBlockSize; 
-  uint64_t subBlockAlign;
+  CharUnits subBlockAlign;
   llvm::SmallVector<const Expr *, 8> subBlockDeclRefDecls;
   bool subBlockHasCopyDispose = false;
   llvm::DenseMap<const Decl*, llvm::Value*> LocalDeclMap;
@@ -678,7 +695,7 @@ CodeGenFunction::GenerateBlockFunction(const BlockExpr *BExpr,
                                        const Decl *OuterFuncDecl,
                                   llvm::DenseMap<const Decl*, llvm::Value*> ldm,
                                        CharUnits &Size,
-                                       uint64_t &Align,
+                                       CharUnits &Align,
                        llvm::SmallVector<const Expr *, 8> &subBlockDeclRefDecls,
                                        bool &subBlockHasCopyDispose) {
 
@@ -698,13 +715,14 @@ CodeGenFunction::GenerateBlockFunction(const BlockExpr *BExpr,
       LocalDeclMap[VD] = i->second;
   }
 
-  BlockOffset = CharUnits::fromQuantity(
-      CGM.getTargetData()
-        .getTypeStoreSizeInBits(CGM.getGenericBlockLiteralType()) / 8);
-  BlockAlign = getContext().getTypeAlign(getContext().VoidPtrTy) / 8;
+  BlockOffset = 
+      CGM.GetTargetTypeStoreSize(CGM.getGenericBlockLiteralType());
+  BlockAlign = getContext().getTypeAlignInChars(getContext().VoidPtrTy);
 
   const FunctionType *BlockFunctionType = BExpr->getFunctionType();
   QualType ResultType;
+  CallingConv CC = BlockFunctionType->getCallConv();
+  bool NoReturn = BlockFunctionType->getNoReturnAttr();
   bool IsVariadic;
   if (const FunctionProtoType *FTy =
       dyn_cast<FunctionProtoType>(BlockFunctionType)) {
@@ -743,7 +761,7 @@ CodeGenFunction::GenerateBlockFunction(const BlockExpr *BExpr,
     Args.push_back(std::make_pair(*i, (*i)->getType()));
 
   const CGFunctionInfo &FI =
-    CGM.getTypes().getFunctionInfo(ResultType, Args);
+    CGM.getTypes().getFunctionInfo(ResultType, Args, CC, NoReturn);
 
   CodeGenTypes &Types = CGM.getTypes();
   const llvm::FunctionType *LTy = Types.GetFunctionType(FI, IsVariadic);
@@ -777,10 +795,9 @@ CodeGenFunction::GenerateBlockFunction(const BlockExpr *BExpr,
 
   if (CGDebugInfo *DI = getDebugInfo()) {
     // Emit debug information for all the BlockDeclRefDecls.
-    for (unsigned i=0; i < BlockDeclRefDecls.size(); ++i) {
-      const Expr *E = BlockDeclRefDecls[i];
-      const BlockDeclRefExpr *BDRE = dyn_cast<BlockDeclRefExpr>(E);
-      if (BDRE) {
+    for (unsigned i = 0, e = BlockDeclRefDecls.size(); i != e; ++i) {
+      if (const BlockDeclRefExpr *BDRE = 
+            dyn_cast<BlockDeclRefExpr>(BlockDeclRefDecls[i])) {
         const ValueDecl *D = BDRE->getDecl();
         DI->setLocation(D->getLocation());
         DI->EmitDeclareOfBlockDeclRefVariable(BDRE,
@@ -798,9 +815,10 @@ CodeGenFunction::GenerateBlockFunction(const BlockExpr *BExpr,
   FinishFunction(cast<CompoundStmt>(BExpr->getBody())->getRBracLoc());
 
   // The runtime needs a minimum alignment of a void *.
-  uint64_t MinAlign = getContext().getTypeAlign(getContext().VoidPtrTy) / 8;
+  CharUnits MinAlign = getContext().getTypeAlignInChars(getContext().VoidPtrTy);
   BlockOffset = CharUnits::fromQuantity(
-      llvm::RoundUpToAlignment(BlockOffset.getQuantity(), MinAlign));
+      llvm::RoundUpToAlignment(BlockOffset.getQuantity(), 
+                               MinAlign.getQuantity()));
 
   Size = BlockOffset;
   Align = BlockAlign;
@@ -813,20 +831,20 @@ CharUnits BlockFunction::getBlockOffset(const BlockDeclRefExpr *BDRE) {
   const ValueDecl *D = dyn_cast<ValueDecl>(BDRE->getDecl());
 
   CharUnits Size = getContext().getTypeSizeInChars(D->getType());
-  uint64_t Align = getContext().getDeclAlignInBytes(D);
+  CharUnits Align = getContext().getDeclAlign(D);
 
   if (BDRE->isByRef()) {
     Size = getContext().getTypeSizeInChars(getContext().VoidPtrTy);
-    Align = getContext().getTypeAlign(getContext().VoidPtrTy) / 8;
+    Align = getContext().getTypeAlignInChars(getContext().VoidPtrTy);
   }
 
-  assert ((Align > 0) && "alignment must be 1 byte or more");
+  assert ((Align.isPositive()) && "alignment must be 1 byte or more");
 
   CharUnits OldOffset = BlockOffset;
 
   // Ensure proper alignment, even if it means we have to have a gap
   BlockOffset = CharUnits::fromQuantity(
-      llvm::RoundUpToAlignment(BlockOffset.getQuantity(), Align));
+      llvm::RoundUpToAlignment(BlockOffset.getQuantity(), Align.getQuantity()));
   BlockAlign = std::max(Align, BlockAlign);
 
   CharUnits Pad = BlockOffset - OldOffset;
@@ -868,7 +886,7 @@ GenerateCopyHelperFunction(bool BlockHasCopyDispose, const llvm::StructType *T,
   Args.push_back(std::make_pair(Src, Src->getType()));
 
   const CGFunctionInfo &FI =
-    CGM.getTypes().getFunctionInfo(R, Args);
+    CGM.getTypes().getFunctionInfo(R, Args, CC_Default, false);
 
   // FIXME: We'd like to put these into a mergable by content, with
   // internal linkage.
@@ -949,7 +967,7 @@ GenerateDestroyHelperFunction(bool BlockHasCopyDispose,
   Args.push_back(std::make_pair(Src, Src->getType()));
 
   const CGFunctionInfo &FI =
-    CGM.getTypes().getFunctionInfo(R, Args);
+    CGM.getTypes().getFunctionInfo(R, Args, CC_Default, false);
 
   // FIXME: We'd like to put these into a mergable by content, with
   // internal linkage.
@@ -1033,7 +1051,7 @@ GeneratebyrefCopyHelperFunction(const llvm::Type *T, int flag) {
   Args.push_back(std::make_pair(Src, Src->getType()));
 
   const CGFunctionInfo &FI =
-    CGM.getTypes().getFunctionInfo(R, Args);
+    CGM.getTypes().getFunctionInfo(R, Args, CC_Default, false);
 
   CodeGenTypes &Types = CGM.getTypes();
   const llvm::FunctionType *LTy = Types.GetFunctionType(FI, false);
@@ -1096,7 +1114,7 @@ BlockFunction::GeneratebyrefDestroyHelperFunction(const llvm::Type *T,
   Args.push_back(std::make_pair(Src, Src->getType()));
 
   const CGFunctionInfo &FI =
-    CGM.getTypes().getFunctionInfo(R, Args);
+    CGM.getTypes().getFunctionInfo(R, Args, CC_Default, false);
 
   CodeGenTypes &Types = CGM.getTypes();
   const llvm::FunctionType *LTy = Types.GetFunctionType(FI, false);
diff --git a/lib/CodeGen/CGBlocks.h b/lib/CodeGen/CGBlocks.h
index f42244c..a9f5ae0 100644
--- a/lib/CodeGen/CGBlocks.h
+++ b/lib/CodeGen/CGBlocks.h
@@ -177,8 +177,9 @@ public:
   /// BlockOffset - The offset in bytes for the next allocation of an
   /// imported block variable.
   CharUnits BlockOffset;
-  /// BlockAlign - Maximal alignment needed for the Block expressed in bytes.
-  uint64_t BlockAlign;
+  /// BlockAlign - Maximal alignment needed for the Block expressed in 
+  /// characters.
+  CharUnits BlockAlign;
 
   /// getBlockOffset - Allocate an offset for the ValueDecl from a
   /// BlockDeclRefExpr in a block literal (BlockExpr).
diff --git a/lib/CodeGen/CGBuiltin.cpp b/lib/CodeGen/CGBuiltin.cpp
index f11d52e..beaf7b8 100644
--- a/lib/CodeGen/CGBuiltin.cpp
+++ b/lib/CodeGen/CGBuiltin.cpp
@@ -304,6 +304,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Size = Builder.CreateIntCast(Size, llvm::Type::getInt32Ty(VMContext), false, "tmp");
     return RValue::get(Builder.CreateAlloca(llvm::Type::getInt8Ty(VMContext), Size, "tmp"));
   }
+  case Builtin::BIbzero:
   case Builtin::BI__builtin_bzero: {
     Value *Address = EmitScalarExpr(E->getArg(0));
     Builder.CreateCall4(CGM.getMemSetFn(), Address,
@@ -656,7 +657,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
 
   // Unknown builtin, for now just dump it out and return undef.
   if (hasAggregateLLVMType(E->getType()))
-    return RValue::getAggregate(CreateTempAlloca(ConvertType(E->getType())));
+    return RValue::getAggregate(CreateMemTemp(E->getType()));
   return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
 }
 
diff --git a/lib/CodeGen/CGCXX.cpp b/lib/CodeGen/CGCXX.cpp
index 4323f84..28c4c6b 100644
--- a/lib/CodeGen/CGCXX.cpp
+++ b/lib/CodeGen/CGCXX.cpp
@@ -165,7 +165,8 @@ CodeGenFunction::GenerateCovariantThunk(llvm::Function *Fn,
                                    GlobalDecl GD, bool Extern,
                                    const CovariantThunkAdjustment &Adjustment) {
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
-  QualType ResultType = MD->getType()->getAs<FunctionType>()->getResultType();
+  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
+  QualType ResultType = FPT->getResultType();
 
   FunctionArgList Args;
   ImplicitParamDecl *ThisDecl =
@@ -190,7 +191,6 @@ CodeGenFunction::GenerateCovariantThunk(llvm::Function *Fn,
   StartFunction(FD, ResultType, Fn, Args, SourceLocation());
 
   // generate body
-  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
   const llvm::Type *Ty =
     CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(MD),
                                    FPT->isVariadic());
@@ -232,7 +232,9 @@ CodeGenFunction::GenerateCovariantThunk(llvm::Function *Fn,
     CallArgs.push_back(std::make_pair(EmitCallArg(Arg, ArgType), ArgType));
   }
 
-  RValue RV = EmitCall(CGM.getTypes().getFunctionInfo(ResultType, CallArgs),
+  RValue RV = EmitCall(CGM.getTypes().getFunctionInfo(ResultType, CallArgs,
+                                                      FPT->getCallConv(),
+                                                      FPT->getNoReturnAttr()),
                        Callee, ReturnValueSlot(), CallArgs, MD);
   if (ShouldAdjustReturnPointer && !Adjustment.ReturnAdjustment.isEmpty()) {
     bool CanBeZero = !(ResultType->isReferenceType()
diff --git a/lib/CodeGen/CGCall.cpp b/lib/CodeGen/CGCall.cpp
index 2dda0b8..b064c12 100644
--- a/lib/CodeGen/CGCall.cpp
+++ b/lib/CodeGen/CGCall.cpp
@@ -33,12 +33,19 @@ using namespace CodeGen;
 
 // FIXME: Use iterator and sidestep silly type array creation.
 
+static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) {
+  switch (CC) {
+  default: return llvm::CallingConv::C;
+  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
+  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
+  }
+}
+
 const
 CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionNoProtoType *FTNP) {
-  // FIXME: Set calling convention correctly, it needs to be associated with the
-  // type somehow.
   return getFunctionInfo(FTNP->getResultType(),
-                         llvm::SmallVector<QualType, 16>(), 0);
+                         llvm::SmallVector<QualType, 16>(),
+                         FTNP->getCallConv(), FTNP->getNoReturnAttr());
 }
 
 const
@@ -47,20 +54,19 @@ CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionProtoType *FTP) {
   // FIXME: Kill copy.
   for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
     ArgTys.push_back(FTP->getArgType(i));
-  // FIXME: Set calling convention correctly, it needs to be associated with the
-  // type somehow.
-  return getFunctionInfo(FTP->getResultType(), ArgTys, 0);
+  return getFunctionInfo(FTP->getResultType(), ArgTys,
+                         FTP->getCallConv(), FTP->getNoReturnAttr());
 }
 
-static unsigned getCallingConventionForDecl(const Decl *D) {
+static CallingConv getCallingConventionForDecl(const Decl *D) {
   // Set the appropriate calling convention for the Function.
   if (D->hasAttr<StdCallAttr>())
-    return llvm::CallingConv::X86_StdCall;
+    return CC_X86StdCall;
 
   if (D->hasAttr<FastCallAttr>())
-    return llvm::CallingConv::X86_FastCall;
+    return CC_X86FastCall;
 
-  return llvm::CallingConv::C;
+  return CC_C;
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXRecordDecl *RD,
@@ -75,7 +81,8 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXRecordDecl *RD,
   
   // FIXME: Set calling convention correctly, it needs to be associated with the
   // type somehow.
-  return getFunctionInfo(FTP->getResultType(), ArgTys, 0);
+  return getFunctionInfo(FTP->getResultType(), ArgTys,
+                         FTP->getCallConv(), FTP->getNoReturnAttr());
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
@@ -87,8 +94,8 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
   const FunctionProtoType *FTP = MD->getType()->getAs<FunctionProtoType>();
   for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
     ArgTys.push_back(FTP->getArgType(i));
-  return getFunctionInfo(FTP->getResultType(), ArgTys,
-                         getCallingConventionForDecl(MD));
+  return getFunctionInfo(FTP->getResultType(), ArgTys, FTP->getCallConv(),
+                         FTP->getNoReturnAttr());
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D, 
@@ -105,8 +112,8 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D,
   const FunctionProtoType *FTP = D->getType()->getAs<FunctionProtoType>();
   for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
     ArgTys.push_back(FTP->getArgType(i));
-  return getFunctionInfo(FTP->getResultType(), ArgTys,
-                         getCallingConventionForDecl(D));
+  return getFunctionInfo(FTP->getResultType(), ArgTys, FTP->getCallConv(),
+                         FTP->getNoReturnAttr());
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXDestructorDecl *D,
@@ -123,8 +130,8 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXDestructorDecl *D,
   const FunctionProtoType *FTP = D->getType()->getAs<FunctionProtoType>();
   for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
     ArgTys.push_back(FTP->getArgType(i));
-  return getFunctionInfo(FTP->getResultType(), ArgTys,
-                         getCallingConventionForDecl(D));
+  return getFunctionInfo(FTP->getResultType(), ArgTys, FTP->getCallConv(),
+                         FTP->getNoReturnAttr());
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionDecl *FD) {
@@ -132,19 +139,19 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionDecl *FD) {
     if (MD->isInstance())
       return getFunctionInfo(MD);
 
-  unsigned CallingConvention = getCallingConventionForDecl(FD);
   const FunctionType *FTy = FD->getType()->getAs<FunctionType>();
   if (const FunctionNoProtoType *FNTP = dyn_cast<FunctionNoProtoType>(FTy))
     return getFunctionInfo(FNTP->getResultType(), 
                            llvm::SmallVector<QualType, 16>(),
-                           CallingConvention);
+                           FNTP->getCallConv(), FNTP->getNoReturnAttr());
   
   const FunctionProtoType *FPT = cast<FunctionProtoType>(FTy);
   llvm::SmallVector<QualType, 16> ArgTys;
   // FIXME: Kill copy.
   for (unsigned i = 0, e = FPT->getNumArgs(); i != e; ++i)
     ArgTys.push_back(FPT->getArgType(i));
-  return getFunctionInfo(FPT->getResultType(), ArgTys, CallingConvention);
+  return getFunctionInfo(FPT->getResultType(), ArgTys,
+                         FPT->getCallConv(), FPT->getNoReturnAttr());
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const ObjCMethodDecl *MD) {
@@ -156,37 +163,56 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const ObjCMethodDecl *MD) {
          e = MD->param_end(); i != e; ++i)
     ArgTys.push_back((*i)->getType());
   return getFunctionInfo(MD->getResultType(), ArgTys,
-                         getCallingConventionForDecl(MD));
+                         getCallingConventionForDecl(MD),
+                         /*NoReturn*/ false);
+}
+
+const CGFunctionInfo &CodeGenTypes::getFunctionInfo(GlobalDecl GD) {
+  // FIXME: Do we need to handle ObjCMethodDecl?
+  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
+                                              
+  if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
+    return getFunctionInfo(CD, GD.getCtorType());
+
+  if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD))
+    return getFunctionInfo(DD, GD.getDtorType());
+  
+  return getFunctionInfo(FD);
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
                                                     const CallArgList &Args,
-                                                    unsigned CallingConvention){
+                                                    CallingConv CC,
+                                                    bool NoReturn) {
   // FIXME: Kill copy.
   llvm::SmallVector<QualType, 16> ArgTys;
   for (CallArgList::const_iterator i = Args.begin(), e = Args.end();
        i != e; ++i)
     ArgTys.push_back(i->second);
-  return getFunctionInfo(ResTy, ArgTys, CallingConvention);
+  return getFunctionInfo(ResTy, ArgTys, CC, NoReturn);
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
                                                     const FunctionArgList &Args,
-                                                    unsigned CallingConvention){
+                                                    CallingConv CC,
+                                                    bool NoReturn) {
   // FIXME: Kill copy.
   llvm::SmallVector<QualType, 16> ArgTys;
   for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
        i != e; ++i)
     ArgTys.push_back(i->second);
-  return getFunctionInfo(ResTy, ArgTys, CallingConvention);
+  return getFunctionInfo(ResTy, ArgTys, CC, NoReturn);
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
                                   const llvm::SmallVector<QualType, 16> &ArgTys,
-                                                    unsigned CallingConvention){
+                                                    CallingConv CallConv,
+                                                    bool NoReturn) {
+  unsigned CC = ClangCallConvToLLVMCallConv(CallConv);
+
   // Lookup or create unique function info.
   llvm::FoldingSetNodeID ID;
-  CGFunctionInfo::Profile(ID, CallingConvention, ResTy,
+  CGFunctionInfo::Profile(ID, CC, NoReturn, ResTy,
                           ArgTys.begin(), ArgTys.end());
 
   void *InsertPos = 0;
@@ -195,7 +221,7 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
     return *FI;
 
   // Construct the function info.
-  FI = new CGFunctionInfo(CallingConvention, ResTy, ArgTys);
+  FI = new CGFunctionInfo(CC, NoReturn, ResTy, ArgTys);
   FunctionInfos.InsertNode(FI, InsertPos);
 
   // Compute ABI information.
@@ -205,10 +231,12 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
 }
 
 CGFunctionInfo::CGFunctionInfo(unsigned _CallingConvention,
+                               bool _NoReturn,
                                QualType ResTy,
                                const llvm::SmallVector<QualType, 16> &ArgTys) 
   : CallingConvention(_CallingConvention),
-    EffectiveCallingConvention(_CallingConvention)
+    EffectiveCallingConvention(_CallingConvention),
+    NoReturn(_NoReturn)
 {
   NumArgs = ArgTys.size();
   Args = new ArgInfo[1 + NumArgs];
@@ -258,7 +286,7 @@ CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
     QualType FT = FD->getType();
 
     // FIXME: What are the right qualifiers here?
-    LValue LV = EmitLValueForField(Addr, FD, false, 0);
+    LValue LV = EmitLValueForField(Addr, FD, 0);
     if (CodeGenFunction::hasAggregateLLVMType(FT)) {
       AI = ExpandTypeFromArgs(FT, LV, AI);
     } else {
@@ -285,7 +313,7 @@ CodeGenFunction::ExpandTypeToArgs(QualType Ty, RValue RV,
     QualType FT = FD->getType();
 
     // FIXME: What are the right qualifiers here?
-    LValue LV = EmitLValueForField(Addr, FD, false, 0);
+    LValue LV = EmitLValueForField(Addr, FD, 0);
     if (CodeGenFunction::hasAggregateLLVMType(FT)) {
       ExpandTypeToArgs(FT, RValue::getAggregate(LV.getAddress()), Args);
     } else {
@@ -490,6 +518,9 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
 
   CallingConv = FI.getEffectiveCallingConvention();
 
+  if (FI.isNoReturn())
+    FuncAttrs |= llvm::Attribute::NoReturn;
+
   // FIXME: handle sseregparm someday...
   if (TargetDecl) {
     if (TargetDecl->hasAttr<NoThrowAttr>())
@@ -685,7 +716,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
         // Create a temporary alloca to hold the argument; the rest of
         // codegen expects to access aggregates & complex values by
         // reference.
-        V = CreateTempAlloca(ConvertTypeForMem(Ty));
+        V = CreateMemTemp(Ty);
         Builder.CreateStore(AI, V);
       } else {
         if (!getContext().typesAreCompatible(Ty, Arg->getType())) {
@@ -702,8 +733,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       // If this structure was expanded into multiple arguments then
       // we need to create a temporary and reconstruct it from the
       // arguments.
-      llvm::Value *Temp = CreateTempAlloca(ConvertTypeForMem(Ty),
-                                           Arg->getName() + ".addr");
+      llvm::Value *Temp = CreateMemTemp(Ty, Arg->getName() + ".addr");
       // FIXME: What are the right qualifiers here?
       llvm::Function::arg_iterator End =
         ExpandTypeFromArgs(Ty, LValue::MakeAddr(Temp, Qualifiers()), AI);
@@ -719,7 +749,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
     case ABIArgInfo::Ignore:
       // Initialize the local variable appropriately.
       if (hasAggregateLLVMType(Ty)) {
-        EmitParmDecl(*Arg, CreateTempAlloca(ConvertTypeForMem(Ty)));
+        EmitParmDecl(*Arg, CreateMemTemp(Ty));
       } else {
         EmitParmDecl(*Arg, llvm::UndefValue::get(ConvertType(Arg->getType())));
       }
@@ -732,7 +762,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       // FIXME: This is very wasteful; EmitParmDecl is just going to drop the
       // result in a new alloca anyway, so we could just store into that
       // directly if we broke the abstraction down more.
-      llvm::Value *V = CreateTempAlloca(ConvertTypeForMem(Ty), "coerce");
+      llvm::Value *V = CreateMemTemp(Ty, "coerce");
       CreateCoercedStore(AI, V, /*DestIsVolatile=*/false, *this);
       // Match to what EmitParmDecl is expecting for this type.
       if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
@@ -803,7 +833,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
 
 RValue CodeGenFunction::EmitCallArg(const Expr *E, QualType ArgType) {
   if (ArgType->isReferenceType())
-    return EmitReferenceBindingToExpr(E, ArgType);
+    return EmitReferenceBindingToExpr(E);
 
   return EmitAnyExprToTemp(E);
 }
@@ -827,7 +857,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   if (CGM.ReturnTypeUsesSret(CallInfo)) {
     llvm::Value *Value = ReturnValue.getValue();
     if (!Value)
-      Value = CreateTempAlloca(ConvertTypeForMem(RetTy));
+      Value = CreateMemTemp(RetTy);
     Args.push_back(Value);
   }
 
@@ -843,7 +873,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     case ABIArgInfo::Indirect:
       if (RV.isScalar() || RV.isComplex()) {
         // Make a temporary alloca to pass the argument.
-        Args.push_back(CreateTempAlloca(ConvertTypeForMem(I->second)));
+        Args.push_back(CreateMemTemp(I->second));
         if (RV.isScalar())
           EmitStoreOfScalar(RV.getScalarVal(), Args.back(), false, I->second);
         else
@@ -874,10 +904,10 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       // FIXME: Avoid the conversion through memory if possible.
       llvm::Value *SrcPtr;
       if (RV.isScalar()) {
-        SrcPtr = CreateTempAlloca(ConvertTypeForMem(I->second), "coerce");
+        SrcPtr = CreateMemTemp(I->second, "coerce");
         EmitStoreOfScalar(RV.getScalarVal(), SrcPtr, false, I->second);
       } else if (RV.isComplex()) {
-        SrcPtr = CreateTempAlloca(ConvertTypeForMem(I->second), "coerce");
+        SrcPtr = CreateMemTemp(I->second, "coerce");
         StoreComplexToAddr(RV.getComplexVal(), SrcPtr, false);
       } else
         SrcPtr = RV.getAggregateAddr();
@@ -982,7 +1012,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       bool DestIsVolatile = ReturnValue.isVolatile();
 
       if (!DestPtr) {
-        DestPtr = CreateTempAlloca(ConvertTypeForMem(RetTy), "agg.tmp");
+        DestPtr = CreateMemTemp(RetTy, "agg.tmp");
         DestIsVolatile = false;
       }
       Builder.CreateStore(CI, DestPtr, DestIsVolatile);
@@ -1000,7 +1030,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     bool DestIsVolatile = ReturnValue.isVolatile();
     
     if (!DestPtr) {
-      DestPtr = CreateTempAlloca(ConvertTypeForMem(RetTy), "coerce");
+      DestPtr = CreateMemTemp(RetTy, "coerce");
       DestIsVolatile = false;
     }
     
diff --git a/lib/CodeGen/CGCall.h b/lib/CodeGen/CGCall.h
index 427ab5f..9601e9a 100644
--- a/lib/CodeGen/CGCall.h
+++ b/lib/CodeGen/CGCall.h
@@ -69,6 +69,9 @@ namespace CodeGen {
     /// depend on the ABI.
     unsigned EffectiveCallingConvention;
 
+    /// Whether this function is noreturn.
+    bool NoReturn;
+
     unsigned NumArgs;
     ArgInfo *Args;
 
@@ -77,6 +80,7 @@ namespace CodeGen {
     typedef ArgInfo *arg_iterator;
 
     CGFunctionInfo(unsigned CallingConvention,
+                   bool NoReturn,
                    QualType ResTy,
                    const llvm::SmallVector<QualType, 16> &ArgTys);
     ~CGFunctionInfo() { delete[] Args; }
@@ -88,6 +92,8 @@ namespace CodeGen {
 
     unsigned  arg_size() const { return NumArgs; }
 
+    bool isNoReturn() const { return NoReturn; }
+
     /// getCallingConvention - Return the user specified calling
     /// convention.
     unsigned getCallingConvention() const { return CallingConvention; }
@@ -108,6 +114,7 @@ namespace CodeGen {
 
     void Profile(llvm::FoldingSetNodeID &ID) {
       ID.AddInteger(getCallingConvention());
+      ID.AddBoolean(NoReturn);
       getReturnType().Profile(ID);
       for (arg_iterator it = arg_begin(), ie = arg_end(); it != ie; ++it)
         it->type.Profile(ID);
@@ -115,10 +122,12 @@ namespace CodeGen {
     template<class Iterator>
     static void Profile(llvm::FoldingSetNodeID &ID,
                         unsigned CallingConvention,
+                        bool NoReturn,
                         QualType ResTy,
                         Iterator begin,
                         Iterator end) {
       ID.AddInteger(CallingConvention);
+      ID.AddBoolean(NoReturn);
       ResTy.Profile(ID);
       for (; begin != end; ++begin)
         begin->Profile(ID);
diff --git a/lib/CodeGen/CGClass.cpp b/lib/CodeGen/CGClass.cpp
index a822ca2..fa5a47f 100644
--- a/lib/CodeGen/CGClass.cpp
+++ b/lib/CodeGen/CGClass.cpp
@@ -19,11 +19,11 @@ using namespace clang;
 using namespace CodeGen;
 
 static uint64_t 
-ComputeNonVirtualBaseClassOffset(ASTContext &Context, CXXBasePaths &Paths,
+ComputeNonVirtualBaseClassOffset(ASTContext &Context,
+                                 const CXXBasePath &Path,
                                  unsigned Start) {
   uint64_t Offset = 0;
 
-  const CXXBasePath &Path = Paths.front();
   for (unsigned i = Start, e = Path.size(); i != e; ++i) {
     const CXXBasePathElement& Element = Path[i];
 
@@ -44,20 +44,21 @@ ComputeNonVirtualBaseClassOffset(ASTContext &Context, CXXBasePaths &Paths,
 }
 
 llvm::Constant *
-CodeGenModule::GetCXXBaseClassOffset(const CXXRecordDecl *ClassDecl,
-                                     const CXXRecordDecl *BaseClassDecl) {
-  if (ClassDecl == BaseClassDecl)
+CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *Class,
+                                            const CXXRecordDecl *BaseClass) {
+  if (Class == BaseClass)
     return 0;
 
   CXXBasePaths Paths(/*FindAmbiguities=*/false,
                      /*RecordPaths=*/true, /*DetectVirtual=*/false);
-  if (!const_cast<CXXRecordDecl *>(ClassDecl)->
-        isDerivedFrom(const_cast<CXXRecordDecl *>(BaseClassDecl), Paths)) {
+  if (!const_cast<CXXRecordDecl *>(Class)->
+        isDerivedFrom(const_cast<CXXRecordDecl *>(BaseClass), Paths)) {
     assert(false && "Class must be derived from the passed in base class!");
     return 0;
   }
 
-  uint64_t Offset = ComputeNonVirtualBaseClassOffset(getContext(), Paths, 0);
+  uint64_t Offset = ComputeNonVirtualBaseClassOffset(getContext(),
+                                                     Paths.front(), 0);
   if (!Offset)
     return 0;
 
@@ -67,51 +68,6 @@ CodeGenModule::GetCXXBaseClassOffset(const CXXRecordDecl *ClassDecl,
   return llvm::ConstantInt::get(PtrDiffTy, Offset);
 }
 
-static llvm::Value *GetCXXBaseClassOffset(CodeGenFunction &CGF,
-                                          llvm::Value *BaseValue,
-                                          const CXXRecordDecl *ClassDecl,
-                                          const CXXRecordDecl *BaseClassDecl) {
-  CXXBasePaths Paths(/*FindAmbiguities=*/false,
-                     /*RecordPaths=*/true, /*DetectVirtual=*/false);
-  if (!const_cast<CXXRecordDecl *>(ClassDecl)->
-        isDerivedFrom(const_cast<CXXRecordDecl *>(BaseClassDecl), Paths)) {
-    assert(false && "Class must be derived from the passed in base class!");
-    return 0;
-  }
-
-  unsigned Start = 0;
-  llvm::Value *VirtualOffset = 0;
-
-  const CXXBasePath &Path = Paths.front();
-  const CXXRecordDecl *VBase = 0;
-  for (unsigned i = 0, e = Path.size(); i != e; ++i) {
-    const CXXBasePathElement& Element = Path[i];
-    if (Element.Base->isVirtual()) {
-      Start = i+1;
-      QualType VBaseType = Element.Base->getType();
-      VBase = cast<CXXRecordDecl>(VBaseType->getAs<RecordType>()->getDecl());
-    }
-  }
-  if (VBase)
-    VirtualOffset = 
-      CGF.GetVirtualCXXBaseClassOffset(BaseValue, ClassDecl, VBase);
-  
-  uint64_t Offset = 
-    ComputeNonVirtualBaseClassOffset(CGF.getContext(), Paths, Start);
-  
-  if (!Offset)
-    return VirtualOffset;
-  
-  const llvm::Type *PtrDiffTy = 
-    CGF.ConvertType(CGF.getContext().getPointerDiffType());
-  llvm::Value *NonVirtualOffset = llvm::ConstantInt::get(PtrDiffTy, Offset);
-  
-  if (VirtualOffset)
-    return CGF.Builder.CreateAdd(VirtualOffset, NonVirtualOffset);
-                    
-  return NonVirtualOffset;
-}
-
 // FIXME: This probably belongs in CGVtable, but it relies on 
 // the static function ComputeNonVirtualBaseClassOffset, so we should make that
 // a CodeGenModule member function as well.
@@ -144,25 +100,91 @@ CodeGenModule::ComputeThunkAdjustment(const CXXRecordDecl *ClassDecl,
       getVtableInfo().getVirtualBaseOffsetIndex(ClassDecl, BaseClassDecl);
   
   uint64_t Offset = 
-    ComputeNonVirtualBaseClassOffset(getContext(), Paths, Start);
+    ComputeNonVirtualBaseClassOffset(getContext(), Paths.front(), Start);
   return ThunkAdjustment(Offset, VirtualOffset);
 }
 
+/// Gets the address of a virtual base class within a complete object.
+/// This should only be used for (1) non-virtual bases or (2) virtual bases
+/// when the type is known to be complete (e.g. in complete destructors).
+///
+/// The object pointed to by 'This' is assumed to be non-null.
+llvm::Value *
+CodeGenFunction::GetAddressOfBaseOfCompleteClass(llvm::Value *This,
+                                                 bool isBaseVirtual,
+                                                 const CXXRecordDecl *Derived,
+                                                 const CXXRecordDecl *Base) {
+  // 'this' must be a pointer (in some address space) to Derived.
+  assert(This->getType()->isPointerTy() &&
+         cast<llvm::PointerType>(This->getType())->getElementType()
+           == ConvertType(Derived));
+
+  // Compute the offset of the virtual base.
+  uint64_t Offset;
+  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
+  if (isBaseVirtual)
+    Offset = Layout.getVBaseClassOffset(Base);
+  else
+    Offset = Layout.getBaseClassOffset(Base);
+
+  // Shift and cast down to the base type.
+  // TODO: for complete types, this should be possible with a GEP.
+  llvm::Value *V = This;
+  if (Offset) {
+    const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(getLLVMContext());
+    V = Builder.CreateBitCast(V, Int8PtrTy);
+    V = Builder.CreateConstInBoundsGEP1_64(V, Offset / 8);
+  }
+  V = Builder.CreateBitCast(V, ConvertType(Base)->getPointerTo());
+
+  return V;
+}                                      
+
 llvm::Value *
 CodeGenFunction::GetAddressOfBaseClass(llvm::Value *Value,
-                                       const CXXRecordDecl *ClassDecl,
-                                       const CXXRecordDecl *BaseClassDecl,
+                                       const CXXRecordDecl *Class,
+                                       const CXXRecordDecl *BaseClass,
                                        bool NullCheckValue) {
   QualType BTy =
     getContext().getCanonicalType(
-      getContext().getTypeDeclType(const_cast<CXXRecordDecl*>(BaseClassDecl)));
+      getContext().getTypeDeclType(BaseClass));
   const llvm::Type *BasePtrTy = llvm::PointerType::getUnqual(ConvertType(BTy));
 
-  if (ClassDecl == BaseClassDecl) {
+  if (Class == BaseClass) {
     // Just cast back.
     return Builder.CreateBitCast(Value, BasePtrTy);
   }
+
+  CXXBasePaths Paths(/*FindAmbiguities=*/false,
+                     /*RecordPaths=*/true, /*DetectVirtual=*/false);
+  if (!const_cast<CXXRecordDecl *>(Class)->
+        isDerivedFrom(const_cast<CXXRecordDecl *>(BaseClass), Paths)) {
+    assert(false && "Class must be derived from the passed in base class!");
+    return 0;
+  }
+
+  unsigned Start = 0;
+  llvm::Value *VirtualOffset = 0;
+
+  const CXXBasePath &Path = Paths.front();
+  const CXXRecordDecl *VBase = 0;
+  for (unsigned i = 0, e = Path.size(); i != e; ++i) {
+    const CXXBasePathElement& Element = Path[i];
+    if (Element.Base->isVirtual()) {
+      Start = i+1;
+      QualType VBaseType = Element.Base->getType();
+      VBase = cast<CXXRecordDecl>(VBaseType->getAs<RecordType>()->getDecl());
+    }
+  }
+
+  uint64_t Offset = 
+    ComputeNonVirtualBaseClassOffset(getContext(), Paths.front(), Start);
   
+  if (!Offset && !VBase) {
+    // Just cast back.
+    return Builder.CreateBitCast(Value, BasePtrTy);
+  }    
+
   llvm::BasicBlock *CastNull = 0;
   llvm::BasicBlock *CastNotNull = 0;
   llvm::BasicBlock *CastEnd = 0;
@@ -179,16 +201,27 @@ CodeGenFunction::GetAddressOfBaseClass(llvm::Value *Value,
     EmitBlock(CastNotNull);
   }
   
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
+  if (VBase)
+    VirtualOffset = GetVirtualBaseClassOffset(Value, Class, VBase);
 
-  llvm::Value *Offset = 
-    GetCXXBaseClassOffset(*this, Value, ClassDecl, BaseClassDecl);
+  const llvm::Type *PtrDiffTy = ConvertType(getContext().getPointerDiffType());
+  llvm::Value *NonVirtualOffset = 0;
+  if (Offset)
+    NonVirtualOffset = llvm::ConstantInt::get(PtrDiffTy, Offset);
   
-  if (Offset) {
-    // Apply the offset.
-    Value = Builder.CreateBitCast(Value, Int8PtrTy);
-    Value = Builder.CreateGEP(Value, Offset, "add.ptr");
-  }
+  llvm::Value *BaseOffset;
+  if (VBase) {
+    if (NonVirtualOffset)
+      BaseOffset = Builder.CreateAdd(VirtualOffset, NonVirtualOffset);
+    else
+      BaseOffset = VirtualOffset;
+  } else
+    BaseOffset = NonVirtualOffset;
+  
+  // Apply the base offset.
+  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(getLLVMContext());
+  Value = Builder.CreateBitCast(Value, Int8PtrTy);
+  Value = Builder.CreateGEP(Value, BaseOffset, "add.ptr");
   
   // Cast back.
   Value = Builder.CreateBitCast(Value, BasePtrTy);
@@ -212,19 +245,27 @@ CodeGenFunction::GetAddressOfBaseClass(llvm::Value *Value,
 
 llvm::Value *
 CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value,
-                                          const CXXRecordDecl *ClassDecl,
-                                          const CXXRecordDecl *DerivedClassDecl,
+                                          const CXXRecordDecl *Class,
+                                          const CXXRecordDecl *DerivedClass,
                                           bool NullCheckValue) {
   QualType DerivedTy =
     getContext().getCanonicalType(
-    getContext().getTypeDeclType(const_cast<CXXRecordDecl*>(DerivedClassDecl)));
+    getContext().getTypeDeclType(const_cast<CXXRecordDecl*>(DerivedClass)));
   const llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo();
   
-  if (ClassDecl == DerivedClassDecl) {
+  if (Class == DerivedClass) {
     // Just cast back.
     return Builder.CreateBitCast(Value, DerivedPtrTy);
   }
 
+  llvm::Value *NonVirtualOffset =
+    CGM.GetNonVirtualBaseClassOffset(DerivedClass, Class);
+  
+  if (!NonVirtualOffset) {
+    // No offset, we can just cast back.
+    return Builder.CreateBitCast(Value, DerivedPtrTy);
+  }
+  
   llvm::BasicBlock *CastNull = 0;
   llvm::BasicBlock *CastNotNull = 0;
   llvm::BasicBlock *CastEnd = 0;
@@ -241,17 +282,13 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value,
     EmitBlock(CastNotNull);
   }
   
-  llvm::Value *Offset = GetCXXBaseClassOffset(*this, Value, DerivedClassDecl,
-                                              ClassDecl);
-  if (Offset) {
-    // Apply the offset.
-    Value = Builder.CreatePtrToInt(Value, Offset->getType());
-    Value = Builder.CreateSub(Value, Offset);
-    Value = Builder.CreateIntToPtr(Value, DerivedPtrTy);
-  } else {
-    // Just cast.
-    Value = Builder.CreateBitCast(Value, DerivedPtrTy);
-  }
+  // Apply the offset.
+  Value = Builder.CreatePtrToInt(Value, NonVirtualOffset->getType());
+  Value = Builder.CreateSub(Value, NonVirtualOffset);
+  Value = Builder.CreateIntToPtr(Value, DerivedPtrTy);
+
+  // Just cast.
+  Value = Builder.CreateBitCast(Value, DerivedPtrTy);
 
   if (NullCheckValue) {
     Builder.CreateBr(CastEnd);
@@ -327,9 +364,9 @@ void CodeGenFunction::EmitClassAggrMemberwiseCopy(llvm::Value *Dest,
     // Push the Src ptr.
     CallArgs.push_back(std::make_pair(RValue::get(Src),
                                      BaseCopyCtor->getParamDecl(0)->getType()));
-    QualType ResultType =
-      BaseCopyCtor->getType()->getAs<FunctionType>()->getResultType();
-    EmitCall(CGM.getTypes().getFunctionInfo(ResultType, CallArgs),
+    const FunctionProtoType *FPT
+      = BaseCopyCtor->getType()->getAs<FunctionProtoType>();
+    EmitCall(CGM.getTypes().getFunctionInfo(CallArgs, FPT),
              Callee, ReturnValueSlot(), CallArgs, BaseCopyCtor);
   }
   EmitBlock(ContinueBlock);
@@ -412,8 +449,7 @@ void CodeGenFunction::EmitClassAggrCopyAssignment(llvm::Value *Dest,
     RValue SrcValue = SrcTy->isReferenceType() ? RValue::get(Src) :
                                                  RValue::getAggregate(Src);
     CallArgs.push_back(std::make_pair(SrcValue, SrcTy));
-    QualType ResultType = MD->getType()->getAs<FunctionType>()->getResultType();
-    EmitCall(CGM.getTypes().getFunctionInfo(ResultType, CallArgs),
+    EmitCall(CGM.getTypes().getFunctionInfo(CallArgs, FPT),
              Callee, ReturnValueSlot(), CallArgs, MD);
   }
   EmitBlock(ContinueBlock);
@@ -503,9 +539,9 @@ void CodeGenFunction::EmitClassMemberwiseCopy(
     // Push the Src ptr.
     CallArgs.push_back(std::make_pair(RValue::get(Src),
                        BaseCopyCtor->getParamDecl(0)->getType()));
-    QualType ResultType =
-    BaseCopyCtor->getType()->getAs<FunctionType>()->getResultType();
-    EmitCall(CGM.getTypes().getFunctionInfo(ResultType, CallArgs),
+    const FunctionProtoType *FPT =
+      BaseCopyCtor->getType()->getAs<FunctionProtoType>();
+    EmitCall(CGM.getTypes().getFunctionInfo(CallArgs, FPT),
              Callee, ReturnValueSlot(), CallArgs, BaseCopyCtor);
   }
 }
@@ -550,9 +586,7 @@ void CodeGenFunction::EmitClassCopyAssignment(
   RValue SrcValue = SrcTy->isReferenceType() ? RValue::get(Src) :
                                                RValue::getAggregate(Src);
   CallArgs.push_back(std::make_pair(SrcValue, SrcTy));
-  QualType ResultType =
-    MD->getType()->getAs<FunctionType>()->getResultType();
-  EmitCall(CGM.getTypes().getFunctionInfo(ResultType, CallArgs),
+  EmitCall(CGM.getTypes().getFunctionInfo(CallArgs, FPT),
            Callee, ReturnValueSlot(), CallArgs, MD);
 }
 
@@ -629,8 +663,8 @@ CodeGenFunction::SynthesizeCXXCopyConstructor(const CXXConstructorDecl *Ctor,
     if (const RecordType *FieldClassType = FieldType->getAs<RecordType>()) {
       CXXRecordDecl *FieldClassDecl
         = cast<CXXRecordDecl>(FieldClassType->getDecl());
-      LValue LHS = EmitLValueForField(LoadOfThis, Field, false, 0);
-      LValue RHS = EmitLValueForField(LoadOfSrc, Field, false, 0);
+      LValue LHS = EmitLValueForField(LoadOfThis, Field, 0);
+      LValue RHS = EmitLValueForField(LoadOfSrc, Field, 0);
       if (Array) {
         const llvm::Type *BasePtr = ConvertType(FieldType);
         BasePtr = llvm::PointerType::getUnqual(BasePtr);
@@ -647,26 +681,9 @@ CodeGenFunction::SynthesizeCXXCopyConstructor(const CXXConstructorDecl *Ctor,
       continue;
     }
     
-    if (Field->getType()->isReferenceType()) {
-      unsigned FieldIndex = CGM.getTypes().getLLVMFieldNo(Field);
- 
-      llvm::Value *LHS = Builder.CreateStructGEP(LoadOfThis, FieldIndex,
-                                                 "lhs.ref");
-      
-      llvm::Value *RHS = Builder.CreateStructGEP(LoadOfThis, FieldIndex,
-                                                 "rhs.ref");
-
-      // Load the value in RHS.
-      RHS = Builder.CreateLoad(RHS);
-      
-      // And store it in the LHS
-      Builder.CreateStore(RHS, LHS);
-
-      continue;
-    }
     // Do a built-in assignment of scalar data members.
-    LValue LHS = EmitLValueForField(LoadOfThis, Field, false, 0);
-    LValue RHS = EmitLValueForField(LoadOfSrc, Field, false, 0);
+    LValue LHS = EmitLValueForFieldInitialization(LoadOfThis, Field, 0);
+    LValue RHS = EmitLValueForFieldInitialization(LoadOfSrc, Field, 0);
 
     if (!hasAggregateLLVMType(Field->getType())) {
       RValue RVRHS = EmitLoadOfLValue(RHS, Field->getType());
@@ -745,8 +762,8 @@ void CodeGenFunction::SynthesizeCXXCopyAssignment(const CXXMethodDecl *CD,
     if (const RecordType *FieldClassType = FieldType->getAs<RecordType>()) {
       CXXRecordDecl *FieldClassDecl
       = cast<CXXRecordDecl>(FieldClassType->getDecl());
-      LValue LHS = EmitLValueForField(LoadOfThis, *Field, false, 0);
-      LValue RHS = EmitLValueForField(LoadOfSrc, *Field, false, 0);
+      LValue LHS = EmitLValueForField(LoadOfThis, *Field, 0);
+      LValue RHS = EmitLValueForField(LoadOfSrc, *Field, 0);
       if (Array) {
         const llvm::Type *BasePtr = ConvertType(FieldType);
         BasePtr = llvm::PointerType::getUnqual(BasePtr);
@@ -763,8 +780,8 @@ void CodeGenFunction::SynthesizeCXXCopyAssignment(const CXXMethodDecl *CD,
       continue;
     }
     // Do a built-in assignment of scalar data members.
-    LValue LHS = EmitLValueForField(LoadOfThis, *Field, false, 0);
-    LValue RHS = EmitLValueForField(LoadOfSrc, *Field, false, 0);
+    LValue LHS = EmitLValueForField(LoadOfThis, *Field, 0);
+    LValue RHS = EmitLValueForField(LoadOfSrc, *Field, 0);
     if (!hasAggregateLLVMType(Field->getType())) {
       RValue RVRHS = EmitLoadOfLValue(RHS, Field->getType());
       EmitStoreThroughLValue(RVRHS, LHS, Field->getType());
@@ -810,29 +827,21 @@ static void EmitBaseInitializer(CodeGenFunction &CGF,
   if (CtorType == Ctor_Base && isBaseVirtual)
     return;
 
-  // Compute the offset to the base; we do this directly instead of using
-  // GetAddressOfBaseClass because the class doesn't have a vtable pointer
-  // at this point.
-  // FIXME: This could be refactored back into GetAddressOfBaseClass if it took
-  // an extra parameter for whether the derived class is the complete object
-  // class.
-  const ASTRecordLayout &Layout =
-      CGF.getContext().getASTRecordLayout(ClassDecl);
-  uint64_t Offset;
-  if (isBaseVirtual)
-    Offset = Layout.getVBaseClassOffset(BaseClassDecl);
-  else
-    Offset = Layout.getBaseClassOffset(BaseClassDecl);
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
-  const llvm::Type *BaseClassType = CGF.ConvertType(QualType(BaseType, 0));
-  llvm::Value *V = CGF.Builder.CreateBitCast(ThisPtr, Int8PtrTy);
-  V = CGF.Builder.CreateConstInBoundsGEP1_64(V, Offset/8);
-  V = CGF.Builder.CreateBitCast(V, BaseClassType->getPointerTo());
-
-  CGF.EmitCXXConstructorCall(BaseInit->getConstructor(),
-                             Ctor_Base, V,
-                             BaseInit->const_arg_begin(),
-                             BaseInit->const_arg_end());
+  // We can pretend to be a complete class because it only matters for
+  // virtual bases, and we only do virtual bases for complete ctors.
+  llvm::Value *V = ThisPtr;
+  V = CGF.GetAddressOfBaseOfCompleteClass(V, isBaseVirtual,
+                                          ClassDecl, BaseClassDecl);
+
+  CGF.EmitAggExpr(BaseInit->getInit(), V, false, false, true);
+  
+  if (CGF.Exceptions && !BaseClassDecl->hasTrivialDestructor()) {
+    // FIXME: Is this OK for C++0x delegating constructors?
+    CodeGenFunction::EHCleanupBlock Cleanup(CGF);
+
+    CXXDestructorDecl *DD = BaseClassDecl->getDestructor(CGF.getContext());
+    CGF.EmitCXXDestructorCall(DD, Dtor_Base, V);
+  }
 }
 
 static void EmitMemberInitializer(CodeGenFunction &CGF,
@@ -846,91 +855,62 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
   QualType FieldType = CGF.getContext().getCanonicalType(Field->getType());
 
   llvm::Value *ThisPtr = CGF.LoadCXXThis();
-  LValue LHS;
-  if (FieldType->isReferenceType()) {
-    // FIXME: This is really ugly; should be refactored somehow
-    unsigned idx = CGF.CGM.getTypes().getLLVMFieldNo(Field);
-    llvm::Value *V = CGF.Builder.CreateStructGEP(ThisPtr, idx, "tmp");
-    assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
-    LHS = LValue::MakeAddr(V, CGF.MakeQualifiers(FieldType));
-  } else {
-    LHS = CGF.EmitLValueForField(ThisPtr, Field, ClassDecl->isUnion(), 0);
-  }
-
+  LValue LHS = CGF.EmitLValueForFieldInitialization(ThisPtr, Field, 0);
+  
   // If we are initializing an anonymous union field, drill down to the field.
   if (MemberInit->getAnonUnionMember()) {
     Field = MemberInit->getAnonUnionMember();
-    LHS = CGF.EmitLValueForField(LHS.getAddress(), Field,
-                                 /*IsUnion=*/true, 0);
+    LHS = CGF.EmitLValueForField(LHS.getAddress(), Field, 0);
     FieldType = Field->getType();
   }
 
-  // If the field is an array, branch based on the element type.
-  const ConstantArrayType *Array =
-    CGF.getContext().getAsConstantArrayType(FieldType);
-  if (Array)
-    FieldType = CGF.getContext().getBaseElementType(FieldType);
-
-  // We lose the constructor for anonymous union members, so handle them
-  // explicitly.
-  // FIXME: This is somwhat ugly.
-  if (MemberInit->getAnonUnionMember() && FieldType->getAs<RecordType>()) {
-    if (MemberInit->getNumArgs())
-      CGF.EmitAggExpr(*MemberInit->arg_begin(), LHS.getAddress(),
-                      LHS.isVolatileQualified());
-    else
-      CGF.EmitAggregateClear(LHS.getAddress(), Field->getType());
-    return;
-  }
-
-  if (FieldType->getAs<RecordType>()) {
-    assert(MemberInit->getConstructor() &&
-           "EmitCtorPrologue - no constructor to initialize member");
-    if (Array) {
-      const llvm::Type *BasePtr = CGF.ConvertType(FieldType);
-      BasePtr = llvm::PointerType::getUnqual(BasePtr);
-      llvm::Value *BaseAddrPtr =
-        CGF.Builder.CreateBitCast(LHS.getAddress(), BasePtr);
-      CGF.EmitCXXAggrConstructorCall(MemberInit->getConstructor(),
-                                     Array, BaseAddrPtr,
-                                     MemberInit->const_arg_begin(),
-                                     MemberInit->const_arg_end());
-    }
-    else
-      CGF.EmitCXXConstructorCall(MemberInit->getConstructor(),
-                                 Ctor_Complete, LHS.getAddress(),
-                                 MemberInit->const_arg_begin(),
-                                 MemberInit->const_arg_end());
-    return;
-  }
-
-  assert(MemberInit->getNumArgs() == 1 && "Initializer count must be 1 only");
-  Expr *RhsExpr = *MemberInit->arg_begin();
+  // FIXME: If there's no initializer and the CXXBaseOrMemberInitializer
+  // was implicitly generated, we shouldn't be zeroing memory.
   RValue RHS;
   if (FieldType->isReferenceType()) {
-    RHS = CGF.EmitReferenceBindingToExpr(RhsExpr, FieldType,
-                                    /*IsInitializer=*/true);
+    RHS = CGF.EmitReferenceBindingToExpr(MemberInit->getInit(),
+                                         /*IsInitializer=*/true);
     CGF.EmitStoreThroughLValue(RHS, LHS, FieldType);
-  } else if (Array) {
+  } else if (FieldType->isArrayType() && !MemberInit->getInit()) {
     CGF.EmitMemSetToZero(LHS.getAddress(), Field->getType());
-  } else if (!CGF.hasAggregateLLVMType(RhsExpr->getType())) {
-    RHS = RValue::get(CGF.EmitScalarExpr(RhsExpr, true));
+  } else if (!CGF.hasAggregateLLVMType(Field->getType())) {
+    RHS = RValue::get(CGF.EmitScalarExpr(MemberInit->getInit(), true));
     CGF.EmitStoreThroughLValue(RHS, LHS, FieldType);
-  } else if (RhsExpr->getType()->isAnyComplexType()) {
-    CGF.EmitComplexExprIntoAddr(RhsExpr, LHS.getAddress(),
+  } else if (MemberInit->getInit()->getType()->isAnyComplexType()) {
+    CGF.EmitComplexExprIntoAddr(MemberInit->getInit(), LHS.getAddress(),
                                 LHS.isVolatileQualified());
   } else {
-    // Handle member function pointers; other aggregates shouldn't get this far.
-    CGF.EmitAggExpr(RhsExpr, LHS.getAddress(), LHS.isVolatileQualified());
+    CGF.EmitAggExpr(MemberInit->getInit(), LHS.getAddress(), 
+                    LHS.isVolatileQualified(), false, true);
+    
+    if (!CGF.Exceptions)
+      return;
+
+    const RecordType *RT = FieldType->getAs<RecordType>();
+    if (!RT)
+      return;
+    
+    CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+    if (!RD->hasTrivialDestructor()) {
+      // FIXME: Is this OK for C++0x delegating constructors?
+      CodeGenFunction::EHCleanupBlock Cleanup(CGF);
+      
+      llvm::Value *ThisPtr = CGF.LoadCXXThis();
+      LValue LHS = CGF.EmitLValueForField(ThisPtr, Field, 0);
+
+      CXXDestructorDecl *DD = RD->getDestructor(CGF.getContext());
+      CGF.EmitCXXDestructorCall(DD, Dtor_Complete, LHS.getAddress());
+    }
   }
 }
 
 /// EmitCtorPrologue - This routine generates necessary code to initialize
 /// base classes and non-static data members belonging to this constructor.
-/// FIXME: This needs to take a CXXCtorType.
 void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
                                        CXXCtorType CtorType) {
   const CXXRecordDecl *ClassDecl = CD->getParent();
+
+  llvm::SmallVector<CXXBaseOrMemberInitializer *, 8> MemberInitializers;
   
   // FIXME: Add vbase initialization
   
@@ -945,14 +925,17 @@ void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
     if (Member->isBaseInitializer())
       EmitBaseInitializer(*this, ClassDecl, Member, CtorType);
     else
-      EmitMemberInitializer(*this, ClassDecl, Member);
-
-    // Pop any live temporaries that the initializers might have pushed.
-    while (!LiveTemporaries.empty())
-      PopCXXTemporary();
+      MemberInitializers.push_back(Member);
   }
 
   InitializeVtablePtrs(ClassDecl);
+
+  for (unsigned I = 0, E = MemberInitializers.size(); I != E; ++I) {
+    assert(LiveTemporaries.empty() &&
+           "Should not have any live temporaries at initializer start!");
+    
+    EmitMemberInitializer(*this, ClassDecl, MemberInitializers[I]);
+  }
 }
 
 /// EmitDtorEpilogue - Emit all code that comes at the end of class's
@@ -1002,7 +985,6 @@ void CodeGenFunction::EmitDtorEpilogue(const CXXDestructorDecl *DD,
     llvm::Value *ThisPtr = LoadCXXThis();
 
     LValue LHS = EmitLValueForField(ThisPtr, Field, 
-                                    /*isUnion=*/false,
                                     // FIXME: Qualifiers?
                                     /*CVRQualifiers=*/0);
     if (Array) {
@@ -1050,15 +1032,16 @@ void CodeGenFunction::EmitDtorEpilogue(const CXXDestructorDecl *DD,
        ClassDecl->vbases_rbegin(), E = ClassDecl->vbases_rend(); I != E; ++I) {
     const CXXBaseSpecifier &Base = *I;
     CXXRecordDecl *BaseClassDecl
-    = cast<CXXRecordDecl>(Base.getType()->getAs<RecordType>()->getDecl());
+      = cast<CXXRecordDecl>(Base.getType()->getAs<RecordType>()->getDecl());
     
     // Ignore trivial destructors.
     if (BaseClassDecl->hasTrivialDestructor())
       continue;
     const CXXDestructorDecl *D = BaseClassDecl->getDestructor(getContext());
-    llvm::Value *V = GetAddressOfBaseClass(LoadCXXThis(),
-                                           ClassDecl, BaseClassDecl, 
-                                           /*NullCheckValue=*/false);
+    llvm::Value *V = GetAddressOfBaseOfCompleteClass(LoadCXXThis(),
+                                                     true,
+                                                     ClassDecl,
+                                                     BaseClassDecl);
     EmitCXXDestructorCall(D, Dtor_Base, V);
   }
     
@@ -1080,7 +1063,7 @@ void CodeGenFunction::SynthesizeDefaultDestructor(const CXXDestructorDecl *Dtor,
 
   StartFunction(GlobalDecl(Dtor, DtorType), Dtor->getResultType(), Fn, Args, 
                 SourceLocation());
-
+  InitializeVtablePtrs(Dtor->getParent());
   EmitDtorEpilogue(Dtor, DtorType);
   FinishFunction();
 }
@@ -1263,7 +1246,8 @@ CodeGenFunction::GenerateCXXAggrDestructorHelper(const CXXDestructorDecl *D,
   llvm::SmallString<16> Name;
   llvm::raw_svector_ostream(Name) << "__tcf_" << (++UniqueAggrDestructorCount);
   QualType R = getContext().VoidTy;
-  const CGFunctionInfo &FI = CGM.getTypes().getFunctionInfo(R, Args);
+  const CGFunctionInfo &FI
+    = CGM.getTypes().getFunctionInfo(R, Args, CC_Default, false);
   const llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI, false);
   llvm::Function *Fn =
     llvm::Function::Create(FTy, llvm::GlobalValue::InternalLinkage,
@@ -1295,20 +1279,21 @@ CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
                                         llvm::Value *This,
                                         CallExpr::const_arg_iterator ArgBeg,
                                         CallExpr::const_arg_iterator ArgEnd) {
-  if (D->isCopyConstructor()) {
-    const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(D->getDeclContext());
-    if (ClassDecl->hasTrivialCopyConstructor()) {
-      assert(!ClassDecl->hasUserDeclaredCopyConstructor() &&
-             "EmitCXXConstructorCall - user declared copy constructor");
-      const Expr *E = (*ArgBeg);
-      QualType Ty = E->getType();
-      llvm::Value *Src = EmitLValue(E).getAddress();
-      EmitAggregateCopy(This, Src, Ty);
+  if (D->isTrivial()) {
+    if (ArgBeg == ArgEnd) {
+      // Trivial default constructor, no codegen required.
+      assert(D->isDefaultConstructor() &&
+             "trivial 0-arg ctor not a default ctor");
       return;
     }
-  } else if (D->isTrivial()) {
-    // FIXME: Track down why we're trying to generate calls to the trivial
-    // default constructor!
+
+    assert(ArgBeg + 1 == ArgEnd && "unexpected argcount for trivial ctor");
+    assert(D->isCopyConstructor() && "trivial 1-arg ctor not a copy ctor");
+
+    const Expr *E = (*ArgBeg);
+    QualType Ty = E->getType();
+    llvm::Value *Src = EmitLValue(E).getAddress();
+    EmitAggregateCopy(This, Src, Ty);
     return;
   }
 
@@ -1328,8 +1313,8 @@ void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
 }
 
 llvm::Value *
-CodeGenFunction::GetVirtualCXXBaseClassOffset(llvm::Value *This,
-                                              const CXXRecordDecl *ClassDecl,
+CodeGenFunction::GetVirtualBaseClassOffset(llvm::Value *This,
+                                           const CXXRecordDecl *ClassDecl,
                                            const CXXRecordDecl *BaseClassDecl) {
   const llvm::Type *Int8PtrTy = 
     llvm::Type::getInt8Ty(VMContext)->getPointerTo();
diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp
index 1ffad3e..5b9c6b0 100644
--- a/lib/CodeGen/CGDebugInfo.cpp
+++ b/lib/CodeGen/CGDebugInfo.cpp
@@ -49,19 +49,21 @@ void CGDebugInfo::setLocation(SourceLocation Loc) {
     CurLoc = CGM.getContext().getSourceManager().getInstantiationLoc(Loc);
 }
 
-/// getContext - Get context info for the decl.
-llvm::DIDescriptor CGDebugInfo::getContext(const VarDecl *Decl,
-                                           llvm::DIDescriptor &CompileUnit) {
-  if (Decl->isFileVarDecl())
+/// getContextDescriptor - Get context info for the decl.
+llvm::DIDescriptor CGDebugInfo::getContextDescriptor(const Decl *Context,
+                                              llvm::DIDescriptor &CompileUnit) {
+  if (!Context)
     return CompileUnit;
-  if (Decl->getDeclContext()->isFunctionOrMethod()) {
-    // Find the last subprogram in region stack.
-    for (unsigned RI = RegionStack.size(), RE = 0; RI != RE; --RI) {
-      llvm::DIDescriptor R(RegionStack[RI - 1]);
-      if (R.isSubprogram())
-        return R;
-    }
-  }
+
+  llvm::DenseMap<const Decl *, llvm::WeakVH>::iterator
+    I = RegionMap.find(Context);
+  if (I != RegionMap.end())
+    return llvm::DIDescriptor(dyn_cast_or_null<llvm::MDNode>(I->second));
+
+  // Check namespace.
+  if (const NamespaceDecl *NSDecl = dyn_cast<NamespaceDecl>(Context))
+    return llvm::DIDescriptor(getOrCreateNameSpace(NSDecl, CompileUnit));
+  
   return CompileUnit;
 }
 
@@ -78,10 +80,9 @@ llvm::StringRef CGDebugInfo::getFunctionName(const FunctionDecl *FD) {
   std::string NS = FD->getNameAsString();
 
   // Copy this name on the side and use its reference.
-  unsigned Length = NS.length() + 1;
-  char *StrPtr = FunctionNames.Allocate<char>(Length);
-  strncpy(StrPtr, NS.c_str(), Length);
-  return llvm::StringRef(StrPtr);
+  char *StrPtr = DebugInfoNames.Allocate<char>(NS.length());
+  memcpy(StrPtr, NS.data(), NS.length());
+  return llvm::StringRef(StrPtr, NS.length());
 }
 
 /// getOrCreateCompileUnit - Get the compile unit from the cache or create a new
@@ -90,16 +91,15 @@ llvm::DICompileUnit CGDebugInfo::getOrCreateCompileUnit(SourceLocation Loc) {
   // Get source file information.
   const char *FileName =  "<unknown>";
   SourceManager &SM = CGM.getContext().getSourceManager();
-  unsigned FID = 0;
   if (Loc.isValid()) {
     PresumedLoc PLoc = SM.getPresumedLoc(Loc);
     FileName = PLoc.getFilename();
-    FID = PLoc.getIncludeLoc().getRawEncoding();
-  }
+    unsigned FID = PLoc.getIncludeLoc().getRawEncoding();
 
-  // See if this compile unit has been used before.
-  llvm::DICompileUnit &Unit = CompileUnitCache[FID];
-  if (!Unit.isNull()) return Unit;
+    // See if this compile unit has been used before for this valid location.
+    llvm::DICompileUnit &Unit = CompileUnitCache[FID];
+    if (!Unit.isNull()) return Unit;
+  }
 
   // Get absolute path name.
   llvm::sys::Path AbsFileName(FileName);
@@ -149,9 +149,16 @@ llvm::DICompileUnit CGDebugInfo::getOrCreateCompileUnit(SourceLocation Loc) {
     RuntimeVers = LO.ObjCNonFragileABI ? 2 : 1;
 
   // Create new compile unit.
-  return Unit = DebugFactory.CreateCompileUnit(
+  llvm::DICompileUnit Unit = DebugFactory.CreateCompileUnit(
     LangTag, AbsFileName.getLast(), AbsFileName.getDirname(), Producer, isMain,
     LO.Optimize, CGM.getCodeGenOpts().DwarfDebugFlags, RuntimeVers);
+
+  if (Loc.isValid()) {
+    PresumedLoc PLoc = SM.getPresumedLoc(Loc);
+    unsigned FID = PLoc.getIncludeLoc().getRawEncoding();
+    CompileUnitCache[FID] = Unit;
+  }
+  return Unit;
 }
 
 /// CreateType - Get the Basic type from the cache or create a new
@@ -419,17 +426,18 @@ llvm::DIType CGDebugInfo::CreateType(const TypedefType *Ty,
 
   // We don't set size information, but do specify where the typedef was
   // declared.
-  SourceLocation DefLoc = Ty->getDecl()->getLocation();
-  llvm::DICompileUnit DefUnit = getOrCreateCompileUnit(DefLoc);
-
   SourceManager &SM = CGM.getContext().getSourceManager();
-  PresumedLoc PLoc = SM.getPresumedLoc(DefLoc);
+  PresumedLoc PLoc = SM.getPresumedLoc(Ty->getDecl()->getLocation());
   unsigned Line = PLoc.isInvalid() ? 0 : PLoc.getLine();
 
+  llvm::DIDescriptor TyContext 
+    = getContextDescriptor(dyn_cast<Decl>(Ty->getDecl()->getDeclContext()),
+                           Unit);
   llvm::DIType DbgTy = 
-    DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_typedef, Unit,
-                                   Ty->getDecl()->getName(),
-                                   DefUnit, Line, 0, 0, 0, 0, Src);
+    DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_typedef, 
+                                   TyContext,
+                                   Ty->getDecl()->getName(), Unit,
+                                   Line, 0, 0, 0, 0, Src);
   return DbgTy;
 }
 
@@ -463,22 +471,21 @@ llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty,
 /// CollectRecordFields - A helper function to collect debug info for
 /// record fields. This is used while creating debug info entry for a Record.
 void CGDebugInfo::
-CollectRecordFields(const RecordDecl *Decl,
-                    llvm::DICompileUnit Unit,
+CollectRecordFields(const RecordDecl *RD, llvm::DICompileUnit Unit,
                     llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys) {
   unsigned FieldNo = 0;
   SourceManager &SM = CGM.getContext().getSourceManager();
-  const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(Decl);
-  for (RecordDecl::field_iterator I = Decl->field_begin(),
-                                  E = Decl->field_end();
+  const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
+  for (RecordDecl::field_iterator I = RD->field_begin(),
+                                  E = RD->field_end();
        I != E; ++I, ++FieldNo) {
     FieldDecl *Field = *I;
     llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit);
 
     llvm::StringRef FieldName = Field->getName();
 
-    // Ignore unnamed fields.
-    if (FieldName.empty())
+    // Ignore unnamed fields. Do not ignore unnamed records.
+    if (FieldName.empty() && !isa<RecordType>(Field->getType()))
       continue;
 
     // Get the location for the field.
@@ -519,87 +526,264 @@ CollectRecordFields(const RecordDecl *Decl,
   }
 }
 
+/// getOrCreateMethodType - CXXMethodDecl's type is a FunctionType. This
+/// function type is not updated to include implicit "this" pointer. Use this
+/// routine to get a method type which includes "this" pointer.
+llvm::DIType
+CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
+                                   llvm::DICompileUnit Unit) {
+  llvm::DIType FnTy = getOrCreateType(Method->getType(), Unit);
+  
+  // Static methods do not need "this" pointer argument.
+  if (Method->isStatic())
+    return FnTy;
+
+  // Add "this" pointer.
+
+  llvm::DIArray Args = llvm::DICompositeType(FnTy.getNode()).getTypeArray();
+  assert (Args.getNumElements() && "Invalid number of arguments!");
+
+  llvm::SmallVector<llvm::DIDescriptor, 16> Elts;
+
+  // First element is always return type. For 'void' functions it is NULL.
+  Elts.push_back(Args.getElement(0));
+
+  // "this" pointer is always first argument.
+  ASTContext &Context = CGM.getContext();
+  QualType ThisPtr = 
+    Context.getPointerType(Context.getTagDeclType(Method->getParent()));
+  llvm::DIType ThisPtrType = 
+    DebugFactory.CreateArtificialType(getOrCreateType(ThisPtr, Unit));
+  TypeCache[ThisPtr.getAsOpaquePtr()] = ThisPtrType.getNode();  
+  Elts.push_back(ThisPtrType);
+
+  // Copy rest of the arguments.
+  for (unsigned i = 1, e = Args.getNumElements(); i != e; ++i)
+    Elts.push_back(Args.getElement(i));
+
+  llvm::DIArray EltTypeArray =
+    DebugFactory.GetOrCreateArray(Elts.data(), Elts.size());
+
+  return
+    DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_subroutine_type,
+                                     Unit, "", llvm::DICompileUnit(),
+                                     0, 0, 0, 0, 0,
+                                     llvm::DIType(), EltTypeArray);
+}
+
+/// CreateCXXMemberFunction - A helper function to create a DISubprogram for
+/// a single member function GlobalDecl.
+llvm::DISubprogram
+CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
+                                     llvm::DICompileUnit Unit,
+                                     llvm::DICompositeType &RecordTy) {
+  bool IsCtorOrDtor = 
+    isa<CXXConstructorDecl>(Method) || isa<CXXDestructorDecl>(Method);
+  
+  llvm::StringRef MethodName = getFunctionName(Method);
+  llvm::StringRef MethodLinkageName;
+  llvm::DIType MethodTy = getOrCreateMethodType(Method, Unit);
+  
+  // Since a single ctor/dtor corresponds to multiple functions, it doesn't
+  // make sense to give a single ctor/dtor a linkage name.
+  if (!IsCtorOrDtor)
+    MethodLinkageName = CGM.getMangledName(Method);
+
+  SourceManager &SM = CGM.getContext().getSourceManager();
+
+  // Get the location for the method.
+  SourceLocation MethodDefLoc = Method->getLocation();
+  PresumedLoc PLoc = SM.getPresumedLoc(MethodDefLoc);
+  llvm::DICompileUnit MethodDefUnit;
+  unsigned MethodLine = 0;
+
+  if (!PLoc.isInvalid()) {
+    MethodDefUnit = getOrCreateCompileUnit(MethodDefLoc);
+    MethodLine = PLoc.getLine();
+  }
+
+  // Collect virtual method info.
+  llvm::DIType ContainingType;
+  unsigned Virtuality = 0; 
+  unsigned VIndex = 0;
+  
+  if (Method->isVirtual()) {
+    if (Method->isPure())
+      Virtuality = llvm::dwarf::DW_VIRTUALITY_pure_virtual;
+    else
+      Virtuality = llvm::dwarf::DW_VIRTUALITY_virtual;
+    
+    // It doesn't make sense to give a virtual destructor a vtable index,
+    // since a single destructor has two entries in the vtable.
+    if (!isa<CXXDestructorDecl>(Method))
+      VIndex = CGM.getVtableInfo().getMethodVtableIndex(Method);
+    ContainingType = RecordTy;
+  }
+
+  llvm::DISubprogram SP =
+    DebugFactory.CreateSubprogram(RecordTy , MethodName, MethodName, 
+                                  MethodLinkageName,
+                                  MethodDefUnit, MethodLine,
+                                  MethodTy, /*isLocalToUnit=*/false, 
+                                  Method->isThisDeclarationADefinition(),
+                                  Virtuality, VIndex, ContainingType);
+  
+  // Don't cache ctors or dtors since we have to emit multiple functions for
+  // a single ctor or dtor.
+  if (!IsCtorOrDtor && Method->isThisDeclarationADefinition())
+    SPCache[Method] = llvm::WeakVH(SP.getNode());
+
+  return SP;
+}
+
 /// CollectCXXMemberFunctions - A helper function to collect debug info for
 /// C++ member functions.This is used while creating debug info entry for 
 /// a Record.
 void CGDebugInfo::
-CollectCXXMemberFunctions(const CXXRecordDecl *Decl,
-                          llvm::DICompileUnit Unit,
+CollectCXXMemberFunctions(const CXXRecordDecl *RD, llvm::DICompileUnit Unit,
                           llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
                           llvm::DICompositeType &RecordTy) {
-  SourceManager &SM = CGM.getContext().getSourceManager();
-  for(CXXRecordDecl::method_iterator I = Decl->method_begin(),
-        E = Decl->method_end(); I != E; ++I) {
-    CXXMethodDecl *Method = *I;
-    llvm::StringRef MethodName;
-    llvm::StringRef MethodLinkageName;
-    llvm::DIType MethodTy = getOrCreateType(Method->getType(), Unit);
-    if (CXXConstructorDecl *CDecl = dyn_cast<CXXConstructorDecl>(Method)) {
-      if (CDecl->isImplicit())
-        continue;
-      MethodName = Decl->getName();
-      // FIXME : Find linkage name.
-    } else if (CXXDestructorDecl *DDecl = dyn_cast<CXXDestructorDecl>(Method)) {
-      if (DDecl->isImplicit())
-        continue;
-      MethodName = getFunctionName(Method);
-      // FIXME : Find linkage name.
-    } else {
-      if (Method->isImplicit())
-        continue;
-      // regular method
-      MethodName = getFunctionName(Method);
-      MethodLinkageName = CGM.getMangledName(Method);
-    }
-
-    // Get the location for the method.
-    SourceLocation MethodDefLoc = Method->getLocation();
-    PresumedLoc PLoc = SM.getPresumedLoc(MethodDefLoc);
-    llvm::DICompileUnit MethodDefUnit;
-    unsigned MethodLine = 0;
-
-    if (!PLoc.isInvalid()) {
-      MethodDefUnit = getOrCreateCompileUnit(MethodDefLoc);
-      MethodLine = PLoc.getLine();
-    }
+  for(CXXRecordDecl::method_iterator I = RD->method_begin(),
+        E = RD->method_end(); I != E; ++I) {
+    const CXXMethodDecl *Method = *I;
+    
+    if (Method->isImplicit() && !Method->isUsed())
+      continue;
 
-    llvm::DISubprogram SP =
-      DebugFactory.CreateSubprogram(RecordTy , MethodName, MethodName, 
-                                    MethodLinkageName,
-                                    MethodDefUnit, MethodLine,
-                                    MethodTy, false, 
-                                    Method->isThisDeclarationADefinition(),
-                                    0 /*Virtuality*/, 0 /*VIndex*/, 
-                                    llvm::DIType() /*ContainingType*/);
-    if (Method->isThisDeclarationADefinition())
-      SPCache[cast<FunctionDecl>(Method)] = llvm::WeakVH(SP.getNode());
-    EltTys.push_back(SP);
+    EltTys.push_back(CreateCXXMemberFunction(Method, Unit, RecordTy));
   }
 }                                 
 
+/// CollectCXXBases - A helper function to collect debug info for
+/// C++ base classes. This is used while creating debug info entry for 
+/// a Record.
+void CGDebugInfo::
+CollectCXXBases(const CXXRecordDecl *RD, llvm::DICompileUnit Unit,
+                llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
+                llvm::DICompositeType &RecordTy) {
+
+  const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
+  for (CXXRecordDecl::base_class_const_iterator BI = RD->bases_begin(),
+         BE = RD->bases_end(); BI != BE; ++BI) {
+    unsigned BFlags = 0;
+    uint64_t BaseOffset;
+    
+    const CXXRecordDecl *Base =
+      cast<CXXRecordDecl>(BI->getType()->getAs<RecordType>()->getDecl());
+    
+    if (BI->isVirtual()) {
+      // virtual base offset index is -ve. The code generator emits dwarf
+      // expression where it expects +ve number.
+      BaseOffset = 0 - CGM.getVtableInfo().getVirtualBaseOffsetIndex(RD, Base);
+      BFlags = llvm::DIType::FlagVirtual;
+    } else
+      BaseOffset = RL.getBaseClassOffset(Base);
+    
+    AccessSpecifier Access = BI->getAccessSpecifier();
+    if (Access == clang::AS_private)
+      BFlags |= llvm::DIType::FlagPrivate;
+    else if (Access == clang::AS_protected)
+      BFlags |= llvm::DIType::FlagProtected;
+    
+    llvm::DIType DTy =
+      DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_inheritance,
+                                     RecordTy, llvm::StringRef(), 
+                                     llvm::DICompileUnit(), 0, 0, 0,
+                                     BaseOffset, BFlags,
+                                     getOrCreateType(BI->getType(),
+                                                     Unit));
+    EltTys.push_back(DTy);
+  }
+}
+
+/// getOrCreateVTablePtrType - Return debug info descriptor for vtable.
+llvm::DIType CGDebugInfo::getOrCreateVTablePtrType(llvm::DICompileUnit Unit) {
+  if (!VTablePtrType.isNull())
+    return VTablePtrType;
+
+  ASTContext &Context = CGM.getContext();
+
+  /* Function type */
+  llvm::SmallVector<llvm::DIDescriptor, 16> STys;
+  STys.push_back(getOrCreateType(Context.IntTy, Unit));
+  llvm::DIArray SElements =
+    DebugFactory.GetOrCreateArray(STys.data(), STys.size());
+  llvm::DIType SubTy =
+    DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_subroutine_type,
+                                     Unit, "", llvm::DICompileUnit(),
+                                     0, 0, 0, 0, 0, llvm::DIType(), SElements);
+
+  unsigned Size = Context.getTypeSize(Context.VoidPtrTy);
+  llvm::DIType vtbl_ptr_type 
+    = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_pointer_type,
+                                     Unit, "__vtbl_ptr_type", llvm::DICompileUnit(),
+                                     0, Size, 0, 0, 0, SubTy);
+
+  VTablePtrType = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_pointer_type,
+                                          Unit, "", llvm::DICompileUnit(),
+                                          0, Size, 0, 0, 0, vtbl_ptr_type);
+  return VTablePtrType;
+}
+
+/// getVtableName - Get vtable name for the given Class.
+llvm::StringRef CGDebugInfo::getVtableName(const CXXRecordDecl *RD) {
+  // Otherwise construct gdb compatible name name.
+  std::string Name = "_vptr$" + RD->getNameAsString();
+
+  // Copy this name on the side and use its reference.
+  char *StrPtr = DebugInfoNames.Allocate<char>(Name.length());
+  memcpy(StrPtr, Name.data(), Name.length());
+  return llvm::StringRef(StrPtr, Name.length());
+}
+
+
+/// CollectVtableInfo - If the C++ class has vtable info then insert appropriate
+/// debug info entry in EltTys vector.
+void CGDebugInfo::
+CollectVtableInfo(const CXXRecordDecl *RD, llvm::DICompileUnit Unit,
+                  llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys) {
+  const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
+
+  // If there is a primary base then it will hold vtable info.
+  if (RL.getPrimaryBase())
+    return;
+
+  // If this class is not dynamic then there is not any vtable info to collect.
+  if (!RD->isDynamicClass())
+    return;
+
+  unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy);
+  llvm::DIType VPTR
+    = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
+                                     getVtableName(RD), llvm::DICompileUnit(),
+                                     0, Size, 0, 0, 0, 
+                                     getOrCreateVTablePtrType(Unit));
+  EltTys.push_back(VPTR);
+}
+
 /// CreateType - get structure or union type.
 llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
                                      llvm::DICompileUnit Unit) {
-  RecordDecl *Decl = Ty->getDecl();
+  RecordDecl *RD = Ty->getDecl();
 
   unsigned Tag;
-  if (Decl->isStruct())
+  if (RD->isStruct())
     Tag = llvm::dwarf::DW_TAG_structure_type;
-  else if (Decl->isUnion())
+  else if (RD->isUnion())
     Tag = llvm::dwarf::DW_TAG_union_type;
   else {
-    assert(Decl->isClass() && "Unknown RecordType!");
+    assert(RD->isClass() && "Unknown RecordType!");
     Tag = llvm::dwarf::DW_TAG_class_type;
   }
 
   SourceManager &SM = CGM.getContext().getSourceManager();
 
   // Get overall information about the record type for the debug info.
-  PresumedLoc PLoc = SM.getPresumedLoc(Decl->getLocation());
+  PresumedLoc PLoc = SM.getPresumedLoc(RD->getLocation());
   llvm::DICompileUnit DefUnit;
   unsigned Line = 0;
   if (!PLoc.isInvalid()) {
-    DefUnit = getOrCreateCompileUnit(Decl->getLocation());
+    DefUnit = getOrCreateCompileUnit(RD->getLocation());
     Line = PLoc.getLine();
   }
 
@@ -610,19 +794,19 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
   // may refer to the forward decl if the struct is recursive) and replace all
   // uses of the forward declaration with the final definition.
 
-  // A Decl->getName() is not unique. However, the debug info descriptors 
-  // are uniqued. The debug info descriptor describing record's context is
-  // necessary to keep two Decl's descriptor unique if their name match.
-  // FIXME : Use RecordDecl's DeclContext's descriptor. As a temp. step
-  // use type's name in FwdDecl.
+  // A RD->getName() is not unique. However, the debug info descriptors 
+  // are uniqued so use type name to ensure uniquness.
   std::string STy = QualType(Ty, 0).getAsString();
+  llvm::DIDescriptor FDContext = 
+    getContextDescriptor(dyn_cast<Decl>(RD->getDeclContext()), Unit);
   llvm::DICompositeType FwdDecl =
-    DebugFactory.CreateCompositeType(Tag, Unit, STy.c_str(),
+    DebugFactory.CreateCompositeType(Tag, FDContext,
+                                     STy.c_str(),
                                      DefUnit, Line, 0, 0, 0, 0,
                                      llvm::DIType(), llvm::DIArray());
 
   // If this is just a forward declaration, return it.
-  if (!Decl->getDefinition(CGM.getContext()))
+  if (!RD->getDefinition())
     return FwdDecl;
 
   llvm::TrackingVH<llvm::MDNode> FwdDeclNode = FwdDecl.getNode();
@@ -633,10 +817,25 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
   // Convert all the elements.
   llvm::SmallVector<llvm::DIDescriptor, 16> EltTys;
 
-  CollectRecordFields(Decl, Unit, EltTys);
-  if (CXXRecordDecl *CXXDecl = dyn_cast<CXXRecordDecl>(Decl))
+  const CXXRecordDecl *CXXDecl = dyn_cast<CXXRecordDecl>(RD);
+  if (CXXDecl) {
+    CollectCXXBases(CXXDecl, Unit, EltTys, FwdDecl);
+    CollectVtableInfo(CXXDecl, Unit, EltTys);
+  }
+  CollectRecordFields(RD, Unit, EltTys);
+  llvm::MDNode *ContainingType = NULL;
+  if (CXXDecl) {
     CollectCXXMemberFunctions(CXXDecl, Unit, EltTys, FwdDecl);
 
+    // A class's primary base or the class itself contains the vtable.
+    const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
+    if (const CXXRecordDecl *PBase = RL.getPrimaryBase())
+      ContainingType = 
+        getOrCreateType(QualType(PBase->getTypeForDecl(), 0), Unit).getNode();
+    else if (CXXDecl->isDynamicClass()) 
+      ContainingType = FwdDecl.getNode();
+  }
+
   llvm::DIArray Elements =
     DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
 
@@ -644,10 +843,14 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
   uint64_t Size = CGM.getContext().getTypeSize(Ty);
   uint64_t Align = CGM.getContext().getTypeAlign(Ty);
 
+  llvm::DIDescriptor RDContext =  
+    getContextDescriptor(dyn_cast<Decl>(RD->getDeclContext()), Unit);
   llvm::DICompositeType RealDecl =
-    DebugFactory.CreateCompositeType(Tag, Unit, Decl->getName(),
+    DebugFactory.CreateCompositeType(Tag, RDContext,
+                                     RD->getName(),
                                      DefUnit, Line, Size, Align, 0, 0, 
-                                     llvm::DIType(), Elements);
+                                     llvm::DIType(), Elements, 
+                                     0, ContainingType);
 
   // Now that we have a real decl for the struct, replace anything using the
   // old decl with the new one.  This will recursively update the debug info.
@@ -659,14 +862,14 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
 /// CreateType - get objective-c interface type.
 llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
                                      llvm::DICompileUnit Unit) {
-  ObjCInterfaceDecl *Decl = Ty->getDecl();
+  ObjCInterfaceDecl *ID = Ty->getDecl();
 
   unsigned Tag = llvm::dwarf::DW_TAG_structure_type;
   SourceManager &SM = CGM.getContext().getSourceManager();
 
   // Get overall information about the record type for the debug info.
-  llvm::DICompileUnit DefUnit = getOrCreateCompileUnit(Decl->getLocation());
-  PresumedLoc PLoc = SM.getPresumedLoc(Decl->getLocation());
+  llvm::DICompileUnit DefUnit = getOrCreateCompileUnit(ID->getLocation());
+  PresumedLoc PLoc = SM.getPresumedLoc(ID->getLocation());
   unsigned Line = PLoc.isInvalid() ? 0 : PLoc.getLine();
 
 
@@ -679,13 +882,13 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   // may refer to the forward decl if the struct is recursive) and replace all
   // uses of the forward declaration with the final definition.
   llvm::DICompositeType FwdDecl =
-    DebugFactory.CreateCompositeType(Tag, Unit, Decl->getName(),
+    DebugFactory.CreateCompositeType(Tag, Unit, ID->getName(),
                                      DefUnit, Line, 0, 0, 0, 0,
                                      llvm::DIType(), llvm::DIArray(),
                                      RuntimeLang);
 
   // If this is just a forward declaration, return it.
-  if (Decl->isForwardDecl())
+  if (ID->isForwardDecl())
     return FwdDecl;
 
   llvm::TrackingVH<llvm::MDNode> FwdDeclNode = FwdDecl.getNode();
@@ -696,7 +899,7 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   // Convert all the elements.
   llvm::SmallVector<llvm::DIDescriptor, 16> EltTys;
 
-  ObjCInterfaceDecl *SClass = Decl->getSuperClass();
+  ObjCInterfaceDecl *SClass = ID->getSuperClass();
   if (SClass) {
     llvm::DIType SClassTy =
       getOrCreateType(CGM.getContext().getObjCInterfaceType(SClass), Unit);
@@ -707,11 +910,11 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
     EltTys.push_back(InhTag);
   }
 
-  const ASTRecordLayout &RL = CGM.getContext().getASTObjCInterfaceLayout(Decl);
+  const ASTRecordLayout &RL = CGM.getContext().getASTObjCInterfaceLayout(ID);
 
   unsigned FieldNo = 0;
-  for (ObjCInterfaceDecl::ivar_iterator I = Decl->ivar_begin(),
-         E = Decl->ivar_end();  I != E; ++I, ++FieldNo) {
+  for (ObjCInterfaceDecl::ivar_iterator I = ID->ivar_begin(),
+         E = ID->ivar_end();  I != E; ++I, ++FieldNo) {
     ObjCIvarDecl *Field = *I;
     llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit);
 
@@ -769,7 +972,7 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   uint64_t Align = CGM.getContext().getTypeAlign(Ty);
 
   llvm::DICompositeType RealDecl =
-    DebugFactory.CreateCompositeType(Tag, Unit, Decl->getName(), DefUnit,
+    DebugFactory.CreateCompositeType(Tag, Unit, ID->getName(), DefUnit,
                                      Line, Size, Align, 0, 0, llvm::DIType(), 
                                      Elements, RuntimeLang);
 
@@ -782,13 +985,13 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
 
 llvm::DIType CGDebugInfo::CreateType(const EnumType *Ty,
                                      llvm::DICompileUnit Unit) {
-  EnumDecl *Decl = Ty->getDecl();
+  EnumDecl *ED = Ty->getDecl();
 
   llvm::SmallVector<llvm::DIDescriptor, 32> Enumerators;
 
   // Create DIEnumerator elements for each enumerator.
   for (EnumDecl::enumerator_iterator
-         Enum = Decl->enumerator_begin(), EnumEnd = Decl->enumerator_end();
+         Enum = ED->enumerator_begin(), EnumEnd = ED->enumerator_end();
        Enum != EnumEnd; ++Enum) {
     Enumerators.push_back(DebugFactory.CreateEnumerator(Enum->getName(),
                                             Enum->getInitVal().getZExtValue()));
@@ -798,7 +1001,7 @@ llvm::DIType CGDebugInfo::CreateType(const EnumType *Ty,
   llvm::DIArray EltArray =
     DebugFactory.GetOrCreateArray(Enumerators.data(), Enumerators.size());
 
-  SourceLocation DefLoc = Decl->getLocation();
+  SourceLocation DefLoc = ED->getLocation();
   llvm::DICompileUnit DefUnit = getOrCreateCompileUnit(DefLoc);
   SourceManager &SM = CGM.getContext().getSourceManager();
   PresumedLoc PLoc = SM.getPresumedLoc(DefLoc);
@@ -815,7 +1018,7 @@ llvm::DIType CGDebugInfo::CreateType(const EnumType *Ty,
 
   llvm::DIType DbgTy = 
     DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_enumeration_type,
-                                     Unit, Decl->getName(), DefUnit, Line,
+                                     Unit, ED->getName(), DefUnit, Line,
                                      Size, Align, 0, 0,
                                      llvm::DIType(), EltArray);
   return DbgTy;
@@ -1083,6 +1286,7 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
       llvm::DISubprogram SP(dyn_cast_or_null<llvm::MDNode>(FI->second));
       if (!SP.isNull() && SP.isSubprogram() && SP.isDefinition()) {
         RegionStack.push_back(SP.getNode());
+        RegionMap[D] = llvm::WeakVH(SP.getNode());
         return;
       }
     }
@@ -1113,6 +1317,7 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
 
   // Push function on region stack.
   RegionStack.push_back(SP.getNode());
+  RegionMap[D] = llvm::WeakVH(SP.getNode());
 }
 
 
@@ -1164,160 +1369,172 @@ void CGDebugInfo::EmitRegionEnd(llvm::Function *Fn, CGBuilderTy &Builder) {
   RegionStack.pop_back();
 }
 
-/// EmitDeclare - Emit local variable declaration debug info.
-void CGDebugInfo::EmitDeclare(const VarDecl *Decl, unsigned Tag,
-                              llvm::Value *Storage, CGBuilderTy &Builder) {
-  assert(!RegionStack.empty() && "Region stack mismatch, stack empty!");
-
-  // Do not emit variable debug information while generating optimized code.
-  // The llvm optimizer and code generator are not yet ready to support
-  // optimized code debugging.
-  const CodeGenOptions &CGO = CGM.getCodeGenOpts();
-  if (CGO.OptimizationLevel)
-    return;
-
-  llvm::DICompileUnit Unit = getOrCreateCompileUnit(Decl->getLocation());
-  QualType Type = Decl->getType();
-  llvm::DIType Ty = getOrCreateType(Type, Unit);
-  if (Decl->hasAttr<BlocksAttr>()) {
-    llvm::DICompileUnit DefUnit;
-    unsigned Tag = llvm::dwarf::DW_TAG_structure_type;
+// EmitTypeForVarWithBlocksAttr - Build up structure info for the byref.  
+// See BuildByRefType.
+llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const ValueDecl *VD,
+                                                       uint64_t *XOffset) {
 
-    llvm::SmallVector<llvm::DIDescriptor, 5> EltTys;
-
-    llvm::DIType FieldTy;
+  llvm::SmallVector<llvm::DIDescriptor, 5> EltTys;
 
-    QualType FType;
-    uint64_t FieldSize, FieldOffset;
-    unsigned FieldAlign;
+  QualType FType;
+  uint64_t FieldSize, FieldOffset;
+  unsigned FieldAlign;
+  
+  llvm::DICompileUnit Unit = getOrCreateCompileUnit(VD->getLocation());
+  QualType Type = VD->getType();  
 
-    llvm::DIArray Elements;
-    llvm::DIType EltTy;
-    
-    // Build up structure for the byref.  See BuildByRefType.
-    FieldOffset = 0;
+  FieldOffset = 0;
+  FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
+  llvm::DIType FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
+  FieldSize = CGM.getContext().getTypeSize(FType);
+  FieldAlign = CGM.getContext().getTypeAlign(FType);
+  FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
+                                           "__isa", llvm::DICompileUnit(),
+                                           0, FieldSize, FieldAlign,
+                                           FieldOffset, 0, FieldTy);
+  EltTys.push_back(FieldTy);
+  FieldOffset += FieldSize;
+  
+  FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
+  FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
+  FieldSize = CGM.getContext().getTypeSize(FType);
+  FieldAlign = CGM.getContext().getTypeAlign(FType);
+  FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
+                                           "__forwarding", llvm::DICompileUnit(),
+                                           0, FieldSize, FieldAlign,
+                                           FieldOffset, 0, FieldTy);
+  EltTys.push_back(FieldTy);
+  FieldOffset += FieldSize;
+  
+  FType = CGM.getContext().IntTy;
+  FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
+  FieldSize = CGM.getContext().getTypeSize(FType);
+  FieldAlign = CGM.getContext().getTypeAlign(FType);
+  FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
+                                           "__flags", llvm::DICompileUnit(),
+                                           0, FieldSize, FieldAlign,
+                                           FieldOffset, 0, FieldTy);
+  EltTys.push_back(FieldTy);
+  FieldOffset += FieldSize;
+  
+  FType = CGM.getContext().IntTy;
+  FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
+  FieldSize = CGM.getContext().getTypeSize(FType);
+  FieldAlign = CGM.getContext().getTypeAlign(FType);
+  FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
+                                           "__size", llvm::DICompileUnit(),
+                                           0, FieldSize, FieldAlign,
+                                           FieldOffset, 0, FieldTy);
+  EltTys.push_back(FieldTy);
+  FieldOffset += FieldSize;
+  
+  bool HasCopyAndDispose = CGM.BlockRequiresCopying(Type);
+  if (HasCopyAndDispose) {
     FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
     FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
     FieldSize = CGM.getContext().getTypeSize(FType);
     FieldAlign = CGM.getContext().getTypeAlign(FType);
     FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             "__isa", DefUnit,
+                                             "__copy_helper", 
+                                             llvm::DICompileUnit(),
                                              0, FieldSize, FieldAlign,
                                              FieldOffset, 0, FieldTy);
     EltTys.push_back(FieldTy);
     FieldOffset += FieldSize;
-
+    
     FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
     FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
     FieldSize = CGM.getContext().getTypeSize(FType);
     FieldAlign = CGM.getContext().getTypeAlign(FType);
     FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             "__forwarding", DefUnit,
-                                             0, FieldSize, FieldAlign,
-                                             FieldOffset, 0, FieldTy);
-    EltTys.push_back(FieldTy);
-    FieldOffset += FieldSize;
-
-    FType = CGM.getContext().IntTy;
-    FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-    FieldSize = CGM.getContext().getTypeSize(FType);
-    FieldAlign = CGM.getContext().getTypeAlign(FType);
-    FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             "__flags", DefUnit,
-                                             0, FieldSize, FieldAlign,
-                                             FieldOffset, 0, FieldTy);
-    EltTys.push_back(FieldTy);
-    FieldOffset += FieldSize;
-
-    FType = CGM.getContext().IntTy;
-    FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-    FieldSize = CGM.getContext().getTypeSize(FType);
-    FieldAlign = CGM.getContext().getTypeAlign(FType);
-    FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             "__size", DefUnit,
+                                             "__destroy_helper", 
+                                             llvm::DICompileUnit(),
                                              0, FieldSize, FieldAlign,
                                              FieldOffset, 0, FieldTy);
     EltTys.push_back(FieldTy);
     FieldOffset += FieldSize;
+  }
+  
+  CharUnits Align = CGM.getContext().getDeclAlign(VD);
+  if (Align > CharUnits::fromQuantity(
+        CGM.getContext().Target.getPointerAlign(0) / 8)) {
+    unsigned AlignedOffsetInBytes
+      = llvm::RoundUpToAlignment(FieldOffset/8, Align.getQuantity());
+    unsigned NumPaddingBytes
+      = AlignedOffsetInBytes - FieldOffset/8;
     
-    bool HasCopyAndDispose = CGM.BlockRequiresCopying(Type);
-    if (HasCopyAndDispose) {
-      FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
-      FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-      FieldSize = CGM.getContext().getTypeSize(FType);
-      FieldAlign = CGM.getContext().getTypeAlign(FType);
-      FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                               "__copy_helper", DefUnit,
-                                               0, FieldSize, FieldAlign,
-                                               FieldOffset, 0, FieldTy);
-      EltTys.push_back(FieldTy);
-      FieldOffset += FieldSize;
-
-      FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
+    if (NumPaddingBytes > 0) {
+      llvm::APInt pad(32, NumPaddingBytes);
+      FType = CGM.getContext().getConstantArrayType(CGM.getContext().CharTy,
+                                                    pad, ArrayType::Normal, 0);
       FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
       FieldSize = CGM.getContext().getTypeSize(FType);
       FieldAlign = CGM.getContext().getTypeAlign(FType);
-      FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                               "__destroy_helper", DefUnit,
+      FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member,
+                                               Unit, "", llvm::DICompileUnit(),
                                                0, FieldSize, FieldAlign,
                                                FieldOffset, 0, FieldTy);
       EltTys.push_back(FieldTy);
       FieldOffset += FieldSize;
     }
-    
-    unsigned Align = CGM.getContext().getDeclAlignInBytes(Decl);
-    if (Align > CGM.getContext().Target.getPointerAlign(0) / 8) {
-      unsigned AlignedOffsetInBytes
-        = llvm::RoundUpToAlignment(FieldOffset/8, Align);
-      unsigned NumPaddingBytes
-        = AlignedOffsetInBytes - FieldOffset/8;
-
-      if (NumPaddingBytes > 0) {
-        llvm::APInt pad(32, NumPaddingBytes);
-        FType = CGM.getContext().getConstantArrayType(CGM.getContext().CharTy,
-                                                     pad, ArrayType::Normal, 0);
-        FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-        FieldSize = CGM.getContext().getTypeSize(FType);
-        FieldAlign = CGM.getContext().getTypeAlign(FType);
-        FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member,
-                                                 Unit, "", DefUnit,
-                                                 0, FieldSize, FieldAlign,
-                                                 FieldOffset, 0, FieldTy);
-        EltTys.push_back(FieldTy);
-        FieldOffset += FieldSize;
-      }
-    }
-
-    FType = Type;
-    FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-    FieldSize = CGM.getContext().getTypeSize(FType);
-    FieldAlign = Align*8;
-    
-    FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             Decl->getName(), DefUnit,
-                                             0, FieldSize, FieldAlign,
-                                             FieldOffset, 0, FieldTy);
-    EltTys.push_back(FieldTy);
-    FieldOffset += FieldSize;
-
-    Elements = DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
-
-    unsigned Flags = llvm::DIType::FlagBlockByrefStruct;
+  }
+  
+  FType = Type;
+  FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
+  FieldSize = CGM.getContext().getTypeSize(FType);
+  FieldAlign = Align.getQuantity()*8;
 
-    Ty = DebugFactory.CreateCompositeType(Tag, Unit, "",
+  *XOffset = FieldOffset;  
+  FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
+                                           VD->getName(), llvm::DICompileUnit(),
+                                           0, FieldSize, FieldAlign,
+                                           FieldOffset, 0, FieldTy);
+  EltTys.push_back(FieldTy);
+  FieldOffset += FieldSize;
+  
+  llvm::DIArray Elements = 
+    DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
+  
+  unsigned Flags = llvm::DIType::FlagBlockByrefStruct;
+  
+  return DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_structure_type, 
+                                          Unit, "",
                                           llvm::DICompileUnit(),
                                           0, FieldOffset, 0, 0, Flags,
                                           llvm::DIType(), Elements);
-  }
+  
+}
+/// EmitDeclare - Emit local variable declaration debug info.
+void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
+                              llvm::Value *Storage, CGBuilderTy &Builder) {
+  assert(!RegionStack.empty() && "Region stack mismatch, stack empty!");
+
+  // Do not emit variable debug information while generating optimized code.
+  // The llvm optimizer and code generator are not yet ready to support
+  // optimized code debugging.
+  const CodeGenOptions &CGO = CGM.getCodeGenOpts();
+  if (CGO.OptimizationLevel)
+    return;
+
+  llvm::DICompileUnit Unit = getOrCreateCompileUnit(VD->getLocation());
+  llvm::DIType Ty;
+  uint64_t XOffset = 0;
+  if (VD->hasAttr<BlocksAttr>())
+    Ty = EmitTypeForVarWithBlocksAttr(VD, &XOffset);
+  else 
+    Ty = getOrCreateType(VD->getType(), Unit);
 
   // Get location information.
   SourceManager &SM = CGM.getContext().getSourceManager();
-  PresumedLoc PLoc = SM.getPresumedLoc(Decl->getLocation());
+  PresumedLoc PLoc = SM.getPresumedLoc(VD->getLocation());
   unsigned Line = 0;
   unsigned Column = 0;
-  if (!PLoc.isInvalid()) {
+  if (PLoc.isInvalid())
+    PLoc = SM.getPresumedLoc(CurLoc);
+  if (PLoc.isValid()) {
     Line = PLoc.getLine();
     Column = PLoc.getColumn();
+    Unit = getOrCreateCompileUnit(CurLoc);
   } else {
     Unit = llvm::DICompileUnit();
   }
@@ -1325,7 +1542,7 @@ void CGDebugInfo::EmitDeclare(const VarDecl *Decl, unsigned Tag,
   // Create the descriptor for the variable.
   llvm::DIVariable D =
     DebugFactory.CreateVariable(Tag, llvm::DIDescriptor(RegionStack.back()),
-                                Decl->getName(),
+                                VD->getName(),
                                 Unit, Line, Ty);
   // Insert an llvm.dbg.declare into the current block.
   llvm::Instruction *Call =
@@ -1342,7 +1559,7 @@ void CGDebugInfo::EmitDeclare(const VarDecl *Decl, unsigned Tag,
 void CGDebugInfo::EmitDeclare(const BlockDeclRefExpr *BDRE, unsigned Tag,
                               llvm::Value *Storage, CGBuilderTy &Builder,
                               CodeGenFunction *CGF) {
-  const ValueDecl *Decl = BDRE->getDecl();
+  const ValueDecl *VD = BDRE->getDecl();
   assert(!RegionStack.empty() && "Region stack mismatch, stack empty!");
 
   // Do not emit variable debug information while generating optimized code.
@@ -1353,186 +1570,50 @@ void CGDebugInfo::EmitDeclare(const BlockDeclRefExpr *BDRE, unsigned Tag,
     return;
 
   uint64_t XOffset = 0;
-  llvm::DICompileUnit Unit = getOrCreateCompileUnit(Decl->getLocation());
-  QualType Type = Decl->getType();
-  llvm::DIType Ty = getOrCreateType(Type, Unit);
-  if (Decl->hasAttr<BlocksAttr>()) {
-    llvm::DICompileUnit DefUnit;
-    unsigned Tag = llvm::dwarf::DW_TAG_structure_type;
-
-    llvm::SmallVector<llvm::DIDescriptor, 5> EltTys;
-
-    llvm::DIType FieldTy;
-
-    QualType FType;
-    uint64_t FieldSize, FieldOffset;
-    unsigned FieldAlign;
-
-    llvm::DIArray Elements;
-    llvm::DIType EltTy;
-    
-    // Build up structure for the byref.  See BuildByRefType.
-    FieldOffset = 0;
-    FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
-    FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-    FieldSize = CGM.getContext().getTypeSize(FType);
-    FieldAlign = CGM.getContext().getTypeAlign(FType);
-    FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             "__isa", DefUnit,
-                                             0, FieldSize, FieldAlign,
-                                             FieldOffset, 0, FieldTy);
-    EltTys.push_back(FieldTy);
-    FieldOffset += FieldSize;
-
-    FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
-    FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-    FieldSize = CGM.getContext().getTypeSize(FType);
-    FieldAlign = CGM.getContext().getTypeAlign(FType);
-    FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             "__forwarding", DefUnit,
-                                             0, FieldSize, FieldAlign,
-                                             FieldOffset, 0, FieldTy);
-    EltTys.push_back(FieldTy);
-    FieldOffset += FieldSize;
-
-    FType = CGM.getContext().IntTy;
-    FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-    FieldSize = CGM.getContext().getTypeSize(FType);
-    FieldAlign = CGM.getContext().getTypeAlign(FType);
-    FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             "__flags", DefUnit,
-                                             0, FieldSize, FieldAlign,
-                                             FieldOffset, 0, FieldTy);
-    EltTys.push_back(FieldTy);
-    FieldOffset += FieldSize;
-
-    FType = CGM.getContext().IntTy;
-    FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-    FieldSize = CGM.getContext().getTypeSize(FType);
-    FieldAlign = CGM.getContext().getTypeAlign(FType);
-    FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             "__size", DefUnit,
-                                             0, FieldSize, FieldAlign,
-                                             FieldOffset, 0, FieldTy);
-    EltTys.push_back(FieldTy);
-    FieldOffset += FieldSize;
-    
-    bool HasCopyAndDispose = CGM.BlockRequiresCopying(Type);
-    if (HasCopyAndDispose) {
-      FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
-      FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-      FieldSize = CGM.getContext().getTypeSize(FType);
-      FieldAlign = CGM.getContext().getTypeAlign(FType);
-      FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                               "__copy_helper", DefUnit,
-                                               0, FieldSize, FieldAlign,
-                                               FieldOffset, 0, FieldTy);
-      EltTys.push_back(FieldTy);
-      FieldOffset += FieldSize;
-
-      FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
-      FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-      FieldSize = CGM.getContext().getTypeSize(FType);
-      FieldAlign = CGM.getContext().getTypeAlign(FType);
-      FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                               "__destroy_helper", DefUnit,
-                                               0, FieldSize, FieldAlign,
-                                               FieldOffset, 0, FieldTy);
-      EltTys.push_back(FieldTy);
-      FieldOffset += FieldSize;
-    }
-    
-    unsigned Align = CGM.getContext().getDeclAlignInBytes(Decl);
-    if (Align > CGM.getContext().Target.getPointerAlign(0) / 8) {
-      unsigned AlignedOffsetInBytes
-        = llvm::RoundUpToAlignment(FieldOffset/8, Align);
-      unsigned NumPaddingBytes
-        = AlignedOffsetInBytes - FieldOffset/8;
-
-      if (NumPaddingBytes > 0) {
-        llvm::APInt pad(32, NumPaddingBytes);
-        FType = CGM.getContext().getConstantArrayType(CGM.getContext().CharTy,
-                                                     pad, ArrayType::Normal, 0);
-        FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-        FieldSize = CGM.getContext().getTypeSize(FType);
-        FieldAlign = CGM.getContext().getTypeAlign(FType);
-        FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member,
-                                                 Unit, "", DefUnit,
-                                                 0, FieldSize, FieldAlign,
-                                                 FieldOffset, 0, FieldTy);
-        EltTys.push_back(FieldTy);
-        FieldOffset += FieldSize;
-      }
-    }
-
-    FType = Type;
-    FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
-    FieldSize = CGM.getContext().getTypeSize(FType);
-    FieldAlign = Align*8;
-    
-    XOffset = FieldOffset;
-    FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             Decl->getName(), DefUnit,
-                                             0, FieldSize, FieldAlign,
-                                             FieldOffset, 0, FieldTy);
-    EltTys.push_back(FieldTy);
-    FieldOffset += FieldSize;
-
-    Elements = DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
-
-    unsigned Flags = llvm::DIType::FlagBlockByrefStruct;
-
-    Ty = DebugFactory.CreateCompositeType(Tag, Unit, "",
-                                          llvm::DICompileUnit(),
-                                          0, FieldOffset, 0, 0, Flags,
-                                          llvm::DIType(), Elements);
-  }
+  llvm::DICompileUnit Unit = getOrCreateCompileUnit(VD->getLocation());
+  llvm::DIType Ty;
+  if (VD->hasAttr<BlocksAttr>())
+    Ty = EmitTypeForVarWithBlocksAttr(VD, &XOffset);
+  else 
+    Ty = getOrCreateType(VD->getType(), Unit);
 
   // Get location information.
   SourceManager &SM = CGM.getContext().getSourceManager();
-  PresumedLoc PLoc = SM.getPresumedLoc(Decl->getLocation());
+  PresumedLoc PLoc = SM.getPresumedLoc(VD->getLocation());
   unsigned Line = 0;
   if (!PLoc.isInvalid())
     Line = PLoc.getLine();
   else
     Unit = llvm::DICompileUnit();
 
-  CharUnits offset = CGF->BlockDecls[Decl];
+  CharUnits offset = CGF->BlockDecls[VD];
   llvm::SmallVector<llvm::Value *, 9> addr;
-  llvm::LLVMContext &VMContext = CGM.getLLVMContext();
-  addr.push_back(llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
-                                        llvm::DIFactory::OpDeref));
-  addr.push_back(llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
-                                        llvm::DIFactory::OpPlus));
-  addr.push_back(llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
-                                        offset.getQuantity()));
+  const llvm::Type *Int64Ty = llvm::Type::getInt64Ty(CGM.getLLVMContext());
+  addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpDeref));
+  addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpPlus));
+  addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
   if (BDRE->isByRef()) {
-    addr.push_back(llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
-                                          llvm::DIFactory::OpDeref));
-    addr.push_back(llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
-                                          llvm::DIFactory::OpPlus));
+    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpDeref));
+    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpPlus));
     // offset of __forwarding field
     offset = CharUnits::fromQuantity(CGF->LLVMPointerWidth/8);
-    addr.push_back(llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
-                                          offset.getQuantity()));
-    addr.push_back(llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
-                                          llvm::DIFactory::OpDeref));
-    addr.push_back(llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
-                                          llvm::DIFactory::OpPlus));
+    addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
+    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpDeref));
+    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpPlus));
     // offset of x field
     offset = CharUnits::fromQuantity(XOffset/8);
-    addr.push_back(llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
-                                          offset.getQuantity()));
+    addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
   }
 
   // Create the descriptor for the variable.
   llvm::DIVariable D =
-    DebugFactory.CreateComplexVariable(Tag, llvm::DIDescriptor(RegionStack.back()),
-                                       Decl->getName(), Unit, Line, Ty,
+    DebugFactory.CreateComplexVariable(Tag,
+                                       llvm::DIDescriptor(RegionStack.back()),
+                                       VD->getName(), Unit, Line, Ty,
                                        addr);
   // Insert an llvm.dbg.declare into the current block.
   llvm::Instruction *Call = 
-    DebugFactory.InsertDeclare(Storage, D, Builder.GetInsertPoint());
+    DebugFactory.InsertDeclare(Storage, D, Builder.GetInsertBlock());
 
   llvm::DIScope DS(RegionStack.back());
   llvm::DILocation DO(NULL);
@@ -1542,10 +1623,10 @@ void CGDebugInfo::EmitDeclare(const BlockDeclRefExpr *BDRE, unsigned Tag,
   Call->setMetadata("dbg", DL.getNode());
 }
 
-void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *Decl,
+void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD,
                                             llvm::Value *Storage,
                                             CGBuilderTy &Builder) {
-  EmitDeclare(Decl, llvm::dwarf::DW_TAG_auto_variable, Storage, Builder);
+  EmitDeclare(VD, llvm::dwarf::DW_TAG_auto_variable, Storage, Builder);
 }
 
 void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
@@ -1556,24 +1637,24 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
 
 /// EmitDeclareOfArgVariable - Emit call to llvm.dbg.declare for an argument
 /// variable declaration.
-void CGDebugInfo::EmitDeclareOfArgVariable(const VarDecl *Decl, llvm::Value *AI,
+void CGDebugInfo::EmitDeclareOfArgVariable(const VarDecl *VD, llvm::Value *AI,
                                            CGBuilderTy &Builder) {
-  EmitDeclare(Decl, llvm::dwarf::DW_TAG_arg_variable, AI, Builder);
+  EmitDeclare(VD, llvm::dwarf::DW_TAG_arg_variable, AI, Builder);
 }
 
 
 
 /// EmitGlobalVariable - Emit information about a global variable.
 void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
-                                     const VarDecl *Decl) {
-
+                                     const VarDecl *D) {
+  
   // Create global variable debug descriptor.
-  llvm::DICompileUnit Unit = getOrCreateCompileUnit(Decl->getLocation());
+  llvm::DICompileUnit Unit = getOrCreateCompileUnit(D->getLocation());
   SourceManager &SM = CGM.getContext().getSourceManager();
-  PresumedLoc PLoc = SM.getPresumedLoc(Decl->getLocation());
+  PresumedLoc PLoc = SM.getPresumedLoc(D->getLocation());
   unsigned LineNo = PLoc.isInvalid() ? 0 : PLoc.getLine();
 
-  QualType T = Decl->getType();
+  QualType T = D->getType();
   if (T->isIncompleteArrayType()) {
 
     // CodeGen turns int[] into int[1] so we'll do the same here.
@@ -1585,9 +1666,11 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
     T = CGM.getContext().getConstantArrayType(ET, ConstVal,
                                            ArrayType::Normal, 0);
   }
-  llvm::StringRef DeclName = Decl->getName();
-  DebugFactory.CreateGlobalVariable(getContext(Decl, Unit), DeclName, DeclName,
-                                    llvm::StringRef(), Unit, LineNo,
+  llvm::StringRef DeclName = D->getName();
+  llvm::DIDescriptor DContext = 
+    getContextDescriptor(dyn_cast<Decl>(D->getDeclContext()), Unit);
+  DebugFactory.CreateGlobalVariable(DContext, DeclName,
+                                    DeclName, llvm::StringRef(), Unit, LineNo,
                                     getOrCreateType(T, Unit),
                                     Var->hasInternalLinkage(),
                                     true/*definition*/, Var);
@@ -1595,16 +1678,16 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
 
 /// EmitGlobalVariable - Emit information about an objective-c interface.
 void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
-                                     ObjCInterfaceDecl *Decl) {
+                                     ObjCInterfaceDecl *ID) {
   // Create global variable debug descriptor.
-  llvm::DICompileUnit Unit = getOrCreateCompileUnit(Decl->getLocation());
+  llvm::DICompileUnit Unit = getOrCreateCompileUnit(ID->getLocation());
   SourceManager &SM = CGM.getContext().getSourceManager();
-  PresumedLoc PLoc = SM.getPresumedLoc(Decl->getLocation());
+  PresumedLoc PLoc = SM.getPresumedLoc(ID->getLocation());
   unsigned LineNo = PLoc.isInvalid() ? 0 : PLoc.getLine();
 
-  llvm::StringRef Name = Decl->getName();
+  llvm::StringRef Name = ID->getName();
 
-  QualType T = CGM.getContext().getObjCInterfaceType(Decl);
+  QualType T = CGM.getContext().getObjCInterfaceType(ID);
   if (T->isIncompleteArrayType()) {
 
     // CodeGen turns int[] into int[1] so we'll do the same here.
@@ -1622,3 +1705,26 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
                                     Var->hasInternalLinkage(),
                                     true/*definition*/, Var);
 }
+
+/// getOrCreateNamesSpace - Return namespace descriptor for the given
+/// namespace decl.
+llvm::DINameSpace 
+CGDebugInfo::getOrCreateNameSpace(const NamespaceDecl *NSDecl, 
+                                  llvm::DIDescriptor Unit) {
+  llvm::DenseMap<const NamespaceDecl *, llvm::WeakVH>::iterator I = 
+    NameSpaceCache.find(NSDecl);
+  if (I != NameSpaceCache.end())
+    return llvm::DINameSpace(cast<llvm::MDNode>(I->second));
+  
+  SourceManager &SM = CGM.getContext().getSourceManager();
+  PresumedLoc PLoc = SM.getPresumedLoc(NSDecl->getLocation());
+  unsigned LineNo = PLoc.isInvalid() ? 0 : PLoc.getLine();
+
+  llvm::DIDescriptor Context = 
+    getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext()), Unit);
+  llvm::DINameSpace NS =
+    DebugFactory.CreateNameSpace(Context, NSDecl->getName(), 
+	llvm::DICompileUnit(Unit.getNode()), LineNo);
+  NameSpaceCache[NSDecl] = llvm::WeakVH(NS.getNode());
+  return NS;
+}
diff --git a/lib/CodeGen/CGDebugInfo.h b/lib/CodeGen/CGDebugInfo.h
index fddd23b..b2d3a1f 100644
--- a/lib/CodeGen/CGDebugInfo.h
+++ b/lib/CodeGen/CGDebugInfo.h
@@ -47,6 +47,8 @@ class CGDebugInfo {
   llvm::DIFactory DebugFactory;
 
   SourceLocation CurLoc, PrevLoc;
+  
+  llvm::DIType VTablePtrType;
 
   /// CompileUnitCache - Cache of previously constructed CompileUnits.
   llvm::DenseMap<unsigned, llvm::DICompileUnit> CompileUnitCache;
@@ -59,12 +61,14 @@ class CGDebugInfo {
   llvm::DIType BlockLiteralGeneric;
 
   std::vector<llvm::TrackingVH<llvm::MDNode> > RegionStack;
+  llvm::DenseMap<const Decl *, llvm::WeakVH> RegionMap;
 
-  /// FunctionNames - This is a storage for function names that are
+  /// DebugInfoNames - This is a storage for names that are
   /// constructed on demand. For example, C++ destructors, C++ operators etc..
-  llvm::BumpPtrAllocator FunctionNames;
+  llvm::BumpPtrAllocator DebugInfoNames;
 
   llvm::DenseMap<const FunctionDecl *, llvm::WeakVH> SPCache;
+  llvm::DenseMap<const NamespaceDecl *, llvm::WeakVH> NameSpaceCache;
 
   /// Helper functions for getOrCreateType.
   llvm::DIType CreateType(const BuiltinType *Ty, llvm::DICompileUnit U);
@@ -83,16 +87,37 @@ class CGDebugInfo {
   llvm::DIType CreateType(const ArrayType *Ty, llvm::DICompileUnit U);
   llvm::DIType CreateType(const LValueReferenceType *Ty, llvm::DICompileUnit U);
   llvm::DIType CreateType(const MemberPointerType *Ty, llvm::DICompileUnit U);
-  
+  llvm::DIType getOrCreateMethodType(const CXXMethodDecl *Method,
+                                     llvm::DICompileUnit Unit);
+  llvm::DIType getOrCreateVTablePtrType(llvm::DICompileUnit Unit);
+  llvm::DINameSpace getOrCreateNameSpace(const NamespaceDecl *N, 
+                                         llvm::DIDescriptor Unit);
+
   llvm::DIType CreatePointerLikeType(unsigned Tag,
                                      const Type *Ty, QualType PointeeTy,
                                      llvm::DICompileUnit U);
+  
+  llvm::DISubprogram CreateCXXMemberFunction(const CXXMethodDecl *Method,
+                                             llvm::DICompileUnit Unit,
+                                             llvm::DICompositeType &RecordTy);
+  
   void CollectCXXMemberFunctions(const CXXRecordDecl *Decl,
                                  llvm::DICompileUnit U,
                                  llvm::SmallVectorImpl<llvm::DIDescriptor> &E,
                                  llvm::DICompositeType &T);
+  void CollectCXXBases(const CXXRecordDecl *Decl,
+                       llvm::DICompileUnit Unit,
+                       llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
+                       llvm::DICompositeType &RecordTy);
+
+
   void CollectRecordFields(const RecordDecl *Decl, llvm::DICompileUnit U,
                            llvm::SmallVectorImpl<llvm::DIDescriptor> &E);
+
+  void CollectVtableInfo(const CXXRecordDecl *Decl,
+                         llvm::DICompileUnit Unit,
+                         llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys);
+
 public:
   CGDebugInfo(CodeGenModule &CGM);
   ~CGDebugInfo();
@@ -150,8 +175,14 @@ private:
   void EmitDeclare(const BlockDeclRefExpr *BDRE, unsigned Tag, llvm::Value *AI,
                    CGBuilderTy &Builder, CodeGenFunction *CGF);
 
-  /// getContext - Get context info for the decl.
-  llvm::DIDescriptor getContext(const VarDecl *Decl,llvm::DIDescriptor &CU);
+  // EmitTypeForVarWithBlocksAttr - Build up structure info for the byref.  
+  // See BuildByRefType.
+  llvm::DIType EmitTypeForVarWithBlocksAttr(const ValueDecl *VD, 
+                                            uint64_t *OffSet);
+
+  /// getContextDescriptor - Get context info for the decl.
+  llvm::DIDescriptor getContextDescriptor(const Decl *Decl,
+                                          llvm::DIDescriptor &CU);
 
   /// getOrCreateCompileUnit - Get the compile unit from the cache or create a
   /// new one if necessary.
@@ -168,6 +199,10 @@ private:
   /// name is constructred on demand (e.g. C++ destructor) then the name
   /// is stored on the side.
   llvm::StringRef getFunctionName(const FunctionDecl *FD);
+
+  /// getVtableName - Get vtable name for the given Class.
+  llvm::StringRef getVtableName(const CXXRecordDecl *Decl);
+
 };
 } // namespace CodeGen
 } // namespace clang
diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp
index 9606a71..793a220 100644
--- a/lib/CodeGen/CGDecl.cpp
+++ b/lib/CodeGen/CGDecl.cpp
@@ -76,8 +76,21 @@ void CodeGenFunction::EmitBlockVarDecl(const VarDecl &D) {
   case VarDecl::Auto:
   case VarDecl::Register:
     return EmitLocalBlockVarDecl(D);
-  case VarDecl::Static:
-    return EmitStaticBlockVarDecl(D);
+  case VarDecl::Static: {
+    llvm::GlobalValue::LinkageTypes Linkage = 
+      llvm::GlobalValue::InternalLinkage;
+
+    // If this is a static declaration inside an inline function, it must have
+    // weak linkage so that the linker will merge multiple definitions of it.
+    if (getContext().getLangOptions().CPlusPlus) {
+      if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(CurFuncDecl)) {
+        if (FD->isInlined())
+          Linkage = llvm::GlobalValue::WeakAnyLinkage;
+      }
+    }
+    
+    return EmitStaticBlockVarDecl(D, Linkage);
+  }
   case VarDecl::Extern:
   case VarDecl::PrivateExtern:
     // Don't emit it now, allow it to be emitted lazily on its first use.
@@ -120,7 +133,7 @@ CodeGenFunction::CreateStaticBlockVarDecl(const VarDecl &D,
                              Ty.isConstant(getContext()), Linkage,
                              CGM.EmitNullConstant(D.getType()), Name, 0,
                              D.isThreadSpecified(), Ty.getAddressSpace());
-  GV->setAlignment(getContext().getDeclAlignInBytes(&D));
+  GV->setAlignment(getContext().getDeclAlign(&D).getQuantity());
   return GV;
 }
 
@@ -138,8 +151,13 @@ CodeGenFunction::AddInitializerToGlobalBlockVarDecl(const VarDecl &D,
   if (!Init) {
     if (!getContext().getLangOptions().CPlusPlus)
       CGM.ErrorUnsupported(D.getInit(), "constant l-value expression");
-    else
+    else {
+      // Since we have a static initializer, this global variable can't 
+      // be constant.
+      GV->setConstant(false);
+      
       EmitStaticCXXBlockVarDeclInit(D, GV);
+    }
     return GV;
   }
   
@@ -172,12 +190,12 @@ CodeGenFunction::AddInitializerToGlobalBlockVarDecl(const VarDecl &D,
   return GV;
 }
 
-void CodeGenFunction::EmitStaticBlockVarDecl(const VarDecl &D) {
+void CodeGenFunction::EmitStaticBlockVarDecl(const VarDecl &D,
+                                      llvm::GlobalValue::LinkageTypes Linkage) {
   llvm::Value *&DMEntry = LocalDeclMap[&D];
   assert(DMEntry == 0 && "Decl already exists in localdeclmap!");
 
-  llvm::GlobalVariable *GV =
-    CreateStaticBlockVarDecl(D, ".", llvm::GlobalValue::InternalLinkage);
+  llvm::GlobalVariable *GV = CreateStaticBlockVarDecl(D, ".", Linkage);
 
   // Store into LocalDeclMap before generating initializer to handle
   // circular references.
@@ -281,8 +299,8 @@ const llvm::Type *CodeGenFunction::BuildByRefType(const ValueDecl *D) {
   }
 
   bool Packed = false;
-  unsigned Align = getContext().getDeclAlignInBytes(D);
-  if (Align > Target.getPointerAlign(0) / 8) {
+  CharUnits Align = getContext().getDeclAlign(D);
+  if (Align > CharUnits::fromQuantity(Target.getPointerAlign(0) / 8)) {
     // We have to insert padding.
     
     // The struct above has 2 32-bit integers.
@@ -294,7 +312,7 @@ const llvm::Type *CodeGenFunction::BuildByRefType(const ValueDecl *D) {
     
     // Align the offset.
     unsigned AlignedOffsetInBytes = 
-      llvm::RoundUpToAlignment(CurrentOffsetInBytes, Align);
+      llvm::RoundUpToAlignment(CurrentOffsetInBytes, Align.getQuantity());
     
     unsigned NumPaddingBytes = AlignedOffsetInBytes - CurrentOffsetInBytes;
     if (NumPaddingBytes > 0) {
@@ -334,7 +352,7 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D) {
   QualType Ty = D.getType();
   bool isByRef = D.hasAttr<BlocksAttr>();
   bool needsDispose = false;
-  unsigned Align = 0;
+  CharUnits Align = CharUnits::Zero();
   bool IsSimpleConstantInitializer = false;
 
   llvm::Value *DeclPtr;
@@ -350,7 +368,7 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D) {
         // If this variable is marked 'const', emit the value as a global.
         if (CGM.getCodeGenOpts().MergeAllConstants &&
             Ty.isConstant(getContext())) {
-          EmitStaticBlockVarDecl(D);
+          EmitStaticBlockVarDecl(D, llvm::GlobalValue::InternalLinkage);
           return;
         }
         
@@ -364,10 +382,11 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D) {
       llvm::AllocaInst *Alloc = CreateTempAlloca(LTy);
       Alloc->setName(D.getNameAsString());
 
-      Align = getContext().getDeclAlignInBytes(&D);
+      Align = getContext().getDeclAlign(&D);
       if (isByRef)
-        Align = std::max(Align, unsigned(Target.getPointerAlign(0) / 8));
-      Alloc->setAlignment(Align);
+        Align = std::max(Align, 
+            CharUnits::fromQuantity(Target.getPointerAlign(0) / 8));
+      Alloc->setAlignment(Align.getQuantity());
       DeclPtr = Alloc;
     } else {
       // Targets that don't support recursion emit locals as globals.
@@ -420,7 +439,7 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D) {
     // Allocate memory for the array.
     llvm::AllocaInst *VLA = 
       Builder.CreateAlloca(llvm::Type::getInt8Ty(VMContext), VLASize, "vla");
-    VLA->setAlignment(getContext().getDeclAlignInBytes(&D));
+    VLA->setAlignment(getContext().getDeclAlign(&D).getQuantity());
 
     DeclPtr = Builder.CreateBitCast(VLA, LElemPtrTy, "tmp");
   }
@@ -468,7 +487,8 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D) {
       assert(Init != 0 && "Wasn't a simple constant init?");
       
       llvm::Value *AlignVal = 
-        llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), Align);
+        llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), 
+            Align.getQuantity());
       const llvm::Type *IntPtr =
         llvm::IntegerType::get(VMContext, LLVMPointerWidth);
       llvm::Value *SizeVal =
@@ -492,7 +512,7 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D) {
           new llvm::GlobalVariable(CGM.getModule(), Init->getType(), true,
                                    llvm::GlobalValue::InternalLinkage,
                                    Init, Name, 0, false, 0);
-        GV->setAlignment(Align);
+        GV->setAlignment(Align.getQuantity());
 
         llvm::Value *SrcPtr = GV;
         if (SrcPtr->getType() != BP)
@@ -501,7 +521,7 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D) {
         Builder.CreateCall4(CGM.getMemCpyFn(), Loc, SrcPtr, SizeVal, AlignVal);
       }
     } else if (Ty->isReferenceType()) {
-      RValue RV = EmitReferenceBindingToExpr(Init, Ty, /*IsInitializer=*/true);
+      RValue RV = EmitReferenceBindingToExpr(Init, /*IsInitializer=*/true);
       EmitStoreOfScalar(RV.getScalarVal(), Loc, false, Ty);
     } else if (!hasAggregateLLVMType(Init->getType())) {
       llvm::Value *V = EmitScalarExpr(Init);
@@ -554,19 +574,19 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D) {
     const llvm::Type *V1;
     V1 = cast<llvm::PointerType>(DeclPtr->getType())->getElementType();
     V = llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext),
-                               (CGM.getTargetData().getTypeStoreSizeInBits(V1)
-                                / 8));
+                               CGM.GetTargetTypeStoreSize(V1).getQuantity());
     Builder.CreateStore(V, size_field);
 
     if (flags & BLOCK_HAS_COPY_DISPOSE) {
       BlockHasCopyDispose = true;
       llvm::Value *copy_helper = Builder.CreateStructGEP(DeclPtr, 4);
-      Builder.CreateStore(BuildbyrefCopyHelper(DeclPtr->getType(), flag, Align),
+      Builder.CreateStore(BuildbyrefCopyHelper(DeclPtr->getType(), flag, 
+                                               Align.getQuantity()),
                           copy_helper);
 
       llvm::Value *destroy_helper = Builder.CreateStructGEP(DeclPtr, 5);
       Builder.CreateStore(BuildbyrefDestroyHelper(DeclPtr->getType(), flag,
-                                                  Align),
+                                                  Align.getQuantity()),
                           destroy_helper);
     }
   }
@@ -683,25 +703,18 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg) {
   CanQualType CTy = getContext().getCanonicalType(Ty);
 
   llvm::Value *DeclPtr;
-  if (!Ty->isConstantSizeType()) {
-    // Variable sized values always are passed by-reference.
+  // If this is an aggregate or variable sized value, reuse the input pointer.
+  if (!Ty->isConstantSizeType() ||
+      CodeGenFunction::hasAggregateLLVMType(Ty)) {
     DeclPtr = Arg;
   } else {
-    // A fixed sized single-value variable becomes an alloca in the entry block.
-    const llvm::Type *LTy = ConvertTypeForMem(Ty);
-    if (LTy->isSingleValueType()) {
-      // TODO: Alignment
-      DeclPtr = CreateTempAlloca(LTy);
-      DeclPtr->setName(D.getNameAsString() + llvm::StringRef(".addr"));
-
-      // Store the initial value into the alloca.
-      EmitStoreOfScalar(Arg, DeclPtr, CTy.isVolatileQualified(), Ty);
-    } else {
-      // Otherwise, if this is an aggregate, just use the input pointer.
-      DeclPtr = Arg;
-    }
-    Arg->setName(D.getNameAsString());
+    // Otherwise, create a temporary to hold the value.
+    DeclPtr = CreateMemTemp(Ty, D.getName() + ".addr");
+
+    // Store the initial value into the alloca.
+    EmitStoreOfScalar(Arg, DeclPtr, CTy.isVolatileQualified(), Ty);
   }
+  Arg->setName(D.getName());
 
   llvm::Value *&DMEntry = LocalDeclMap[&D];
   assert(DMEntry == 0 && "Decl already exists in localdeclmap!");
diff --git a/lib/CodeGen/CGDeclCXX.cpp b/lib/CodeGen/CGDeclCXX.cpp
index 47773a0..0de3b0b 100644
--- a/lib/CodeGen/CGDeclCXX.cpp
+++ b/lib/CodeGen/CGDeclCXX.cpp
@@ -80,8 +80,13 @@ void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
     EmitDeclInit(*this, D, DeclPtr);
     return;
   }
-
-  ErrorUnsupported(Init, "global variable that binds to a reference");
+  if (Init->isLvalue(getContext()) == Expr::LV_Valid) {
+    RValue RV = EmitReferenceBindingToExpr(Init, /*IsInitializer=*/true);
+    EmitStoreOfScalar(RV.getScalarVal(), DeclPtr, false, T);
+    return;
+  }
+  ErrorUnsupported(Init, 
+                   "global variable that binds reference to a non-lvalue");
 }
 
 void
@@ -179,57 +184,120 @@ void CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
   FinishFunction();
 }
 
+static llvm::Constant *getGuardAcquireFn(CodeGenFunction &CGF) {
+  // int __cxa_guard_acquire(__int64_t *guard_object);
+  
+  const llvm::Type *Int64PtrTy = 
+    llvm::Type::getInt64PtrTy(CGF.getLLVMContext());
+  
+  std::vector<const llvm::Type*> Args(1, Int64PtrTy);
+  
+  const llvm::FunctionType *FTy =
+    llvm::FunctionType::get(CGF.ConvertType(CGF.getContext().IntTy),
+                            Args, /*isVarArg=*/false);
+  
+  return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_guard_acquire");
+}
+
+static llvm::Constant *getGuardReleaseFn(CodeGenFunction &CGF) {
+  // void __cxa_guard_release(__int64_t *guard_object);
+  
+  const llvm::Type *Int64PtrTy = 
+    llvm::Type::getInt64PtrTy(CGF.getLLVMContext());
+  
+  std::vector<const llvm::Type*> Args(1, Int64PtrTy);
+  
+  const llvm::FunctionType *FTy =
+  llvm::FunctionType::get(llvm::Type::getVoidTy(CGF.getLLVMContext()),
+                          Args, /*isVarArg=*/false);
+  
+  return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_guard_release");
+}
+
+static llvm::Constant *getGuardAbortFn(CodeGenFunction &CGF) {
+  // void __cxa_guard_abort(__int64_t *guard_object);
+  
+  const llvm::Type *Int64PtrTy = 
+    llvm::Type::getInt64PtrTy(CGF.getLLVMContext());
+  
+  std::vector<const llvm::Type*> Args(1, Int64PtrTy);
+  
+  const llvm::FunctionType *FTy =
+  llvm::FunctionType::get(llvm::Type::getVoidTy(CGF.getLLVMContext()),
+                          Args, /*isVarArg=*/false);
+  
+  return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_guard_abort");
+}
+
 void
 CodeGenFunction::EmitStaticCXXBlockVarDeclInit(const VarDecl &D,
                                                llvm::GlobalVariable *GV) {
-  // FIXME: This should use __cxa_guard_{acquire,release}?
-
-  assert(!getContext().getLangOptions().ThreadsafeStatics &&
-         "thread safe statics are currently not supported!");
-
+  bool ThreadsafeStatics = getContext().getLangOptions().ThreadsafeStatics;
+  
   llvm::SmallString<256> GuardVName;
   CGM.getMangleContext().mangleGuardVariable(&D, GuardVName);
 
   // Create the guard variable.
-  llvm::GlobalValue *GuardV =
-    new llvm::GlobalVariable(CGM.getModule(), llvm::Type::getInt64Ty(VMContext),
+  const llvm::Type *Int64Ty = llvm::Type::getInt64Ty(VMContext);
+  llvm::GlobalValue *GuardVariable =
+    new llvm::GlobalVariable(CGM.getModule(), Int64Ty,
                              false, GV->getLinkage(),
-                llvm::Constant::getNullValue(llvm::Type::getInt64Ty(VMContext)),
+                             llvm::Constant::getNullValue(Int64Ty),
                              GuardVName.str());
 
   // Load the first byte of the guard variable.
   const llvm::Type *PtrTy
     = llvm::PointerType::get(llvm::Type::getInt8Ty(VMContext), 0);
-  llvm::Value *V = Builder.CreateLoad(Builder.CreateBitCast(GuardV, PtrTy),
-                                      "tmp");
-
-  // Compare it against 0.
-  llvm::Value *nullValue
-    = llvm::Constant::getNullValue(llvm::Type::getInt8Ty(VMContext));
-  llvm::Value *ICmp = Builder.CreateICmpEQ(V, nullValue , "tobool");
+  llvm::Value *V = 
+    Builder.CreateLoad(Builder.CreateBitCast(GuardVariable, PtrTy), "tmp");
 
-  llvm::BasicBlock *InitBlock = createBasicBlock("init");
+  llvm::BasicBlock *InitCheckBlock = createBasicBlock("init.check");
   llvm::BasicBlock *EndBlock = createBasicBlock("init.end");
 
-  // If the guard variable is 0, jump to the initializer code.
-  Builder.CreateCondBr(ICmp, InitBlock, EndBlock);
+  // Check if the first byte of the guard variable is zero.
+  Builder.CreateCondBr(Builder.CreateIsNull(V, "tobool"), 
+                       InitCheckBlock, EndBlock);
 
-  EmitBlock(InitBlock);
+  EmitBlock(InitCheckBlock);
+
+  if (ThreadsafeStatics) {
+    // Call __cxa_guard_acquire.
+    V = Builder.CreateCall(getGuardAcquireFn(*this), GuardVariable);
+               
+    llvm::BasicBlock *InitBlock = createBasicBlock("init");
+  
+    Builder.CreateCondBr(Builder.CreateIsNotNull(V, "tobool"),
+                         InitBlock, EndBlock);
+  
+    EmitBlock(InitBlock);
+
+    if (Exceptions) {
+      EHCleanupBlock Cleanup(*this);
+    
+      // Call __cxa_guard_abort.
+      Builder.CreateCall(getGuardAbortFn(*this), GuardVariable);
+    }
+  }
 
   if (D.getType()->isReferenceType()) {
     QualType T = D.getType();
     // We don't want to pass true for IsInitializer here, because a static
     // reference to a temporary does not extend its lifetime.
-    RValue RV = EmitReferenceBindingToExpr(D.getInit(), T,
+    RValue RV = EmitReferenceBindingToExpr(D.getInit(),
                                            /*IsInitializer=*/false);
     EmitStoreOfScalar(RV.getScalarVal(), GV, /*Volatile=*/false, T);
 
   } else
     EmitDeclInit(*this, D, GV);
 
-  Builder.CreateStore(llvm::ConstantInt::get(llvm::Type::getInt8Ty(VMContext),
-                                             1),
-                      Builder.CreateBitCast(GuardV, PtrTy));
+  if (ThreadsafeStatics) {
+    // Call __cxa_guard_release.
+    Builder.CreateCall(getGuardReleaseFn(*this), GuardVariable);
+  } else {
+    llvm::Value *One = 
+      llvm::ConstantInt::get(llvm::Type::getInt8Ty(VMContext), 1);
+    Builder.CreateStore(One, Builder.CreateBitCast(GuardVariable, PtrTy));
+  }
 
   EmitBlock(EndBlock);
 }
diff --git a/lib/CodeGen/CGException.cpp b/lib/CodeGen/CGException.cpp
index bd0461f..d956c1c 100644
--- a/lib/CodeGen/CGException.cpp
+++ b/lib/CodeGen/CGException.cpp
@@ -100,10 +100,6 @@ static llvm::Constant *getUnexpectedFn(CodeGenFunction &CGF) {
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_call_unexpected");
 }
 
-// FIXME: Eventually this will all go into the backend.  Set from the target for
-// now.
-static int using_sjlj_exceptions = 0;
-
 static llvm::Constant *getUnwindResumeOrRethrowFn(CodeGenFunction &CGF) {
   const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
   std::vector<const llvm::Type*> Args(1, Int8PtrTy);
@@ -112,7 +108,7 @@ static llvm::Constant *getUnwindResumeOrRethrowFn(CodeGenFunction &CGF) {
     llvm::FunctionType::get(llvm::Type::getVoidTy(CGF.getLLVMContext()), Args,
                             false);
 
-  if (using_sjlj_exceptions)
+  if (CGF.CGM.getLangOptions().SjLjExceptions)
     return CGF.CGM.CreateRuntimeFunction(FTy, "_Unwind_SjLj_Resume");
   return CGF.CGM.CreateRuntimeFunction(FTy, "_Unwind_Resume_or_Rethrow");
 }
@@ -194,9 +190,9 @@ static void CopyObject(CodeGenFunction &CGF, const Expr *E,
       // Push the Src ptr.
       CallArgs.push_back(std::make_pair(RValue::get(Src),
                                         CopyCtor->getParamDecl(0)->getType()));
-      QualType ResultType =
-        CopyCtor->getType()->getAs<FunctionType>()->getResultType();
-      CGF.EmitCall(CGF.CGM.getTypes().getFunctionInfo(ResultType, CallArgs),
+      const FunctionProtoType *FPT
+        = CopyCtor->getType()->getAs<FunctionProtoType>();
+      CGF.EmitCall(CGF.CGM.getTypes().getFunctionInfo(CallArgs, FPT),
                    Callee, ReturnValueSlot(), CallArgs, CopyCtor);
       CGF.setInvokeDest(PrevLandingPad);
     } else
@@ -244,9 +240,10 @@ static void CopyObject(CodeGenFunction &CGF, QualType ObjectType,
       // Push the Src ptr.
       CallArgs.push_back(std::make_pair(RValue::get(Src),
                                         CopyCtor->getParamDecl(0)->getType()));
-      QualType ResultType =
-        CopyCtor->getType()->getAs<FunctionType>()->getResultType();
-      CGF.EmitCall(CGF.CGM.getTypes().getFunctionInfo(ResultType, CallArgs),
+
+      const FunctionProtoType *FPT
+        = CopyCtor->getType()->getAs<FunctionProtoType>();
+      CGF.EmitCall(CGF.CGM.getTypes().getFunctionInfo(CallArgs, FPT),
                    Callee, ReturnValueSlot(), CallArgs, CopyCtor);
     } else
       llvm_unreachable("uncopyable object");
@@ -316,6 +313,9 @@ void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E) {
 }
 
 void CodeGenFunction::EmitStartEHSpec(const Decl *D) {
+  if (!Exceptions)
+    return;
+  
   const FunctionDecl* FD = dyn_cast_or_null<FunctionDecl>(D);
   if (FD == 0)
     return;
@@ -410,6 +410,9 @@ void CodeGenFunction::EmitStartEHSpec(const Decl *D) {
 }
 
 void CodeGenFunction::EmitEndEHSpec(const Decl *D) {
+  if (!Exceptions)
+    return;
+  
   const FunctionDecl* FD = dyn_cast_or_null<FunctionDecl>(D);
   if (FD == 0)
     return;
@@ -466,6 +469,7 @@ void CodeGenFunction::EmitCXXTryStmt(const CXXTryStmt &S) {
       llvm::BasicBlock *DtorEpilogue  = createBasicBlock("dtor.epilogue");
       PushCleanupBlock(DtorEpilogue);
 
+      InitializeVtablePtrs(DD->getParent());
       EmitStmt(S.getTryBlock());
 
       CleanupBlockInfo Info = PopCleanupBlock();
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
index 2358bb3..830954f 100644
--- a/lib/CodeGen/CGExpr.cpp
+++ b/lib/CodeGen/CGExpr.cpp
@@ -36,16 +36,24 @@ llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(const llvm::Type *Ty,
   return new llvm::AllocaInst(Ty, 0, Name, AllocaInsertPt);
 }
 
+llvm::Value *CodeGenFunction::CreateMemTemp(QualType Ty, const llvm::Twine &Name) {
+  llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty), Name);
+  // FIXME: Should we prefer the preferred type alignment here?
+  CharUnits Align = getContext().getTypeAlignInChars(Ty);
+  Alloc->setAlignment(Align.getQuantity());
+  return Alloc;
+}
+
 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
 /// expression and compare the result against zero, returning an Int1Ty value.
 llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
   QualType BoolTy = getContext().BoolTy;
   if (E->getType()->isMemberFunctionPointerType()) {
-    llvm::Value *Ptr = CreateTempAlloca(ConvertType(E->getType()));
-    EmitAggExpr(E, Ptr, /*VolatileDest=*/false);
+    LValue LV = EmitAggExprToLValue(E);
 
     // Get the pointer.
-    llvm::Value *FuncPtr = Builder.CreateStructGEP(Ptr, 0, "src.ptr");
+    llvm::Value *FuncPtr = Builder.CreateStructGEP(LV.getAddress(), 0,
+                                                   "src.ptr");
     FuncPtr = Builder.CreateLoad(FuncPtr);
 
     llvm::Value *IsNotNull = 
@@ -87,13 +95,12 @@ RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E,
 
   if (hasAggregateLLVMType(E->getType()) &&
       !E->getType()->isAnyComplexType())
-    AggLoc = CreateTempAlloca(ConvertType(E->getType()), "agg.tmp");
+    AggLoc = CreateMemTemp(E->getType(), "agg.tmp");
   return EmitAnyExpr(E, AggLoc, IsAggLocVolatile, /*IgnoreResult=*/false,
                      IsInitializer);
 }
 
 RValue CodeGenFunction::EmitReferenceBindingToExpr(const Expr* E,
-                                                   QualType DestType,
                                                    bool IsInitializer) {
   bool ShouldDestroyTemporaries = false;
   unsigned OldNumLiveTemporaries = 0;
@@ -114,8 +121,16 @@ RValue CodeGenFunction::EmitReferenceBindingToExpr(const Expr* E,
   if (E->isLvalue(getContext()) == Expr::LV_Valid) {
     // Emit the expr as an lvalue.
     LValue LV = EmitLValue(E);
-    if (LV.isSimple())
+    if (LV.isSimple()) {
+      if (ShouldDestroyTemporaries) {
+        // Pop temporaries.
+        while (LiveTemporaries.size() > OldNumLiveTemporaries)
+          PopCXXTemporary();
+      }
+      
       return RValue::get(LV.getAddress());
+    }
+    
     Val = EmitLoadOfLValue(LV, E->getType());
     
     if (ShouldDestroyTemporaries) {
@@ -188,8 +203,7 @@ RValue CodeGenFunction::EmitReferenceBindingToExpr(const Expr* E,
     Val = RValue::get(Val.getAggregateAddr());
   } else {
     // Create a temporary variable that we can bind the reference to.
-    llvm::Value *Temp = CreateTempAlloca(ConvertTypeForMem(E->getType()),
-                                         "reftmp");
+    llvm::Value *Temp = CreateMemTemp(E->getType(), "reftmp");
     if (Val.isScalar())
       EmitStoreOfScalar(Val.getScalarVal(), Temp, false, E->getType());
     else
@@ -546,6 +560,8 @@ RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, QualType ExprType) {
       cast<llvm::PointerType>(Ptr->getType())->getElementType();
 
     // Simple scalar l-value.
+    //
+    // FIXME: We shouldn't have to use isSingleValueType here.
     if (EltTy->isSingleValueType())
       return RValue::get(EmitLoadOfScalar(Ptr, LV.isVolatileQualified(),
                                           ExprType));
@@ -1069,9 +1085,12 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(ND))
     return EmitFunctionDeclLValue(*this, E, FD);
   
+  // FIXME: the qualifier check does not seem sufficient here
   if (E->getQualifier()) {
-    // FIXME: the qualifier check does not seem sufficient here
-    return EmitPointerToDataMemberLValue(cast<FieldDecl>(ND));
+    const FieldDecl *FD = cast<FieldDecl>(ND);
+    llvm::Value *V = CGM.EmitPointerToDataMember(FD);
+
+    return LValue::MakeAddr(V, MakeQualifiers(FD->getType()));
   }
   
   assert(false && "Unhandled DeclRefExpr");
@@ -1166,8 +1185,7 @@ LValue CodeGenFunction::EmitPredefinedFunctionName(unsigned Type) {
   GlobalVarName += FnName;
 
   std::string FunctionName =
-    PredefinedExpr::ComputeName(getContext(), (PredefinedExpr::IdentType)Type,
-                                CurCodeDecl);
+    PredefinedExpr::ComputeName((PredefinedExpr::IdentType)Type, CurCodeDecl);
 
   llvm::Constant *C =
     CGM.GetAddrOfConstantCString(FunctionName, GlobalVarName.c_str());
@@ -1350,7 +1368,7 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
     llvm::Value *Vec = EmitScalarExpr(E->getBase());
     
     // Store the vector to memory (because LValue wants an address).
-    llvm::Value *VecMem =CreateTempAlloca(ConvertType(E->getBase()->getType()));
+    llvm::Value *VecMem = CreateMemTemp(E->getBase()->getType());
     Builder.CreateStore(Vec, VecMem);
     Base = LValue::MakeAddr(VecMem, Qualifiers());
   }
@@ -1381,7 +1399,6 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
 }
 
 LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
-  bool isUnion = false;
   bool isNonGC = false;
   Expr *BaseExpr = E->getBase();
   llvm::Value *BaseValue = NULL;
@@ -1392,16 +1409,12 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
     BaseValue = EmitScalarExpr(BaseExpr);
     const PointerType *PTy =
       BaseExpr->getType()->getAs<PointerType>();
-    if (PTy->getPointeeType()->isUnionType())
-      isUnion = true;
     BaseQuals = PTy->getPointeeType().getQualifiers();
   } else if (isa<ObjCPropertyRefExpr>(BaseExpr->IgnoreParens()) ||
              isa<ObjCImplicitSetterGetterRefExpr>(
                BaseExpr->IgnoreParens())) {
     RValue RV = EmitObjCPropertyGet(BaseExpr);
     BaseValue = RV.getAggregateAddr();
-    if (BaseExpr->getType()->isUnionType())
-      isUnion = true;
     BaseQuals = BaseExpr->getType().getQualifiers();
   } else {
     LValue BaseLV = EmitLValue(BaseExpr);
@@ -1410,14 +1423,12 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
     // FIXME: this isn't right for bitfields.
     BaseValue = BaseLV.getAddress();
     QualType BaseTy = BaseExpr->getType();
-    if (BaseTy->isUnionType())
-      isUnion = true;
     BaseQuals = BaseTy.getQualifiers();
   }
 
   NamedDecl *ND = E->getMemberDecl();
   if (FieldDecl *Field = dyn_cast<FieldDecl>(ND)) {
-    LValue LV = EmitLValueForField(BaseValue, Field, isUnion,
+    LValue LV = EmitLValueForField(BaseValue, Field, 
                                    BaseQuals.getCVRQualifiers());
     LValue::SetObjCNonGC(LV, isNonGC);
     setObjCGCLValueClass(getContext(), E, LV);
@@ -1461,7 +1472,6 @@ LValue CodeGenFunction::EmitLValueForBitfield(llvm::Value* BaseValue,
 
 LValue CodeGenFunction::EmitLValueForField(llvm::Value* BaseValue,
                                            const FieldDecl* Field,
-                                           bool isUnion,
                                            unsigned CVRQualifiers) {
   if (Field->isBitField())
     return EmitLValueForBitfield(BaseValue, Field, CVRQualifiers);
@@ -1470,7 +1480,7 @@ LValue CodeGenFunction::EmitLValueForField(llvm::Value* BaseValue,
   llvm::Value *V = Builder.CreateStructGEP(BaseValue, idx, "tmp");
 
   // Match union field type.
-  if (isUnion) {
+  if (Field->getParent()->isUnion()) {
     const llvm::Type *FieldTy =
       CGM.getTypes().ConvertTypeForMem(Field->getType());
     const llvm::PointerType * BaseTy =
@@ -1492,9 +1502,26 @@ LValue CodeGenFunction::EmitLValueForField(llvm::Value* BaseValue,
   return LValue::MakeAddr(V, Quals);
 }
 
+LValue 
+CodeGenFunction::EmitLValueForFieldInitialization(llvm::Value* BaseValue, 
+                                                  const FieldDecl* Field,
+                                                  unsigned CVRQualifiers) {
+  QualType FieldType = Field->getType();
+  
+  if (!FieldType->isReferenceType())
+    return EmitLValueForField(BaseValue, Field, CVRQualifiers);
+
+  unsigned idx = CGM.getTypes().getLLVMFieldNo(Field);
+  llvm::Value *V = Builder.CreateStructGEP(BaseValue, idx, "tmp");
+
+  assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
+
+  return LValue::MakeAddr(V, MakeQualifiers(FieldType));
+}
+
 LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr* E){
-  const llvm::Type *LTy = ConvertType(E->getType());
-  llvm::Value *DeclPtr = CreateTempAlloca(LTy, ".compoundliteral");
+  llvm::Value *DeclPtr = CreateTempAlloca(ConvertTypeForMem(E->getType()),
+                                          ".compoundliteral");
 
   const Expr* InitExpr = E->getInitializer();
   LValue Result = LValue::MakeAddr(DeclPtr, MakeQualifiers(E->getType()));
@@ -1527,18 +1554,25 @@ CodeGenFunction::EmitConditionalOperatorLValue(const ConditionalOperator* E) {
     
     EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock);
     
+    // Any temporaries created here are conditional.
+    BeginConditionalBranch();
     EmitBlock(LHSBlock);
-
     LValue LHS = EmitLValue(E->getLHS());
+    EndConditionalBranch();
+    
     if (!LHS.isSimple())
       return EmitUnsupportedLValue(E, "conditional operator");
 
+    // FIXME: We shouldn't need an alloca for this.
     llvm::Value *Temp = CreateTempAlloca(LHS.getAddress()->getType(),"condtmp");
     Builder.CreateStore(LHS.getAddress(), Temp);
     EmitBranch(ContBlock);
     
+    // Any temporaries created here are conditional.
+    BeginConditionalBranch();
     EmitBlock(RHSBlock);
     LValue RHS = EmitLValue(E->getRHS());
+    EndConditionalBranch();
     if (!RHS.isSimple())
       return EmitUnsupportedLValue(E, "conditional operator");
 
@@ -1556,10 +1590,7 @@ CodeGenFunction::EmitConditionalOperatorLValue(const ConditionalOperator* E) {
           !E->getType()->isAnyComplexType()) &&
          "Unexpected conditional operator!");
 
-  llvm::Value *Temp = CreateTempAlloca(ConvertType(E->getType()));
-  EmitAggExpr(E, Temp, false);
-
-  return LValue::MakeAddr(Temp, MakeQualifiers(E->getType()));
+  return EmitAggExprToLValue(E);
 }
 
 /// EmitCastLValue - Casts are never lvalues unless that cast is a dynamic_cast.
@@ -1606,12 +1637,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
     
     return LValue::MakeAddr(Base, MakeQualifiers(E->getType()));
   }
-  case CastExpr::CK_ToUnion: {
-    llvm::Value *Temp = CreateTempAlloca(ConvertType(E->getType()));
-    EmitAnyExpr(E->getSubExpr(), Temp, false);
-
-    return LValue::MakeAddr(Temp, MakeQualifiers(E->getType()));
-  }
+  case CastExpr::CK_ToUnion:
+    return EmitAggExprToLValue(E);
   case CastExpr::CK_BaseToDerived: {
     const RecordType *BaseClassTy = 
       E->getSubExpr()->getType()->getAs<RecordType>();
@@ -1646,13 +1673,9 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
 LValue CodeGenFunction::EmitNullInitializationLValue(
                                               const CXXZeroInitValueExpr *E) {
   QualType Ty = E->getType();
-  const llvm::Type *LTy = ConvertTypeForMem(Ty);
-  llvm::AllocaInst *Alloc = CreateTempAlloca(LTy);
-  unsigned Align = getContext().getTypeAlign(Ty)/8;
-  Alloc->setAlignment(Align);
-  LValue lvalue = LValue::MakeAddr(Alloc, Qualifiers());
-  EmitMemSetToZero(lvalue.getAddress(), Ty);
-  return lvalue;
+  LValue LV = LValue::MakeAddr(CreateMemTemp(Ty), MakeQualifiers(Ty));
+  EmitMemSetToZero(LV.getAddress(), Ty);
+  return LV;
 }
 
 //===--------------------------------------------------------------------===//
@@ -1725,10 +1748,7 @@ LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
     return LV;
   }
   
-  llvm::Value *Temp = CreateTempAlloca(ConvertType(E->getType()));
-  EmitAggExpr(E, Temp, false);
-  // FIXME: Are these qualifiers correct?
-  return LValue::MakeAddr(Temp, MakeQualifiers(E->getType()));
+  return EmitAggExprToLValue(E);
 }
 
 LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
@@ -1746,13 +1766,11 @@ LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
 
 LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
   // FIXME: This shouldn't require another copy.
-  llvm::Value *Temp = CreateTempAlloca(ConvertType(E->getType()));
-  EmitAggExpr(E, Temp, false);
-  return LValue::MakeAddr(Temp, MakeQualifiers(E->getType()));
+  return EmitAggExprToLValue(E);
 }
 
 LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
-  llvm::Value *Temp = CreateTempAlloca(ConvertTypeForMem(E->getType()), "tmp");
+  llvm::Value *Temp = CreateMemTemp(E->getType(), "tmp");
   EmitCXXConstructExpr(Temp, E);
   return LValue::MakeAddr(Temp, MakeQualifiers(E->getType()));
 }
@@ -1840,21 +1858,6 @@ LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
   return LValue::MakeAddr(RV.getAggregateAddr(), MakeQualifiers(E->getType()));
 }
 
-
-LValue CodeGenFunction::EmitPointerToDataMemberLValue(const FieldDecl *Field) {
-  const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(Field->getDeclContext());
-  QualType NNSpecTy = 
-    getContext().getCanonicalType(
-      getContext().getTypeDeclType(const_cast<CXXRecordDecl*>(ClassDecl)));
-  NNSpecTy = getContext().getPointerType(NNSpecTy);
-  llvm::Value *V = llvm::Constant::getNullValue(ConvertType(NNSpecTy));
-  LValue MemExpLV = EmitLValueForField(V, Field, /*isUnion=*/false, 
-                                       /*Qualifiers=*/0);
-  const llvm::Type *ResultType = ConvertType(getContext().getPointerDiffType());
-  V = Builder.CreatePtrToInt(MemExpLV.getAddress(), ResultType, "datamember");
-  return LValue::MakeAddr(V, MakeQualifiers(Field->getType()));
-}
-
 RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
                                  ReturnValueSlot ReturnValue,
                                  CallExpr::const_arg_iterator ArgBeg,
@@ -1867,20 +1870,14 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
 
   CalleeType = getContext().getCanonicalType(CalleeType);
 
-  QualType FnType = cast<PointerType>(CalleeType)->getPointeeType();
-  QualType ResultType = cast<FunctionType>(FnType)->getResultType();
+  const FunctionType *FnType
+    = cast<FunctionType>(cast<PointerType>(CalleeType)->getPointeeType());
+  QualType ResultType = FnType->getResultType();
 
   CallArgList Args;
   EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), ArgBeg, ArgEnd);
 
-  // FIXME: We should not need to do this, it should be part of the function
-  // type.
-  unsigned CallingConvention = 0;
-  if (const llvm::Function *F =
-      dyn_cast<llvm::Function>(Callee->stripPointerCasts()))
-    CallingConvention = F->getCallingConv();
-  return EmitCall(CGM.getTypes().getFunctionInfo(ResultType, Args,
-                                                 CallingConvention),
+  return EmitCall(CGM.getTypes().getFunctionInfo(Args, FnType),
                   Callee, ReturnValue, Args, TargetDecl);
 }
 
diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp
index c852d65..97455c7 100644
--- a/lib/CodeGen/CGExprAgg.cpp
+++ b/lib/CodeGen/CGExprAgg.cpp
@@ -118,7 +118,7 @@ public:
 
   void VisitVAArgExpr(VAArgExpr *E);
 
-  void EmitInitializationToLValue(Expr *E, LValue Address);
+  void EmitInitializationToLValue(Expr *E, LValue Address, QualType T);
   void EmitNullInitializationToLValue(LValue Address, QualType T);
   //  case Expr::ChooseExprClass:
   void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
@@ -147,7 +147,7 @@ void AggExprEmitter::EmitFinalDestCopy(const Expr *E, RValue Src, bool Ignore) {
       return;
     // If the source is volatile, we must read from it; to do that, we need
     // some place to put it.
-    DestPtr = CGF.CreateTempAlloca(CGF.ConvertType(E->getType()), "agg.tmp");
+    DestPtr = CGF.CreateMemTemp(E->getType(), "agg.tmp");
   }
 
   if (RequiresGCollection) {
@@ -188,7 +188,8 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
     llvm::Value *CastPtr = Builder.CreateBitCast(DestPtr,
                                                  CGF.ConvertType(PtrTy));
     EmitInitializationToLValue(E->getSubExpr(),
-                               LValue::MakeAddr(CastPtr, Qualifiers()));
+                               LValue::MakeAddr(CastPtr, Qualifiers()), 
+                               E->getType());
     break;
   }
 
@@ -227,8 +228,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
   case CastExpr::CK_BaseToDerivedMemberPointer: {
     QualType SrcType = E->getSubExpr()->getType();
     
-    llvm::Value *Src = CGF.CreateTempAlloca(CGF.ConvertTypeForMem(SrcType), 
-                                            "tmp");
+    llvm::Value *Src = CGF.CreateMemTemp(SrcType, "tmp");
     CGF.EmitAggExpr(E->getSubExpr(), Src, SrcType.isVolatileQualified());
     
     llvm::Value *SrcPtr = Builder.CreateStructGEP(Src, 0, "src.ptr");
@@ -252,8 +252,8 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
     if (E->getCastKind() == CastExpr::CK_DerivedToBaseMemberPointer)
       std::swap(DerivedDecl, BaseDecl);
 
-    llvm::Constant *Adj = CGF.CGM.GetCXXBaseClassOffset(DerivedDecl, BaseDecl);
-    if (Adj) {
+    if (llvm::Constant *Adj = 
+          CGF.CGM.GetNonVirtualBaseClassOffset(DerivedDecl, BaseDecl)) {
       if (E->getCastKind() == CastExpr::CK_DerivedToBaseMemberPointer)
         SrcAdj = Builder.CreateSub(SrcAdj, Adj, "adj");
       else
@@ -326,10 +326,18 @@ void AggExprEmitter::VisitUnaryAddrOf(const UnaryOperator *E) {
     int64_t Index = 
       CGF.CGM.getVtableInfo().getMethodVtableIndex(MD);
     
-    FuncPtr = llvm::ConstantInt::get(PtrDiffTy, Index + 1);
+    // Itanium C++ ABI 2.3:
+    //   For a non-virtual function, this field is a simple function pointer. 
+    //   For a virtual function, it is 1 plus the virtual table offset 
+    //   (in bytes) of the function, represented as a ptrdiff_t. 
+    FuncPtr = llvm::ConstantInt::get(PtrDiffTy, (Index * 8) + 1);
   } else {
-    FuncPtr = llvm::ConstantExpr::getPtrToInt(CGF.CGM.GetAddrOfFunction(MD), 
-                                              PtrDiffTy);
+    const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
+    const llvm::Type *Ty =
+      CGF.CGM.getTypes().GetFunctionType(CGF.CGM.getTypes().getFunctionInfo(MD),
+                                         FPT->isVariadic());
+    llvm::Constant *Fn = CGF.CGM.GetAddrOfFunction(MD, Ty);
+    FuncPtr = llvm::ConstantExpr::getPtrToInt(Fn, PtrDiffTy);
   }
   Builder.CreateStore(FuncPtr, DstPtr, VolatileDest);
 
@@ -371,14 +379,14 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
   if (LHS.isPropertyRef()) {
     llvm::Value *AggLoc = DestPtr;
     if (!AggLoc)
-      AggLoc = CGF.CreateTempAlloca(CGF.ConvertType(E->getRHS()->getType()));
+      AggLoc = CGF.CreateMemTemp(E->getRHS()->getType());
     CGF.EmitAggExpr(E->getRHS(), AggLoc, VolatileDest);
     CGF.EmitObjCPropertySet(LHS.getPropertyRefExpr(),
                             RValue::getAggregate(AggLoc, VolatileDest));
   } else if (LHS.isKVCRef()) {
     llvm::Value *AggLoc = DestPtr;
     if (!AggLoc)
-      AggLoc = CGF.CreateTempAlloca(CGF.ConvertType(E->getRHS()->getType()));
+      AggLoc = CGF.CreateMemTemp(E->getRHS()->getType());
     CGF.EmitAggExpr(E->getRHS(), AggLoc, VolatileDest);
     CGF.EmitObjCPropertySet(LHS.getKVCRefExpr(),
                             RValue::getAggregate(AggLoc, VolatileDest));
@@ -408,21 +416,21 @@ void AggExprEmitter::VisitConditionalOperator(const ConditionalOperator *E) {
 
   CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock);
 
-  CGF.StartConditionalBranch();
+  CGF.BeginConditionalBranch();
   CGF.EmitBlock(LHSBlock);
 
   // Handle the GNU extension for missing LHS.
   assert(E->getLHS() && "Must have LHS for aggregate value");
 
   Visit(E->getLHS());
-  CGF.FinishConditionalBranch();
+  CGF.EndConditionalBranch();
   CGF.EmitBranch(ContBlock);
 
-  CGF.StartConditionalBranch();
+  CGF.BeginConditionalBranch();
   CGF.EmitBlock(RHSBlock);
 
   Visit(E->getRHS());
-  CGF.FinishConditionalBranch();
+  CGF.EndConditionalBranch();
   CGF.EmitBranch(ContBlock);
 
   CGF.EmitBlock(ContBlock);
@@ -449,7 +457,7 @@ void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
 
   if (!Val) {
     // Create a temporary variable.
-    Val = CGF.CreateTempAlloca(CGF.ConvertTypeForMem(E->getType()), "tmp");
+    Val = CGF.CreateMemTemp(E->getType(), "tmp");
 
     // FIXME: volatile
     CGF.EmitAggExpr(E->getSubExpr(), Val, false);
@@ -467,7 +475,7 @@ AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
 
   if (!Val) {
     // Create a temporary variable.
-    Val = CGF.CreateTempAlloca(CGF.ConvertTypeForMem(E->getType()), "tmp");
+    Val = CGF.CreateMemTemp(E->getType(), "tmp");
   }
 
   if (E->requiresZeroInitialization())
@@ -484,7 +492,7 @@ void AggExprEmitter::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
 
   if (!Val) {
     // Create a temporary variable.
-    Val = CGF.CreateTempAlloca(CGF.ConvertTypeForMem(E->getType()), "tmp");
+    Val = CGF.CreateMemTemp(E->getType(), "tmp");
   }
   CGF.EmitCXXExprWithTemporaries(E, Val, VolatileDest, IsInitializer);
 }
@@ -494,7 +502,7 @@ void AggExprEmitter::VisitCXXZeroInitValueExpr(CXXZeroInitValueExpr *E) {
 
   if (!Val) {
     // Create a temporary variable.
-    Val = CGF.CreateTempAlloca(CGF.ConvertTypeForMem(E->getType()), "tmp");
+    Val = CGF.CreateMemTemp(E->getType(), "tmp");
   }
   LValue LV = LValue::MakeAddr(Val, Qualifiers());
   EmitNullInitializationToLValue(LV, E->getType());
@@ -505,23 +513,27 @@ void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
 
   if (!Val) {
     // Create a temporary variable.
-    Val = CGF.CreateTempAlloca(CGF.ConvertTypeForMem(E->getType()), "tmp");
+    Val = CGF.CreateMemTemp(E->getType(), "tmp");
   }
   LValue LV = LValue::MakeAddr(Val, Qualifiers());
   EmitNullInitializationToLValue(LV, E->getType());
 }
 
-void AggExprEmitter::EmitInitializationToLValue(Expr* E, LValue LV) {
+void 
+AggExprEmitter::EmitInitializationToLValue(Expr* E, LValue LV, QualType T) {
   // FIXME: Ignore result?
   // FIXME: Are initializers affected by volatile?
   if (isa<ImplicitValueInitExpr>(E)) {
-    EmitNullInitializationToLValue(LV, E->getType());
-  } else if (E->getType()->isComplexType()) {
+    EmitNullInitializationToLValue(LV, T);
+  } else if (T->isReferenceType()) {
+    RValue RV = CGF.EmitReferenceBindingToExpr(E, /*IsInitializer=*/false);
+    CGF.EmitStoreThroughLValue(RV, LV, T);
+  } else if (T->isAnyComplexType()) {
     CGF.EmitComplexExprIntoAddr(E, LV.getAddress(), false);
-  } else if (CGF.hasAggregateLLVMType(E->getType())) {
+  } else if (CGF.hasAggregateLLVMType(T)) {
     CGF.EmitAnyExpr(E, LV.getAddress(), false);
   } else {
-    CGF.EmitStoreThroughLValue(CGF.EmitAnyExpr(E), LV, E->getType());
+    CGF.EmitStoreThroughLValue(CGF.EmitAnyExpr(E), LV, T);
   }
 }
 
@@ -590,7 +602,8 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
       llvm::Value *NextVal = Builder.CreateStructGEP(DestPtr, i, ".array");
       if (i < NumInitElements)
         EmitInitializationToLValue(E->getInit(i),
-                                   LValue::MakeAddr(NextVal, Quals));
+                                   LValue::MakeAddr(NextVal, Quals), 
+                                   ElementType);
       else
         EmitNullInitializationToLValue(LValue::MakeAddr(NextVal, Quals),
                                        ElementType);
@@ -627,11 +640,11 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
 
     // FIXME: volatility
     FieldDecl *Field = E->getInitializedFieldInUnion();
-    LValue FieldLoc = CGF.EmitLValueForField(DestPtr, Field, true, 0);
+    LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestPtr, Field, 0);
 
     if (NumInitElements) {
       // Store the initializer into the field
-      EmitInitializationToLValue(E->getInit(0), FieldLoc);
+      EmitInitializationToLValue(E->getInit(0), FieldLoc, Field->getType());
     } else {
       // Default-initialize to null
       EmitNullInitializationToLValue(FieldLoc, Field->getType());
@@ -653,12 +666,13 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
       continue;
 
     // FIXME: volatility
-    LValue FieldLoc = CGF.EmitLValueForField(DestPtr, *Field, false, 0);
+    LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestPtr, *Field, 0);
     // We never generate write-barries for initialized fields.
     LValue::SetObjCNonGC(FieldLoc, true);
     if (CurInitVal < NumInitElements) {
       // Store the initializer into the field
-      EmitInitializationToLValue(E->getInit(CurInitVal++), FieldLoc);
+      EmitInitializationToLValue(E->getInit(CurInitVal++), FieldLoc, 
+                                 Field->getType());
     } else {
       // We're out of initalizers; default-initialize to null
       EmitNullInitializationToLValue(FieldLoc, Field->getType());
@@ -674,6 +688,8 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
 /// type.  The result is computed into DestPtr.  Note that if DestPtr is null,
 /// the value of the aggregate expression is not needed.  If VolatileDest is
 /// true, DestPtr cannot be 0.
+//
+// FIXME: Take Qualifiers object.
 void CodeGenFunction::EmitAggExpr(const Expr *E, llvm::Value *DestPtr,
                                   bool VolatileDest, bool IgnoreResult,
                                   bool IsInitializer,
@@ -688,6 +704,14 @@ void CodeGenFunction::EmitAggExpr(const Expr *E, llvm::Value *DestPtr,
     .Visit(const_cast<Expr*>(E));
 }
 
+LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
+  assert(hasAggregateLLVMType(E->getType()) && "Invalid argument!");
+  Qualifiers Q = MakeQualifiers(E->getType());
+  llvm::Value *Temp = CreateMemTemp(E->getType());
+  EmitAggExpr(E, Temp, Q.hasVolatile());
+  return LValue::MakeAddr(Temp, Q);
+}
+
 void CodeGenFunction::EmitAggregateClear(llvm::Value *DestPtr, QualType Ty) {
   assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
 
diff --git a/lib/CodeGen/CGExprCXX.cpp b/lib/CodeGen/CGExprCXX.cpp
index e264109..0328621 100644
--- a/lib/CodeGen/CGExprCXX.cpp
+++ b/lib/CodeGen/CGExprCXX.cpp
@@ -42,8 +42,10 @@ RValue CodeGenFunction::EmitCXXMemberCall(const CXXMethodDecl *MD,
   // And the rest of the call args
   EmitCallArgs(Args, FPT, ArgBeg, ArgEnd);
 
-  QualType ResultType = MD->getType()->getAs<FunctionType>()->getResultType();
-  return EmitCall(CGM.getTypes().getFunctionInfo(ResultType, Args), Callee, 
+  QualType ResultType = FPT->getResultType();
+  return EmitCall(CGM.getTypes().getFunctionInfo(ResultType, Args,
+                                                 FPT->getCallConv(),
+                                                 FPT->getNoReturnAttr()), Callee, 
                   ReturnValue, Args, MD);
 }
 
@@ -60,7 +62,7 @@ static bool canDevirtualizeMemberFunctionCalls(const Expr *Base) {
   }
   
   // We can always devirtualize calls on temporary object expressions.
-  if (isa<CXXTemporaryObjectExpr>(Base))
+  if (isa<CXXConstructExpr>(Base))
     return true;
   
   // And calls on bound temporaries.
@@ -159,12 +161,10 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
     CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(RD, FPT),
                                    FPT->isVariadic());
 
-  const llvm::Type *Int8PtrTy = 
-    llvm::Type::getInt8Ty(VMContext)->getPointerTo();
+  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
 
   // Get the member function pointer.
-  llvm::Value *MemFnPtr = 
-    CreateTempAlloca(ConvertType(MemFnExpr->getType()), "mem.fn");
+  llvm::Value *MemFnPtr = CreateMemTemp(MemFnExpr->getType(), "mem.fn");
   EmitAggExpr(MemFnExpr, MemFnPtr, /*VolatileDest=*/false);
 
   // Emit the 'this' pointer.
@@ -206,19 +206,20 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
   Builder.CreateCondBr(IsVirtual, FnVirtual, FnNonVirtual);
   EmitBlock(FnVirtual);
   
-  const llvm::Type *VTableTy = 
-    FTy->getPointerTo()->getPointerTo()->getPointerTo();
+  const llvm::Type *VtableTy = 
+    FTy->getPointerTo()->getPointerTo();
 
-  llvm::Value *VTable = Builder.CreateBitCast(This, VTableTy);
-  VTable = Builder.CreateLoad(VTable);
+  llvm::Value *Vtable = Builder.CreateBitCast(This, VtableTy->getPointerTo());
+  Vtable = Builder.CreateLoad(Vtable);
   
-  VTable = Builder.CreateGEP(VTable, FnAsInt, "fn");
+  Vtable = Builder.CreateBitCast(Vtable, Int8PtrTy);
+  llvm::Value *VtableOffset = 
+    Builder.CreateSub(FnAsInt, llvm::ConstantInt::get(PtrDiffTy, 1));
   
-  // Since the function pointer is 1 plus the virtual table offset, we
-  // subtract 1 by using a GEP.
-  VTable = Builder.CreateConstGEP1_64(VTable, (uint64_t)-1);
+  Vtable = Builder.CreateGEP(Vtable, VtableOffset, "fn");
+  Vtable = Builder.CreateBitCast(Vtable, VtableTy);
   
-  llvm::Value *VirtualFn = Builder.CreateLoad(VTable, "virtualfn");
+  llvm::Value *VirtualFn = Builder.CreateLoad(Vtable, "virtualfn");
   
   EmitBranch(FnEnd);
   EmitBlock(FnNonVirtual);
@@ -244,8 +245,8 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
   
   // And the rest of the call args
   EmitCallArgs(Args, FPT, E->arg_begin(), E->arg_end());
-  QualType ResultType = BO->getType()->getAs<FunctionType>()->getResultType();
-  return EmitCall(CGM.getTypes().getFunctionInfo(ResultType, Args), Callee, 
+  const FunctionType *BO_FPT = BO->getType()->getAs<FunctionProtoType>();
+  return EmitCall(CGM.getTypes().getFunctionInfo(Args, BO_FPT), Callee, 
                   ReturnValue, Args);
 }
 
@@ -339,18 +340,20 @@ CodeGenFunction::EmitCXXConstructExpr(llvm::Value *Dest,
   }
   else
     // Call the constructor.
-    EmitCXXConstructorCall(CD, Ctor_Complete, Dest,
+    EmitCXXConstructorCall(CD, 
+                           E->isBaseInitialization()? Ctor_Base : Ctor_Complete, 
+                           Dest,
                            E->arg_begin(), E->arg_end());
 }
 
-static uint64_t CalculateCookiePadding(ASTContext &Ctx, QualType ElementType) {
+static CharUnits CalculateCookiePadding(ASTContext &Ctx, QualType ElementType) {
   const RecordType *RT = ElementType->getAs<RecordType>();
   if (!RT)
-    return 0;
+    return CharUnits::Zero();
   
   const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
   if (!RD)
-    return 0;
+    return CharUnits::Zero();
   
   // Check if the class has a trivial destructor.
   if (RD->hasTrivialDestructor()) {
@@ -372,25 +375,25 @@ static uint64_t CalculateCookiePadding(ASTContext &Ctx, QualType ElementType) {
     
     // No usual deallocation function, we don't need a cookie.
     if (!UsualDeallocationFunction)
-      return 0;
+      return CharUnits::Zero();
     
     // The usual deallocation function doesn't take a size_t argument, so we
     // don't need a cookie.
     if (UsualDeallocationFunction->getNumParams() == 1)
-      return 0;
+      return CharUnits::Zero();
         
     assert(UsualDeallocationFunction->getNumParams() == 2 && 
            "Unexpected deallocation function type!");
   }  
   
   // Padding is the maximum of sizeof(size_t) and alignof(ElementType)
-  return std::max(Ctx.getTypeSize(Ctx.getSizeType()),
-                  static_cast<uint64_t>(Ctx.getTypeAlign(ElementType))) / 8;
+  return std::max(Ctx.getTypeSizeInChars(Ctx.getSizeType()),
+                  Ctx.getTypeAlignInChars(ElementType));
 }
 
-static uint64_t CalculateCookiePadding(ASTContext &Ctx, const CXXNewExpr *E) {
+static CharUnits CalculateCookiePadding(ASTContext &Ctx, const CXXNewExpr *E) {
   if (!E->isArray())
-    return 0;
+    return CharUnits::Zero();
 
   // No cookie is required if the new operator being used is 
   // ::operator new[](size_t, void*).
@@ -401,7 +404,7 @@ static uint64_t CalculateCookiePadding(ASTContext &Ctx, const CXXNewExpr *E) {
         Ctx.getCanonicalType(OperatorNew->getParamDecl(1)->getType());
       
       if (ParamType == Ctx.VoidPtrTy)
-        return 0;
+        return CharUnits::Zero();
     }
   }
       
@@ -412,25 +415,25 @@ static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF,
                                         const CXXNewExpr *E,
                                         llvm::Value *& NumElements) {
   QualType Type = E->getAllocatedType();
-  uint64_t TypeSizeInBytes = CGF.getContext().getTypeSize(Type) / 8;
+  CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(Type);
   const llvm::Type *SizeTy = CGF.ConvertType(CGF.getContext().getSizeType());
   
   if (!E->isArray())
-    return llvm::ConstantInt::get(SizeTy, TypeSizeInBytes);
+    return llvm::ConstantInt::get(SizeTy, TypeSize.getQuantity());
 
-  uint64_t CookiePadding = CalculateCookiePadding(CGF.getContext(), E);
+  CharUnits CookiePadding = CalculateCookiePadding(CGF.getContext(), E);
   
   Expr::EvalResult Result;
   if (E->getArraySize()->Evaluate(Result, CGF.getContext()) &&
       !Result.HasSideEffects && Result.Val.isInt()) {
 
-    uint64_t AllocSize = 
-      Result.Val.getInt().getZExtValue() * TypeSizeInBytes + CookiePadding;
+    CharUnits AllocSize = 
+      Result.Val.getInt().getZExtValue() * TypeSize + CookiePadding;
     
     NumElements = 
       llvm::ConstantInt::get(SizeTy, Result.Val.getInt().getZExtValue());
     
-    return llvm::ConstantInt::get(SizeTy, AllocSize);
+    return llvm::ConstantInt::get(SizeTy, AllocSize.getQuantity());
   }
   
   // Emit the array size expression.
@@ -439,11 +442,13 @@ static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF,
   // Multiply with the type size.
   llvm::Value *V = 
     CGF.Builder.CreateMul(NumElements, 
-                          llvm::ConstantInt::get(SizeTy, TypeSizeInBytes));
+                          llvm::ConstantInt::get(SizeTy, 
+                                                 TypeSize.getQuantity()));
 
   // And add the cookie padding if necessary.
-  if (CookiePadding)
-    V = CGF.Builder.CreateAdd(V, llvm::ConstantInt::get(SizeTy, CookiePadding));
+  if (!CookiePadding.isZero())
+    V = CGF.Builder.CreateAdd(V, 
+        llvm::ConstantInt::get(SizeTy, CookiePadding.getQuantity()));
   
   return V;
 }
@@ -538,7 +543,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
 
   // Emit the call to new.
   RValue RV =
-    EmitCall(CGM.getTypes().getFunctionInfo(NewFTy->getResultType(), NewArgs),
+    EmitCall(CGM.getTypes().getFunctionInfo(NewArgs, NewFTy),
              CGM.GetAddrOfFunction(NewFD), ReturnValueSlot(), NewArgs, NewFD);
 
   // If an allocation function is declared with an empty exception specification
@@ -567,20 +572,22 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
     Builder.CreateCondBr(IsNull, NewNull, NewNotNull);
     EmitBlock(NewNotNull);
   }
-
-  if (uint64_t CookiePadding = CalculateCookiePadding(getContext(), E)) {
-    uint64_t CookieOffset = 
-      CookiePadding - getContext().getTypeSize(SizeTy) / 8;
+  
+  CharUnits CookiePadding = CalculateCookiePadding(getContext(), E);
+  if (!CookiePadding.isZero()) {
+    CharUnits CookieOffset = 
+      CookiePadding - getContext().getTypeSizeInChars(SizeTy);
     
     llvm::Value *NumElementsPtr = 
-      Builder.CreateConstInBoundsGEP1_64(NewPtr, CookieOffset);
+      Builder.CreateConstInBoundsGEP1_64(NewPtr, CookieOffset.getQuantity());
     
     NumElementsPtr = Builder.CreateBitCast(NumElementsPtr, 
                                            ConvertType(SizeTy)->getPointerTo());
     Builder.CreateStore(NumElements, NumElementsPtr);
 
     // Now add the padding to the new ptr.
-    NewPtr = Builder.CreateConstInBoundsGEP1_64(NewPtr, CookiePadding);
+    NewPtr = Builder.CreateConstInBoundsGEP1_64(NewPtr, 
+                                                CookiePadding.getQuantity());
   }
   
   NewPtr = Builder.CreateBitCast(NewPtr, ConvertType(E->getType()));
@@ -611,23 +618,24 @@ GetAllocatedObjectPtrAndNumElements(CodeGenFunction &CGF,
   QualType SizeTy = CGF.getContext().getSizeType();
   const llvm::Type *SizeLTy = CGF.ConvertType(SizeTy);
   
-  uint64_t DeleteTypeAlign = CGF.getContext().getTypeAlign(DeleteTy);
-  uint64_t CookiePadding = std::max(CGF.getContext().getTypeSize(SizeTy),
-                                    DeleteTypeAlign) / 8;
-  assert(CookiePadding && "CookiePadding should not be 0.");
+  CharUnits DeleteTypeAlign = CGF.getContext().getTypeAlignInChars(DeleteTy);
+  CharUnits CookiePadding = 
+    std::max(CGF.getContext().getTypeSizeInChars(SizeTy),
+             DeleteTypeAlign);
+  assert(!CookiePadding.isZero() && "CookiePadding should not be 0.");
 
   const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
-  uint64_t CookieOffset = 
-    CookiePadding - CGF.getContext().getTypeSize(SizeTy) / 8;
+  CharUnits CookieOffset = 
+    CookiePadding - CGF.getContext().getTypeSizeInChars(SizeTy);
 
   llvm::Value *AllocatedObjectPtr = CGF.Builder.CreateBitCast(Ptr, Int8PtrTy);
   AllocatedObjectPtr = 
     CGF.Builder.CreateConstInBoundsGEP1_64(AllocatedObjectPtr,
-                                           -CookiePadding);
+                                           -CookiePadding.getQuantity());
 
   llvm::Value *NumElementsPtr =
     CGF.Builder.CreateConstInBoundsGEP1_64(AllocatedObjectPtr, 
-                                           CookieOffset);
+                                           CookieOffset.getQuantity());
   NumElementsPtr = 
     CGF.Builder.CreateBitCast(NumElementsPtr, SizeLTy->getPointerTo());
   
@@ -651,13 +659,13 @@ void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
   QualType SizeTy;
   if (DeleteFTy->getNumArgs() == 2) {
     SizeTy = DeleteFTy->getArgType(1);
-    uint64_t DeleteTypeSize = getContext().getTypeSize(DeleteTy) / 8;
-    Size = llvm::ConstantInt::get(ConvertType(SizeTy), DeleteTypeSize);
+    CharUnits DeleteTypeSize = getContext().getTypeSizeInChars(DeleteTy);
+    Size = llvm::ConstantInt::get(ConvertType(SizeTy), 
+                                  DeleteTypeSize.getQuantity());
   }
   
   if (DeleteFD->getOverloadedOperator() == OO_Array_Delete &&
-      
-      CalculateCookiePadding(getContext(), DeleteTy)) {
+      !CalculateCookiePadding(getContext(), DeleteTy).isZero()) {
     // We need to get the number of elements in the array from the cookie.
     llvm::Value *AllocatedObjectPtr;
     llvm::Value *NumElements;
@@ -679,8 +687,7 @@ void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
     DeleteArgs.push_back(std::make_pair(RValue::get(Size), SizeTy));
 
   // Emit the call to delete.
-  EmitCall(CGM.getTypes().getFunctionInfo(DeleteFTy->getResultType(),
-                                          DeleteArgs),
+  EmitCall(CGM.getTypes().getFunctionInfo(DeleteArgs, DeleteFTy),
            CGM.GetAddrOfFunction(DeleteFD), ReturnValueSlot(), 
            DeleteArgs, DeleteFD);
 }
diff --git a/lib/CodeGen/CGExprComplex.cpp b/lib/CodeGen/CGExprComplex.cpp
index 5ec336c..5915340 100644
--- a/lib/CodeGen/CGExprComplex.cpp
+++ b/lib/CodeGen/CGExprComplex.cpp
@@ -366,7 +366,7 @@ ComplexPairTy ComplexExprEmitter::VisitUnaryMinus(const UnaryOperator *E) {
   ComplexPairTy Op = Visit(E->getSubExpr());
 
   llvm::Value *ResR, *ResI;
-  if (Op.first->getType()->isFloatingPoint()) {
+  if (Op.first->getType()->isFloatingPointTy()) {
     ResR = Builder.CreateFNeg(Op.first,  "neg.r");
     ResI = Builder.CreateFNeg(Op.second, "neg.i");
   } else {
@@ -384,7 +384,7 @@ ComplexPairTy ComplexExprEmitter::VisitUnaryNot(const UnaryOperator *E) {
   // ~(a+ib) = a + i*-b
   ComplexPairTy Op = Visit(E->getSubExpr());
   llvm::Value *ResI;
-  if (Op.second->getType()->isFloatingPoint())
+  if (Op.second->getType()->isFloatingPointTy())
     ResI = Builder.CreateFNeg(Op.second, "conj.i");
   else
     ResI = Builder.CreateNeg(Op.second, "conj.i");
@@ -395,7 +395,7 @@ ComplexPairTy ComplexExprEmitter::VisitUnaryNot(const UnaryOperator *E) {
 ComplexPairTy ComplexExprEmitter::EmitBinAdd(const BinOpInfo &Op) {
   llvm::Value *ResR, *ResI;
 
-  if (Op.LHS.first->getType()->isFloatingPoint()) {
+  if (Op.LHS.first->getType()->isFloatingPointTy()) {
     ResR = Builder.CreateFAdd(Op.LHS.first,  Op.RHS.first,  "add.r");
     ResI = Builder.CreateFAdd(Op.LHS.second, Op.RHS.second, "add.i");
   } else {
@@ -407,7 +407,7 @@ ComplexPairTy ComplexExprEmitter::EmitBinAdd(const BinOpInfo &Op) {
 
 ComplexPairTy ComplexExprEmitter::EmitBinSub(const BinOpInfo &Op) {
   llvm::Value *ResR, *ResI;
-  if (Op.LHS.first->getType()->isFloatingPoint()) {
+  if (Op.LHS.first->getType()->isFloatingPointTy()) {
     ResR = Builder.CreateFSub(Op.LHS.first,  Op.RHS.first,  "sub.r");
     ResI = Builder.CreateFSub(Op.LHS.second, Op.RHS.second, "sub.i");
   } else {
@@ -422,7 +422,7 @@ ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) {
   using llvm::Value;
   Value *ResR, *ResI;
 
-  if (Op.LHS.first->getType()->isFloatingPoint()) {
+  if (Op.LHS.first->getType()->isFloatingPointTy()) {
     Value *ResRl = Builder.CreateFMul(Op.LHS.first, Op.RHS.first, "mul.rl");
     Value *ResRr = Builder.CreateFMul(Op.LHS.second, Op.RHS.second,"mul.rr");
     ResR  = Builder.CreateFSub(ResRl, ResRr, "mul.r");
@@ -448,7 +448,7 @@ ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) {
 
 
   llvm::Value *DSTr, *DSTi;
-  if (Op.LHS.first->getType()->isFloatingPoint()) {
+  if (Op.LHS.first->getType()->isFloatingPointTy()) {
     // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd))
     llvm::Value *Tmp1 = Builder.CreateFMul(LHSr, RHSr, "tmp"); // a*c
     llvm::Value *Tmp2 = Builder.CreateFMul(LHSi, RHSi, "tmp"); // b*d
diff --git a/lib/CodeGen/CGExprConstant.cpp b/lib/CodeGen/CGExprConstant.cpp
index 7d5b3da..3df552d 100644
--- a/lib/CodeGen/CGExprConstant.cpp
+++ b/lib/CodeGen/CGExprConstant.cpp
@@ -418,12 +418,19 @@ public:
     // Get the function pointer (or index if this is a virtual function).
     if (MD->isVirtual()) {
       uint64_t Index = CGM.getVtableInfo().getMethodVtableIndex(MD);
-      
-      // The pointer is 1 + the virtual table offset in bytes.
+
+      // Itanium C++ ABI 2.3:
+      //   For a non-virtual function, this field is a simple function pointer. 
+      //   For a virtual function, it is 1 plus the virtual table offset 
+      //   (in bytes) of the function, represented as a ptrdiff_t. 
       Values[0] = llvm::ConstantInt::get(PtrDiffTy, (Index * 8) + 1);
     } else {
-      llvm::Constant *FuncPtr = CGM.GetAddrOfFunction(MD);
+      const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
+      const llvm::Type *Ty =
+        CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(MD),
+                                       FPT->isVariadic());
 
+      llvm::Constant *FuncPtr = CGM.GetAddrOfFunction(MD, Ty);
       Values[0] = llvm::ConstantExpr::getPtrToInt(FuncPtr, PtrDiffTy);
     } 
     
@@ -438,16 +445,16 @@ public:
     if (const MemberPointerType *MPT = 
         E->getType()->getAs<MemberPointerType>()) {
       QualType T = MPT->getPointeeType();
-      if (T->isFunctionProtoType()) {
-        DeclRefExpr *DRE = cast<DeclRefExpr>(E->getSubExpr());
-        
-        return EmitMemberFunctionPointer(cast<CXXMethodDecl>(DRE->getDecl()));
-      }
+      DeclRefExpr *DRE = cast<DeclRefExpr>(E->getSubExpr());
+
+      NamedDecl *ND = DRE->getDecl();
+      if (T->isFunctionProtoType())
+        return EmitMemberFunctionPointer(cast<CXXMethodDecl>(ND));
       
-      // FIXME: Should we handle other member pointer types here too,
-      // or should they be handled by Expr::Evaluate?
+      // We have a pointer to data member.
+      return CGM.EmitPointerToDataMember(cast<FieldDecl>(ND));
     }
-    
+
     return 0;
   }
     
@@ -534,8 +541,8 @@ public:
         llvm::ConstantStruct *CS = cast<llvm::ConstantStruct>(C);
         
         // Check if we need to update the adjustment.
-        if (llvm::Constant *Offset = CGM.GetCXXBaseClassOffset(DerivedClass,
-                                                               BaseClass)) {
+        if (llvm::Constant *Offset = 
+              CGM.GetNonVirtualBaseClassOffset(DerivedClass, BaseClass)) {
           llvm::Constant *Values[2];
         
           Values[0] = CS->getOperand(0);
@@ -668,6 +675,40 @@ public:
     return 0;
   }
 
+  llvm::Constant *VisitCXXConstructExpr(CXXConstructExpr *E) {
+    if (!E->getConstructor()->isTrivial())
+      return 0;
+
+    QualType Ty = E->getType();
+
+    // FIXME: We should not have to call getBaseElementType here.
+    const RecordType *RT = 
+      CGM.getContext().getBaseElementType(Ty)->getAs<RecordType>();
+    const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+    
+    // If the class doesn't have a trivial destructor, we can't emit it as a
+    // constant expr.
+    if (!RD->hasTrivialDestructor())
+      return 0;
+    
+    // Only copy and default constructors can be trivial.
+
+
+    if (E->getNumArgs()) {
+      assert(E->getNumArgs() == 1 && "trivial ctor with > 1 argument");
+      assert(E->getConstructor()->isCopyConstructor() &&
+             "trivial ctor has argument but isn't a copy ctor");
+
+      Expr *Arg = E->getArg(0);
+      assert(CGM.getContext().hasSameUnqualifiedType(Ty, Arg->getType()) &&
+             "argument to copy ctor is of wrong type");
+
+      return Visit(Arg);
+    }
+
+    return CGM.EmitNullConstant(Ty);
+  }
+
   llvm::Constant *VisitStringLiteral(StringLiteral *E) {
     assert(!E->getType()->isPointerType() && "Strings are always arrays");
 
@@ -911,7 +952,24 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
   return C;
 }
 
-static inline bool isDataMemberPointerType(QualType T) {
+static bool containsPointerToDataMember(CodeGenTypes &Types, QualType T) {
+  // No need to check for member pointers when not compiling C++.
+  if (!Types.getContext().getLangOptions().CPlusPlus)
+    return false;
+  
+  T = Types.getContext().getBaseElementType(T);
+  
+  if (const RecordType *RT = T->getAs<RecordType>()) {
+    const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+    
+    // FIXME: It would be better if there was a way to explicitly compute the
+    // record layout instead of converting to a type.
+    Types.ConvertTagDeclType(RD);
+    
+    const CGRecordLayout &Layout = Types.getCGRecordLayout(RD);
+    return Layout.containsPointerToDataMember();
+  }
+    
   if (const MemberPointerType *MPT = T->getAs<MemberPointerType>())
     return !MPT->getPointeeType()->isFunctionType();
   
@@ -919,43 +977,80 @@ static inline bool isDataMemberPointerType(QualType T) {
 }
 
 llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
-  // No need to check for member pointers when not compiling C++.
-  if (!getContext().getLangOptions().CPlusPlus)
+  if (!containsPointerToDataMember(getTypes(), T))
     return llvm::Constant::getNullValue(getTypes().ConvertTypeForMem(T));
-
+    
   if (const ConstantArrayType *CAT = Context.getAsConstantArrayType(T)) {
 
     QualType ElementTy = CAT->getElementType();
 
-    // FIXME: Handle arrays of structs that contain member pointers.
-    if (isDataMemberPointerType(Context.getBaseElementType(ElementTy))) {
-      llvm::Constant *Element = EmitNullConstant(ElementTy);
-      uint64_t NumElements = CAT->getSize().getZExtValue();
-      std::vector<llvm::Constant *> Array(NumElements);
-      for (uint64_t i = 0; i != NumElements; ++i)
-        Array[i] = Element;
+    llvm::Constant *Element = EmitNullConstant(ElementTy);
+    unsigned NumElements = CAT->getSize().getZExtValue();
+    std::vector<llvm::Constant *> Array(NumElements);
+    for (unsigned i = 0; i != NumElements; ++i)
+      Array[i] = Element;
 
-      const llvm::ArrayType *ATy =
-        cast<llvm::ArrayType>(getTypes().ConvertTypeForMem(T));
-      return llvm::ConstantArray::get(ATy, Array);
-    }
+    const llvm::ArrayType *ATy =
+      cast<llvm::ArrayType>(getTypes().ConvertTypeForMem(T));
+    return llvm::ConstantArray::get(ATy, Array);
   }
 
   if (const RecordType *RT = T->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
-    // FIXME: It would be better if there was a way to explicitly compute the
-    // record layout instead of converting to a type.
-    Types.ConvertTagDeclType(RD);
-
-    const CGRecordLayout &Layout = Types.getCGRecordLayout(RD);
-    if (Layout.containsMemberPointer()) {
-      assert(0 && "FIXME: No support for structs with member pointers yet!");
+    const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+    assert(!RD->getNumBases() && 
+           "FIXME: Handle zero-initializing structs with bases and "
+           "pointers to data members.");
+    const llvm::StructType *STy =
+      cast<llvm::StructType>(getTypes().ConvertTypeForMem(T));
+    unsigned NumElements = STy->getNumElements();
+    std::vector<llvm::Constant *> Elements(NumElements);
+
+    for (RecordDecl::field_iterator I = RD->field_begin(),
+         E = RD->field_end(); I != E; ++I) {
+      const FieldDecl *FD = *I;
+      
+      unsigned FieldNo = getTypes().getLLVMFieldNo(FD);
+      Elements[FieldNo] = EmitNullConstant(FD->getType());
+    }
+    
+    // Now go through all other fields and zero them out.
+    for (unsigned i = 0; i != NumElements; ++i) {
+      if (!Elements[i])
+        Elements[i] = llvm::Constant::getNullValue(STy->getElementType(i));
     }
+    
+    return llvm::ConstantStruct::get(STy, Elements);
   }
 
-  // FIXME: Handle structs that contain member pointers.
-  if (isDataMemberPointerType(T))
-    return llvm::Constant::getAllOnesValue(getTypes().ConvertTypeForMem(T));
+  assert(!T->getAs<MemberPointerType>()->getPointeeType()->isFunctionType() &&
+         "Should only see pointers to data members here!");
+  
+  // Itanium C++ ABI 2.3:
+  //   A NULL pointer is represented as -1.
+  return llvm::ConstantInt::get(getTypes().ConvertTypeForMem(T), -1ULL, 
+                                /*isSigned=*/true);
+}
+
+llvm::Constant *
+CodeGenModule::EmitPointerToDataMember(const FieldDecl *FD) {
+
+  // Itanium C++ ABI 2.3:
+  //   A pointer to data member is an offset from the base address of the class
+  //   object containing it, represented as a ptrdiff_t
+
+  const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(FD->getParent());
+  QualType ClassType = 
+    getContext().getTypeDeclType(const_cast<CXXRecordDecl *>(ClassDecl));
+  
+  const llvm::StructType *ClassLTy =
+    cast<llvm::StructType>(getTypes().ConvertType(ClassType));
+
+  unsigned FieldNo = getTypes().getLLVMFieldNo(FD);
+  uint64_t Offset = 
+    getTargetData().getStructLayout(ClassLTy)->getElementOffset(FieldNo);
+
+  const llvm::Type *PtrDiffTy = 
+    getTypes().ConvertType(getContext().getPointerDiffType());
 
-  return llvm::Constant::getNullValue(getTypes().ConvertTypeForMem(T));
+  return llvm::ConstantInt::get(PtrDiffTy, Offset);
 }
diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp
index 690a7dc..db0998b 100644
--- a/lib/CodeGen/CGExprScalar.cpp
+++ b/lib/CodeGen/CGExprScalar.cpp
@@ -290,7 +290,7 @@ public:
     if (CGF.getContext().getLangOptions().OverflowChecking
         && Ops.Ty->isSignedIntegerType())
       return EmitOverflowCheckedBinOp(Ops);
-    if (Ops.LHS->getType()->isFPOrFPVector())
+    if (Ops.LHS->getType()->isFPOrFPVectorTy())
       return Builder.CreateFMul(Ops.LHS, Ops.RHS, "mul");
     return Builder.CreateMul(Ops.LHS, Ops.RHS, "mul");
   }
@@ -388,8 +388,6 @@ Value *ScalarExprEmitter::EmitConversionToBool(Value *Src, QualType SrcType) {
   }
 
   if (SrcType->isMemberPointerType()) {
-    // FIXME: This is ABI specific.
-
     // Compare against -1.
     llvm::Value *NegativeOne = llvm::Constant::getAllOnesValue(Src->getType());
     return Builder.CreateICmpNE(Src, NegativeOne, "tobool");
@@ -507,7 +505,7 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
       return Builder.CreateUIToFP(Src, DstTy, "conv");
   }
 
-  assert(Src->getType()->isFloatingPoint() && "Unknown real conversion");
+  assert(Src->getType()->isFloatingPointTy() && "Unknown real conversion");
   if (isa<llvm::IntegerType>(DstTy)) {
     if (DstType->isSignedIntegerType())
       return Builder.CreateFPToSI(Src, DstTy, "conv");
@@ -515,7 +513,7 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
       return Builder.CreateFPToUI(Src, DstTy, "conv");
   }
 
-  assert(DstTy->isFloatingPoint() && "Unknown real conversion");
+  assert(DstTy->isFloatingPointTy() && "Unknown real conversion");
   if (DstTy->getTypeID() < Src->getType()->getTypeID())
     return Builder.CreateFPTrunc(Src, DstTy, "conv");
   else
@@ -891,8 +889,8 @@ Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
     if (CE->getCastKind() == CastExpr::CK_DerivedToBaseMemberPointer)
       std::swap(DerivedDecl, BaseDecl);
 
-    llvm::Constant *Adj = CGF.CGM.GetCXXBaseClassOffset(DerivedDecl, BaseDecl);
-    if (Adj) {
+    if (llvm::Constant *Adj = 
+          CGF.CGM.GetNonVirtualBaseClassOffset(DerivedDecl, BaseDecl)) {
       if (CE->getCastKind() == CastExpr::CK_DerivedToBaseMemberPointer)
         Src = Builder.CreateSub(Src, Adj, "adj");
       else
@@ -1009,7 +1007,7 @@ Value *ScalarExprEmitter::VisitBlockDeclRefExpr(const BlockDeclRefExpr *E) {
 Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E) {
   TestAndClearIgnoreResultAssign();
   Value *Op = Visit(E->getSubExpr());
-  if (Op->getType()->isFPOrFPVector())
+  if (Op->getType()->isFPOrFPVectorTy())
     return Builder.CreateFNeg(Op, "neg");
   return Builder.CreateNeg(Op, "neg");
 }
@@ -1154,7 +1152,7 @@ Value *ScalarExprEmitter::EmitCompoundAssign(const CompoundAssignOperator *E,
 
 
 Value *ScalarExprEmitter::EmitDiv(const BinOpInfo &Ops) {
-  if (Ops.LHS->getType()->isFPOrFPVector())
+  if (Ops.LHS->getType()->isFPOrFPVectorTy())
     return Builder.CreateFDiv(Ops.LHS, Ops.RHS, "div");
   else if (Ops.Ty->isUnsignedIntegerType())
     return Builder.CreateUDiv(Ops.LHS, Ops.RHS, "div");
@@ -1261,7 +1259,7 @@ Value *ScalarExprEmitter::EmitAdd(const BinOpInfo &Ops) {
         Ops.Ty->isSignedIntegerType())
       return EmitOverflowCheckedBinOp(Ops);
 
-    if (Ops.LHS->getType()->isFPOrFPVector())
+    if (Ops.LHS->getType()->isFPOrFPVectorTy())
       return Builder.CreateFAdd(Ops.LHS, Ops.RHS, "add");
 
     // Signed integer overflow is undefined behavior.
@@ -1337,7 +1335,7 @@ Value *ScalarExprEmitter::EmitSub(const BinOpInfo &Ops) {
         && Ops.Ty->isSignedIntegerType())
       return EmitOverflowCheckedBinOp(Ops);
 
-    if (Ops.LHS->getType()->isFPOrFPVector())
+    if (Ops.LHS->getType()->isFPOrFPVectorTy())
       return Builder.CreateFSub(Ops.LHS, Ops.RHS, "sub");
     return Builder.CreateSub(Ops.LHS, Ops.RHS, "sub");
   }
@@ -1505,7 +1503,7 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc,
     Value *LHS = Visit(E->getLHS());
     Value *RHS = Visit(E->getRHS());
 
-    if (LHS->getType()->isFPOrFPVector()) {
+    if (LHS->getType()->isFPOrFPVectorTy()) {
       Result = Builder.CreateFCmp((llvm::CmpInst::Predicate)FCmpOpc,
                                   LHS, RHS, "cmp");
     } else if (LHSTy->isSignedIntegerType()) {
@@ -1615,10 +1613,10 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
        PI != PE; ++PI)
     PN->addIncoming(llvm::ConstantInt::getFalse(VMContext), *PI);
 
-  CGF.StartConditionalBranch();
+  CGF.BeginConditionalBranch();
   CGF.EmitBlock(RHSBlock);
   Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
-  CGF.FinishConditionalBranch();
+  CGF.EndConditionalBranch();
 
   // Reaquire the RHS block, as there may be subblocks inserted.
   RHSBlock = Builder.GetInsertBlock();
@@ -1665,13 +1663,13 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
        PI != PE; ++PI)
     PN->addIncoming(llvm::ConstantInt::getTrue(VMContext), *PI);
 
-  CGF.StartConditionalBranch();
+  CGF.BeginConditionalBranch();
 
   // Emit the RHS condition as a bool value.
   CGF.EmitBlock(RHSBlock);
   Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
 
-  CGF.FinishConditionalBranch();
+  CGF.EndConditionalBranch();
 
   // Reaquire the RHS block, as there may be subblocks inserted.
   RHSBlock = Builder.GetInsertBlock();
@@ -1785,7 +1783,7 @@ VisitConditionalOperator(const ConditionalOperator *E) {
     Builder.CreateCondBr(CondBoolVal, LHSBlock, RHSBlock);
   }
 
-  CGF.StartConditionalBranch();
+  CGF.BeginConditionalBranch();
   CGF.EmitBlock(LHSBlock);
 
   // Handle the GNU extension for missing LHS.
@@ -1795,15 +1793,15 @@ VisitConditionalOperator(const ConditionalOperator *E) {
   else    // Perform promotions, to handle cases like "short ?: int"
     LHS = EmitScalarConversion(CondVal, E->getCond()->getType(), E->getType());
 
-  CGF.FinishConditionalBranch();
+  CGF.EndConditionalBranch();
   LHSBlock = Builder.GetInsertBlock();
   CGF.EmitBranch(ContBlock);
 
-  CGF.StartConditionalBranch();
+  CGF.BeginConditionalBranch();
   CGF.EmitBlock(RHSBlock);
 
   Value *RHS = Visit(E->getRHS());
-  CGF.FinishConditionalBranch();
+  CGF.EndConditionalBranch();
   RHSBlock = Builder.GetInsertBlock();
   CGF.EmitBranch(ContBlock);
 
@@ -1882,14 +1880,23 @@ LValue CodeGenFunction::EmitObjCIsaExpr(const ObjCIsaExpr *E) {
   llvm::Value *V;
   // object->isa or (*object).isa
   // Generate code as for: *(Class*)object
+  // build Class* type
+  const llvm::Type *ClassPtrTy = ConvertType(E->getType());
+
   Expr *BaseExpr = E->getBase();
-  if (E->isArrow())
-    V = ScalarExprEmitter(*this).EmitLoadOfLValue(BaseExpr);
-  else
-    V  = EmitLValue(BaseExpr).getAddress();
+  if (BaseExpr->isLvalue(getContext()) != Expr::LV_Valid) {
+    V = CreateTempAlloca(ClassPtrTy, "resval");
+    llvm::Value *Src = EmitScalarExpr(BaseExpr);
+    Builder.CreateStore(Src, V);
+  }
+  else {
+      if (E->isArrow())
+        V = ScalarExprEmitter(*this).EmitLoadOfLValue(BaseExpr);
+      else
+        V  = EmitLValue(BaseExpr).getAddress();
+  }
   
   // build Class* type
-  const llvm::Type *ClassPtrTy = ConvertType(E->getType());
   ClassPtrTy = ClassPtrTy->getPointerTo();
   V = Builder.CreateBitCast(V, ClassPtrTy);
   LValue LV = LValue::MakeAddr(V, MakeQualifiers(E->getType()));
diff --git a/lib/CodeGen/CGObjC.cpp b/lib/CodeGen/CGObjC.cpp
index 896d220..b62e6ed 100644
--- a/lib/CodeGen/CGObjC.cpp
+++ b/lib/CodeGen/CGObjC.cpp
@@ -122,7 +122,7 @@ void CodeGenFunction::StartObjCMethod(const ObjCMethodDecl *OMD,
        E = OMD->param_end(); PI != E; ++PI)
     Args.push_back(std::make_pair(*PI, (*PI)->getType()));
 
-  StartFunction(OMD, OMD->getResultType(), Fn, Args, OMD->getLocEnd());
+  StartFunction(OMD, OMD->getResultType(), Fn, Args, OMD->getLocStart());
 }
 
 /// Generate an Objective-C method.  An Objective-C method is a C function with
@@ -190,7 +190,8 @@ void CodeGenFunction::GenerateObjCGetter(ObjCImplementationDecl *IMP,
     Args.push_back(std::make_pair(RValue::get(True), getContext().BoolTy));
     // FIXME: We shouldn't need to get the function info here, the
     // runtime already should have computed it to build the function.
-    RValue RV = EmitCall(Types.getFunctionInfo(PD->getType(), Args),
+    RValue RV = EmitCall(Types.getFunctionInfo(PD->getType(), Args,
+                                               CC_Default, false),
                          GetPropertyFn, ReturnValueSlot(), Args);
     // We need to fix the type here. Ivars with copy & retain are
     // always objects so we don't need to worry about complex or
@@ -278,7 +279,8 @@ void CodeGenFunction::GenerateObjCSetter(ObjCImplementationDecl *IMP,
                                   getContext().BoolTy));
     // FIXME: We shouldn't need to get the function info here, the runtime
     // already should have computed it to build the function.
-    EmitCall(Types.getFunctionInfo(getContext().VoidTy, Args), SetPropertyFn, 
+    EmitCall(Types.getFunctionInfo(getContext().VoidTy, Args,
+                                   CC_Default, false), SetPropertyFn,
              ReturnValueSlot(), Args);
   } else {
     // FIXME: Find a clean way to avoid AST node creation.
@@ -450,9 +452,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
 
   // Fast enumeration state.
   QualType StateTy = getContext().getObjCFastEnumerationStateType();
-  llvm::AllocaInst *StatePtr = CreateTempAlloca(ConvertType(StateTy),
-                                                "state.ptr");
-  StatePtr->setAlignment(getContext().getTypeAlign(StateTy) >> 3);
+  llvm::Value *StatePtr = CreateMemTemp(StateTy, "state.ptr");
   EmitMemSetToZero(StatePtr, StateTy);
 
   // Number of elements in the items array.
@@ -470,7 +470,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
     getContext().getConstantArrayType(getContext().getObjCIdType(),
                                       llvm::APInt(32, NumItems),
                                       ArrayType::Normal, 0);
-  llvm::Value *ItemsPtr = CreateTempAlloca(ConvertType(ItemsTy), "items.ptr");
+  llvm::Value *ItemsPtr = CreateMemTemp(ItemsTy, "items.ptr");
 
   llvm::Value *Collection = EmitScalarExpr(S.getCollection());
 
@@ -492,7 +492,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
                                              FastEnumSel,
                                              Collection, false, Args);
 
-  llvm::Value *LimitPtr = CreateTempAlloca(UnsignedLongLTy, "limit.ptr");
+  llvm::Value *LimitPtr = CreateMemTemp(getContext().UnsignedLongTy,
+                                        "limit.ptr");
   Builder.CreateStore(CountRV.getScalarVal(), LimitPtr);
 
   llvm::BasicBlock *NoElements = createBasicBlock("noelements");
@@ -506,8 +507,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
 
   EmitBlock(SetStartMutations);
 
-  llvm::Value *StartMutationsPtr =
-    CreateTempAlloca(UnsignedLongLTy);
+  llvm::Value *StartMutationsPtr = CreateMemTemp(getContext().UnsignedLongTy);
 
   llvm::Value *StateMutationsPtrPtr =
     Builder.CreateStructGEP(StatePtr, 2, "mutationsptr.ptr");
@@ -522,7 +522,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
   llvm::BasicBlock *LoopStart = createBasicBlock("loopstart");
   EmitBlock(LoopStart);
 
-  llvm::Value *CounterPtr = CreateTempAlloca(UnsignedLongLTy, "counter.ptr");
+  llvm::Value *CounterPtr = CreateMemTemp(getContext().UnsignedLongTy,
+                                       "counter.ptr");
   Builder.CreateStore(Zero, CounterPtr);
 
   llvm::BasicBlock *LoopBody = createBasicBlock("loopbody");
@@ -553,7 +554,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
                                 getContext().getObjCIdType()));
   // FIXME: We shouldn't need to get the function info here, the runtime already
   // should have computed it to build the function.
-  EmitCall(CGM.getTypes().getFunctionInfo(getContext().VoidTy, Args2),
+  EmitCall(CGM.getTypes().getFunctionInfo(getContext().VoidTy, Args2,
+                                          CC_Default, false),
            EnumerationMutationFn, ReturnValueSlot(), Args2);
 
   EmitBlock(WasNotMutated);
diff --git a/lib/CodeGen/CGObjCGNU.cpp b/lib/CodeGen/CGObjCGNU.cpp
index 77be9fb..1d38ef9 100644
--- a/lib/CodeGen/CGObjCGNU.cpp
+++ b/lib/CodeGen/CGObjCGNU.cpp
@@ -55,6 +55,7 @@ private:
   const llvm::PointerType *PtrToInt8Ty;
   const llvm::FunctionType *IMPTy;
   const llvm::PointerType *IdTy;
+  const llvm::PointerType *PtrToIdTy;
   QualType ASTIdTy;
   const llvm::IntegerType *IntTy;
   const llvm::PointerType *PtrTy;
@@ -65,11 +66,17 @@ private:
   std::vector<llvm::Constant*> Classes;
   std::vector<llvm::Constant*> Categories;
   std::vector<llvm::Constant*> ConstantStrings;
+  llvm::StringMap<llvm::Constant*> ObjCStrings;
   llvm::Function *LoadFunction;
   llvm::StringMap<llvm::Constant*> ExistingProtocols;
   typedef std::pair<std::string, std::string> TypedSelector;
   std::map<TypedSelector, llvm::GlobalAlias*> TypedSelectors;
   llvm::StringMap<llvm::GlobalAlias*> UntypedSelectors;
+  // Selectors that we don't emit in GC mode
+  Selector RetainSel, ReleaseSel, AutoreleaseSel;
+  // Functions used for GC.
+  llvm::Constant *IvarAssignFn, *StrongCastAssignFn, *MemMoveFn, *WeakReadFn, 
+    *WeakAssignFn, *GlobalAssignFn;
   // Some zeros used for GEPs in lots of places.
   llvm::Constant *Zeros[2];
   llvm::Constant *NULLPtr;
@@ -114,14 +121,18 @@ private:
   llvm::Constant *ExportUniqueString(const std::string &Str, const std::string
           prefix);
   llvm::Constant *MakeGlobal(const llvm::StructType *Ty,
-    std::vector<llvm::Constant*> &V, const std::string &Name="",
+    std::vector<llvm::Constant*> &V, llvm::StringRef Name="",
     llvm::GlobalValue::LinkageTypes linkage=llvm::GlobalValue::InternalLinkage);
   llvm::Constant *MakeGlobal(const llvm::ArrayType *Ty,
-    std::vector<llvm::Constant*> &V, const std::string &Name="",
+    std::vector<llvm::Constant*> &V, llvm::StringRef Name="",
     llvm::GlobalValue::LinkageTypes linkage=llvm::GlobalValue::InternalLinkage);
   llvm::GlobalVariable *ObjCIvarOffsetVariable(const ObjCInterfaceDecl *ID,
       const ObjCIvarDecl *Ivar);
   void EmitClassRef(const std::string &className);
+  llvm::Value* EnforceType(CGBuilderTy B, llvm::Value *V, const llvm::Type *Ty){
+    if (V->getType() == Ty) return V;
+    return B.CreateBitCast(V, Ty);
+  }
 public:
   CGObjCGNU(CodeGen::CodeGenModule &cgm);
   virtual llvm::Constant *GenerateConstantString(const StringLiteral *);
@@ -257,12 +268,54 @@ CGObjCGNU::CGObjCGNU(CodeGen::CodeGenModule &cgm)
   } else {
     IdTy = cast<llvm::PointerType>(CGM.getTypes().ConvertType(ASTIdTy));
   }
+  PtrToIdTy = llvm::PointerType::getUnqual(IdTy);
 
   // IMP type
   std::vector<const llvm::Type*> IMPArgs;
   IMPArgs.push_back(IdTy);
   IMPArgs.push_back(SelectorTy);
   IMPTy = llvm::FunctionType::get(IdTy, IMPArgs, true);
+
+  if (CGM.getLangOptions().getGCMode() != LangOptions::NonGC) {
+    // Get selectors needed in GC mode
+    RetainSel = GetNullarySelector("retain", CGM.getContext());
+    ReleaseSel = GetNullarySelector("release", CGM.getContext());
+    AutoreleaseSel = GetNullarySelector("autorelease", CGM.getContext());
+
+    // Get functions needed in GC mode
+
+    // id objc_assign_ivar(id, id, ptrdiff_t);
+    std::vector<const llvm::Type*> Args(1, IdTy);
+    Args.push_back(PtrToIdTy);
+    // FIXME: ptrdiff_t
+    Args.push_back(LongTy);
+    llvm::FunctionType *FTy = llvm::FunctionType::get(IdTy, Args, false);
+    IvarAssignFn = CGM.CreateRuntimeFunction(FTy, "objc_assign_ivar");
+    // id objc_assign_strongCast (id, id*)
+    Args.pop_back();
+    FTy = llvm::FunctionType::get(IdTy, Args, false);
+    StrongCastAssignFn = 
+        CGM.CreateRuntimeFunction(FTy, "objc_assign_strongCast");
+    // id objc_assign_global(id, id*);
+    FTy = llvm::FunctionType::get(IdTy, Args, false);
+    GlobalAssignFn = CGM.CreateRuntimeFunction(FTy, "objc_assign_global");
+    // id objc_assign_weak(id, id*);
+    FTy = llvm::FunctionType::get(IdTy, Args, false);
+    WeakAssignFn = CGM.CreateRuntimeFunction(FTy, "objc_assign_weak");
+    // id objc_read_weak(id*);
+    Args.clear();
+    Args.push_back(PtrToIdTy);
+    FTy = llvm::FunctionType::get(IdTy, Args, false);
+    WeakReadFn = CGM.CreateRuntimeFunction(FTy, "objc_read_weak");
+    // void *objc_memmove_collectable(void*, void *, size_t);
+    Args.clear();
+    Args.push_back(PtrToInt8Ty);
+    Args.push_back(PtrToInt8Ty);
+    // FIXME: size_t
+    Args.push_back(LongTy);
+    FTy = llvm::FunctionType::get(IdTy, Args, false);
+    MemMoveFn = CGM.CreateRuntimeFunction(FTy, "objc_memmove_collectable");
+  }
 }
 
 // This has to perform the lookup every time, since posing and related
@@ -340,7 +393,7 @@ llvm::Constant *CGObjCGNU::ExportUniqueString(const std::string &Str,
 }
 
 llvm::Constant *CGObjCGNU::MakeGlobal(const llvm::StructType *Ty,
-    std::vector<llvm::Constant*> &V, const std::string &Name,
+    std::vector<llvm::Constant*> &V, llvm::StringRef Name,
     llvm::GlobalValue::LinkageTypes linkage) {
   llvm::Constant *C = llvm::ConstantStruct::get(Ty, V);
   return new llvm::GlobalVariable(TheModule, Ty, false,
@@ -348,7 +401,7 @@ llvm::Constant *CGObjCGNU::MakeGlobal(const llvm::StructType *Ty,
 }
 
 llvm::Constant *CGObjCGNU::MakeGlobal(const llvm::ArrayType *Ty,
-    std::vector<llvm::Constant*> &V, const std::string &Name,
+    std::vector<llvm::Constant*> &V, llvm::StringRef Name,
     llvm::GlobalValue::LinkageTypes linkage) {
   llvm::Constant *C = llvm::ConstantArray::get(Ty, V);
   return new llvm::GlobalVariable(TheModule, Ty, false,
@@ -357,8 +410,14 @@ llvm::Constant *CGObjCGNU::MakeGlobal(const llvm::ArrayType *Ty,
 
 /// Generate an NSConstantString object.
 llvm::Constant *CGObjCGNU::GenerateConstantString(const StringLiteral *SL) {
+
   std::string Str(SL->getStrData(), SL->getByteLength());
 
+  // Look for an existing one
+  llvm::StringMap<llvm::Constant*>::iterator old = ObjCStrings.find(Str);
+  if (old != ObjCStrings.end())
+    return old->getValue();
+
   std::vector<llvm::Constant*> Ivars;
   Ivars.push_back(NULLPtr);
   Ivars.push_back(MakeConstantString(Str));
@@ -366,8 +425,9 @@ llvm::Constant *CGObjCGNU::GenerateConstantString(const StringLiteral *SL) {
   llvm::Constant *ObjCStr = MakeGlobal(
     llvm::StructType::get(VMContext, PtrToInt8Ty, PtrToInt8Ty, IntTy, NULL),
     Ivars, ".objc_str");
-  ConstantStrings.push_back(
-      llvm::ConstantExpr::getBitCast(ObjCStr, PtrToInt8Ty));
+  ObjCStr = llvm::ConstantExpr::getBitCast(ObjCStr, PtrToInt8Ty);
+  ObjCStrings[Str] = ObjCStr;
+  ConstantStrings.push_back(ObjCStr);
   return ObjCStr;
 }
 
@@ -384,6 +444,14 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
                                     bool IsClassMessage,
                                     const CallArgList &CallArgs,
                                     const ObjCMethodDecl *Method) {
+  if (CGM.getLangOptions().getGCMode() != LangOptions::NonGC) {
+    if (Sel == RetainSel || Sel == AutoreleaseSel) {
+      return RValue::get(Receiver);
+    }
+    if (Sel == ReleaseSel) {
+      return RValue::get(0);
+    }
+  }
   llvm::Value *cmd = GetSelector(CGF.Builder, Sel);
 
   CallArgList ActualArgs;
@@ -396,7 +464,8 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
   ActualArgs.insert(ActualArgs.end(), CallArgs.begin(), CallArgs.end());
 
   CodeGenTypes &Types = CGM.getTypes();
-  const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, ActualArgs);
+  const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, ActualArgs,
+                                                       CC_Default, false);
   const llvm::FunctionType *impType =
     Types.GetFunctionType(FnInfo, Method ? Method->isVariadic() : false);
 
@@ -478,6 +547,14 @@ CGObjCGNU::GenerateMessageSend(CodeGen::CodeGenFunction &CGF,
                                bool IsClassMessage,
                                const CallArgList &CallArgs,
                                const ObjCMethodDecl *Method) {
+  if (CGM.getLangOptions().getGCMode() != LangOptions::NonGC) {
+    if (Sel == RetainSel || Sel == AutoreleaseSel) {
+      return RValue::get(Receiver);
+    }
+    if (Sel == ReleaseSel) {
+      return RValue::get(0);
+    }
+  }
   CGBuilderTy &Builder = CGF.Builder;
   IdTy = cast<llvm::PointerType>(CGM.getTypes().ConvertType(ASTIdTy));
   llvm::Value *cmd;
@@ -495,7 +572,8 @@ CGObjCGNU::GenerateMessageSend(CodeGen::CodeGenFunction &CGF,
   ActualArgs.insert(ActualArgs.end(), CallArgs.begin(), CallArgs.end());
 
   CodeGenTypes &Types = CGM.getTypes();
-  const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, ActualArgs);
+  const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, ActualArgs,
+                                                       CC_Default, false);
   const llvm::FunctionType *impType =
     Types.GetFunctionType(FnInfo, Method ? Method->isVariadic() : false);
 
@@ -1517,7 +1595,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
     Elements.push_back(llvm::ConstantInt::get(LongTy, RuntimeVersion));
   }
   // sizeof(ModuleTy)
-  llvm::TargetData td = llvm::TargetData::TargetData(&TheModule);
+  llvm::TargetData td(&TheModule);
   Elements.push_back(llvm::ConstantInt::get(LongTy,
                      td.getTypeSizeInBits(ModuleTy)/8));
   //FIXME: Should be the path to the file where this module was declared
@@ -1610,7 +1688,8 @@ llvm::Constant *CGObjCGNU::EnumerationMutationFunction() {
   llvm::SmallVector<QualType,16> Params;
   Params.push_back(ASTIdTy);
   const llvm::FunctionType *FTy =
-    Types.GetFunctionType(Types.getFunctionInfo(Ctx.VoidTy, Params), false);
+    Types.GetFunctionType(Types.getFunctionInfo(Ctx.VoidTy, Params,
+                                                CC_Default, false), false);
   return CGM.CreateRuntimeFunction(FTy, "objc_enumerationMutation");
 }
 
@@ -1668,7 +1747,6 @@ void CGObjCGNU::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
   CGF.EmitBlock(TryHandler);
 
   // Get the correct versions of the exception handling intrinsics
-  llvm::TargetData td = llvm::TargetData::TargetData(&TheModule);
   llvm::Value *llvm_eh_exception =
     CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_exception);
   llvm::Value *llvm_eh_selector =
@@ -1899,35 +1977,58 @@ void CGObjCGNU::EmitThrowStmt(CodeGen::CodeGenFunction &CGF,
 
 llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF,
                                           llvm::Value *AddrWeakObj) {
-  return 0;
+  CGBuilderTy B = CGF.Builder;
+  AddrWeakObj = EnforceType(B, AddrWeakObj, IdTy);
+  return B.CreateCall(WeakReadFn, AddrWeakObj);
 }
 
 void CGObjCGNU::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
                                    llvm::Value *src, llvm::Value *dst) {
-  return;
+  CGBuilderTy B = CGF.Builder;
+  src = EnforceType(B, src, IdTy);
+  dst = EnforceType(B, dst, PtrToIdTy);
+  B.CreateCall2(WeakAssignFn, src, dst);
 }
 
 void CGObjCGNU::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
                                      llvm::Value *src, llvm::Value *dst) {
-  return;
+  CGBuilderTy B = CGF.Builder;
+  src = EnforceType(B, src, IdTy);
+  dst = EnforceType(B, dst, PtrToIdTy);
+  B.CreateCall2(GlobalAssignFn, src, dst);
 }
 
 void CGObjCGNU::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
                                    llvm::Value *src, llvm::Value *dst,
                                    llvm::Value *ivarOffset) {
-  return;
+  CGBuilderTy B = CGF.Builder;
+  src = EnforceType(B, src, IdTy);
+  dst = EnforceType(B, dst, PtrToIdTy);
+  B.CreateCall3(IvarAssignFn, src, dst, ivarOffset);
 }
 
 void CGObjCGNU::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
                                          llvm::Value *src, llvm::Value *dst) {
-  return;
+  CGBuilderTy B = CGF.Builder;
+  src = EnforceType(B, src, IdTy);
+  dst = EnforceType(B, dst, PtrToIdTy);
+  B.CreateCall2(StrongCastAssignFn, src, dst);
 }
 
 void CGObjCGNU::EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF,
                                          llvm::Value *DestPtr,
                                          llvm::Value *SrcPtr,
                                          QualType Ty) {
-  return;
+  CGBuilderTy B = CGF.Builder;
+  DestPtr = EnforceType(B, DestPtr, IdTy);
+  SrcPtr = EnforceType(B, SrcPtr, PtrToIdTy);
+
+  std::pair<uint64_t, unsigned> TypeInfo = CGM.getContext().getTypeInfo(Ty);
+  unsigned long size = TypeInfo.first/8;
+  // FIXME: size_t
+  llvm::Value *N = llvm::ConstantInt::get(LongTy, size);
+
+  B.CreateCall3(MemMoveFn, DestPtr, SrcPtr, N);
 }
 
 llvm::GlobalVariable *CGObjCGNU::ObjCIvarOffsetVariable(
diff --git a/lib/CodeGen/CGObjCMac.cpp b/lib/CodeGen/CGObjCMac.cpp
index 137ea51..b16a510 100644
--- a/lib/CodeGen/CGObjCMac.cpp
+++ b/lib/CodeGen/CGObjCMac.cpp
@@ -21,6 +21,7 @@
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/StmtObjC.h"
 #include "clang/Basic/LangOptions.h"
+#include "clang/CodeGen/CodeGenOptions.h"
 
 #include "llvm/Intrinsics.h"
 #include "llvm/LLVMContext.h"
@@ -304,7 +305,8 @@ public:
     Params.push_back(Ctx.LongTy);
     Params.push_back(Ctx.BoolTy);
     const llvm::FunctionType *FTy =
-      Types.GetFunctionType(Types.getFunctionInfo(IdType, Params), false);
+      Types.GetFunctionType(Types.getFunctionInfo(IdType, Params,
+                                                  CC_Default, false), false);
     return CGM.CreateRuntimeFunction(FTy, "objc_getProperty");
   }
 
@@ -322,7 +324,8 @@ public:
     Params.push_back(Ctx.BoolTy);
     Params.push_back(Ctx.BoolTy);
     const llvm::FunctionType *FTy =
-      Types.GetFunctionType(Types.getFunctionInfo(Ctx.VoidTy, Params), false);
+      Types.GetFunctionType(Types.getFunctionInfo(Ctx.VoidTy, Params,
+                                                  CC_Default, false), false);
     return CGM.CreateRuntimeFunction(FTy, "objc_setProperty");
   }
 
@@ -333,7 +336,8 @@ public:
     llvm::SmallVector<QualType,16> Params;
     Params.push_back(Ctx.getObjCIdType());
     const llvm::FunctionType *FTy =
-      Types.GetFunctionType(Types.getFunctionInfo(Ctx.VoidTy, Params), false);
+      Types.GetFunctionType(Types.getFunctionInfo(Ctx.VoidTy, Params,
+                                                  CC_Default, false), false);
     return CGM.CreateRuntimeFunction(FTy, "objc_enumerationMutation");
   }
 
@@ -734,7 +738,8 @@ public:
     return CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext),
                               Params, false),
-      "_Unwind_Resume_or_Rethrow");
+      (CGM.getLangOptions().SjLjExceptions ? "_Unwind_SjLj_Resume" :
+       "_Unwind_Resume_or_Rethrow"));
   }
 
   llvm::Constant *getObjCEndCatchFn() {
@@ -1553,7 +1558,8 @@ CGObjCCommonMac::EmitLegacyMessageSend(CodeGen::CodeGenFunction &CGF,
   ActualArgs.insert(ActualArgs.end(), CallArgs.begin(), CallArgs.end());
 
   CodeGenTypes &Types = CGM.getTypes();
-  const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, ActualArgs);
+  const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, ActualArgs,
+                                                       CC_Default, false);
   const llvm::FunctionType *FTy =
     Types.GetFunctionType(FnInfo, Method ? Method->isVariadic() : false);
 
@@ -3625,7 +3631,7 @@ ObjCCommonTypesHelper::ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm)
                                 Ctx.getObjCIdType(), 0, 0, false));
   RD->addDecl(FieldDecl::Create(Ctx, RD, SourceLocation(), 0,
                                 Ctx.getObjCClassType(), 0, 0, false));
-  RD->completeDefinition(Ctx);
+  RD->completeDefinition();
 
   SuperCTy = Ctx.getTagDeclType(RD);
   SuperPtrCTy = Ctx.getPointerType(SuperCTy);
@@ -4086,7 +4092,7 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul
                                 Ctx.VoidPtrTy, 0, 0, false));
   RD->addDecl(FieldDecl::Create(Ctx, RD, SourceLocation(), 0,
                                 Ctx.getObjCSelType(), 0, 0, false));
-  RD->completeDefinition(Ctx);
+  RD->completeDefinition();
 
   MessageRefCTy = Ctx.getTagDeclType(RD);
   MessageRefCPtrTy = Ctx.getPointerType(MessageRefCTy);
@@ -4224,6 +4230,9 @@ void CGObjCNonFragileABIMac::FinishNonFragileABIModule() {
 /// message dispatch call for all the rest.
 ///
 bool CGObjCNonFragileABIMac::LegacyDispatchedSelector(Selector Sel) {
+  if (CGM.getCodeGenOpts().ObjCLegacyDispatch)
+    return true;
+
   if (NonLegacyDispatchMethods.empty()) {
     NonLegacyDispatchMethods.insert(GetNullarySelector("alloc"));
     NonLegacyDispatchMethods.insert(GetNullarySelector("class"));
@@ -5085,7 +5094,8 @@ CodeGen::RValue CGObjCNonFragileABIMac::EmitMessageSend(
   // FIXME. This is too much work to get the ABI-specific result type needed to
   // find the message name.
   const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType,
-                                                       llvm::SmallVector<QualType, 16>());
+                                                       llvm::SmallVector<QualType, 16>(),
+                                                       CC_Default, false);
   llvm::Constant *Fn = 0;
   std::string Name("\01l_");
   if (CGM.ReturnTypeUsesSret(FnInfo)) {
@@ -5159,7 +5169,8 @@ CodeGen::RValue CGObjCNonFragileABIMac::EmitMessageSend(
   ActualArgs.push_back(std::make_pair(RValue::get(Arg1),
                                       ObjCTypes.MessageRefCPtrTy));
   ActualArgs.insert(ActualArgs.end(), CallArgs.begin(), CallArgs.end());
-  const CGFunctionInfo &FnInfo1 = Types.getFunctionInfo(ResultType, ActualArgs);
+  const CGFunctionInfo &FnInfo1 = Types.getFunctionInfo(ResultType, ActualArgs,
+                                                        CC_Default, false);
   llvm::Value *Callee = CGF.Builder.CreateStructGEP(Arg1, 0);
   Callee = CGF.Builder.CreateLoad(Callee);
   const llvm::FunctionType *FTy = Types.GetFunctionType(FnInfo1, true);
diff --git a/lib/CodeGen/CGRTTI.cpp b/lib/CodeGen/CGRTTI.cpp
index 29552ce..5236d20 100644
--- a/lib/CodeGen/CGRTTI.cpp
+++ b/lib/CodeGen/CGRTTI.cpp
@@ -256,6 +256,9 @@ bool ShouldUseExternalRTTIDescriptor(QualType Ty) {
 
   if (const RecordType *RecordTy = dyn_cast<RecordType>(Ty)) {
     const CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getDecl());
+    if (!RD->hasDefinition())
+      return false;
+
     if (!RD->isDynamicClass())
       return false;
 
@@ -469,7 +472,7 @@ void RTTIBuilder::BuildVtablePointer(const Type *Ty) {
     const CXXRecordDecl *RD = 
       cast<CXXRecordDecl>(cast<RecordType>(Ty)->getDecl());
     
-    if (!RD->getNumBases()) {
+    if (!RD->hasDefinition() || !RD->getNumBases()) {
       // abi::__class_type_info.
       VtableName = "_ZTVN10__cxxabiv117__class_type_infoE";
     } else if (CanUseSingleInheritance(RD)) {
@@ -566,7 +569,7 @@ llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty) {
   case Type::Record: {
     const CXXRecordDecl *RD = 
       cast<CXXRecordDecl>(cast<RecordType>(Ty)->getDecl());
-    if (!RD->getNumBases()) {
+    if (!RD->hasDefinition() || !RD->getNumBases()) {
       // We don't need to emit any fields.
       break;
     }
diff --git a/lib/CodeGen/CGRecordLayoutBuilder.cpp b/lib/CodeGen/CGRecordLayoutBuilder.cpp
index 9f90ec5..baafd68 100644
--- a/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -108,6 +108,9 @@ bool CGRecordLayoutBuilder::LayoutField(const FieldDecl *D,
     return true;
   }
 
+  // Check if we have a pointer to data member in this field.
+  CheckForPointerToDataMember(D->getType());
+  
   assert(FieldOffset % 8 == 0 && "FieldOffset is not on a byte boundary!");
   uint64_t FieldOffsetInBytes = FieldOffset / 8;
 
@@ -162,6 +165,8 @@ void CGRecordLayoutBuilder::LayoutUnion(const RecordDecl *D) {
   uint64_t Size = 0;
   unsigned Align = 0;
 
+  bool HasOnlyZeroSizedBitFields = true;
+  
   unsigned FieldNo = 0;
   for (RecordDecl::field_iterator Field = D->field_begin(),
        FieldEnd = D->field_end(); Field != FieldEnd; ++Field, ++FieldNo) {
@@ -181,6 +186,8 @@ void CGRecordLayoutBuilder::LayoutUnion(const RecordDecl *D) {
     } else
       Types.addFieldInfo(*Field, 0);
 
+    HasOnlyZeroSizedBitFields = false;
+    
     const llvm::Type *FieldTy =
       Types.ConvertTypeForMemRecursive(Field->getType());
     unsigned FieldAlign = Types.getTargetData().getABITypeAlignment(FieldTy);
@@ -207,7 +214,8 @@ void CGRecordLayoutBuilder::LayoutUnion(const RecordDecl *D) {
     }
   }
   if (!Align) {
-    assert((D->field_begin() == D->field_end()) && "LayoutUnion - Align 0");
+    assert(HasOnlyZeroSizedBitFields &&
+           "0-align record did not have all zero-sized bit-fields!");
     Align = 1;
   }
   
@@ -333,23 +341,34 @@ uint64_t CGRecordLayoutBuilder::getTypeSizeInBytes(const llvm::Type *Ty) const {
   return Types.getTargetData().getTypeAllocSize(Ty);
 }
 
-void CGRecordLayoutBuilder::CheckForMemberPointer(const FieldDecl *FD) {
+void CGRecordLayoutBuilder::CheckForPointerToDataMember(QualType T) {
   // This record already contains a member pointer.
-  if (ContainsMemberPointer)
+  if (ContainsPointerToDataMember)
     return;
 
   // Can only have member pointers if we're compiling C++.
   if (!Types.getContext().getLangOptions().CPlusPlus)
     return;
 
-  QualType Ty = FD->getType();
-
-  if (Ty->isMemberPointerType()) {
-    // We have a member pointer!
-    ContainsMemberPointer = true;
-    return;
-  }
+  T = Types.getContext().getBaseElementType(T);
 
+  if (const MemberPointerType *MPT = T->getAs<MemberPointerType>()) {
+    if (!MPT->getPointeeType()->isFunctionType()) {
+      // We have a pointer to data member.
+      ContainsPointerToDataMember = true;
+    }
+  } else if (const RecordType *RT = T->getAs<RecordType>()) {
+    const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+    
+    // FIXME: It would be better if there was a way to explicitly compute the
+    // record layout instead of converting to a type.
+    Types.ConvertTagDeclType(RD);
+    
+    const CGRecordLayout &Layout = Types.getCGRecordLayout(RD);
+    
+    if (Layout.containsPointerToDataMember())
+      ContainsPointerToDataMember = true;
+  }    
 }
 
 CGRecordLayout *
@@ -381,5 +400,5 @@ CGRecordLayoutBuilder::ComputeLayout(CodeGenTypes &Types,
     Types.addBitFieldInfo(Info.FD, Info.FieldNo, Info.Start, Info.Size);
   }
 
-  return new CGRecordLayout(Ty, Builder.ContainsMemberPointer);
+  return new CGRecordLayout(Ty, Builder.ContainsPointerToDataMember);
 }
diff --git a/lib/CodeGen/CGRecordLayoutBuilder.h b/lib/CodeGen/CGRecordLayoutBuilder.h
index cf84053..eb60ed7 100644
--- a/lib/CodeGen/CGRecordLayoutBuilder.h
+++ b/lib/CodeGen/CGRecordLayoutBuilder.h
@@ -27,6 +27,7 @@ namespace clang {
   class CXXRecordDecl;
   class FieldDecl;
   class RecordDecl;
+  class QualType;
 
 namespace CodeGen {
   class CGRecordLayout;
@@ -38,9 +39,10 @@ class CGRecordLayoutBuilder {
   /// Packed - Whether the resulting LLVM struct will be packed or not.
   bool Packed;
 
-  /// ContainsMemberPointer - Whether one of the fields is a member pointer
-  /// or is a struct that contains a member pointer.
-  bool ContainsMemberPointer;
+  /// ContainsPointerToDataMember - Whether one of the fields in this record 
+  /// layout is a pointer to data member, or a struct that contains pointer to
+  /// data member.
+  bool ContainsPointerToDataMember;
 
   /// Alignment - Contains the alignment of the RecordDecl.
   unsigned Alignment;
@@ -78,7 +80,7 @@ class CGRecordLayoutBuilder {
   llvm::SmallVector<LLVMBitFieldInfo, 16> LLVMBitFields;
 
   CGRecordLayoutBuilder(CodeGenTypes &Types)
-    : Types(Types), Packed(false), ContainsMemberPointer(false)
+    : Types(Types), Packed(false), ContainsPointerToDataMember(false)
     , Alignment(0), AlignmentAsLLVMStruct(1)
     , BitsAvailableInLastField(0), NextFieldOffsetInBytes(0) { }
 
@@ -123,8 +125,9 @@ class CGRecordLayoutBuilder {
   unsigned getTypeAlignment(const llvm::Type *Ty) const;
   uint64_t getTypeSizeInBytes(const llvm::Type *Ty) const;
 
-  /// CheckForMemberPointer - Check if the field contains a member pointer.
-  void CheckForMemberPointer(const FieldDecl *FD);
+  /// CheckForPointerToDataMember - Check if the given type contains a pointer 
+  /// to data member.
+  void CheckForPointerToDataMember(QualType T);
 
 public:
   /// ComputeLayout - Return the right record layout for a given record decl.
diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp
index bbd5462..008a480 100644
--- a/lib/CodeGen/CGStmt.cpp
+++ b/lib/CodeGen/CGStmt.cpp
@@ -861,14 +861,13 @@ llvm::Value* CodeGenFunction::EmitAsmInput(const AsmStmt &S,
                                            std::string &ConstraintStr) {
   llvm::Value *Arg;
   if (Info.allowsRegister() || !Info.allowsMemory()) {
-    const llvm::Type *Ty = ConvertType(InputExpr->getType());
-
-    if (Ty->isSingleValueType()) {
+    if (!CodeGenFunction::hasAggregateLLVMType(InputExpr->getType())) {
       Arg = EmitScalarExpr(InputExpr);
     } else {
       InputExpr = InputExpr->IgnoreParenNoopCasts(getContext());
       LValue Dest = EmitLValue(InputExpr);
 
+      const llvm::Type *Ty = ConvertType(InputExpr->getType());
       uint64_t Size = CGM.getTargetData().getTypeSizeInBits(Ty);
       if (Size <= 64 && llvm::isPowerOf2_64(Size)) {
         Ty = llvm::IntegerType::get(VMContext, Size);
@@ -916,18 +915,18 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
   for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {
     TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i),
                                     S.getOutputName(i));
-    bool result = Target.validateOutputConstraint(Info);
-    assert(result && "Failed to parse output constraint"); result=result;
+    assert(Target.validateOutputConstraint(Info) && 
+           "Failed to parse output constraint");
     OutputConstraintInfos.push_back(Info);
   }
 
   for (unsigned i = 0, e = S.getNumInputs(); i != e; i++) {
     TargetInfo::ConstraintInfo Info(S.getInputConstraint(i),
                                     S.getInputName(i));
-    bool result = Target.validateInputConstraint(OutputConstraintInfos.data(),
-                                                 S.getNumOutputs(),
-                                                 Info); result=result;
-    assert(result && "Failed to parse input constraint");
+    assert(Target.validateInputConstraint(OutputConstraintInfos.data(),
+                                          S.getNumOutputs(),
+                                          Info) &&
+           "Failed to parse input constraint");
     InputConstraintInfos.push_back(Info);
   }
 
@@ -1066,10 +1065,9 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
 
   // Clobbers
   for (unsigned i = 0, e = S.getNumClobbers(); i != e; i++) {
-    std::string Clobber(S.getClobber(i)->getStrData(),
-                        S.getClobber(i)->getByteLength());
+    llvm::StringRef Clobber = S.getClobber(i)->getString();
 
-    Clobber = Target.getNormalizedGCCRegisterName(Clobber.c_str());
+    Clobber = Target.getNormalizedGCCRegisterName(Clobber);
 
     if (i != 0 || NumConstraints != 0)
       Constraints += ',';
diff --git a/lib/CodeGen/CGVtable.cpp b/lib/CodeGen/CGVtable.cpp
index e5abfc6..970bbd7 100644
--- a/lib/CodeGen/CGVtable.cpp
+++ b/lib/CodeGen/CGVtable.cpp
@@ -16,14 +16,1245 @@
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/RecordLayout.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/Format.h"
 #include <cstdio>
 
 using namespace clang;
 using namespace CodeGen;
 
 namespace {
+
+/// FinalOverriders - Contains the final overrider member functions for all
+/// member functions in the base subobjects of a class.
+class FinalOverriders {
+public:
+  /// BaseOffset - Represents an offset from a derived class to a direct or
+  /// indirect base class.
+  struct BaseOffset {
+    /// DerivedClass - The derived class.
+    const CXXRecordDecl *DerivedClass;
+    
+    /// VirtualBase - If the path from the derived class to the base class
+    /// involves a virtual base class, this holds its declaration.
+    const CXXRecordDecl *VirtualBase;
+
+    /// NonVirtualOffset - The offset from the derived class to the base class.
+    /// Or the offset from the virtual base class to the base class, if the path
+    /// from the derived class to the base class involves a virtual base class.
+    int64_t NonVirtualOffset;
+    
+    BaseOffset() : DerivedClass(0), VirtualBase(0), NonVirtualOffset(0) { }
+    BaseOffset(const CXXRecordDecl *DerivedClass,
+               const CXXRecordDecl *VirtualBase, int64_t NonVirtualOffset)
+      : DerivedClass(DerivedClass), VirtualBase(VirtualBase), 
+      NonVirtualOffset(NonVirtualOffset) { }
+
+    bool isEmpty() const { return !NonVirtualOffset && !VirtualBase; }
+  };
+  
+  /// OverriderInfo - Information about a final overrider.
+  struct OverriderInfo {
+    /// Method - The method decl of the overrider.
+    const CXXMethodDecl *Method;
+    
+    OverriderInfo() : Method(0) { }
+  };
+
+private:
+  /// MostDerivedClass - The most derived class for which the final overriders
+  /// are stored.
+  const CXXRecordDecl *MostDerivedClass;
+  
+  ASTContext &Context;
+  
+  /// MostDerivedClassLayout - the AST record layout of the most derived class.
+  const ASTRecordLayout &MostDerivedClassLayout;
+
+  /// BaseSubobjectMethodPairTy - Uniquely identifies a member function
+  /// in a base subobject.
+  typedef std::pair<BaseSubobject, const CXXMethodDecl *>
+    BaseSubobjectMethodPairTy;
+  
+  typedef llvm::DenseMap<BaseSubobjectMethodPairTy,
+                         OverriderInfo> OverridersMapTy;
+  
+  /// OverridersMap - The final overriders for all virtual member functions of 
+  /// all the base subobjects of the most derived class.
+  OverridersMapTy OverridersMap;
+  
+  /// VisitedVirtualBases - A set of all the visited virtual bases, used to
+  /// avoid visiting virtual bases more than once.
+  llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBases;
+
+  typedef llvm::DenseMap<BaseSubobjectMethodPairTy, BaseOffset>
+    AdjustmentOffsetsMapTy;
+
+  /// ReturnAdjustments - Holds return adjustments for all the overriders that 
+  /// need to perform return value adjustments.
+  AdjustmentOffsetsMapTy ReturnAdjustments;
+  
+  /// ThisAdjustments - Holds 'this' adjustments for all the overriders that
+  /// need them.
+  AdjustmentOffsetsMapTy ThisAdjustments;
+
+  typedef llvm::SmallVector<uint64_t, 1> OffsetVectorTy;
+  
+  /// SubobjectOffsetsMapTy - This map is used for keeping track of all the
+  /// base subobject offsets that a single class declaration might refer to.
+  ///
+  /// For example, in:
+  ///
+  /// struct A { virtual void f(); };
+  /// struct B1 : A { };
+  /// struct B2 : A { };
+  /// struct C : B1, B2 { virtual void f(); };
+  ///
+  /// when we determine that C::f() overrides A::f(), we need to update the
+  /// overriders map for both A-in-B1 and A-in-B2 and the subobject offsets map
+  /// will have the subobject offsets for both A copies.
+  typedef llvm::DenseMap<const CXXRecordDecl *, OffsetVectorTy>
+    SubobjectOffsetsMapTy;
+  
+  /// ComputeFinalOverriders - Compute the final overriders for a given base
+  /// subobject (and all its direct and indirect bases).
+  void ComputeFinalOverriders(BaseSubobject Base,
+                              SubobjectOffsetsMapTy &Offsets);
+  
+  /// AddOverriders - Add the final overriders for this base subobject to the
+  /// map of final overriders.  
+  void AddOverriders(BaseSubobject Base, SubobjectOffsetsMapTy &Offsets);
+
+  /// PropagateOverrider - Propagate the NewMD overrider to all the functions 
+  /// that OldMD overrides. For example, if we have:
+  ///
+  /// struct A { virtual void f(); };
+  /// struct B : A { virtual void f(); };
+  /// struct C : B { virtual void f(); };
+  ///
+  /// and we want to override B::f with C::f, we also need to override A::f with
+  /// C::f.
+  void PropagateOverrider(const CXXMethodDecl *OldMD,
+                          BaseSubobject NewBase,
+                          const CXXMethodDecl *NewMD,
+                          SubobjectOffsetsMapTy &Offsets);
+  
+  /// ComputeThisAdjustmentBaseOffset - Compute the base offset for adjusting
+  /// the 'this' pointer from the base subobject to the derived subobject.
+  BaseOffset ComputeThisAdjustmentBaseOffset(BaseSubobject Base,
+                                             BaseSubobject Derived);
+                                             
+  static void MergeSubobjectOffsets(const SubobjectOffsetsMapTy &NewOffsets,
+                                    SubobjectOffsetsMapTy &Offsets);
+
+public:
+  explicit FinalOverriders(const CXXRecordDecl *MostDerivedClass);
+  
+  /// getOverrider - Get the final overrider for the given method declaration in
+  /// the given base subobject.
+  OverriderInfo getOverrider(BaseSubobject Base,
+                             const CXXMethodDecl *MD) const {
+    assert(OverridersMap.count(std::make_pair(Base, MD)) && 
+           "Did not find overrider!");
+    
+    return OverridersMap.lookup(std::make_pair(Base, MD));
+  }
+  
+  /// getReturnAdjustmentOffset - Get the return adjustment offset for the
+  /// method decl in the given base subobject. Returns an empty base offset if
+  /// no adjustment is needed.
+  BaseOffset getReturnAdjustmentOffset(BaseSubobject Base,
+                                       const CXXMethodDecl *MD) const {
+    return ReturnAdjustments.lookup(std::make_pair(Base, MD));
+  }
+
+  /// getThisAdjustmentOffset - Get the 'this' pointer adjustment offset for the
+  /// method decl in the given base subobject. Returns an empty base offset if
+  /// no adjustment is needed.
+  BaseOffset getThisAdjustmentOffset(BaseSubobject Base,
+                                     const CXXMethodDecl *MD) const {
+    return ThisAdjustments.lookup(std::make_pair(Base, MD));
+  }
+  
+  /// dump - dump the final overriders.
+  void dump() {
+    assert(VisitedVirtualBases.empty() &&
+           "Visited virtual bases aren't empty!");
+    dump(llvm::errs(), BaseSubobject(MostDerivedClass, 0)); 
+    VisitedVirtualBases.clear();
+  }
+  
+  /// dump - dump the final overriders for a base subobject, and all its direct
+  /// and indirect base subobjects.
+  void dump(llvm::raw_ostream &Out, BaseSubobject Base);
+};
+
+FinalOverriders::FinalOverriders(const CXXRecordDecl *MostDerivedClass)
+  : MostDerivedClass(MostDerivedClass), 
+  Context(MostDerivedClass->getASTContext()),
+  MostDerivedClassLayout(Context.getASTRecordLayout(MostDerivedClass)) {
+    
+  // Compute the final overriders.
+  SubobjectOffsetsMapTy Offsets;
+  ComputeFinalOverriders(BaseSubobject(MostDerivedClass, 0), Offsets);
+    
+  // And dump them (for now).
+  dump();
+    
+  // Also dump the base offsets (for now).
+  for (SubobjectOffsetsMapTy::const_iterator I = Offsets.begin(),
+       E = Offsets.end(); I != E; ++I) {
+    const OffsetVectorTy& OffsetVector = I->second;
+
+    llvm::errs() << "Base offsets for ";
+    llvm::errs() << I->first->getQualifiedNameAsString() << '\n';
+
+    for (unsigned I = 0, E = OffsetVector.size(); I != E; ++I)
+      llvm::errs() << "  " << I << " - " << OffsetVector[I] << '\n';
+  }
+}
+
+void FinalOverriders::AddOverriders(BaseSubobject Base,
+                                    SubobjectOffsetsMapTy &Offsets) {
+  const CXXRecordDecl *RD = Base.getBase();
+
+  for (CXXRecordDecl::method_iterator I = RD->method_begin(), 
+       E = RD->method_end(); I != E; ++I) {
+    const CXXMethodDecl *MD = *I;
+    
+    if (!MD->isVirtual())
+      continue;
+
+    // First, propagate the overrider.
+    PropagateOverrider(MD, Base, MD, Offsets);
+
+    // Add the overrider as the final overrider of itself.
+    OverriderInfo& Overrider = OverridersMap[std::make_pair(Base, MD)];
+    assert(!Overrider.Method && "Overrider should not exist yet!");
+
+    Overrider.Method = MD;
+  }
+}
+
+static FinalOverriders::BaseOffset
+ComputeBaseOffset(ASTContext &Context, const CXXRecordDecl *DerivedRD,
+                  const CXXBasePath &Path) {
+  int64_t NonVirtualOffset = 0;
+
+  unsigned NonVirtualStart = 0;
+  const CXXRecordDecl *VirtualBase = 0;
+  
+  // First, look for the virtual base class.
+  for (unsigned I = 0, E = Path.size(); I != E; ++I) {
+    const CXXBasePathElement &Element = Path[I];
+    
+    if (Element.Base->isVirtual()) {
+      // FIXME: Can we break when we find the first virtual base?
+      // (If we can't, can't we just iterate over the path in reverse order?)
+      NonVirtualStart = I + 1;
+      QualType VBaseType = Element.Base->getType();
+      VirtualBase = 
+        cast<CXXRecordDecl>(VBaseType->getAs<RecordType>()->getDecl());
+    }
+  }
+  
+  // Now compute the non-virtual offset.
+  for (unsigned I = NonVirtualStart, E = Path.size(); I != E; ++I) {
+    const CXXBasePathElement &Element = Path[I];
+    
+    // Check the base class offset.
+    const ASTRecordLayout &Layout = Context.getASTRecordLayout(Element.Class);
+    
+    const RecordType *BaseType = Element.Base->getType()->getAs<RecordType>();
+    const CXXRecordDecl *Base = cast<CXXRecordDecl>(BaseType->getDecl());
+    
+    NonVirtualOffset += Layout.getBaseClassOffset(Base);
+  }
+  
+  // FIXME: This should probably use CharUnits or something. Maybe we should
+  // even change the base offsets in ASTRecordLayout to be specified in 
+  // CharUnits.
+  return FinalOverriders::BaseOffset(DerivedRD, VirtualBase, 
+                                     NonVirtualOffset / 8);
+  
+}
+
+static FinalOverriders::BaseOffset 
+ComputeBaseOffset(ASTContext &Context, const CXXRecordDecl *BaseRD,
+                  const CXXRecordDecl *DerivedRD) {
+  CXXBasePaths Paths(/*FindAmbiguities=*/false,
+                     /*RecordPaths=*/true, /*DetectVirtual=*/false);
+  
+  if (!const_cast<CXXRecordDecl *>(DerivedRD)->
+      isDerivedFrom(const_cast<CXXRecordDecl *>(BaseRD), Paths)) {
+    assert(false && "Class must be derived from the passed in base class!");
+    return FinalOverriders::BaseOffset();
+  }
+
+  return ComputeBaseOffset(Context, DerivedRD, Paths.front());
+}
+
+static FinalOverriders::BaseOffset
+ComputeReturnAdjustmentBaseOffset(ASTContext &Context, 
+                                  const CXXMethodDecl *DerivedMD,
+                                  const CXXMethodDecl *BaseMD) {
+  const FunctionType *BaseFT = BaseMD->getType()->getAs<FunctionType>();
+  const FunctionType *DerivedFT = DerivedMD->getType()->getAs<FunctionType>();
+  
+  // Canonicalize the return types.
+  CanQualType CanDerivedReturnType = 
+    Context.getCanonicalType(DerivedFT->getResultType());
+  CanQualType CanBaseReturnType = 
+    Context.getCanonicalType(BaseFT->getResultType());
+  
+  assert(CanDerivedReturnType->getTypeClass() == 
+         CanBaseReturnType->getTypeClass() && 
+         "Types must have same type class!");
+  
+  if (CanDerivedReturnType == CanBaseReturnType) {
+    // No adjustment needed.
+    return FinalOverriders::BaseOffset();
+  }
+  
+  if (isa<ReferenceType>(CanDerivedReturnType)) {
+    CanDerivedReturnType = 
+      CanDerivedReturnType->getAs<ReferenceType>()->getPointeeType();
+    CanBaseReturnType = 
+      CanBaseReturnType->getAs<ReferenceType>()->getPointeeType();
+  } else if (isa<PointerType>(CanDerivedReturnType)) {
+    CanDerivedReturnType = 
+      CanDerivedReturnType->getAs<PointerType>()->getPointeeType();
+    CanBaseReturnType = 
+      CanBaseReturnType->getAs<PointerType>()->getPointeeType();
+  } else {
+    assert(false && "Unexpected return type!");
+  }
+  
+  // We need to compare unqualified types here; consider
+  //   const T *Base::foo();
+  //   T *Derived::foo();
+  if (CanDerivedReturnType.getUnqualifiedType() == 
+      CanBaseReturnType.getUnqualifiedType()) {
+    // No adjustment needed.
+    return FinalOverriders::BaseOffset();
+  }
+  
+  const CXXRecordDecl *DerivedRD = 
+    cast<CXXRecordDecl>(cast<RecordType>(CanDerivedReturnType)->getDecl());
+  
+  const CXXRecordDecl *BaseRD = 
+    cast<CXXRecordDecl>(cast<RecordType>(CanBaseReturnType)->getDecl());
+
+  return ComputeBaseOffset(Context, BaseRD, DerivedRD);
+}
+
+FinalOverriders::BaseOffset
+FinalOverriders::ComputeThisAdjustmentBaseOffset(BaseSubobject Base,
+                                                 BaseSubobject Derived) {
+  const CXXRecordDecl *BaseRD = Base.getBase();
+  const CXXRecordDecl *DerivedRD = Derived.getBase();
+  
+  CXXBasePaths Paths(/*FindAmbiguities=*/true,
+                     /*RecordPaths=*/true, /*DetectVirtual=*/true);
+
+  if (!const_cast<CXXRecordDecl *>(DerivedRD)->
+      isDerivedFrom(const_cast<CXXRecordDecl *>(BaseRD), Paths)) {
+    assert(false && "Class must be derived from the passed in base class!");
+    return FinalOverriders::BaseOffset();
+  }
+
+  assert(!Paths.getDetectedVirtual() && "FIXME: Handle virtual bases!");
+
+  BaseOffset Offset;
+  
+  // FIXME: This is not going to be enough with virtual bases.
+  // FIXME: We should not use / 8 here.
+  int64_t DerivedToBaseOffset = 
+    (Base.getBaseOffset() - Derived.getBaseOffset()) / 8;
+  
+  Offset.NonVirtualOffset = -DerivedToBaseOffset;
+
+  return Offset;
+}
+  
+void FinalOverriders::PropagateOverrider(const CXXMethodDecl *OldMD,
+                                         BaseSubobject NewBase,
+                                         const CXXMethodDecl *NewMD,
+                                         SubobjectOffsetsMapTy &Offsets) {
+  for (CXXMethodDecl::method_iterator I = OldMD->begin_overridden_methods(),
+       E = OldMD->end_overridden_methods(); I != E; ++I) {
+    const CXXMethodDecl *OverriddenMD = *I;
+    const CXXRecordDecl *OverriddenRD = OverriddenMD->getParent();
+
+    // We want to override OverriddenMD in all subobjects, for example:
+    //
+    /// struct A { virtual void f(); };
+    /// struct B1 : A { };
+    /// struct B2 : A { };
+    /// struct C : B1, B2 { virtual void f(); };
+    ///
+    /// When overriding A::f with C::f we need to do so in both A subobjects.
+    const OffsetVectorTy &OffsetVector = Offsets[OverriddenRD];
+    
+    // Go through all the subobjects.
+    for (unsigned I = 0, E = OffsetVector.size(); I != E; ++I) {
+      uint64_t Offset = OffsetVector[I];
+
+      BaseSubobject OverriddenSubobject = BaseSubobject(OverriddenRD, Offset);
+      BaseSubobjectMethodPairTy SubobjectAndMethod =
+        std::make_pair(OverriddenSubobject, OverriddenMD);
+      
+      OverriderInfo &Overrider = OverridersMap[SubobjectAndMethod];
+
+      assert(Overrider.Method && "Did not find existing overrider!");
+
+      // Check if we need return adjustments or base adjustments.
+      // (We don't want to do this for pure virtual member functions).
+      if (!NewMD->isPure()) {
+        // Get the return adjustment base offset.
+        BaseOffset ReturnBaseOffset =
+          ComputeReturnAdjustmentBaseOffset(Context, NewMD, OverriddenMD);
+
+        if (!ReturnBaseOffset.isEmpty()) {
+          // Store the return adjustment base offset.
+          ReturnAdjustments[SubobjectAndMethod] = ReturnBaseOffset;
+        }
+        
+        // Check if we need a 'this' adjustment base offset as well.
+        if (Offset != NewBase.getBaseOffset()) {
+          BaseOffset ThisBaseOffset =
+            ComputeThisAdjustmentBaseOffset(OverriddenSubobject,
+                                            NewBase);
+          assert(!ThisBaseOffset.isEmpty() && 
+                 "Should not get an empty 'this' adjustment!");
+          
+          ThisAdjustments[SubobjectAndMethod] = ThisBaseOffset;
+        }
+      }
+
+      // Set the new overrider.
+      Overrider.Method = NewMD;
+      
+      // And propagate it further.
+      PropagateOverrider(OverriddenMD, NewBase, NewMD, Offsets);
+    }
+  }
+}
+
+void 
+FinalOverriders::MergeSubobjectOffsets(const SubobjectOffsetsMapTy &NewOffsets,
+                                       SubobjectOffsetsMapTy &Offsets) {
+  // Iterate over the new offsets.
+  for (SubobjectOffsetsMapTy::const_iterator I = NewOffsets.begin(),
+       E = NewOffsets.end(); I != E; ++I) {
+    const CXXRecordDecl *NewRD = I->first;
+    const OffsetVectorTy& NewOffsetVector = I->second;
+    
+    OffsetVectorTy &OffsetVector = Offsets[NewRD];
+    if (OffsetVector.empty()) {
+      // There were no previous offsets in this vector, just insert all entries
+      // from the new offset vector.
+      OffsetVector.append(NewOffsetVector.begin(), NewOffsetVector.end());
+      continue;
+    }
+    
+    // We need to merge the new offsets vector into the old, but we don't want
+    // to have duplicate entries. Do this by inserting the old offsets in a set
+    // so they'll be unique. After this, we iterate over the new offset vector
+    // and only append elements that aren't in the set.
+    
+    // First, add the existing offsets to the set.
+    llvm::SmallSet<uint64_t, 4> OffsetSet;
+    for (unsigned I = 0, E = OffsetVector.size(); I != E; ++I) {
+      bool Inserted = OffsetSet.insert(OffsetVector[I]);
+      if (!Inserted)
+        assert(false && "Set of offsets should be unique!");
+    }
+    
+    // Next, only add the new offsets if they are not already in the set.
+    for (unsigned I = 0, E = NewOffsetVector.size(); I != E; ++I) {
+      uint64_t Offset = NewOffsetVector[I];
+
+      if (OffsetSet.count(Offset)) {
+        // Ignore the offset.
+        continue;
+      }
+      
+      // Otherwise, add it to the offsets vector.
+      OffsetVector.push_back(Offset);
+    }
+  }
+}
+
+void FinalOverriders::ComputeFinalOverriders(BaseSubobject Base,
+                                             SubobjectOffsetsMapTy &Offsets) {
+  const CXXRecordDecl *RD = Base.getBase();
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+  
+  SubobjectOffsetsMapTy NewOffsets;
+  
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl = 
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+    
+    // Ignore bases that don't have any virtual member functions.
+    if (!BaseDecl->isPolymorphic())
+      continue;
+    
+    uint64_t BaseOffset;
+    if (I->isVirtual()) {
+      BaseOffset = MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+    } else {
+      BaseOffset = Layout.getBaseClassOffset(BaseDecl) + Base.getBaseOffset();
+    }
+    
+    // Compute the final overriders for this base.
+    ComputeFinalOverriders(BaseSubobject(BaseDecl, BaseOffset), NewOffsets);
+  }
+
+  /// Now add the overriders for this particular subobject.
+  AddOverriders(Base, NewOffsets);
+  
+  // And merge the newly discovered subobject offsets.
+  MergeSubobjectOffsets(NewOffsets, Offsets);
+
+  /// Finally, add the offset for our own subobject.
+  Offsets[RD].push_back(Base.getBaseOffset());
+}
+
+void FinalOverriders::dump(llvm::raw_ostream &Out, BaseSubobject Base) {
+  const CXXRecordDecl *RD = Base.getBase();
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl = 
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+    
+    // Ignore bases that don't have any virtual member functions.
+    if (!BaseDecl->isPolymorphic())
+      continue;
+
+    uint64_t BaseOffset;
+    if (I->isVirtual()) {
+      if (!VisitedVirtualBases.insert(BaseDecl)) {
+        // We've visited this base before.
+        continue;
+      }
+      
+      BaseOffset = MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+    } else {
+      BaseOffset = Layout.getBaseClassOffset(BaseDecl) + 
+        Base.getBaseOffset();
+    }
+
+    dump(Out, BaseSubobject(BaseDecl, BaseOffset));
+  }
+
+  Out << "Final overriders for (" << RD->getQualifiedNameAsString() << ", ";
+  Out << Base.getBaseOffset() << ")\n";
+
+  // Now dump the overriders for this base subobject.
+  for (CXXRecordDecl::method_iterator I = RD->method_begin(), 
+       E = RD->method_end(); I != E; ++I) {
+    const CXXMethodDecl *MD = *I;
+
+    if (!MD->isVirtual())
+      continue;
+  
+    OverriderInfo Overrider = getOverrider(Base, MD);
+
+    Out << "  " << MD->getQualifiedNameAsString() << " - ";
+    Out << Overrider.Method->getQualifiedNameAsString();
+    
+    AdjustmentOffsetsMapTy::const_iterator AI =
+      ReturnAdjustments.find(std::make_pair(Base, MD));
+    if (AI != ReturnAdjustments.end()) {
+      const BaseOffset &Offset = AI->second;
+
+      Out << " [ret-adj: ";
+      if (Offset.VirtualBase)
+        Out << Offset.VirtualBase->getQualifiedNameAsString() << " vbase, ";
+             
+      Out << Offset.NonVirtualOffset << " nv]";
+    }
+    
+    AI = ThisAdjustments.find(std::make_pair(Base, MD));
+    if (AI != ThisAdjustments.end()) {
+      const BaseOffset &Offset = AI->second;
+      
+      Out << " [this-adj: ";
+      if (Offset.VirtualBase)
+        Out << Offset.VirtualBase->getQualifiedNameAsString() << " vbase, ";
+      
+      Out << Offset.NonVirtualOffset << " nv]";
+    }
+    
+    Out << "\n";
+  }  
+}
+
+/// VtableComponent - Represents a single component in a vtable.
+class VtableComponent {
+public:
+  enum Kind {
+    CK_VCallOffset,
+    CK_VBaseOffset,
+    CK_OffsetToTop,
+    CK_RTTI,
+    CK_FunctionPointer,
+    
+    /// CK_CompleteDtorPointer - A pointer to the complete destructor.
+    CK_CompleteDtorPointer,
+    
+    /// CK_DeletingDtorPointer - A pointer to the deleting destructor.
+    CK_DeletingDtorPointer
+  };
+
+  static VtableComponent MakeVBaseOffset(int64_t Offset) {
+    return VtableComponent(CK_VBaseOffset, Offset);
+  }
+
+  static VtableComponent MakeOffsetToTop(int64_t Offset) {
+    return VtableComponent(CK_OffsetToTop, Offset);
+  }
+  
+  static VtableComponent MakeRTTI(const CXXRecordDecl *RD) {
+    return VtableComponent(CK_RTTI, reinterpret_cast<uintptr_t>(RD));
+  }
+
+  static VtableComponent MakeFunction(const CXXMethodDecl *MD) {
+    assert(!isa<CXXDestructorDecl>(MD) && 
+           "Don't use MakeFunction with destructors!");
+
+    return VtableComponent(CK_FunctionPointer, 
+                           reinterpret_cast<uintptr_t>(MD));
+  }
+  
+  static VtableComponent MakeCompleteDtor(const CXXDestructorDecl *DD) {
+    return VtableComponent(CK_CompleteDtorPointer,
+                           reinterpret_cast<uintptr_t>(DD));
+  }
+
+  static VtableComponent MakeDeletingDtor(const CXXDestructorDecl *DD) {
+    return VtableComponent(CK_DeletingDtorPointer, 
+                           reinterpret_cast<uintptr_t>(DD));
+  }
+
+  /// getKind - Get the kind of this vtable component.
+  Kind getKind() const {
+    return (Kind)(Value & 0x7);
+  }
+
+  int64_t getVBaseOffset() const {
+    assert(getKind() == CK_VBaseOffset && "Invalid component kind!");
+    
+    return getOffset();
+  }
+
+  int64_t getOffsetToTop() const {
+    assert(getKind() == CK_OffsetToTop && "Invalid component kind!");
+    
+    return getOffset();
+  }
+  
+  const CXXRecordDecl *getRTTIDecl() const {
+    assert(getKind() == CK_RTTI && "Invalid component kind!");
+    
+    return reinterpret_cast<CXXRecordDecl *>(getPointer());
+  }
+  
+  const CXXMethodDecl *getFunctionDecl() const {
+    assert(getKind() == CK_FunctionPointer);
+    
+    return reinterpret_cast<CXXMethodDecl *>(getPointer());
+  }
+
+  const CXXDestructorDecl *getDestructorDecl() const {
+    assert((getKind() == CK_CompleteDtorPointer ||
+            getKind() == CK_DeletingDtorPointer) && "Invalid component kind!");
+    
+    return reinterpret_cast<CXXDestructorDecl *>(getPointer());
+  }
+
+private:
+  VtableComponent(Kind ComponentKind, int64_t Offset) {
+    assert((ComponentKind == CK_VCallOffset || 
+            ComponentKind == CK_VBaseOffset ||
+            ComponentKind == CK_OffsetToTop) && "Invalid component kind!");
+    assert(Offset <= ((1LL << 56) - 1) && "Offset is too big!");
+    
+    Value = ((Offset << 3) | ComponentKind);
+  }
+
+  VtableComponent(Kind ComponentKind, uintptr_t Ptr) {
+    assert((ComponentKind == CK_RTTI || 
+            ComponentKind == CK_FunctionPointer ||
+            ComponentKind == CK_CompleteDtorPointer ||
+            ComponentKind == CK_DeletingDtorPointer) &&
+            "Invalid component kind!");
+    
+    assert((Ptr & 7) == 0 && "Pointer not sufficiently aligned!");
+    
+    Value = Ptr | ComponentKind;
+  }
+  
+  int64_t getOffset() const {
+    assert((getKind() == CK_VCallOffset || getKind() == CK_VBaseOffset ||
+            getKind() == CK_OffsetToTop) && "Invalid component kind!");
+    
+    return Value >> 3;
+  }
+
+  uintptr_t getPointer() const {
+    assert((getKind() == CK_RTTI || 
+            getKind() == CK_FunctionPointer ||
+            getKind() == CK_CompleteDtorPointer ||
+            getKind() == CK_DeletingDtorPointer) &&
+           "Invalid component kind!");
+    
+    return static_cast<uintptr_t>(Value & ~7ULL);
+  }
+  
+  /// The kind is stored in the lower 3 bits of the value. For offsets, we
+  /// make use of the facts that classes can't be larger than 2^55 bytes,
+  /// so we store the offset in the lower part of the 61 bytes that remain.
+  /// (The reason that we're not simply using a PointerIntPair here is that we
+  /// need the offsets to be 64-bit, even when on a 32-bit machine).
+  int64_t Value;
+};
+
+/// VtableBuilder - Class for building vtable layout information.
 class VtableBuilder {
 public:
+  /// PrimaryBasesSetTy - A set of direct and indirect primary bases.
+  typedef llvm::SmallPtrSet<const CXXRecordDecl *, 8> PrimaryBasesSetTy;
+
+private:
+  /// VtableInfo - Global vtable information.
+  CGVtableInfo &VtableInfo;
+  
+  /// MostDerivedClass - The most derived class for which we're building this
+  /// vtable.
+  const CXXRecordDecl *MostDerivedClass;
+
+  /// Context - The ASTContext which we will use for layout information.
+  ASTContext &Context;
+  
+  /// FinalOverriders - The final overriders of the most derived class.
+  FinalOverriders Overriders;
+
+  /// VCallAndVBaseOffsets - The vcall and vbase offset, of the vtable we're
+  // building (in reverse order).
+  llvm::SmallVector<VtableComponent, 64> VCallAndVBaseOffsets;
+
+  /// Components - The components of the vtable being built.
+  llvm::SmallVector<VtableComponent, 64> Components;
+
+  /// AddressPoints - Address points for the vtable being built.
+  CGVtableInfo::AddressPointsMapTy AddressPoints;
+
+  /// ReturnAdjustment - A return adjustment.
+  struct ReturnAdjustment {
+    /// NonVirtual - The non-virtual adjustment from the derived object to its
+    /// nearest virtual base.
+    int64_t NonVirtual;
+    
+    /// VBaseOffsetOffset - The offset, in bytes, relative to the address point 
+    /// of the virtual base class offset.
+    int64_t VBaseOffsetOffset;
+    
+    ReturnAdjustment() : NonVirtual(0), VBaseOffsetOffset(0) { }
+    
+    bool isEmpty() const { return !NonVirtual && !VBaseOffsetOffset; }
+  };
+  
+  /// ReturnAdjustments - The return adjustments needed in this vtable.
+  llvm::SmallVector<std::pair<uint64_t, ReturnAdjustment>, 16> 
+    ReturnAdjustments;
+
+  /// ThisAdjustment - A 'this' pointer adjustment thunk.
+  struct ThisAdjustment {
+    /// NonVirtual - The non-virtual adjustment from the derived object to its
+    /// nearest virtual base.
+    int64_t NonVirtual;
+
+    /// FIXME: Add VCallOffsetOffset here.
+    
+    ThisAdjustment() : NonVirtual(0) { }
+
+    bool isEmpty() const { return !NonVirtual; }
+  };
+  
+  /// ThisAdjustments - The 'this' pointer adjustments needed in this vtable.
+  llvm::SmallVector<std::pair<uint64_t, ThisAdjustment>, 16> 
+    ThisAdjustments;
+  
+  typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
+
+  /// AddVCallAndVBaseOffsets - Add vcall offsets and vbase offsets for the
+  /// given class.
+  void AddVCallAndVBaseOffsets(const CXXRecordDecl *RD, int64_t OffsetToTop,
+                               VisitedVirtualBasesSetTy &VBases);
+
+  /// AddVBaseOffsets - Add vbase offsets for the given class.
+  void AddVBaseOffsets(const CXXRecordDecl *RD, int64_t OffsetToTop,
+                       VisitedVirtualBasesSetTy &VBases);
+
+  /// ComputeReturnAdjustment - Compute the return adjustment given a return
+  /// adjustment base offset.
+  ReturnAdjustment ComputeReturnAdjustment(FinalOverriders::BaseOffset Offset);
+  
+  /// ComputeThisAdjustment - Compute the 'this' pointer  adjustment given a 
+  /// 'this' pointer adjustment base offset.
+  ThisAdjustment ComputeThisAdjustment(FinalOverriders::BaseOffset Offset);
+  
+  /// AddMethod - Add a single virtual member function to the vtable
+  /// components vector.
+  void AddMethod(const CXXMethodDecl *MD, ReturnAdjustment ReturnAdjustment,
+                 ThisAdjustment ThisAdjustment);
+
+  /// AddMethods - Add the methods of this base subobject and all its
+  /// primary bases to the vtable components vector.
+  void AddMethods(BaseSubobject Base, PrimaryBasesSetTy &PrimaryBases);
+  
+  /// LayoutVtable - Layout a vtable and all its secondary vtables.
+  void LayoutVtable(BaseSubobject Base);
+  
+public:
+  VtableBuilder(CGVtableInfo &VtableInfo, const CXXRecordDecl *MostDerivedClass)
+    : VtableInfo(VtableInfo), MostDerivedClass(MostDerivedClass), 
+    Context(MostDerivedClass->getASTContext()), Overriders(MostDerivedClass) { 
+
+    LayoutVtable(BaseSubobject(MostDerivedClass, 0));
+  }
+
+  /// dumpLayout - Dump the vtable layout.
+  void dumpLayout(llvm::raw_ostream&);
+};
+
+/// OverridesMethodInPrimaryBase - Checks whether whether this virtual member 
+/// function overrides a member function in a direct or indirect primary base.
+/// Returns the overridden member function, or null if none was found.
+static const CXXMethodDecl * 
+OverridesMethodInPrimaryBase(const CXXMethodDecl *MD,
+                             VtableBuilder::PrimaryBasesSetTy &PrimaryBases) {
+  for (CXXMethodDecl::method_iterator I = MD->begin_overridden_methods(),
+       E = MD->end_overridden_methods(); I != E; ++I) {
+    const CXXMethodDecl *OverriddenMD = *I;
+    const CXXRecordDecl *OverriddenRD = OverriddenMD->getParent();
+    assert(OverriddenMD->isCanonicalDecl() &&
+           "Should have the canonical decl of the overridden RD!");
+    
+    if (PrimaryBases.count(OverriddenRD))
+      return OverriddenMD;
+  }
+      
+  return 0;
+}
+
+VtableBuilder::ReturnAdjustment 
+VtableBuilder::ComputeReturnAdjustment(FinalOverriders::BaseOffset Offset) {
+  ReturnAdjustment Adjustment;
+  
+  if (!Offset.isEmpty()) {
+    if (Offset.VirtualBase) {
+      // Get the virtual base offset offset.
+      Adjustment.VBaseOffsetOffset = 
+        VtableInfo.getVirtualBaseOffsetIndex(Offset.DerivedClass,
+                                             Offset.VirtualBase);
+      // FIXME: Once the assert in getVirtualBaseOffsetIndex is back again,
+      // we can get rid of this assert.
+      assert(Adjustment.VBaseOffsetOffset != 0 && 
+             "Invalid base offset offset!");
+    }
+
+    Adjustment.NonVirtual = Offset.NonVirtualOffset;
+  }
+  
+  return Adjustment;
+}
+
+VtableBuilder::ThisAdjustment
+VtableBuilder::ComputeThisAdjustment(FinalOverriders::BaseOffset Offset) {
+  ThisAdjustment Adjustment;
+  
+  if (!Offset.isEmpty()) {
+    assert(!Offset.VirtualBase && "FIXME: Handle virtual bases!");
+    Adjustment.NonVirtual = Offset.NonVirtualOffset;
+  }
+  
+  return Adjustment;
+}
+
+void 
+VtableBuilder::AddVCallAndVBaseOffsets(const CXXRecordDecl *RD,
+                                       int64_t OffsetToTop,
+                                       VisitedVirtualBasesSetTy &VBases) {
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+  
+  // Itanium C++ ABI 2.5.2:
+  //   ..in classes sharing a virtual table with a primary base class, the vcall
+  //   and vbase offsets added by the derived class all come before the vcall
+  //   and vbase offsets required by the base class, so that the latter may be
+  //   laid out as required by the base class without regard to additions from
+  //   the derived class(es).
+
+  // (Since we're emitting the vcall and vbase offsets in reverse order, we'll
+  // emit them for the primary base first).
+  if (const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase())
+    AddVCallAndVBaseOffsets(PrimaryBase, OffsetToTop, VBases);
+
+  AddVBaseOffsets(RD, OffsetToTop, VBases);
+}
+
+void VtableBuilder::AddVBaseOffsets(const CXXRecordDecl *RD,
+                                    int64_t OffsetToTop,
+                                    VisitedVirtualBasesSetTy &VBases) {
+  const ASTRecordLayout &MostDerivedClassLayout = 
+    Context.getASTRecordLayout(MostDerivedClass);
+
+  // Add vbase offsets.
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl =
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    // Check if this is a virtual base that we haven't visited before.
+    if (I->isVirtual() && VBases.insert(BaseDecl)) {
+      // FIXME: We shouldn't use / 8 here.
+      uint64_t Offset = 
+        OffsetToTop + MostDerivedClassLayout.getVBaseClassOffset(BaseDecl) / 8;
+    
+      VCallAndVBaseOffsets.push_back(VtableComponent::MakeVBaseOffset(Offset));
+    }
+    
+    // Check the base class looking for more vbase offsets.
+    AddVBaseOffsets(BaseDecl, OffsetToTop, VBases);
+  }
+}
+
+void 
+VtableBuilder::AddMethod(const CXXMethodDecl *MD,
+                         ReturnAdjustment ReturnAdjustment,
+                         ThisAdjustment ThisAdjustment) {
+  if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
+    assert(ReturnAdjustment.isEmpty() && 
+           "Destructor can't have return adjustment!");
+    // Add the 'this' pointer adjustments if necessary.
+    if (!ThisAdjustment.isEmpty()) {
+      ThisAdjustments.push_back(std::make_pair(Components.size(),
+                                               ThisAdjustment));
+      ThisAdjustments.push_back(std::make_pair(Components.size() + 1,
+                                               ThisAdjustment));
+    }
+
+    // Add both the complete destructor and the deleting destructor.
+    Components.push_back(VtableComponent::MakeCompleteDtor(DD));
+    Components.push_back(VtableComponent::MakeDeletingDtor(DD));
+  } else {
+    // Add the return adjustment if necessary.
+    if (!ReturnAdjustment.isEmpty())
+      ReturnAdjustments.push_back(std::make_pair(Components.size(),
+                                                 ReturnAdjustment));
+
+    // Add the 'this' pointer adjustment if necessary.
+    if (!ThisAdjustment.isEmpty())
+      ThisAdjustments.push_back(std::make_pair(Components.size(),
+                                               ThisAdjustment));
+
+    // Add the function.
+    Components.push_back(VtableComponent::MakeFunction(MD));
+  }
+}
+
+void 
+VtableBuilder::AddMethods(BaseSubobject Base, PrimaryBasesSetTy &PrimaryBases) {
+  const CXXRecordDecl *RD = Base.getBase();
+
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+
+  if (const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase()) {
+    if (Layout.getPrimaryBaseWasVirtual())
+      assert(false && "FIXME: Handle vbases here.");
+    else
+      assert(Layout.getBaseClassOffset(PrimaryBase) == 0 &&
+             "Primary base should have a zero offset!");
+    
+    AddMethods(BaseSubobject(PrimaryBase, Base.getBaseOffset()), PrimaryBases);
+    
+    if (!PrimaryBases.insert(PrimaryBase))
+      assert(false && "Found a duplicate primary base!");
+  }
+
+  // Now go through all virtual member functions and add them.
+  for (CXXRecordDecl::method_iterator I = RD->method_begin(),
+       E = RD->method_end(); I != E; ++I) {
+    const CXXMethodDecl *MD = *I;
+  
+    if (!MD->isVirtual())
+      continue;
+
+    // Get the final overrider.
+    FinalOverriders::OverriderInfo Overrider = 
+      Overriders.getOverrider(Base, MD);
+
+    // Check if this virtual member function overrides a method in a primary
+    // base. If this is the case, and the return type doesn't require adjustment
+    // then we can just use the member function from the primary base.
+    if (const CXXMethodDecl *OverriddenMD = 
+        OverridesMethodInPrimaryBase(MD, PrimaryBases)) {
+      if (ComputeReturnAdjustmentBaseOffset(Context, MD, 
+                                            OverriddenMD).isEmpty())
+        continue;
+    }
+
+    // Check if this overrider needs a return adjustment.
+    FinalOverriders::BaseOffset ReturnAdjustmentOffset = 
+      Overriders.getReturnAdjustmentOffset(Base, MD);
+
+    ReturnAdjustment ReturnAdjustment = 
+      ComputeReturnAdjustment(ReturnAdjustmentOffset);
+    
+    // Check if this overrider needs a 'this' pointer adjustment.
+    FinalOverriders::BaseOffset ThisAdjustmentOffset =
+      Overriders.getThisAdjustmentOffset(Base, MD);
+    
+    ThisAdjustment ThisAdjustment = ComputeThisAdjustment(ThisAdjustmentOffset);
+    
+    AddMethod(Overrider.Method, ReturnAdjustment, ThisAdjustment);
+  }
+}
+
+void VtableBuilder::LayoutVtable(BaseSubobject Base) {
+  const CXXRecordDecl *RD = Base.getBase();
+  assert(RD->isDynamicClass() && "class does not have a vtable!");
+
+  int64_t OffsetToTop = -(int64_t)Base.getBaseOffset() / 8;
+
+  // Add vcall and vbase offsets for this vtable.
+  VisitedVirtualBasesSetTy VBases;
+  AddVCallAndVBaseOffsets(RD, OffsetToTop, VBases);
+
+  // Reverse them and add them to the vtable components.
+  std::reverse(VCallAndVBaseOffsets.begin(), VCallAndVBaseOffsets.end());
+  Components.append(VCallAndVBaseOffsets.begin(), VCallAndVBaseOffsets.end());
+  VCallAndVBaseOffsets.clear();
+  
+  // Add the offset to top.
+  // FIXME: This is not going to be right for construction vtables.
+  // FIXME: We should not use / 8 here.
+  Components.push_back(VtableComponent::MakeOffsetToTop(OffsetToTop));
+  
+  // Next, add the RTTI.
+  Components.push_back(VtableComponent::MakeRTTI(MostDerivedClass));
+  
+  uint64_t AddressPoint = Components.size();
+
+  // Now go through all virtual member functions and add them.
+  PrimaryBasesSetTy PrimaryBases;
+  AddMethods(Base, PrimaryBases);
+
+  // Record the address point.
+  AddressPoints.insert(std::make_pair(Base, AddressPoint));
+  
+  // Record the address points for all primary bases.
+  for (PrimaryBasesSetTy::const_iterator I = PrimaryBases.begin(),
+       E = PrimaryBases.end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl = *I;
+    
+    // We know that all the primary bases have the same offset as the base
+    // subobject.
+    BaseSubobject PrimaryBase(BaseDecl, Base.getBaseOffset());
+    AddressPoints.insert(std::make_pair(PrimaryBase, AddressPoint));
+  }
+
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+  const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
+
+  // Layout secondary vtables.
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl = 
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    // Ignore bases that don't have a vtable.
+    if (!BaseDecl->isDynamicClass())
+      continue;
+    
+    // Ignore the primary base.
+    if (BaseDecl == PrimaryBase)
+      continue;
+
+    // Ignore virtual bases, we'll emit them later.
+    if (I->isVirtual())
+      continue;
+    
+    // Get the base offset of this base.
+    uint64_t BaseOffset = Base.getBaseOffset() + 
+      Layout.getBaseClassOffset(BaseDecl);
+
+    // Layout this secondary vtable.
+    LayoutVtable(BaseSubobject(BaseDecl, BaseOffset));
+  }
+  
+  // FIXME: Emit vtables for virtual bases here.
+}
+
+/// dumpLayout - Dump the vtable layout.
+void VtableBuilder::dumpLayout(llvm::raw_ostream& Out) {
+  
+  Out << "Vtable for '" << MostDerivedClass->getQualifiedNameAsString();
+  Out << "' (" << Components.size() << " entries).\n";
+
+  // Iterate through the address points and insert them into a new map where
+  // they are keyed by the index and not the base object.
+  // Since an address point can be shared by multiple subobjects, we use an
+  // STL multimap.
+  std::multimap<uint64_t, BaseSubobject> AddressPointsByIndex;
+  for (CGVtableInfo::AddressPointsMapTy::const_iterator I = 
+       AddressPoints.begin(), E = AddressPoints.end(); I != E; ++I) {
+    const BaseSubobject& Base = I->first;
+    uint64_t Index = I->second;
+    
+    AddressPointsByIndex.insert(std::make_pair(Index, Base));
+  }
+  
+  unsigned NextReturnAdjustmentIndex = 0;
+  unsigned NextThisAdjustmentIndex = 0;
+  for (unsigned I = 0, E = Components.size(); I != E; ++I) {
+    uint64_t Index = I;
+    
+    if (AddressPointsByIndex.count(I)) {
+      if (AddressPointsByIndex.count(Index) == 1) {
+        const BaseSubobject &Base = AddressPointsByIndex.find(Index)->second;
+        
+        // FIXME: Instead of dividing by 8, we should be using CharUnits.
+        Out << "       -- (" << Base.getBase()->getQualifiedNameAsString();
+        Out << ", " << Base.getBaseOffset() / 8 << ") vtable address --\n";
+      } else {
+        uint64_t BaseOffset = 
+          AddressPointsByIndex.lower_bound(Index)->second.getBaseOffset();
+        
+        // We store the class names in a set to get a stable order.
+        std::set<std::string> ClassNames;
+        for (std::multimap<uint64_t, BaseSubobject>::const_iterator I =
+             AddressPointsByIndex.lower_bound(Index), E =
+             AddressPointsByIndex.upper_bound(Index); I != E; ++I) {
+          assert(I->second.getBaseOffset() == BaseOffset &&
+                 "Invalid base offset!");
+          const CXXRecordDecl *RD = I->second.getBase();
+          ClassNames.insert(RD->getQualifiedNameAsString());
+        }
+        
+        for (std::set<std::string>::const_iterator I = ClassNames.begin(),
+             E = ClassNames.end(); I != E; ++I) {
+          // FIXME: Instead of dividing by 8, we should be using CharUnits.
+          Out << "       -- (" << *I;
+          Out << ", " << BaseOffset / 8 << ") vtable address --\n";
+        }
+      }
+    }
+
+    Out << llvm::format("%4d | ", I);
+
+    const VtableComponent &Component = Components[I];
+
+    // Dump the component.
+    switch (Component.getKind()) {
+    // FIXME: Remove this default case.
+    default:
+      assert(false && "Unhandled component kind!");
+      break;
+
+    case VtableComponent::CK_VBaseOffset:
+      Out << "vbase_offset (" << Component.getVBaseOffset() << ")";
+      break;
+
+    case VtableComponent::CK_OffsetToTop:
+      Out << "offset_to_top (" << Component.getOffsetToTop() << ")";
+      break;
+    
+    case VtableComponent::CK_RTTI:
+      Out << Component.getRTTIDecl()->getQualifiedNameAsString() << " RTTI";
+      break;
+    
+    case VtableComponent::CK_FunctionPointer: {
+      const CXXMethodDecl *MD = Component.getFunctionDecl();
+
+      std::string Str = 
+        PredefinedExpr::ComputeName(PredefinedExpr::PrettyFunctionNoVirtual, 
+                                    MD);
+      Out << Str;
+      if (MD->isPure())
+        Out << " [pure]";
+
+      // If this function pointer has a return adjustment, dump it.
+      if (NextReturnAdjustmentIndex < ReturnAdjustments.size() && 
+          ReturnAdjustments[NextReturnAdjustmentIndex].first == I) {
+        const ReturnAdjustment Adjustment = 
+          ReturnAdjustments[NextReturnAdjustmentIndex].second;
+        
+        Out << "\n       [return adjustment: ";
+        Out << Adjustment.NonVirtual << " non-virtual";
+        
+        if (Adjustment.VBaseOffsetOffset)
+          Out << ", " << Adjustment.VBaseOffsetOffset << " vbase offset offset";
+        Out << ']';
+
+        NextReturnAdjustmentIndex++;
+      }
+      
+      // If this function pointer has a 'this' pointer adjustment, dump it.
+      if (NextThisAdjustmentIndex < ThisAdjustments.size() && 
+          ThisAdjustments[NextThisAdjustmentIndex].first == I) {
+        const ThisAdjustment Adjustment = 
+          ThisAdjustments[NextThisAdjustmentIndex].second;
+        
+        Out << "\n       [this adjustment: ";
+        Out << Adjustment.NonVirtual << " non-virtual";
+        
+        Out << ']';
+        
+        NextThisAdjustmentIndex++;
+      }
+
+      break;
+    }
+
+    case VtableComponent::CK_CompleteDtorPointer: {
+      const CXXDestructorDecl *DD = Component.getDestructorDecl();
+      
+      Out << DD->getQualifiedNameAsString() << "() [complete]";
+      if (DD->isPure())
+        Out << " [pure]";
+
+      break;
+    }
+
+    case VtableComponent::CK_DeletingDtorPointer: {
+      const CXXDestructorDecl *DD = Component.getDestructorDecl();
+      
+      Out << DD->getQualifiedNameAsString() << "() [deleting]";
+      if (DD->isPure())
+        Out << " [pure]";
+
+      break;
+    }
+
+    }
+
+    Out << '\n';
+  }
+  
+}
+  
+}
+
+namespace {
+class OldVtableBuilder {
+public:
   /// Index_t - Vtable index type.
   typedef uint64_t Index_t;
   typedef std::vector<std::pair<GlobalDecl,
@@ -57,10 +1288,11 @@ private:
   llvm::LLVMContext &VMContext;
   CodeGenModule &CGM;  // Per-module state.
   
-  llvm::DenseMap<GlobalDecl, Index_t> VCall;
+  llvm::DenseMap<const CXXMethodDecl *, Index_t> VCall;
   llvm::DenseMap<GlobalDecl, Index_t> VCallOffset;
+  llvm::DenseMap<GlobalDecl, Index_t> VCallOffsetForVCall;
   // This is the offset to the nearest virtual base
-  llvm::DenseMap<GlobalDecl, Index_t> NonVirtualOffset;
+  llvm::DenseMap<const CXXMethodDecl *, Index_t> NonVirtualOffset;
   llvm::DenseMap<const CXXRecordDecl *, Index_t> VBIndex;
 
   /// PureVirtualFunction - Points to __cxa_pure_virtual.
@@ -180,6 +1412,196 @@ private:
     return *ref;
   }
   
+  bool DclIsSame(const FunctionDecl *New, const FunctionDecl *Old) {
+    FunctionTemplateDecl *OldTemplate = Old->getDescribedFunctionTemplate();
+    FunctionTemplateDecl *NewTemplate = New->getDescribedFunctionTemplate();
+
+    // C++ [temp.fct]p2:
+    //   A function template can be overloaded with other function templates
+    //   and with normal (non-template) functions.
+    if ((OldTemplate == 0) != (NewTemplate == 0))
+      return false;
+
+    // Is the function New an overload of the function Old?
+    QualType OldQType = CGM.getContext().getCanonicalType(Old->getType());
+    QualType NewQType = CGM.getContext().getCanonicalType(New->getType());
+
+    // Compare the signatures (C++ 1.3.10) of the two functions to
+    // determine whether they are overloads. If we find any mismatch
+    // in the signature, they are overloads.
+
+    // If either of these functions is a K&R-style function (no
+    // prototype), then we consider them to have matching signatures.
+    if (isa<FunctionNoProtoType>(OldQType.getTypePtr()) ||
+        isa<FunctionNoProtoType>(NewQType.getTypePtr()))
+      return true;
+
+    FunctionProtoType* OldType = cast<FunctionProtoType>(OldQType);
+    FunctionProtoType* NewType = cast<FunctionProtoType>(NewQType);
+
+    // The signature of a function includes the types of its
+    // parameters (C++ 1.3.10), which includes the presence or absence
+    // of the ellipsis; see C++ DR 357).
+    if (OldQType != NewQType &&
+        (OldType->getNumArgs() != NewType->getNumArgs() ||
+         OldType->isVariadic() != NewType->isVariadic() ||
+         !std::equal(OldType->arg_type_begin(), OldType->arg_type_end(),
+                     NewType->arg_type_begin())))
+      return false;
+
+#if 0
+    // C++ [temp.over.link]p4:
+    //   The signature of a function template consists of its function
+    //   signature, its return type and its template parameter list. The names
+    //   of the template parameters are significant only for establishing the
+    //   relationship between the template parameters and the rest of the
+    //   signature.
+    //
+    // We check the return type and template parameter lists for function
+    // templates first; the remaining checks follow.
+    if (NewTemplate &&
+        (!TemplateParameterListsAreEqual(NewTemplate->getTemplateParameters(),
+                                         OldTemplate->getTemplateParameters(),
+                                         TPL_TemplateMatch) ||
+         OldType->getResultType() != NewType->getResultType()))
+      return false;
+#endif
+
+    // If the function is a class member, its signature includes the
+    // cv-qualifiers (if any) on the function itself.
+    //
+    // As part of this, also check whether one of the member functions
+    // is static, in which case they are not overloads (C++
+    // 13.1p2). While not part of the definition of the signature,
+    // this check is important to determine whether these functions
+    // can be overloaded.
+    const CXXMethodDecl* OldMethod = dyn_cast<CXXMethodDecl>(Old);
+    const CXXMethodDecl* NewMethod = dyn_cast<CXXMethodDecl>(New);
+    if (OldMethod && NewMethod &&
+        !OldMethod->isStatic() && !NewMethod->isStatic() &&
+        OldMethod->getTypeQualifiers() != NewMethod->getTypeQualifiers())
+      return false;
+  
+    // The signatures match; this is not an overload.
+    return true;
+  }
+
+  typedef llvm::DenseMap<const CXXMethodDecl *, const CXXMethodDecl*>
+    ForwardUnique_t;
+  ForwardUnique_t ForwardUnique;
+  llvm::DenseMap<const CXXMethodDecl*, const CXXMethodDecl*> UniqueOverrider;
+
+  void BuildUniqueOverrider(const CXXMethodDecl *U, const CXXMethodDecl *MD) {
+    const CXXMethodDecl *PrevU = UniqueOverrider[MD];
+    assert(U && "no unique overrider");
+    if (PrevU == U)
+      return;
+    if (PrevU != U && PrevU != 0) {
+      // If already set, note the two sets as the same
+      if (0)
+        printf("%s::%s same as %s::%s\n",
+               PrevU->getParent()->getNameAsCString(),
+               PrevU->getNameAsCString(),
+               U->getParent()->getNameAsCString(),
+               U->getNameAsCString());
+      ForwardUnique[PrevU] = U;
+      return;
+    }
+
+    // Not set, set it now
+    if (0)
+      printf("marking %s::%s %p override as %s::%s\n",
+             MD->getParent()->getNameAsCString(),
+             MD->getNameAsCString(),
+             (void*)MD,
+             U->getParent()->getNameAsCString(),
+             U->getNameAsCString());
+    UniqueOverrider[MD] = U;
+
+    for (CXXMethodDecl::method_iterator mi = MD->begin_overridden_methods(),
+           me = MD->end_overridden_methods(); mi != me; ++mi) {
+      BuildUniqueOverrider(U, *mi);
+    }
+  }
+
+  void BuildUniqueOverriders(const CXXRecordDecl *RD) {
+    if (0) printf("walking %s\n", RD->getNameAsCString());
+    for (CXXRecordDecl::method_iterator i = RD->method_begin(),
+           e = RD->method_end(); i != e; ++i) {
+      const CXXMethodDecl *MD = *i;
+      if (!MD->isVirtual())
+        continue;
+
+      if (UniqueOverrider[MD] == 0) {
+        // Only set this, if it hasn't been set yet.
+        BuildUniqueOverrider(MD, MD);
+        if (0)
+          printf("top set is %s::%s %p\n",
+                  MD->getParent()->getNameAsCString(),
+                  MD->getNameAsCString(),
+                  (void*)MD);
+        ForwardUnique[MD] = MD;
+      }
+    }
+    for (CXXRecordDecl::base_class_const_iterator i = RD->bases_begin(),
+           e = RD->bases_end(); i != e; ++i) {
+      const CXXRecordDecl *Base =
+        cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
+      BuildUniqueOverriders(Base);
+    }
+  }
+
+  static int DclCmp(const void *p1, const void *p2) {
+    const CXXMethodDecl *MD1 = *(const CXXMethodDecl *const *)p1;
+    const CXXMethodDecl *MD2 = *(const CXXMethodDecl *const *)p2;
+
+    return (DeclarationName::compare(MD1->getDeclName(), MD2->getDeclName()));
+  }
+  
+  void MergeForwarding() {
+    typedef llvm::SmallVector<const CXXMethodDecl *, 100>  A_t;
+    A_t A;
+    for (ForwardUnique_t::iterator I = ForwardUnique.begin(),
+           E = ForwardUnique.end(); I != E; ++I) {
+      if (I->first == I->second)
+        // Only add the roots of all trees
+        A.push_back(I->first);
+    }
+    llvm::array_pod_sort(A.begin(), A.end(), DclCmp);
+    for (A_t::iterator I = A.begin(),
+           E = A.end(); I != E; ++I) {
+      A_t::iterator J = I;
+      while (++J != E  && DclCmp(I, J) == 0)
+        if (DclIsSame(*I, *J)) {
+          if (0) printf("connecting %s\n", (*I)->getNameAsCString());
+          ForwardUnique[*J] = *I;
+        }
+    }
+  }
+
+  const CXXMethodDecl *getUnique(const CXXMethodDecl *MD) {
+    const CXXMethodDecl *U = UniqueOverrider[MD];
+    assert(U && "unique overrider not found");
+    while (ForwardUnique.count(U)) {
+      const CXXMethodDecl *NU = ForwardUnique[U];
+      if (NU == U) break;
+      U = NU;
+    }
+    return U;
+  }
+
+  GlobalDecl getUnique(GlobalDecl GD) {
+    const CXXMethodDecl *Unique = getUnique(cast<CXXMethodDecl>(GD.getDecl()));
+    
+    if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(Unique))
+      return GlobalDecl(CD, GD.getCtorType());
+    
+    if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(Unique))
+      return GlobalDecl(DD, GD.getDtorType());
+    
+    return Unique;
+  }
+
   /// getPureVirtualFn - Return the __cxa_pure_virtual function.
   llvm::Constant* getPureVirtualFn() {
     if (!PureVirtualFn) {
@@ -193,7 +1615,7 @@ private:
   }
   
 public:
-  VtableBuilder(const CXXRecordDecl *MostDerivedClass,
+  OldVtableBuilder(const CXXRecordDecl *MostDerivedClass,
                 const CXXRecordDecl *l, uint64_t lo, CodeGenModule &cgm,
                 bool build, CGVtableInfo::AddressPointsMapTy& AddressPoints)
     : BuildVtable(build), MostDerivedClass(MostDerivedClass), LayoutClass(l),
@@ -209,6 +1631,8 @@ public:
       QualType ClassType = CGM.getContext().getTagDeclType(MostDerivedClass);
       rtti = CGM.GetAddrOfRTTIDescriptor(ClassType);
     }
+    BuildUniqueOverriders(MostDerivedClass);
+    MergeForwarding();
   }
 
   // getVtableComponents - Returns a reference to the vtable components.
@@ -384,17 +1808,24 @@ public:
     // entry.
     Methods.AddMethod(GD);
 
-    VCallOffset[GD] = Offset/8;
+    VCallOffset[GD] = Offset/8 - CurrentVBaseOffset/8;
+
     if (MorallyVirtual) {
-      Index_t &idx = VCall[GD];
+      GlobalDecl UGD = getUnique(GD);
+      const CXXMethodDecl *UMD = cast<CXXMethodDecl>(UGD.getDecl());
+  
+      assert(UMD && "final overrider not found");
+
+      Index_t &idx = VCall[UMD];
       // Allocate the first one, after that, we reuse the previous one.
       if (idx == 0) {
-        NonVirtualOffset[GD] = CurrentVBaseOffset/8 - Offset/8;
+        VCallOffsetForVCall[UGD] = Offset/8;
+        NonVirtualOffset[UMD] = Offset/8 - CurrentVBaseOffset/8;
         idx = VCalls.size()+1;
-        VCalls.push_back(0);
+        VCalls.push_back(Offset/8 - CurrentVBaseOffset/8);
         D1(printf("  vcall for %s at %d with delta %d\n",
                   dyn_cast<CXXMethodDecl>(GD.getDecl())->getNameAsCString(),
-                  (int)-VCalls.size()-3, 0));
+                  (int)-VCalls.size()-3, (int)VCalls[idx-1]));
       }
     }
   }
@@ -459,6 +1890,9 @@ public:
     }
     VCalls.clear();
     VCall.clear();
+    VCallOffsetForVCall.clear();
+    VCallOffset.clear();
+    NonVirtualOffset.clear();
   }
 
   void AddAddressPoints(const CXXRecordDecl *RD, uint64_t Offset,
@@ -699,84 +2133,7 @@ public:
 };
 } // end anonymous namespace
 
-/// TypeConversionRequiresAdjustment - Returns whether conversion from a 
-/// derived type to a base type requires adjustment.
-static bool
-TypeConversionRequiresAdjustment(ASTContext &Ctx,
-                                 const CXXRecordDecl *DerivedDecl,
-                                 const CXXRecordDecl *BaseDecl) {
-  CXXBasePaths Paths(/*FindAmbiguities=*/false,
-                     /*RecordPaths=*/true, /*DetectVirtual=*/true);
-  if (!const_cast<CXXRecordDecl *>(DerivedDecl)->
-      isDerivedFrom(const_cast<CXXRecordDecl *>(BaseDecl), Paths)) {
-    assert(false && "Class must be derived from the passed in base class!");
-    return false;
-  }
-  
-  // If we found a virtual base we always want to require adjustment.
-  if (Paths.getDetectedVirtual())
-    return true;
-  
-  const CXXBasePath &Path = Paths.front();
-  
-  for (size_t Start = 0, End = Path.size(); Start != End; ++Start) {
-    const CXXBasePathElement &Element = Path[Start];
-    
-    // Check the base class offset.
-    const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(Element.Class);
-    
-    const RecordType *BaseType = Element.Base->getType()->getAs<RecordType>();
-    const CXXRecordDecl *Base = cast<CXXRecordDecl>(BaseType->getDecl());
-    
-    if (Layout.getBaseClassOffset(Base) != 0) {
-      // This requires an adjustment.
-      return true;
-    }
-  }
-  
-  return false;
-}
-
-static bool 
-TypeConversionRequiresAdjustment(ASTContext &Ctx,
-                                 QualType DerivedType, QualType BaseType) {
-  // Canonicalize the types.
-  QualType CanDerivedType = Ctx.getCanonicalType(DerivedType);
-  QualType CanBaseType = Ctx.getCanonicalType(BaseType);
-  
-  assert(CanDerivedType->getTypeClass() == CanBaseType->getTypeClass() && 
-         "Types must have same type class!");
-  
-  if (CanDerivedType == CanBaseType) {
-    // No adjustment needed.
-    return false;
-  }
-  
-  if (const ReferenceType *RT = dyn_cast<ReferenceType>(CanDerivedType)) {
-    CanDerivedType = RT->getPointeeType();
-    CanBaseType = cast<ReferenceType>(CanBaseType)->getPointeeType();
-  } else if (const PointerType *PT = dyn_cast<PointerType>(CanDerivedType)) {
-    CanDerivedType = PT->getPointeeType();
-    CanBaseType = cast<PointerType>(CanBaseType)->getPointeeType();
-  } else {
-    assert(false && "Unexpected return type!");
-  }
-  
-  if (CanDerivedType == CanBaseType) {
-    // No adjustment needed.
-    return false;
-  }
-  
-  const CXXRecordDecl *DerivedDecl = 
-    cast<CXXRecordDecl>(cast<RecordType>(CanDerivedType)->getDecl());
-  
-  const CXXRecordDecl *BaseDecl = 
-    cast<CXXRecordDecl>(cast<RecordType>(CanBaseType)->getDecl());
-  
-  return TypeConversionRequiresAdjustment(Ctx, DerivedDecl, BaseDecl);
-}
-
-bool VtableBuilder::OverrideMethod(GlobalDecl GD, bool MorallyVirtual,
+bool OldVtableBuilder::OverrideMethod(GlobalDecl GD, bool MorallyVirtual,
                                    Index_t OverrideOffset, Index_t Offset,
                                    int64_t CurrentVBaseOffset) {
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
@@ -788,6 +2145,7 @@ bool VtableBuilder::OverrideMethod(GlobalDecl GD, bool MorallyVirtual,
   for (CXXMethodDecl::method_iterator mi = MD->begin_overridden_methods(),
        e = MD->end_overridden_methods(); mi != e; ++mi) {
     GlobalDecl OGD;
+    GlobalDecl OGD2;
     
     const CXXMethodDecl *OMD = *mi;
     if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(OMD))
@@ -801,6 +2159,8 @@ bool VtableBuilder::OverrideMethod(GlobalDecl GD, bool MorallyVirtual,
     if (!Methods.getIndex(OGD, Index))
       continue;
 
+    OGD2 = OGD;
+
     // Get the original method, which we should be computing thunks, etc,
     // against.
     OGD = Methods.getOrigMethod(Index);
@@ -812,8 +2172,8 @@ bool VtableBuilder::OverrideMethod(GlobalDecl GD, bool MorallyVirtual,
       OMD->getType()->getAs<FunctionType>()->getResultType();
     
     // Check if we need a return type adjustment.
-    if (TypeConversionRequiresAdjustment(CGM.getContext(), ReturnType, 
-                                          OverriddenReturnType)) {
+    if (!ComputeReturnAdjustmentBaseOffset(CGM.getContext(), MD, 
+                                           OMD).isEmpty()) {
       CanQualType &BaseReturnType = BaseReturnTypes[Index];
 
       // Insert the base return type.
@@ -824,26 +2184,39 @@ bool VtableBuilder::OverrideMethod(GlobalDecl GD, bool MorallyVirtual,
 
     Methods.OverrideMethod(OGD, GD);
 
+    GlobalDecl UGD = getUnique(GD);
+    const CXXMethodDecl *UMD = cast<CXXMethodDecl>(UGD.getDecl());
+    assert(UGD.getDecl() && "unique overrider not found");
+    assert(UGD == getUnique(OGD) && "unique overrider not unique");
+
     ThisAdjustments.erase(Index);
-    if (MorallyVirtual || VCall.count(OGD)) {
-      Index_t &idx = VCall[OGD];
+    if (MorallyVirtual || VCall.count(UMD)) {
+
+      Index_t &idx = VCall[UMD];
       if (idx == 0) {
-        NonVirtualOffset[GD] = -OverrideOffset/8 + CurrentVBaseOffset/8;
-        VCallOffset[GD] = OverrideOffset/8;
+        VCallOffset[GD] = VCallOffset[OGD];
+        // NonVirtualOffset[UMD] = CurrentVBaseOffset/8 - OverrideOffset/8;
+        NonVirtualOffset[UMD] = VCallOffset[OGD];
+        VCallOffsetForVCall[UMD] = OverrideOffset/8;
         idx = VCalls.size()+1;
-        VCalls.push_back(0);
+        VCalls.push_back(OverrideOffset/8 - CurrentVBaseOffset/8);
         D1(printf("  vcall for %s at %d with delta %d most derived %s\n",
                   MD->getNameAsString().c_str(), (int)-idx-3,
                   (int)VCalls[idx-1], MostDerivedClass->getNameAsCString()));
       } else {
-        NonVirtualOffset[GD] = NonVirtualOffset[OGD];
-        VCallOffset[GD] = VCallOffset[OGD];
-        VCalls[idx-1] = -VCallOffset[OGD] + OverrideOffset/8;
+        VCallOffset[GD] = NonVirtualOffset[UMD];
+        VCalls[idx-1] = -VCallOffsetForVCall[UGD] + OverrideOffset/8;
         D1(printf("  vcall patch for %s at %d with delta %d most derived %s\n",
                   MD->getNameAsString().c_str(), (int)-idx-3,
                   (int)VCalls[idx-1], MostDerivedClass->getNameAsCString()));
       }
-      int64_t NonVirtualAdjustment = NonVirtualOffset[GD];
+      int64_t NonVirtualAdjustment = -VCallOffset[OGD];
+      QualType DerivedType = MD->getThisType(CGM.getContext());
+      QualType BaseType = cast<const CXXMethodDecl>(OGD.getDecl())->getThisType(CGM.getContext());
+      int64_t NonVirtualAdjustment2 = -(getNVOffset(BaseType, DerivedType)/8);
+      if (NonVirtualAdjustment2 != NonVirtualAdjustment) {
+        NonVirtualAdjustment = NonVirtualAdjustment2;
+      }
       int64_t VirtualAdjustment = 
         -((idx + extra + 2) * LLVMPointerWidth / 8);
       
@@ -859,12 +2232,19 @@ bool VtableBuilder::OverrideMethod(GlobalDecl GD, bool MorallyVirtual,
         SavedAdjustments.push_back(
             std::make_pair(GD, std::make_pair(OGD, ThisAdjustment)));
       }
-      VCall[GD] = idx;
       return true;
     }
 
-    int64_t NonVirtualAdjustment = -VCallOffset[OGD] + OverrideOffset/8;
+    VCallOffset[GD] = VCallOffset[OGD2] - OverrideOffset/8;
 
+    int64_t NonVirtualAdjustment = -VCallOffset[GD];
+    QualType DerivedType = MD->getThisType(CGM.getContext());
+    QualType BaseType = cast<const CXXMethodDecl>(OGD.getDecl())->getThisType(CGM.getContext());
+    int64_t NonVirtualAdjustment2 = -(getNVOffset(BaseType, DerivedType)/8);
+    if (NonVirtualAdjustment2 != NonVirtualAdjustment) {
+      NonVirtualAdjustment = NonVirtualAdjustment2;
+    }
+      
     if (NonVirtualAdjustment) {
       ThunkAdjustment ThisAdjustment(NonVirtualAdjustment, 0);
       
@@ -880,7 +2260,7 @@ bool VtableBuilder::OverrideMethod(GlobalDecl GD, bool MorallyVirtual,
   return false;
 }
 
-void VtableBuilder::AppendMethodsToVtable() {
+void OldVtableBuilder::AppendMethodsToVtable() {
   if (!BuildVtable) {
     VtableComponents.insert(VtableComponents.end(), Methods.size(), 
                             (llvm::Constant *)0);
@@ -948,13 +2328,13 @@ void VtableBuilder::AppendMethodsToVtable() {
 void CGVtableInfo::ComputeMethodVtableIndices(const CXXRecordDecl *RD) {
   
   // Itanium C++ ABI 2.5.2:
-  // The order of the virtual function pointers in a virtual table is the 
-  // order of declaration of the corresponding member functions in the class.
+  //   The order of the virtual function pointers in a virtual table is the 
+  //   order of declaration of the corresponding member functions in the class.
   //
-  // There is an entry for any virtual function declared in a class, 
-  // whether it is a new function or overrides a base class function, 
-  // unless it overrides a function from the primary base, and conversion
-  // between their return types does not require an adjustment. 
+  //   There is an entry for any virtual function declared in a class, 
+  //   whether it is a new function or overrides a base class function, 
+  //   unless it overrides a function from the primary base, and conversion
+  //   between their return types does not require an adjustment. 
 
   int64_t CurrentIndex = 0;
   
@@ -972,7 +2352,7 @@ void CGVtableInfo::ComputeMethodVtableIndices(const CXXRecordDecl *RD) {
 
   // Collect all the primary bases, so we can check whether methods override
   // a method from the base.
-  llvm::SmallPtrSet<const CXXRecordDecl *, 5> PrimaryBases;
+  VtableBuilder::PrimaryBasesSetTy PrimaryBases;
   for (ASTRecordLayout::primary_base_info_iterator
        I = Layout.primary_base_begin(), E = Layout.primary_base_end();
        I != E; ++I)
@@ -988,51 +2368,33 @@ void CGVtableInfo::ComputeMethodVtableIndices(const CXXRecordDecl *RD) {
     if (!MD->isVirtual())
       continue;
 
-    bool ShouldAddEntryForMethod = true;
-    
     // Check if this method overrides a method in the primary base.
-    for (CXXMethodDecl::method_iterator i = MD->begin_overridden_methods(),
-         e = MD->end_overridden_methods(); i != e; ++i) {
-      const CXXMethodDecl *OverriddenMD = *i;
-      const CXXRecordDecl *OverriddenRD = OverriddenMD->getParent();
-      assert(OverriddenMD->isCanonicalDecl() &&
-             "Should have the canonical decl of the overridden RD!");
-      
-      if (PrimaryBases.count(OverriddenRD)) {
-        // Check if converting from the return type of the method to the 
-        // return type of the overridden method requires conversion.
-        QualType ReturnType = 
-          MD->getType()->getAs<FunctionType>()->getResultType();
-        QualType OverriddenReturnType =
-          OverriddenMD->getType()->getAs<FunctionType>()->getResultType();
-        
-        if (!TypeConversionRequiresAdjustment(CGM.getContext(), 
-                                            ReturnType, OverriddenReturnType)) {
-          // This index is shared between the index in the vtable of the primary
-          // base class.
-          if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
-            const CXXDestructorDecl *OverriddenDD = 
-              cast<CXXDestructorDecl>(OverriddenMD);
-            
-            // Add both the complete and deleting entries.
-            MethodVtableIndices[GlobalDecl(DD, Dtor_Complete)] = 
-              getMethodVtableIndex(GlobalDecl(OverriddenDD, Dtor_Complete));
-            MethodVtableIndices[GlobalDecl(DD, Dtor_Deleting)] = 
-              getMethodVtableIndex(GlobalDecl(OverriddenDD, Dtor_Deleting));
-          } else {
-            MethodVtableIndices[MD] = getMethodVtableIndex(OverriddenMD);
-          }
+    if (const CXXMethodDecl *OverriddenMD = 
+          OverridesMethodInPrimaryBase(MD, PrimaryBases)) {
+      // Check if converting from the return type of the method to the 
+      // return type of the overridden method requires conversion.
+      if (ComputeReturnAdjustmentBaseOffset(CGM.getContext(), MD, 
+                                            OverriddenMD).isEmpty()) {
+        // This index is shared between the index in the vtable of the primary
+        // base class.
+        if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
+          const CXXDestructorDecl *OverriddenDD = 
+            cast<CXXDestructorDecl>(OverriddenMD);
           
-          // We don't need to add an entry for this method.
-          ShouldAddEntryForMethod = false;
-          break;
-        }        
+          // Add both the complete and deleting entries.
+          MethodVtableIndices[GlobalDecl(DD, Dtor_Complete)] = 
+            getMethodVtableIndex(GlobalDecl(OverriddenDD, Dtor_Complete));
+          MethodVtableIndices[GlobalDecl(DD, Dtor_Deleting)] = 
+            getMethodVtableIndex(GlobalDecl(OverriddenDD, Dtor_Deleting));
+        } else {
+          MethodVtableIndices[MD] = getMethodVtableIndex(OverriddenMD);
+        }
+        
+        // We don't need to add an entry for this method.
+        continue;
       }
     }
     
-    if (!ShouldAddEntryForMethod)
-      continue;
-    
     if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD)) {
       if (MD->isImplicit()) {
         assert(!ImplicitVirtualDtor && 
@@ -1054,8 +2416,8 @@ void CGVtableInfo::ComputeMethodVtableIndices(const CXXRecordDecl *RD) {
 
   if (ImplicitVirtualDtor) {
     // Itanium C++ ABI 2.5.2:
-    // If a class has an implicitly-defined virtual destructor, 
-    // its entries come after the declared virtual function pointers.
+    //   If a class has an implicitly-defined virtual destructor, 
+    //   its entries come after the declared virtual function pointers.
 
     // Add the complete dtor.
     MethodVtableIndices[GlobalDecl(ImplicitVirtualDtor, Dtor_Complete)] = 
@@ -1107,12 +2469,12 @@ CGVtableInfo::getAdjustments(GlobalDecl GD) {
     return 0;
 
   AddressPointsMapTy AddressPoints;
-  VtableBuilder b(RD, RD, 0, CGM, false, AddressPoints);
+  OldVtableBuilder b(RD, RD, 0, CGM, false, AddressPoints);
   D1(printf("vtable %s\n", RD->getNameAsCString()));
   b.GenerateVtableForBase(RD);
   b.GenerateVtableForVBases(RD);
 
-  for (VtableBuilder::SavedAdjustmentsVectorTy::iterator
+  for (OldVtableBuilder::SavedAdjustmentsVectorTy::iterator
        i = b.getSavedAdjustments().begin(),
        e = b.getSavedAdjustments().end(); i != e; i++)
     SavedAdjustments[i->first].push_back(i->second);
@@ -1136,7 +2498,7 @@ int64_t CGVtableInfo::getVirtualBaseOffsetIndex(const CXXRecordDecl *RD,
   // FIXME: This seems expensive.  Can we do a partial job to get
   // just this data.
   AddressPointsMapTy AddressPoints;
-  VtableBuilder b(RD, RD, 0, CGM, false, AddressPoints);
+  OldVtableBuilder b(RD, RD, 0, CGM, false, AddressPoints);
   D1(printf("vtable %s\n", RD->getNameAsCString()));
   b.GenerateVtableForBase(RD);
   b.GenerateVtableForVBases(RD);
@@ -1150,6 +2512,12 @@ int64_t CGVtableInfo::getVirtualBaseOffsetIndex(const CXXRecordDecl *RD,
   }
   
   I = VirtualBaseClassIndicies.find(ClassPair);
+  // FIXME: The assertion below assertion currently fails with the old vtable 
+  /// layout code if there is a non-virtual thunk adjustment in a vtable.
+  // Once the new layout is in place, this return should be removed.
+  if (I == VirtualBaseClassIndicies.end())
+    return 0;
+  
   assert(I != VirtualBaseClassIndicies.end() && "Did not find index!");
   
   return I->second;
@@ -1168,6 +2536,13 @@ CGVtableInfo::GenerateVtable(llvm::GlobalVariable::LinkageTypes Linkage,
                              const CXXRecordDecl *LayoutClass,
                              const CXXRecordDecl *RD, uint64_t Offset,
                              AddressPointsMapTy& AddressPoints) {
+  if (GenerateDefinition && CGM.getLangOptions().DumpVtableLayouts && 
+      LayoutClass == RD) {
+    VtableBuilder Builder(*this, RD);
+    
+    Builder.dumpLayout(llvm::errs());
+  }
+
   llvm::SmallString<256> OutName;
   if (LayoutClass != RD)
     CGM.getMangleContext().mangleCXXCtorVtable(LayoutClass, Offset / 8, 
@@ -1179,8 +2554,8 @@ CGVtableInfo::GenerateVtable(llvm::GlobalVariable::LinkageTypes Linkage,
   llvm::GlobalVariable *GV = CGM.getModule().getGlobalVariable(Name);
   if (GV == 0 || CGM.getVtableInfo().AddressPoints[LayoutClass] == 0 || 
       GV->isDeclaration()) {
-    VtableBuilder b(RD, LayoutClass, Offset, CGM, GenerateDefinition,
-                    AddressPoints);
+    OldVtableBuilder b(RD, LayoutClass, Offset, CGM, GenerateDefinition,
+                       AddressPoints);
 
     D1(printf("vtable %s\n", RD->getNameAsCString()));
     // First comes the vtables for all the non-virtual bases...
diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp
index f0a5c64..5a4f94e 100644
--- a/lib/CodeGen/CodeGenFunction.cpp
+++ b/lib/CodeGen/CodeGenFunction.cpp
@@ -171,6 +171,16 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
   CurFn = Fn;
   assert(CurFn->isDeclaration() && "Function already has body?");
 
+  // Pass inline keyword to optimizer if it appears explicitly on any
+  // declaration.
+  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
+    for (FunctionDecl::redecl_iterator RI = FD->redecls_begin(),
+           RE = FD->redecls_end(); RI != RE; ++RI)
+      if (RI->isInlineSpecified()) {
+        Fn->addFnAttr(llvm::Attribute::InlineHint);
+        break;
+      }
+
   llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
 
   // Create a marker to make it easy to insert allocas into the entryblock
@@ -196,7 +206,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
   }
 
   // FIXME: Leaked.
-  CurFnInfo = &CGM.getTypes().getFunctionInfo(FnRetTy, Args);
+  // CC info is ignored, hopefully?
+  CurFnInfo = &CGM.getTypes().getFunctionInfo(FnRetTy, Args,
+                                              CC_Default, false);
 
   if (RetTy->isVoidType()) {
     // Void type; nothing to return.
@@ -281,6 +293,8 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn) {
       llvm::BasicBlock *DtorEpilogue  = createBasicBlock("dtor.epilogue");
       PushCleanupBlock(DtorEpilogue);
 
+      InitializeVtablePtrs(DD->getParent());
+
       EmitStmt(S);
       
       CleanupBlockInfo Info = PopCleanupBlock();
@@ -431,7 +445,11 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
       EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock);
       EmitBlock(LHSTrue);
 
+      // Any temporaries created here are conditional.
+      BeginConditionalBranch();
       EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock);
+      EndConditionalBranch();
+
       return;
     } else if (CondBOp->getOpcode() == BinaryOperator::LOr) {
       // If we have "0 || X", simplify the code.  "1 || X" would have constant
@@ -454,7 +472,11 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
       EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse);
       EmitBlock(LHSFalse);
 
+      // Any temporaries created here are conditional.
+      BeginConditionalBranch();
       EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock);
+      EndConditionalBranch();
+
       return;
     }
   }
diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h
index 30ad663..fb2e5fe 100644
--- a/lib/CodeGen/CodeGenFunction.h
+++ b/lib/CodeGen/CodeGenFunction.h
@@ -268,7 +268,7 @@ public:
   /// this behavior for branches?
   void EmitBranchThroughCleanup(llvm::BasicBlock *Dest);
 
-  /// StartConditionalBranch - Should be called before a conditional part of an
+  /// BeginConditionalBranch - Should be called before a conditional part of an
   /// expression is emitted. For example, before the RHS of the expression below
   /// is emitted:
   ///
@@ -276,13 +276,16 @@ public:
   ///
   /// This is used to make sure that any temporaries created in the conditional
   /// branch are only destroyed if the branch is taken.
-  void StartConditionalBranch() {
+  void BeginConditionalBranch() {
     ++ConditionalBranchLevel;
   }
 
-  /// FinishConditionalBranch - Should be called after a conditional part of an
+  /// EndConditionalBranch - Should be called after a conditional part of an
   /// expression has been emitted.
-  void FinishConditionalBranch() {
+  void EndConditionalBranch() {
+    assert(ConditionalBranchLevel != 0 &&
+           "Conditional branch mismatch!");
+    
     --ConditionalBranchLevel;
   }
 
@@ -472,7 +475,7 @@ public:
                                         const BlockInfo& Info,
                                         const Decl *OuterFuncDecl,
                                   llvm::DenseMap<const Decl*, llvm::Value*> ldm,
-                                        CharUnits &Size, uint64_t &Align,
+                                        CharUnits &Size, CharUnits &Align,
                       llvm::SmallVector<const Expr *, 8> &subBlockDeclRefDecls,
                                         bool &subBlockHasCopyDispose);
 
@@ -569,6 +572,9 @@ public:
 
   const llvm::Type *ConvertTypeForMem(QualType T);
   const llvm::Type *ConvertType(QualType T);
+  const llvm::Type *ConvertType(const TypeDecl *T) {
+    return ConvertType(getContext().getTypeDeclType(T));
+  }
 
   /// LoadObjCSelf - Load the value of self. This function is only valid while
   /// generating code for an Objective-C method.
@@ -652,10 +658,15 @@ public:
   }
 
   /// CreateTempAlloca - This creates a alloca and inserts it into the entry
-  /// block.
+  /// block. The caller is responsible for setting an appropriate alignment on
+  /// the alloca.
   llvm::AllocaInst *CreateTempAlloca(const llvm::Type *Ty,
                                      const llvm::Twine &Name = "tmp");
 
+  /// CreateMemTemp - Create a temporary memory object of the given type, with
+  /// appropriate alignment.
+  llvm::Value *CreateMemTemp(QualType T, const llvm::Twine &Name = "tmp");
+
   /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
   /// expression and compare the result against zero, returning an Int1Ty value.
   llvm::Value *EvaluateExprAsBool(const Expr *E);
@@ -732,11 +743,16 @@ public:
   /// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
   /// virtual bases.
   llvm::Value *LoadCXXVTT();
+
+  /// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
+  /// complete class down to one of its virtual bases.
+  llvm::Value *GetAddressOfBaseOfCompleteClass(llvm::Value *Value,
+                                               bool IsVirtual,
+                                               const CXXRecordDecl *Derived,
+                                               const CXXRecordDecl *Base);
   
   /// GetAddressOfBaseClass - This function will add the necessary delta to the
   /// load of 'this' and returns address of the base class.
-  // FIXME. This currently only does a derived to non-virtual base conversion.
-  // Other kinds of conversions will come later.
   llvm::Value *GetAddressOfBaseClass(llvm::Value *Value,
                                      const CXXRecordDecl *ClassDecl,
                                      const CXXRecordDecl *BaseClassDecl,
@@ -747,10 +763,9 @@ public:
                                         const CXXRecordDecl *DerivedClassDecl,
                                         bool NullCheckValue);
 
-  llvm::Value *
-  GetVirtualCXXBaseClassOffset(llvm::Value *This,
-                               const CXXRecordDecl *ClassDecl,
-                               const CXXRecordDecl *BaseClassDecl);
+  llvm::Value *GetVirtualBaseClassOffset(llvm::Value *This,
+                                         const CXXRecordDecl *ClassDecl,
+                                         const CXXRecordDecl *BaseClassDecl);
     
   void EmitClassAggrMemberwiseCopy(llvm::Value *DestValue,
                                    llvm::Value *SrcValue,
@@ -843,7 +858,8 @@ public:
   /// This function can be called with a null (unreachable) insert point.
   void EmitLocalBlockVarDecl(const VarDecl &D);
 
-  void EmitStaticBlockVarDecl(const VarDecl &D);
+  void EmitStaticBlockVarDecl(const VarDecl &D,
+                              llvm::GlobalValue::LinkageTypes Linkage);
 
   /// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
   void EmitParmDecl(const VarDecl &D, llvm::Value *Arg);
@@ -1005,12 +1021,18 @@ public:
   LValue EmitCastLValue(const CastExpr *E);
   LValue EmitNullInitializationLValue(const CXXZeroInitValueExpr *E);
   
-  LValue EmitPointerToDataMemberLValue(const FieldDecl *Field);
-  
   llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
                               const ObjCIvarDecl *Ivar);
   LValue EmitLValueForField(llvm::Value* Base, const FieldDecl* Field,
-                            bool isUnion, unsigned CVRQualifiers);
+                            unsigned CVRQualifiers);
+  
+  /// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
+  /// if the Field is a reference, this will return the address of the reference
+  /// and not the address of the value stored in the reference.
+  LValue EmitLValueForFieldInitialization(llvm::Value* Base, 
+                                          const FieldDecl* Field,
+                                          unsigned CVRQualifiers);
+  
   LValue EmitLValueForIvar(QualType ObjectTy,
                            llvm::Value* Base, const ObjCIvarDecl *Ivar,
                            unsigned CVRQualifiers);
@@ -1103,8 +1125,7 @@ public:
 
   /// EmitReferenceBindingToExpr - Emits a reference binding to the passed in
   /// expression. Will emit a temporary variable if E is not an LValue.
-  RValue EmitReferenceBindingToExpr(const Expr* E, QualType DestType,
-                                    bool IsInitializer = false);
+  RValue EmitReferenceBindingToExpr(const Expr* E, bool IsInitializer = false);
 
   //===--------------------------------------------------------------------===//
   //                           Expression Emission
@@ -1135,6 +1156,10 @@ public:
                    bool IgnoreResult = false, bool IsInitializer = false,
                    bool RequiresGCollection = false);
 
+  /// EmitAggExprToLValue - Emit the computation of the specified expression of
+  /// aggregate type into a temporary LValue.
+  LValue EmitAggExprToLValue(const Expr *E);
+
   /// EmitGCMemmoveCollectable - Emit special API for structs with object
   /// pointers.
   void EmitGCMemmoveCollectable(llvm::Value *DestPtr, llvm::Value *SrcPtr,
diff --git a/lib/CodeGen/CodeGenModule.cpp b/lib/CodeGen/CodeGenModule.cpp
index cf504a7..5a552c4 100644
--- a/lib/CodeGen/CodeGenModule.cpp
+++ b/lib/CodeGen/CodeGenModule.cpp
@@ -20,6 +20,7 @@
 #include "TargetInfo.h"
 #include "clang/CodeGen/CodeGenOptions.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/CharUnits.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/RecordLayout.h"
@@ -131,6 +132,10 @@ CodeGenModule::getDeclVisibilityMode(const Decl *D) const {
     }
   }
 
+  // This decl should have the same visibility as its parent.
+  if (const DeclContext *DC = D->getDeclContext()) 
+    return getDeclVisibilityMode(cast<Decl>(DC));
+
   return getLangOptions().getVisibilityMode();
 }
 
@@ -254,34 +259,40 @@ void CodeGenModule::EmitAnnotations() {
 static CodeGenModule::GVALinkage
 GetLinkageForFunction(ASTContext &Context, const FunctionDecl *FD,
                       const LangOptions &Features) {
-  // Everything located semantically within an anonymous namespace is
-  // always internal.
-  if (FD->isInAnonymousNamespace())
-    return CodeGenModule::GVA_Internal;
+  CodeGenModule::GVALinkage External = CodeGenModule::GVA_StrongExternal;
 
-  // "static" functions get internal linkage.
-  if (FD->getStorageClass() == FunctionDecl::Static && !isa<CXXMethodDecl>(FD))
-    return CodeGenModule::GVA_Internal;
+  Linkage L = FD->getLinkage();
+  if (L == ExternalLinkage && Context.getLangOptions().CPlusPlus &&
+      FD->getType()->getLinkage() == UniqueExternalLinkage)
+    L = UniqueExternalLinkage;
   
-  // The kind of external linkage this function will have, if it is not
-  // inline or static.
-  CodeGenModule::GVALinkage External = CodeGenModule::GVA_StrongExternal;
-  if (Context.getLangOptions().CPlusPlus) {
-    TemplateSpecializationKind TSK = FD->getTemplateSpecializationKind();
+  switch (L) {
+  case NoLinkage:
+  case InternalLinkage:
+  case UniqueExternalLinkage:
+    return CodeGenModule::GVA_Internal;
     
-    if (TSK == TSK_ExplicitInstantiationDefinition) {
-      // If a function has been explicitly instantiated, then it should
-      // always have strong external linkage.
+  case ExternalLinkage:
+    switch (FD->getTemplateSpecializationKind()) {
+    case TSK_Undeclared:
+    case TSK_ExplicitSpecialization:
+      External = CodeGenModule::GVA_StrongExternal;
+      break;
+
+    case TSK_ExplicitInstantiationDefinition:
+      // FIXME: explicit instantiation definitions should use weak linkage
       return CodeGenModule::GVA_StrongExternal;
-    } 
-    
-    if (TSK == TSK_ImplicitInstantiation)
+
+    case TSK_ExplicitInstantiationDeclaration:
+    case TSK_ImplicitInstantiation:
       External = CodeGenModule::GVA_TemplateInstantiation;
+      break;
+    }
   }
 
   if (!FD->isInlined())
     return External;
-
+    
   if (!Features.CPlusPlus || FD->hasAttr<GNUInlineAttr>()) {
     // GNU or C99 inline semantics. Determine whether this symbol should be
     // externally visible.
@@ -402,11 +413,13 @@ void CodeGenModule::SetInternalFunctionAttributes(const Decl *D,
   SetCommonAttributes(D, F);
 }
 
-void CodeGenModule::SetFunctionAttributes(const FunctionDecl *FD,
+void CodeGenModule::SetFunctionAttributes(GlobalDecl GD,
                                           llvm::Function *F,
                                           bool IsIncompleteFunction) {
+  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
+
   if (!IsIncompleteFunction)
-    SetLLVMFunctionAttributes(FD, getTypes().getFunctionInfo(FD), F);
+    SetLLVMFunctionAttributes(FD, getTypes().getFunctionInfo(GD), F);
 
   // Only a few attributes are set on declarations; these may later be
   // overridden by a definition.
@@ -580,14 +593,24 @@ bool CodeGenModule::MayDeferGeneration(const ValueDecl *Global) {
       
   // Static data may be deferred, but out-of-line static data members
   // cannot be.
-  if (VD->isInAnonymousNamespace())
-    return true;
-  if (VD->getLinkage() == VarDecl::InternalLinkage) {
+  Linkage L = VD->getLinkage();
+  if (L == ExternalLinkage && getContext().getLangOptions().CPlusPlus &&
+      VD->getType()->getLinkage() == UniqueExternalLinkage)
+    L = UniqueExternalLinkage;
+
+  switch (L) {
+  case NoLinkage:
+  case InternalLinkage:
+  case UniqueExternalLinkage:
     // Initializer has side effects?
     if (VD->getInit() && VD->getInit()->HasSideEffects(Context))
       return false;
     return !(VD->isStaticDataMember() && VD->isOutOfLine());
+
+  case ExternalLinkage:
+    break;
   }
+
   return false;
 }
 
@@ -608,20 +631,7 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
     const VarDecl *VD = cast<VarDecl>(Global);
     assert(VD->isFileVarDecl() && "Cannot emit local var decl as global.");
 
-    if (getLangOptions().CPlusPlus && !VD->getInit()) {
-      // In C++, if this is marked "extern", defer code generation.
-      if (VD->getStorageClass() == VarDecl::Extern || VD->isExternC())
-        return;
-
-      // If this is a declaration of an explicit specialization of a static
-      // data member in a class template, don't emit it.
-      if (VD->isStaticDataMember() && 
-          VD->getTemplateSpecializationKind() == TSK_ExplicitSpecialization)
-        return;
-    }
-
-    // In C, if this isn't a definition, defer code generation.
-    if (!getLangOptions().CPlusPlus && !VD->getInit())
+    if (VD->isThisDeclarationADefinition() != VarDecl::Definition)
       return;
   }
 
@@ -715,8 +725,7 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(const char *MangledName,
                                              "", &getModule());
   F->setName(MangledName);
   if (D.getDecl())
-    SetFunctionAttributes(cast<FunctionDecl>(D.getDecl()), F,
-                          IsIncompleteFunction);
+    SetFunctionAttributes(D, F, IsIncompleteFunction);
   Entry = F;
 
   // This is the first use or definition of a mangled name.  If there is a
@@ -774,7 +783,7 @@ CodeGenModule::CreateRuntimeFunction(const llvm::FunctionType *FTy,
 }
 
 static bool DeclIsConstantGlobal(ASTContext &Context, const VarDecl *D) {
-  if (!D->getType().isConstant(Context))
+  if (!D->getType().isConstant(Context) && !D->getType()->isReferenceType())
     return false;
   if (Context.getLangOptions().CPlusPlus &&
       Context.getBaseElementType(D->getType())->getAs<RecordType>()) {
@@ -941,16 +950,30 @@ CodeGenModule::getVtableLinkage(const CXXRecordDecl *RD) {
 
 static CodeGenModule::GVALinkage
 GetLinkageForVariable(ASTContext &Context, const VarDecl *VD) {
-  // Everything located semantically within an anonymous namespace is
-  // always internal.
-  if (VD->isInAnonymousNamespace())
+  // If this is a static data member, compute the kind of template
+  // specialization. Otherwise, this variable is not part of a
+  // template.
+  TemplateSpecializationKind TSK = TSK_Undeclared;
+  if (VD->isStaticDataMember())
+    TSK = VD->getTemplateSpecializationKind();
+
+  Linkage L = VD->getLinkage();
+  if (L == ExternalLinkage && Context.getLangOptions().CPlusPlus &&
+      VD->getType()->getLinkage() == UniqueExternalLinkage)
+    L = UniqueExternalLinkage;
+
+  switch (L) {
+  case NoLinkage:
+  case InternalLinkage:
+  case UniqueExternalLinkage:
     return CodeGenModule::GVA_Internal;
 
-  // Handle linkage for static data members.
-  if (VD->isStaticDataMember()) {
-    switch (VD->getTemplateSpecializationKind()) {
+  case ExternalLinkage:
+    switch (TSK) {
     case TSK_Undeclared:
     case TSK_ExplicitSpecialization:
+
+      // FIXME: ExplicitInstantiationDefinition should be weak!
     case TSK_ExplicitInstantiationDefinition:
       return CodeGenModule::GVA_StrongExternal;
       
@@ -959,22 +982,26 @@ GetLinkageForVariable(ASTContext &Context, const VarDecl *VD) {
       // Fall through to treat this like any other instantiation.
         
     case TSK_ImplicitInstantiation:
-      return CodeGenModule::GVA_TemplateInstantiation;
+      return CodeGenModule::GVA_TemplateInstantiation;      
     }
   }
 
-  if (VD->getLinkage() == VarDecl::InternalLinkage)
-    return CodeGenModule::GVA_Internal;
-
   return CodeGenModule::GVA_StrongExternal;
 }
 
+CharUnits CodeGenModule::GetTargetTypeStoreSize(const llvm::Type *Ty) const {
+    return CharUnits::fromQuantity(
+      TheTargetData.getTypeStoreSizeInBits(Ty) / Context.getCharWidth());
+}
+
 void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
   llvm::Constant *Init = 0;
   QualType ASTTy = D->getType();
   bool NonConstInit = false;
 
-  if (D->getInit() == 0) {
+  const Expr *InitExpr = D->getAnyInitializer();
+  
+  if (!InitExpr) {
     // This is a tentative definition; tentative definitions are
     // implicitly initialized with { 0 }.
     //
@@ -987,10 +1014,10 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
     assert(!ASTTy->isIncompleteType() && "Unexpected incomplete type");
     Init = EmitNullConstant(D->getType());
   } else {
-    Init = EmitConstantExpr(D->getInit(), D->getType());
+    Init = EmitConstantExpr(InitExpr, D->getType());
 
     if (!Init) {
-      QualType T = D->getInit()->getType();
+      QualType T = InitExpr->getType();
       if (getLangOptions().CPlusPlus) {
         EmitCXXGlobalVarDeclInitFunc(D);
         Init = EmitNullConstant(T);
@@ -1058,7 +1085,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
   if (!NonConstInit && DeclIsConstantGlobal(Context, D))
     GV->setConstant(true);
 
-  GV->setAlignment(getContext().getDeclAlignInBytes(D));
+  GV->setAlignment(getContext().getDeclAlign(D).getQuantity());
 
   // Set the llvm linkage type as appropriate.
   GVALinkage Linkage = GetLinkageForVariable(getContext(), D);
@@ -1256,8 +1283,8 @@ void CodeGenModule::EmitAliasDefinition(const ValueDecl *D) {
   const llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType());
 
   // Unique the name through the identifier table.
-  const char *AliaseeName = AA->getAliasee().c_str();
-  AliaseeName = getContext().Idents.get(AliaseeName).getNameStart();
+  const char *AliaseeName =
+    getContext().Idents.get(AA->getAliasee()).getNameStart();
 
   // Create a reference to the named value.  This ensures that it is emitted
   // if a deferred decl.
@@ -1473,11 +1500,9 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
   // String pointer.
   llvm::Constant *C = llvm::ConstantArray::get(VMContext, Entry.getKey().str());
 
-  const char *Sect = 0;
   llvm::GlobalValue::LinkageTypes Linkage;
   bool isConstant;
   if (isUTF16) {
-    Sect = getContext().Target.getUnicodeStringSection();
     // FIXME: why do utf strings get "_" labels instead of "L" labels?
     Linkage = llvm::GlobalValue::InternalLinkage;
     // Note: -fwritable-strings doesn't make unicode CFStrings writable, but
@@ -1491,11 +1516,9 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
   llvm::GlobalVariable *GV =
     new llvm::GlobalVariable(getModule(), C->getType(), isConstant, Linkage, C,
                              ".str");
-  if (Sect)
-    GV->setSection(Sect);
   if (isUTF16) {
-    unsigned Align = getContext().getTypeAlign(getContext().ShortTy)/8;
-    GV->setAlignment(Align);
+    CharUnits Align = getContext().getTypeAlignInChars(getContext().ShortTy);
+    GV->setAlignment(Align.getQuantity());
   }
   Fields[2] = llvm::ConstantExpr::getGetElementPtr(GV, Zeros, 2);
 
diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h
index 81f3979..8280766 100644
--- a/lib/CodeGen/CodeGenModule.h
+++ b/lib/CodeGen/CodeGenModule.h
@@ -53,6 +53,7 @@ namespace clang {
   class ObjCProtocolDecl;
   class ObjCEncodeExpr;
   class BlockExpr;
+  class CharUnits;
   class Decl;
   class Expr;
   class Stmt;
@@ -238,10 +239,11 @@ public:
   BuildCovariantThunk(const GlobalDecl &GD, bool Extern,
                       const CovariantThunkAdjustment &Adjustment);
 
-  /// GetCXXBaseClassOffset - Returns the offset from a derived class to its
-  /// base class. Returns null if the offset is 0.
-  llvm::Constant *GetCXXBaseClassOffset(const CXXRecordDecl *ClassDecl,
-                                        const CXXRecordDecl *BaseClassDecl);
+  /// GetNonVirtualBaseClassOffset - Returns the offset from a derived class to 
+  /// its base class. Returns null if the offset is 0. 
+  llvm::Constant *
+  GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
+                               const CXXRecordDecl *BaseClassDecl);
 
   /// ComputeThunkAdjustment - Returns the two parts required to compute the
   /// offset for an object.
@@ -346,6 +348,8 @@ public:
   llvm::Constant *EmitAnnotateAttr(llvm::GlobalValue *GV,
                                    const AnnotateAttr *AA, unsigned LineNo);
 
+  llvm::Constant *EmitPointerToDataMember(const FieldDecl *FD);
+
   /// ErrorUnsupported - Print out an error that codegen doesn't support the
   /// specified stmt yet.
   /// \param OmitOnError - If true, then this error should only be emitted if no
@@ -417,6 +421,10 @@ public:
   /// and type information of the given class.
   static llvm::GlobalVariable::LinkageTypes 
   getVtableLinkage(const CXXRecordDecl *RD);
+
+  /// GetTargetTypeStoreSize - Return the store size, in character units, of
+  /// the given LLVM type.
+  CharUnits GetTargetTypeStoreSize(const llvm::Type *Ty) const;
   
 private:
   /// UniqueMangledName - Unique a name by (if necessary) inserting it into the
@@ -443,7 +451,7 @@ private:
 
   /// SetFunctionAttributes - Set function attributes for a function
   /// declaration.
-  void SetFunctionAttributes(const FunctionDecl *FD,
+  void SetFunctionAttributes(GlobalDecl GD,
                              llvm::Function *F,
                              bool IsIncompleteFunction);
 
diff --git a/lib/CodeGen/CodeGenTypes.cpp b/lib/CodeGen/CodeGenTypes.cpp
index 838f62a..3c20934 100644
--- a/lib/CodeGen/CodeGenTypes.cpp
+++ b/lib/CodeGen/CodeGenTypes.cpp
@@ -84,7 +84,7 @@ const llvm::Type *CodeGenTypes::ConvertTypeRecursive(QualType T) {
 
 const llvm::Type *CodeGenTypes::ConvertTypeForMemRecursive(QualType T) {
   const llvm::Type *ResultType = ConvertTypeRecursive(T);
-  if (ResultType->isInteger(1))
+  if (ResultType->isIntegerTy(1))
     return llvm::IntegerType::get(getLLVMContext(),
                                   (unsigned)Context.getTypeSize(T));
   // FIXME: Should assert that the llvm type and AST type has the same size.
@@ -99,7 +99,7 @@ const llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T) {
   const llvm::Type *R = ConvertType(T);
 
   // If this is a non-bool type, don't map it.
-  if (!R->isInteger(1))
+  if (!R->isIntegerTy(1))
     return R;
 
   // Otherwise, return an integer of the target-specified size.
@@ -399,18 +399,6 @@ const llvm::Type *CodeGenTypes::ConvertNewType(QualType T) {
 /// enum.
 const llvm::Type *CodeGenTypes::ConvertTagDeclType(const TagDecl *TD) {
 
-  // FIXME. This may have to move to a better place.
-  if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(TD)) {
-    for (CXXRecordDecl::base_class_const_iterator i = RD->bases_begin(),
-         e = RD->bases_end(); i != e; ++i) {
-      if (!i->isVirtual()) {
-        const CXXRecordDecl *Base =
-          cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
-        ConvertTagDeclType(Base);
-      }
-    }
-  }
-
   // TagDecl's are not necessarily unique, instead use the (clang)
   // type connected to the decl.
   const Type *Key =
@@ -422,8 +410,8 @@ const llvm::Type *CodeGenTypes::ConvertTagDeclType(const TagDecl *TD) {
   if (TDTI != TagDeclTypes.end())
     return TDTI->second;
 
-  // If this is still a forward definition, just define an opaque type to use
-  // for this tagged decl.
+  // If this is still a forward declaration, just define an opaque
+  // type to use for this tagged decl.
   if (!TD->isDefinition()) {
     llvm::Type *ResultType = llvm::OpaqueType::get(getLLVMContext());
     TagDeclTypes.insert(std::make_pair(Key, ResultType));
@@ -446,6 +434,18 @@ const llvm::Type *CodeGenTypes::ConvertTagDeclType(const TagDecl *TD) {
 
   const RecordDecl *RD = cast<const RecordDecl>(TD);
 
+  // Force conversion of non-virtual base classes recursively.
+  if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(TD)) {    
+    for (CXXRecordDecl::base_class_const_iterator i = RD->bases_begin(),
+         e = RD->bases_end(); i != e; ++i) {
+      if (!i->isVirtual()) {
+        const CXXRecordDecl *Base =
+          cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
+        ConvertTagDeclType(Base);
+      }
+    }
+  }
+
   // Layout fields.
   CGRecordLayout *Layout = CGRecordLayoutBuilder::ComputeLayout(*this, RD);
 
diff --git a/lib/CodeGen/CodeGenTypes.h b/lib/CodeGen/CodeGenTypes.h
index 7e34252..87ba0bc 100644
--- a/lib/CodeGen/CodeGenTypes.h
+++ b/lib/CodeGen/CodeGenTypes.h
@@ -20,7 +20,7 @@
 #include <vector>
 
 #include "CGCall.h"
-#include "CGCXX.h"
+#include "GlobalDecl.h"
 
 namespace llvm {
   class FunctionType;
@@ -60,21 +60,24 @@ namespace CodeGen {
     /// LLVMType - The LLVMType corresponding to this record layout.
     const llvm::Type *LLVMType;
 
-    /// ContainsMemberPointer - Whether one of the fields in this record layout
-    /// is a member pointer, or a struct that contains a member pointer.
-    bool ContainsMemberPointer;
+    /// ContainsPointerToDataMember - Whether one of the fields in this record 
+    /// layout is a pointer to data member, or a struct that contains pointer to
+    /// data member.
+    bool ContainsPointerToDataMember;
 
   public:
-    CGRecordLayout(const llvm::Type *T, bool ContainsMemberPointer)
-      : LLVMType(T), ContainsMemberPointer(ContainsMemberPointer) { }
+    CGRecordLayout(const llvm::Type *T, bool ContainsPointerToDataMember)
+      : LLVMType(T), ContainsPointerToDataMember(ContainsPointerToDataMember) { }
 
     /// getLLVMType - Return llvm type associated with this record.
     const llvm::Type *getLLVMType() const {
       return LLVMType;
     }
 
-    bool containsMemberPointer() const {
-      return ContainsMemberPointer;
+    /// containsPointerToDataMember - Whether this struct contains pointers to
+    /// data members.
+    bool containsPointerToDataMember() const {
+      return ContainsPointerToDataMember;
     }
   };
 
@@ -187,6 +190,8 @@ private:
 
 public:
   /// getFunctionInfo - Get the function info for the specified function decl.
+  const CGFunctionInfo &getFunctionInfo(GlobalDecl GD);
+  
   const CGFunctionInfo &getFunctionInfo(const FunctionDecl *FD);
   const CGFunctionInfo &getFunctionInfo(const CXXMethodDecl *MD);
   const CGFunctionInfo &getFunctionInfo(const ObjCMethodDecl *MD);
@@ -195,6 +200,12 @@ public:
   const CGFunctionInfo &getFunctionInfo(const CXXDestructorDecl *D,
                                         CXXDtorType Type);
 
+  const CGFunctionInfo &getFunctionInfo(const CallArgList &Args,
+                                        const FunctionType *Ty) {
+    return getFunctionInfo(Ty->getResultType(), Args,
+                           Ty->getCallConv(), Ty->getNoReturnAttr());
+  }
+
   // getFunctionInfo - Get the function info for a member function.
   const CGFunctionInfo &getFunctionInfo(const CXXRecordDecl *RD,
                                         const FunctionProtoType *FTP);
@@ -204,13 +215,16 @@ public:
   /// specified, the "C" calling convention will be used.
   const CGFunctionInfo &getFunctionInfo(QualType ResTy,
                                         const CallArgList &Args,
-                                        unsigned CallingConvention = 0);
+                                        CallingConv CC,
+                                        bool NoReturn);
   const CGFunctionInfo &getFunctionInfo(QualType ResTy,
                                         const FunctionArgList &Args,
-                                        unsigned CallingConvention = 0);
+                                        CallingConv CC,
+                                        bool NoReturn);
   const CGFunctionInfo &getFunctionInfo(QualType RetTy,
                                   const llvm::SmallVector<QualType, 16> &ArgTys,
-                                        unsigned CallingConvention = 0);
+                                        CallingConv CC,
+                                        bool NoReturn);
 
 public:  // These are internal details of CGT that shouldn't be used externally.
   /// addFieldInfo - Assign field number to field FD.
diff --git a/lib/CodeGen/GlobalDecl.h b/lib/CodeGen/GlobalDecl.h
index b054312..b8a98d7 100644
--- a/lib/CodeGen/GlobalDecl.h
+++ b/lib/CodeGen/GlobalDecl.h
@@ -15,6 +15,10 @@
 #ifndef CLANG_CODEGEN_GLOBALDECL_H
 #define CLANG_CODEGEN_GLOBALDECL_H
 
+#include "CGCXX.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclObjC.h"
+
 namespace clang {
 
 namespace CodeGen {
diff --git a/lib/CodeGen/Makefile b/lib/CodeGen/Makefile
index 108490b..83cb367 100644
--- a/lib/CodeGen/Makefile
+++ b/lib/CodeGen/Makefile
@@ -15,7 +15,6 @@
 LEVEL = ../../../..
 LIBRARYNAME := clangCodeGen
 BUILD_ARCHIVE = 1
-CXXFLAGS = -fno-rtti
 
 CPPFLAGS += -I$(PROJ_SRC_DIR)/../../include -I$(PROJ_OBJ_DIR)/../../include
 ifdef CLANG_VENDOR
diff --git a/lib/CodeGen/Mangle.cpp b/lib/CodeGen/Mangle.cpp
index d873cfe..a302225 100644
--- a/lib/CodeGen/Mangle.cpp
+++ b/lib/CodeGen/Mangle.cpp
@@ -26,6 +26,13 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "CGVtable.h"
+
+#define MANGLE_CHECKER 0
+
+#if MANGLE_CHECKER
+#include <cxxabi.h>
+#endif
+
 using namespace clang;
 using namespace CodeGen;
 
@@ -44,6 +51,8 @@ static const CXXMethodDecl *getStructor(const CXXMethodDecl *MD) {
     
   return MD;
 }
+
+static const unsigned UnknownArity = ~0U;
   
 /// CXXNameMangler - Manage the mangling of a single name.
 class CXXNameMangler {
@@ -55,6 +64,8 @@ class CXXNameMangler {
 
   llvm::DenseMap<uintptr_t, unsigned> Substitutions;
 
+  ASTContext &getASTContext() const { return Context.getASTContext(); }
+
 public:
   CXXNameMangler(MangleContext &C, llvm::SmallVectorImpl<char> &Res)
     : Context(C), Out(Res), Structor(0), StructorType(0) { }
@@ -65,6 +76,17 @@ public:
                  const CXXDestructorDecl *D, CXXDtorType Type)
     : Context(C), Out(Res), Structor(getStructor(D)), StructorType(Type) { }
 
+#if MANGLE_CHECKER
+  ~CXXNameMangler() {
+    if (Out.str()[0] == '\01')
+      return;
+    
+    int status = 0;
+    char *result = abi::__cxa_demangle(Out.str().str().c_str(), 0, 0, &status);
+    assert(status == 0 && "Could not demangle mangled name!");
+    free(result);
+  }
+#endif
   llvm::raw_svector_ostream &getStream() { return Out; }
 
   void mangle(const NamedDecl *D, llvm::StringRef Prefix = "_Z");
@@ -89,10 +111,19 @@ private:
   void addSubstitution(QualType T);
   void addSubstitution(uintptr_t Ptr);
 
+  void mangleUnresolvedScope(NestedNameSpecifier *Qualifier);
+  void mangleUnresolvedName(NestedNameSpecifier *Qualifier,
+                            DeclarationName Name,
+                            unsigned KnownArity = UnknownArity);
+
   void mangleName(const TemplateDecl *TD,
                   const TemplateArgument *TemplateArgs,
                   unsigned NumTemplateArgs);
-  void mangleUnqualifiedName(const NamedDecl *ND);
+  void mangleUnqualifiedName(const NamedDecl *ND) {
+    mangleUnqualifiedName(ND, ND->getDeclName(), UnknownArity);
+  }
+  void mangleUnqualifiedName(const NamedDecl *ND, DeclarationName Name,
+                             unsigned KnownArity);
   void mangleUnscopedName(const NamedDecl *ND);
   void mangleUnscopedTemplateName(const TemplateDecl *ND);
   void mangleSourceName(const IdentifierInfo *II);
@@ -119,6 +150,11 @@ private:
                               bool MangleReturnType);
 
   void mangleIntegerLiteral(QualType T, const llvm::APSInt &Value);
+  void mangleMemberExpr(const Expr *Base, bool IsArrow,
+                        NestedNameSpecifier *Qualifier,
+                        DeclarationName Name,
+                        unsigned KnownArity);
+  void mangleCalledExpression(const Expr *E, unsigned KnownArity);
   void mangleExpression(const Expr *E);
   void mangleCXXCtorType(CXXCtorType T);
   void mangleCXXDtorType(CXXDtorType T);
@@ -171,14 +207,14 @@ bool MangleContext::shouldMangleDeclName(const NamedDecl *D) {
       isInExternCSystemHeader(D->getLocation()))
     return false;
 
-  // Variables at global scope are not mangled.
+  // Variables at global scope with non-internal linkage are not mangled
   if (!FD) {
     const DeclContext *DC = D->getDeclContext();
     // Check for extern variable declared locally.
     if (isa<FunctionDecl>(DC) && D->hasLinkage())
       while (!DC->isNamespace() && !DC->isTranslationUnit())
         DC = DC->getParent();
-    if (DC->isTranslationUnit())
+    if (DC->isTranslationUnit() && D->getLinkage() != InternalLinkage)
       return false;
   }
 
@@ -199,13 +235,10 @@ void CXXNameMangler::mangle(const NamedDecl *D, llvm::StringRef Prefix) {
     return;
   }
 
-  // <mangled-name> ::= _Z [L] <encoding>
+  // <mangled-name> ::= _Z <encoding>
   //            ::= <data name>
   //            ::= <special-name>
   Out << Prefix;
-  if (D->getLinkage() == NamedDecl::InternalLinkage) // match gcc behavior
-    Out << 'L';
-
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D))
     mangleFunctionEncoding(FD);
   else
@@ -367,6 +400,13 @@ void CXXNameMangler::mangleUnscopedTemplateName(const TemplateDecl *ND) {
   if (mangleSubstitution(ND))
     return;
 
+  // <template-template-param> ::= <template-param>
+  if (const TemplateTemplateParmDecl *TTP
+                                     = dyn_cast<TemplateTemplateParmDecl>(ND)) {
+    mangleTemplateParameter(TTP->getIndex());
+    return;
+  } 
+
   mangleUnscopedName(ND->getTemplatedDecl());
   addSubstitution(ND);
 }
@@ -401,28 +441,69 @@ void CXXNameMangler::mangleCallOffset(const ThunkAdjustment &Adjustment) {
   Out << '_';
 }
 
-void CXXNameMangler::mangleUnqualifiedName(const NamedDecl *ND) {
+void CXXNameMangler::mangleUnresolvedScope(NestedNameSpecifier *Qualifier) {
+  Qualifier = getASTContext().getCanonicalNestedNameSpecifier(Qualifier);
+  switch (Qualifier->getKind()) {
+  case NestedNameSpecifier::Global:
+    // nothing
+    break;
+  case NestedNameSpecifier::Namespace:
+    mangleName(Qualifier->getAsNamespace());
+    break;
+  case NestedNameSpecifier::TypeSpec:
+  case NestedNameSpecifier::TypeSpecWithTemplate:
+    mangleType(QualType(Qualifier->getAsType(), 0));
+    break;
+  case NestedNameSpecifier::Identifier:
+    mangleUnresolvedScope(Qualifier->getPrefix());
+    mangleSourceName(Qualifier->getAsIdentifier());
+    break;
+  }
+}
+
+/// Mangles a name which was not resolved to a specific entity.
+void CXXNameMangler::mangleUnresolvedName(NestedNameSpecifier *Qualifier,
+                                          DeclarationName Name,
+                                          unsigned KnownArity) {
+  if (Qualifier)
+    mangleUnresolvedScope(Qualifier);
+  // FIXME: ambiguity of unqualified lookup with ::
+
+  mangleUnqualifiedName(0, Name, KnownArity);
+}
+
+void CXXNameMangler::mangleUnqualifiedName(const NamedDecl *ND,
+                                           DeclarationName Name,
+                                           unsigned KnownArity) {
   //  <unqualified-name> ::= <operator-name>
   //                     ::= <ctor-dtor-name>
   //                     ::= <source-name>
-  DeclarationName Name = ND->getDeclName();
   switch (Name.getNameKind()) {
   case DeclarationName::Identifier: {
+    if (const IdentifierInfo *II = Name.getAsIdentifierInfo()) {
+      // We must avoid conflicts between internally- and externally-
+      // linked variable declaration names in the same TU. 
+      // This naming convention is the same as that followed by GCC, though it
+      // shouldn't actually matter.
+      if (ND && isa<VarDecl>(ND) && ND->getLinkage() == InternalLinkage &&
+          ND->getDeclContext()->isFileContext())
+        Out << 'L';
+
+      mangleSourceName(II);
+      break;
+    }
+
+    // Otherwise, an anonymous entity.  We must have a declaration.
+    assert(ND && "mangling empty name without declaration");
+
     if (const NamespaceDecl *NS = dyn_cast<NamespaceDecl>(ND)) {
       if (NS->isAnonymousNamespace()) {
-        // This is how gcc mangles these names.  It's apparently
-        // always '1', no matter how many different anonymous
-        // namespaces appear in a context.
+        // This is how gcc mangles these names.
         Out << "12_GLOBAL__N_1";
         break;
       }
     }
 
-    if (const IdentifierInfo *II = Name.getAsIdentifierInfo()) {
-      mangleSourceName(II);
-      break;
-    }
-
     // We must have an anonymous struct.
     const TagDecl *TD = cast<TagDecl>(ND);
     if (const TypedefDecl *D = TD->getTypedefForAnonDecl()) {
@@ -484,13 +565,18 @@ void CXXNameMangler::mangleUnqualifiedName(const NamedDecl *ND) {
     break;
 
   case DeclarationName::CXXOperatorName: {
-    unsigned Arity = cast<FunctionDecl>(ND)->getNumParams();
+    unsigned Arity;
+    if (ND) {
+      Arity = cast<FunctionDecl>(ND)->getNumParams();
     
-    // If we have a C++ member function, we need to include the 'this' pointer.
-    // FIXME: This does not make sense for operators that are static, but their
-    // names stay the same regardless of the arity (operator new for instance).
-    if (isa<CXXMethodDecl>(ND))
-      Arity++;
+      // If we have a C++ member function, we need to include the 'this' pointer.
+      // FIXME: This does not make sense for operators that are static, but their
+      // names stay the same regardless of the arity (operator new for instance).
+      if (isa<CXXMethodDecl>(ND))
+        Arity++;
+    } else
+      Arity = KnownArity;
+
     mangleOperatorName(Name.getCXXOverloadedOperator(), Arity);
     break;
   }
@@ -596,15 +682,21 @@ void CXXNameMangler::mangleTemplatePrefix(const TemplateDecl *ND) {
   // <template-prefix> ::= <prefix> <template unqualified-name>
   //                   ::= <template-param>
   //                   ::= <substitution>
+  // <template-template-param> ::= <template-param>
+  //                               <substitution>
 
   if (mangleSubstitution(ND))
     return;
 
-  // FIXME: <template-param>
+  // <template-template-param> ::= <template-param>
+  if (const TemplateTemplateParmDecl *TTP
+                                     = dyn_cast<TemplateTemplateParmDecl>(ND)) {
+    mangleTemplateParameter(TTP->getIndex());
+    return;
+  } 
 
   manglePrefix(ND->getDeclContext());
   mangleUnqualifiedName(ND->getTemplatedDecl());
-
   addSubstitution(ND);
 }
 
@@ -1013,7 +1105,7 @@ void CXXNameMangler::mangleType(const TypenameType *T) {
         dyn_cast<TemplateSpecializationType>(QTy)) {
     if (!mangleSubstitution(QualType(TST, 0))) {
       TemplateDecl *TD = TST->getTemplateName().getAsTemplateDecl();
-
+      assert(TD && "FIXME: Support dependent template names");
       mangleTemplatePrefix(TD);
       mangleTemplateArgs(TST->getArgs(), TST->getNumArgs());
       addSubstitution(QualType(TST, 0));
@@ -1049,23 +1141,151 @@ void CXXNameMangler::mangleIntegerLiteral(QualType T,
   
 }
 
+void CXXNameMangler::mangleCalledExpression(const Expr *E, unsigned Arity) {
+  if (E->getType() != getASTContext().OverloadTy)
+    mangleExpression(E);
+  // propagate arity to dependent overloads?
+
+  llvm::PointerIntPair<OverloadExpr*,1> R
+    = OverloadExpr::find(const_cast<Expr*>(E));
+  if (R.getInt())
+    Out << "an"; // &
+  const OverloadExpr *Ovl = R.getPointer();
+  if (const UnresolvedMemberExpr *ME = dyn_cast<UnresolvedMemberExpr>(Ovl)) {
+    mangleMemberExpr(ME->getBase(), ME->isArrow(), ME->getQualifier(),
+                     ME->getMemberName(), Arity);
+    return;
+  }
+
+  mangleUnresolvedName(Ovl->getQualifier(), Ovl->getName(), Arity);
+}
+
+/// Mangles a member expression.  Implicit accesses are not handled,
+/// but that should be okay, because you shouldn't be able to
+/// make an implicit access in a function template declaration.
+///
+/// The standard ABI does not describe how member expressions should
+/// be mangled, so this is very unstandardized.  We mangle as if it
+/// were a binary operator, except that the RHS is mangled as an
+/// abstract name.
+///
+/// The standard ABI also does not assign a mangling to the dot
+/// operator, so we arbitrarily select 'me'.
+void CXXNameMangler::mangleMemberExpr(const Expr *Base,
+                                      bool IsArrow,
+                                      NestedNameSpecifier *Qualifier,
+                                      DeclarationName Member,
+                                      unsigned Arity) {
+  Out << (IsArrow ? "pt" : "me");
+  mangleExpression(Base);
+  mangleUnresolvedName(Qualifier, Member, Arity);
+}
+
 void CXXNameMangler::mangleExpression(const Expr *E) {
   // <expression> ::= <unary operator-name> <expression>
-	//              ::= <binary operator-name> <expression> <expression>
-	//              ::= <trinary operator-name> <expression> <expression> <expression>
-  //              ::= cl <expression>* E	        # call
+  //              ::= <binary operator-name> <expression> <expression>
+  //              ::= <trinary operator-name> <expression> <expression> <expression>
+  //              ::= cl <expression>* E	     # call
   //              ::= cv <type> expression           # conversion with one argument
   //              ::= cv <type> _ <expression>* E # conversion with a different number of arguments
-  //              ::= st <type>		        # sizeof (a type)
+  //              ::= st <type>		             # sizeof (a type)
   //              ::= at <type>                      # alignof (a type)
   //              ::= <template-param>
   //              ::= <function-param>
   //              ::= sr <type> <unqualified-name>                   # dependent name
   //              ::= sr <type> <unqualified-name> <template-args>   # dependent template-id
   //              ::= sZ <template-param>                            # size of a parameter pack
-	//              ::= <expr-primary>
+  //              ::= <expr-primary>
+  // <expr-primary> ::= L <type> <value number> E    # integer literal
+  //                ::= L <type <value float> E      # floating literal
+  //                ::= L <mangled-name> E           # external name
   switch (E->getStmtClass()) {
-  default: assert(false && "Unhandled expression kind!");
+  default:
+    llvm_unreachable("unexpected statement kind");
+    break;
+
+  case Expr::CallExprClass: {
+    const CallExpr *CE = cast<CallExpr>(E);
+    Out << "cl";
+    mangleCalledExpression(CE->getCallee(), CE->getNumArgs());
+    for (unsigned I = 0, N = CE->getNumArgs(); I != N; ++I)
+      mangleExpression(CE->getArg(I));
+    Out << "E";
+    break;
+  }
+
+  case Expr::MemberExprClass: {
+    const MemberExpr *ME = cast<MemberExpr>(E);
+    mangleMemberExpr(ME->getBase(), ME->isArrow(),
+                     ME->getQualifier(), ME->getMemberDecl()->getDeclName(),
+                     UnknownArity);
+    break;
+  }
+
+  case Expr::UnresolvedMemberExprClass: {
+    const UnresolvedMemberExpr *ME = cast<UnresolvedMemberExpr>(E);
+    mangleMemberExpr(ME->getBase(), ME->isArrow(),
+                     ME->getQualifier(), ME->getMemberName(),
+                     UnknownArity);
+    break;
+  }
+
+  case Expr::CXXDependentScopeMemberExprClass: {
+    const CXXDependentScopeMemberExpr *ME
+      = cast<CXXDependentScopeMemberExpr>(E);
+    mangleMemberExpr(ME->getBase(), ME->isArrow(),
+                     ME->getQualifier(), ME->getMember(),
+                     UnknownArity);
+    break;
+  }
+
+  case Expr::UnresolvedLookupExprClass: {
+    // The ABI doesn't cover how to mangle overload sets, so we mangle
+    // using something as close as possible to the original lookup
+    // expression.
+    const UnresolvedLookupExpr *ULE = cast<UnresolvedLookupExpr>(E);
+    mangleUnresolvedName(ULE->getQualifier(), ULE->getName(), UnknownArity);
+    break;
+  }
+
+  case Expr::CXXUnresolvedConstructExprClass: { 
+    const CXXUnresolvedConstructExpr *CE = cast<CXXUnresolvedConstructExpr>(E);
+    unsigned N = CE->arg_size();
+
+    Out << "cv";
+    mangleType(CE->getType());
+    if (N != 1) Out << "_";
+    for (unsigned I = 0; I != N; ++I) mangleExpression(CE->getArg(I));
+    if (N != 1) Out << "E";
+    break;    
+  }
+
+  case Expr::CXXTemporaryObjectExprClass:
+  case Expr::CXXConstructExprClass: {
+    const CXXConstructExpr *CE = cast<CXXConstructExpr>(E);
+    unsigned N = CE->getNumArgs();
+
+    Out << "cv";
+    mangleType(CE->getType());
+    if (N != 1) Out << "_";
+    for (unsigned I = 0; I != N; ++I) mangleExpression(CE->getArg(I));
+    if (N != 1) Out << "E";
+    break;
+  }
+
+  case Expr::SizeOfAlignOfExprClass: {
+    const SizeOfAlignOfExpr *SAE = cast<SizeOfAlignOfExpr>(E);
+    if (SAE->isSizeOf()) Out << "s";
+    else Out << "a";
+    if (SAE->isArgumentType()) {
+      Out << "t";
+      mangleType(SAE->getArgumentType());
+    } else {
+      Out << "z";
+      mangleExpression(SAE->getArgumentExpr());
+    }
+    break;
+  }
 
   case Expr::UnaryOperatorClass: {
     const UnaryOperator *UO = cast<UnaryOperator>(E);
@@ -1082,7 +1302,7 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     mangleExpression(BO->getLHS());
     mangleExpression(BO->getRHS());                     
     break;
-  }      
+  }
 
   case Expr::ConditionalOperatorClass: {
     const ConditionalOperator *CO = cast<ConditionalOperator>(E);
@@ -1093,6 +1313,24 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     break;
   }
 
+  case Expr::ImplicitCastExprClass: {
+    mangleExpression(cast<ImplicitCastExpr>(E)->getSubExpr());
+    break;
+  }
+
+  case Expr::CStyleCastExprClass:
+  case Expr::CXXStaticCastExprClass:
+  case Expr::CXXDynamicCastExprClass:
+  case Expr::CXXReinterpretCastExprClass:
+  case Expr::CXXConstCastExprClass:
+  case Expr::CXXFunctionalCastExprClass: {
+    const ExplicitCastExpr *ECE = cast<ExplicitCastExpr>(E);
+    Out << "cv";
+    mangleType(ECE->getType());
+    mangleExpression(ECE->getSubExpr());
+    break;
+  }
+    
   case Expr::CXXOperatorCallExprClass: {
     const CXXOperatorCallExpr *CE = cast<CXXOperatorCallExpr>(E);
     unsigned NumArgs = CE->getNumArgs();
@@ -1139,6 +1377,20 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     break;
   }
 
+  case Expr::FloatingLiteralClass: {
+    const FloatingLiteral *FL = cast<FloatingLiteral>(E);
+    Out << "L";
+    mangleType(FL->getType());
+
+    // TODO: avoid this copy with careful stream management.
+    llvm::SmallVector<char,20> Buffer;
+    FL->getValue().bitcastToAPInt().toString(Buffer, 16, false);
+    Out.write(Buffer.data(), Buffer.size());
+
+    Out << "E";
+    break;
+  }
+
   case Expr::IntegerLiteralClass:
     mangleIntegerLiteral(E->getType(), 
                          llvm::APSInt(cast<IntegerLiteral>(E)->getValue()));
@@ -1216,6 +1468,8 @@ void CXXNameMangler::mangleTemplateArg(const TemplateArgument &A) {
     mangleType(A.getAsType());
     break;
   case TemplateArgument::Template:
+    assert(A.getAsTemplate().getAsTemplateDecl() &&
+           "FIXME: Support dependent template names");
     mangleName(A.getAsTemplate().getAsTemplateDecl());
     break;      
   case TemplateArgument::Expression:
diff --git a/lib/CodeGen/TargetInfo.cpp b/lib/CodeGen/TargetInfo.cpp
index 4454662..a7c0caa 100644
--- a/lib/CodeGen/TargetInfo.cpp
+++ b/lib/CodeGen/TargetInfo.cpp
@@ -271,6 +271,10 @@ ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty,
   if (CodeGenFunction::hasAggregateLLVMType(Ty)) {
     return ABIArgInfo::getIndirect(0);
   } else {
+    // Treat an enum type as its underlying type.
+    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
+      Ty = EnumTy->getDecl()->getIntegerType();
+
     return (Ty->isPromotableIntegerType() ?
             ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
   }
@@ -321,6 +325,9 @@ class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
 public:
   X86_32TargetCodeGenInfo(ASTContext &Context, bool d, bool p)
     :TargetCodeGenInfo(new X86_32ABIInfo(Context, d, p)) {}
+
+  void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
+                           CodeGen::CodeGenModule &CGM) const;
 };
 
 }
@@ -358,6 +365,8 @@ bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
   const RecordType *RT = Ty->getAs<RecordType>();
   if (!RT) return false;
 
+  // FIXME: Traverse bases here too.
+
   // Structure types are passed in register if all fields would be
   // passed in a register.
   for (RecordDecl::field_iterator i = RT->getDecl()->field_begin(),
@@ -404,7 +413,7 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
 
     return ABIArgInfo::getDirect();
   } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
-    if (const RecordType *RT = RetTy->getAsStructureType()) {
+    if (const RecordType *RT = RetTy->getAs<RecordType>()) {
       // Structures with either a non-trivial destructor or a non-trivial
       // copy constructor are always indirect.
       if (hasNonTrivialDestructorOrCopyConstructor(RT))
@@ -463,6 +472,10 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
 
     return ABIArgInfo::getIndirect(0);
   } else {
+    // Treat an enum type as its underlying type.
+    if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
+      RetTy = EnumTy->getDecl()->getIntegerType();
+
     return (RetTy->isPromotableIntegerType() ?
             ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
   }
@@ -484,10 +497,16 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
   // FIXME: Set alignment on indirect arguments.
   if (CodeGenFunction::hasAggregateLLVMType(Ty)) {
     // Structures with flexible arrays are always indirect.
-    if (const RecordType *RT = Ty->getAsStructureType())
+    if (const RecordType *RT = Ty->getAs<RecordType>()) {
+      // Structures with either a non-trivial destructor or a non-trivial
+      // copy constructor are always indirect.
+      if (hasNonTrivialDestructorOrCopyConstructor(RT))
+        return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+        
       if (RT->getDecl()->hasFlexibleArrayMember())
         return ABIArgInfo::getIndirect(getIndirectArgumentAlignment(Ty,
                                                                     Context));
+    }
 
     // Ignore empty structs.
     if (Ty->isStructureType() && Context.getTypeSize(Ty) == 0)
@@ -503,6 +522,9 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
 
     return ABIArgInfo::getIndirect(getIndirectArgumentAlignment(Ty, Context));
   } else {
+    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
+      Ty = EnumTy->getDecl()->getIntegerType();
+
     return (Ty->isPromotableIntegerType() ?
             ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
   }
@@ -532,6 +554,20 @@ llvm::Value *X86_32ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
   return AddrTyped;
 }
 
+void X86_32TargetCodeGenInfo::SetTargetAttributes(const Decl *D,
+                                                  llvm::GlobalValue *GV,
+                                            CodeGen::CodeGenModule &CGM) const {
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
+      // Get the LLVM function.
+      llvm::Function *Fn = cast<llvm::Function>(GV);
+
+      // Now add the 'alignstack' attribute with a value of 16.
+      Fn->addFnAttr(llvm::Attribute::constructStackAlignmentFromInt(16));
+    }
+  }
+}
+
 namespace {
 /// X86_64ABIInfo - The X86_64 ABI information.
 class X86_64ABIInfo : public ABIInfo {
@@ -927,6 +963,11 @@ ABIArgInfo X86_64ABIInfo::getCoerceResult(QualType Ty,
   if (CoerceTo == llvm::Type::getInt64Ty(CoerceTo->getContext())) {
     // Integer and pointer types will end up in a general purpose
     // register.
+
+    // Treat an enum type as its underlying type.
+    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
+      Ty = EnumTy->getDecl()->getIntegerType();
+
     if (Ty->isIntegralType() || Ty->hasPointerRepresentation())
       return (Ty->isPromotableIntegerType() ?
               ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
@@ -948,9 +989,14 @@ ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
                                             ASTContext &Context) const {
   // If this is a scalar LLVM value then assume LLVM will pass it in the right
   // place naturally.
-  if (!CodeGenFunction::hasAggregateLLVMType(Ty))
+  if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
+    // Treat an enum type as its underlying type.
+    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
+      Ty = EnumTy->getDecl()->getIntegerType();
+
     return (Ty->isPromotableIntegerType() ?
             ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+  }
 
   bool ByVal = !isRecordWithNonTrivialDestructorOrCopyConstructor(Ty);
 
@@ -1319,14 +1365,14 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
     assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
     const llvm::Type *TyLo = ST->getElementType(0);
     const llvm::Type *TyHi = ST->getElementType(1);
-    assert((TyLo->isFloatingPoint() ^ TyHi->isFloatingPoint()) &&
+    assert((TyLo->isFloatingPointTy() ^ TyHi->isFloatingPointTy()) &&
            "Unexpected ABI info for mixed regs");
     const llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
     const llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
     llvm::Value *GPAddr = CGF.Builder.CreateGEP(RegAddr, gp_offset);
     llvm::Value *FPAddr = CGF.Builder.CreateGEP(RegAddr, fp_offset);
-    llvm::Value *RegLoAddr = TyLo->isFloatingPoint() ? FPAddr : GPAddr;
-    llvm::Value *RegHiAddr = TyLo->isFloatingPoint() ? GPAddr : FPAddr;
+    llvm::Value *RegLoAddr = TyLo->isFloatingPointTy() ? FPAddr : GPAddr;
+    llvm::Value *RegHiAddr = TyLo->isFloatingPointTy() ? GPAddr : FPAddr;
     llvm::Value *V =
       CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegLoAddr, PTyLo));
     CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
@@ -1526,9 +1572,14 @@ void ARMABIInfo::computeInfo(CGFunctionInfo &FI, ASTContext &Context,
 ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty,
                                             ASTContext &Context,
                                           llvm::LLVMContext &VMContext) const {
-  if (!CodeGenFunction::hasAggregateLLVMType(Ty))
+  if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
+    // Treat an enum type as its underlying type.
+    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
+      Ty = EnumTy->getDecl()->getIntegerType();
+
     return (Ty->isPromotableIntegerType() ?
             ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+  }
 
   // Ignore empty records.
   if (isEmptyRecord(Context, Ty, true))
@@ -1577,9 +1628,9 @@ static bool isIntegerLikeType(QualType Ty,
   if (Ty->getAs<BuiltinType>() || Ty->isPointerType())
     return true;
 
-  // Complex types "should" be ok by the definition above, but they are not.
-  if (Ty->isAnyComplexType())
-    return false;
+  // Small complex integer types are "integer like".
+  if (const ComplexType *CT = Ty->getAs<ComplexType>())
+    return isIntegerLikeType(CT->getElementType(), Context, VMContext);
 
   // Single element and zero sized arrays should be allowed, by the definition
   // above, but they are not.
@@ -1603,30 +1654,30 @@ static bool isIntegerLikeType(QualType Ty,
        i != e; ++i, ++idx) {
     const FieldDecl *FD = *i;
 
-    // Check if this field is at offset 0.
-    uint64_t Offset = Layout.getFieldOffset(idx);
-    if (Offset != 0) {
-      // Allow padding bit-fields, but only if they are all at the end of the
-      // structure (despite the wording above, this matches gcc).
-      if (FD->isBitField() && 
-          !FD->getBitWidth()->EvaluateAsInt(Context).getZExtValue()) {
-        for (; i != e; ++i)
-          if (!i->isBitField() ||
-              i->getBitWidth()->EvaluateAsInt(Context).getZExtValue())
-            return false;
-
-        // All remaining fields are padding, allow this.
-        return true;
-      }
+    // Bit-fields are not addressable, we only need to verify they are "integer
+    // like". We still have to disallow a subsequent non-bitfield, for example:
+    //   struct { int : 0; int x }
+    // is non-integer like according to gcc.
+    if (FD->isBitField()) {
+      if (!RD->isUnion())
+        HadField = true;
 
-      return false;
+      if (!isIntegerLikeType(FD->getType(), Context, VMContext))
+        return false;
+
+      continue;
     }
 
+    // Check if this field is at offset 0.
+    if (Layout.getFieldOffset(idx) != 0)
+      return false;
+
     if (!isIntegerLikeType(FD->getType(), Context, VMContext))
       return false;
     
-    // Only allow at most one field in a structure. Again this doesn't match the
-    // wording above, but follows gcc.
+    // Only allow at most one field in a structure. This doesn't match the
+    // wording above, but follows gcc in situations with a field following an
+    // empty structure.
     if (!RD->isUnion()) {
       if (HadField)
         return false;
@@ -1644,15 +1695,28 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy,
   if (RetTy->isVoidType())
     return ABIArgInfo::getIgnore();
 
-  if (!CodeGenFunction::hasAggregateLLVMType(RetTy))
+  if (!CodeGenFunction::hasAggregateLLVMType(RetTy)) {
+    // Treat an enum type as its underlying type.
+    if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
+      RetTy = EnumTy->getDecl()->getIntegerType();
+
     return (RetTy->isPromotableIntegerType() ?
             ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+  }
 
   // Are we following APCS?
   if (getABIKind() == APCS) {
     if (isEmptyRecord(Context, RetTy, false))
       return ABIArgInfo::getIgnore();
 
+    // Complex types are all returned as packed integers.
+    //
+    // FIXME: Consider using 2 x vector types if the back end handles them
+    // correctly.
+    if (RetTy->isAnyComplexType())
+      return ABIArgInfo::getCoerce(llvm::IntegerType::get(
+                                     VMContext, Context.getTypeSize(RetTy)));
+
     // Integer like structures are returned in r0.
     if (isIntegerLikeType(RetTy, Context, VMContext)) {
       // Return in the smallest viable integer type.
@@ -1721,6 +1785,10 @@ ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy,
   } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
     return ABIArgInfo::getIndirect(0);
   } else {
+    // Treat an enum type as its underlying type.
+    if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
+      RetTy = EnumTy->getDecl()->getIntegerType();
+
     return (RetTy->isPromotableIntegerType() ?
             ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
   }