Vendor import of clang trunk r130700:

http://llvm.org/svn/llvm-project/cfe/trunk@130700

1 files changed, 417 insertions, 169 deletions

diff --git a/lib/CodeGen/CGBlocks.cpp b/lib/CodeGen/CGBlocks.cpp
index 9587de2..99a69a4 100644
--- a/lib/CodeGen/CGBlocks.cpp
+++ b/lib/CodeGen/CGBlocks.cpp
@@ -27,13 +27,16 @@ using namespace CodeGen;
 
 CGBlockInfo::CGBlockInfo(const BlockExpr *blockExpr, const char *N)
   : Name(N), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
-    HasCXXObject(false), StructureType(0), Block(blockExpr) {
+    HasCXXObject(false), UsesStret(false), StructureType(0), Block(blockExpr) {
     
   // Skip asm prefix, if any.
   if (Name && Name[0] == '\01')
     ++Name;
 }
 
+// Anchor the vtable to this translation unit.
+CodeGenModule::ByrefHelpers::~ByrefHelpers() {}
+
 /// Build the given block as a global block.
 static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
                                         const CGBlockInfo &blockInfo,
@@ -104,23 +107,6 @@ static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
   return llvm::ConstantExpr::getBitCast(global, CGM.getBlockDescriptorType());
 }
 
-static BlockFlags computeBlockFlag(CodeGenModule &CGM,
-                                   const BlockExpr *BE,
-                                   BlockFlags flags) {
-  const FunctionType *ftype = BE->getFunctionType();
-  
-  // This is a bit overboard.
-  CallArgList args;
-  const CGFunctionInfo &fnInfo =
-    CGM.getTypes().getFunctionInfo(ftype->getResultType(), args,
-                                   ftype->getExtInfo());
-
-  if (CGM.ReturnTypeUsesSRet(fnInfo))
-    flags |= BLOCK_USE_STRET;
-
-  return flags;
-}
-
 /*
   Purely notional variadic template describing the layout of a block.
 
@@ -536,7 +522,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
   BlockFlags flags = BLOCK_HAS_SIGNATURE;
   if (blockInfo.NeedsCopyDispose) flags |= BLOCK_HAS_COPY_DISPOSE;
   if (blockInfo.HasCXXObject) flags |= BLOCK_HAS_CXX_OBJ;
-  flags = computeBlockFlag(CGM, blockInfo.getBlockExpr(), flags);
+  if (blockInfo.UsesStret) flags |= BLOCK_USE_STRET;
 
   // Initialize the block literal.
   Builder.CreateStore(isa, Builder.CreateStructGEP(blockAddr, 0, "block.isa"));
@@ -630,7 +616,9 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
 
       ImplicitCastExpr l2r(ImplicitCastExpr::OnStack, type, CK_LValueToRValue,
                            declRef, VK_RValue);
-      EmitAnyExprToMem(&l2r, blockField, /*volatile*/ false, /*init*/ true);
+      EmitExprAsInit(&l2r, variable, blockField,
+                     getContext().getDeclAlign(variable),
+                     /*captured by init*/ false);
     }
 
     // Push a destructor if necessary.  The semantics for when this
@@ -734,7 +722,7 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr* E,
   // Add the block literal.
   QualType VoidPtrTy = getContext().getPointerType(getContext().VoidTy);
   CallArgList Args;
-  Args.push_back(std::make_pair(RValue::get(BlockLiteral), VoidPtrTy));
+  Args.add(RValue::get(BlockLiteral), VoidPtrTy);
 
   QualType FnType = BPT->getPointeeType();
 
@@ -745,7 +733,7 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr* E,
   // Load the function.
   llvm::Value *Func = Builder.CreateLoad(FuncPtr, "tmp");
 
-  const FunctionType *FuncTy = FnType->getAs<FunctionType>();
+  const FunctionType *FuncTy = FnType->castAs<FunctionType>();
   QualType ResultType = FuncTy->getResultType();
 
   const CGFunctionInfo &FnInfo =
@@ -834,8 +822,9 @@ static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
   fields[0] = CGM.getNSConcreteGlobalBlock();
 
   // __flags
-  BlockFlags flags = computeBlockFlag(CGM, blockInfo.getBlockExpr(),
-                                      BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE);
+  BlockFlags flags = BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE;
+  if (blockInfo.UsesStret) flags |= BLOCK_USE_STRET;
+                                      
   fields[1] = llvm::ConstantInt::get(CGM.IntTy, flags.getBitMask());
 
   // Reserved
@@ -873,7 +862,10 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
                                        const DeclMapTy &ldm) {
   const BlockDecl *blockDecl = blockInfo.getBlockDecl();
 
-  DebugInfo = CGM.getDebugInfo();
+  // Check if we should generate debug info for this block function.
+  if (CGM.getModuleDebugInfo())
+    DebugInfo = CGM.getModuleDebugInfo();
+
   BlockInfo = &blockInfo;
 
   // Arrange for local static and local extern declarations to appear
@@ -897,12 +889,12 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
 
   ImplicitParamDecl selfDecl(const_cast<BlockDecl*>(blockDecl),
                              SourceLocation(), II, selfTy);
-  args.push_back(std::make_pair(&selfDecl, selfTy));
+  args.push_back(&selfDecl);
 
   // Now add the rest of the parameters.
   for (BlockDecl::param_const_iterator i = blockDecl->param_begin(),
        e = blockDecl->param_end(); i != e; ++i)
-    args.push_back(std::make_pair(*i, (*i)->getType()));
+    args.push_back(*i);
 
   // Create the function declaration.
   const FunctionProtoType *fnType =
@@ -910,6 +902,9 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
   const CGFunctionInfo &fnInfo =
     CGM.getTypes().getFunctionInfo(fnType->getResultType(), args,
                                    fnType->getExtInfo());
+  if (CGM.ReturnTypeUsesSRet(fnInfo))
+    blockInfo.UsesStret = true;
+
   const llvm::FunctionType *fnLLVMType =
     CGM.getTypes().GetFunctionType(fnInfo, fnType->isVariadic());
 
@@ -921,8 +916,8 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
   CGM.SetInternalFunctionAttributes(blockDecl, fn, fnInfo);
 
   // Begin generating the function.
-  StartFunction(blockDecl, fnType->getResultType(), fn, args,
-                blockInfo.getBlockExpr()->getBody()->getLocEnd());
+  StartFunction(blockDecl, fnType->getResultType(), fn, fnInfo, args,
+                blockInfo.getBlockExpr()->getBody()->getLocStart());
   CurFuncDecl = outerFnDecl; // StartFunction sets this to blockDecl
 
   // Okay.  Undo some of what StartFunction did.
@@ -1049,13 +1044,10 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
   ASTContext &C = getContext();
 
   FunctionArgList args;
-  // FIXME: This leaks
-  ImplicitParamDecl *dstDecl =
-    ImplicitParamDecl::Create(C, 0, SourceLocation(), 0, C.VoidPtrTy);
-  args.push_back(std::make_pair(dstDecl, dstDecl->getType()));
-  ImplicitParamDecl *srcDecl =
-    ImplicitParamDecl::Create(C, 0, SourceLocation(), 0, C.VoidPtrTy);
-  args.push_back(std::make_pair(srcDecl, srcDecl->getType()));
+  ImplicitParamDecl dstDecl(0, SourceLocation(), 0, C.VoidPtrTy);
+  args.push_back(&dstDecl);
+  ImplicitParamDecl srcDecl(0, SourceLocation(), 0, C.VoidPtrTy);
+  args.push_back(&srcDecl);
 
   const CGFunctionInfo &FI =
       CGM.getTypes().getFunctionInfo(C.VoidTy, args, FunctionType::ExtInfo());
@@ -1073,20 +1065,21 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
 
   FunctionDecl *FD = FunctionDecl::Create(C,
                                           C.getTranslationUnitDecl(),
+                                          SourceLocation(),
                                           SourceLocation(), II, C.VoidTy, 0,
                                           SC_Static,
                                           SC_None,
                                           false,
                                           true);
-  StartFunction(FD, C.VoidTy, Fn, args, SourceLocation());
+  StartFunction(FD, C.VoidTy, Fn, FI, args, SourceLocation());
 
   const llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
 
-  llvm::Value *src = GetAddrOfLocalVar(srcDecl);
+  llvm::Value *src = GetAddrOfLocalVar(&srcDecl);
   src = Builder.CreateLoad(src);
   src = Builder.CreateBitCast(src, structPtrTy, "block.source");
 
-  llvm::Value *dst = GetAddrOfLocalVar(dstDecl);
+  llvm::Value *dst = GetAddrOfLocalVar(&dstDecl);
   dst = Builder.CreateLoad(dst);
   dst = Builder.CreateBitCast(dst, structPtrTy, "block.dest");
 
@@ -1143,10 +1136,8 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
   ASTContext &C = getContext();
 
   FunctionArgList args;
-  // FIXME: This leaks
-  ImplicitParamDecl *srcDecl =
-    ImplicitParamDecl::Create(C, 0, SourceLocation(), 0, C.VoidPtrTy);
-  args.push_back(std::make_pair(srcDecl, srcDecl->getType()));
+  ImplicitParamDecl srcDecl(0, SourceLocation(), 0, C.VoidPtrTy);
+  args.push_back(&srcDecl);
 
   const CGFunctionInfo &FI =
       CGM.getTypes().getFunctionInfo(C.VoidTy, args, FunctionType::ExtInfo());
@@ -1163,15 +1154,16 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
     = &CGM.getContext().Idents.get("__destroy_helper_block_");
 
   FunctionDecl *FD = FunctionDecl::Create(C, C.getTranslationUnitDecl(),
+                                          SourceLocation(),
                                           SourceLocation(), II, C.VoidTy, 0,
                                           SC_Static,
                                           SC_None,
                                           false, true);
-  StartFunction(FD, C.VoidTy, Fn, args, SourceLocation());
+  StartFunction(FD, C.VoidTy, Fn, FI, args, SourceLocation());
 
   const llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
 
-  llvm::Value *src = GetAddrOfLocalVar(srcDecl);
+  llvm::Value *src = GetAddrOfLocalVar(&srcDecl);
   src = Builder.CreateLoad(src);
   src = Builder.CreateBitCast(src, structPtrTy, "block");
 
@@ -1229,99 +1221,158 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
   return llvm::ConstantExpr::getBitCast(Fn, VoidPtrTy);
 }
 
-llvm::Constant *CodeGenFunction::
-GeneratebyrefCopyHelperFunction(const llvm::Type *T, BlockFieldFlags flags,
-                                const VarDecl *variable) {
-  QualType R = getContext().VoidTy;
-
-  FunctionArgList Args;
-  // FIXME: This leaks
-  ImplicitParamDecl *Dst =
-    ImplicitParamDecl::Create(getContext(), 0,
-                              SourceLocation(), 0,
-                              getContext().getPointerType(getContext().VoidTy));
-  Args.push_back(std::make_pair(Dst, Dst->getType()));
-
-  // FIXME: This leaks
-  ImplicitParamDecl *Src =
-    ImplicitParamDecl::Create(getContext(), 0,
-                              SourceLocation(), 0,
-                              getContext().getPointerType(getContext().VoidTy));
-  Args.push_back(std::make_pair(Src, Src->getType()));
+namespace {
+
+/// Emits the copy/dispose helper functions for a __block object of id type.
+class ObjectByrefHelpers : public CodeGenModule::ByrefHelpers {
+  BlockFieldFlags Flags;
+
+public:
+  ObjectByrefHelpers(CharUnits alignment, BlockFieldFlags flags)
+    : ByrefHelpers(alignment), Flags(flags) {}
+
+  void emitCopy(CodeGenFunction &CGF, llvm::Value *destField,
+                llvm::Value *srcField) {
+    destField = CGF.Builder.CreateBitCast(destField, CGF.VoidPtrTy);
+
+    srcField = CGF.Builder.CreateBitCast(srcField, CGF.VoidPtrPtrTy);
+    llvm::Value *srcValue = CGF.Builder.CreateLoad(srcField);
+
+    unsigned flags = (Flags | BLOCK_BYREF_CALLER).getBitMask();
+
+    llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags);
+    llvm::Value *fn = CGF.CGM.getBlockObjectAssign();
+    CGF.Builder.CreateCall3(fn, destField, srcValue, flagsVal);
+  }
+
+  void emitDispose(CodeGenFunction &CGF, llvm::Value *field) {
+    field = CGF.Builder.CreateBitCast(field, CGF.Int8PtrTy->getPointerTo(0));
+    llvm::Value *value = CGF.Builder.CreateLoad(field);
+
+    CGF.BuildBlockRelease(value, Flags | BLOCK_BYREF_CALLER);
+  }
+
+  void profileImpl(llvm::FoldingSetNodeID &id) const {
+    id.AddInteger(Flags.getBitMask());
+  }
+};
+
+/// Emits the copy/dispose helpers for a __block variable with a
+/// nontrivial copy constructor or destructor.
+class CXXByrefHelpers : public CodeGenModule::ByrefHelpers {
+  QualType VarType;
+  const Expr *CopyExpr;
+
+public:
+  CXXByrefHelpers(CharUnits alignment, QualType type,
+                  const Expr *copyExpr)
+    : ByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {}
+
+  bool needsCopy() const { return CopyExpr != 0; }
+  void emitCopy(CodeGenFunction &CGF, llvm::Value *destField,
+                llvm::Value *srcField) {
+    if (!CopyExpr) return;
+    CGF.EmitSynthesizedCXXCopyCtor(destField, srcField, CopyExpr);
+  }
+
+  void emitDispose(CodeGenFunction &CGF, llvm::Value *field) {
+    EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
+    CGF.PushDestructorCleanup(VarType, field);
+    CGF.PopCleanupBlocks(cleanupDepth);
+  }
+
+  void profileImpl(llvm::FoldingSetNodeID &id) const {
+    id.AddPointer(VarType.getCanonicalType().getAsOpaquePtr());
+  }
+};
+} // end anonymous namespace
+
+static llvm::Constant *
+generateByrefCopyHelper(CodeGenFunction &CGF,
+                        const llvm::StructType &byrefType,
+                        CodeGenModule::ByrefHelpers &byrefInfo) {
+  ASTContext &Context = CGF.getContext();
+
+  QualType R = Context.VoidTy;
+
+  FunctionArgList args;
+  ImplicitParamDecl dst(0, SourceLocation(), 0, Context.VoidPtrTy);
+  args.push_back(&dst);
+
+  ImplicitParamDecl src(0, SourceLocation(), 0, Context.VoidPtrTy);
+  args.push_back(&src);
 
   const CGFunctionInfo &FI =
-      CGM.getTypes().getFunctionInfo(R, Args, FunctionType::ExtInfo());
+    CGF.CGM.getTypes().getFunctionInfo(R, args, FunctionType::ExtInfo());
 
-  CodeGenTypes &Types = CGM.getTypes();
+  CodeGenTypes &Types = CGF.CGM.getTypes();
   const llvm::FunctionType *LTy = Types.GetFunctionType(FI, false);
 
   // FIXME: We'd like to put these into a mergable by content, with
   // internal linkage.
   llvm::Function *Fn =
     llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
-                           "__Block_byref_object_copy_", &CGM.getModule());
+                           "__Block_byref_object_copy_", &CGF.CGM.getModule());
 
   IdentifierInfo *II
-    = &CGM.getContext().Idents.get("__Block_byref_object_copy_");
+    = &Context.Idents.get("__Block_byref_object_copy_");
 
-  FunctionDecl *FD = FunctionDecl::Create(getContext(),
-                                          getContext().getTranslationUnitDecl(),
+  FunctionDecl *FD = FunctionDecl::Create(Context,
+                                          Context.getTranslationUnitDecl(),
+                                          SourceLocation(),
                                           SourceLocation(), II, R, 0,
                                           SC_Static,
                                           SC_None,
                                           false, true);
-  StartFunction(FD, R, Fn, Args, SourceLocation());
-
-  // dst->x
-  llvm::Value *V = GetAddrOfLocalVar(Dst);
-  V = Builder.CreateBitCast(V, llvm::PointerType::get(T, 0));
-  V = Builder.CreateLoad(V);
-  V = Builder.CreateStructGEP(V, 6, "x");
-  llvm::Value *DstObj = V;
-
-  // src->x
-  V = GetAddrOfLocalVar(Src);
-  V = Builder.CreateLoad(V);
-  V = Builder.CreateBitCast(V, T);
-  V = Builder.CreateStructGEP(V, 6, "x");
-  
-  if (Expr *copyExpr = getContext().getBlockVarCopyInits(variable)) {
-    llvm::Value *SrcObj = V;
-    EmitSynthesizedCXXCopyCtor(DstObj, SrcObj, copyExpr);
-  } else {
-    DstObj = Builder.CreateBitCast(DstObj, VoidPtrTy);
-    V = Builder.CreateBitCast(V, VoidPtrPtrTy);
-    llvm::Value *SrcObj = Builder.CreateLoad(V);
-    flags |= BLOCK_BYREF_CALLER;
-    llvm::Value *N = llvm::ConstantInt::get(Int32Ty, flags.getBitMask());
-    llvm::Value *F = CGM.getBlockObjectAssign();
-    Builder.CreateCall3(F, DstObj, SrcObj, N);
-  }
-  
-  FinishFunction();
+  CGF.StartFunction(FD, R, Fn, FI, args, SourceLocation());
 
-  return llvm::ConstantExpr::getBitCast(Fn, Int8PtrTy);
+  if (byrefInfo.needsCopy()) {
+    const llvm::Type *byrefPtrType = byrefType.getPointerTo(0);
+
+    // dst->x
+    llvm::Value *destField = CGF.GetAddrOfLocalVar(&dst);
+    destField = CGF.Builder.CreateLoad(destField);
+    destField = CGF.Builder.CreateBitCast(destField, byrefPtrType);
+    destField = CGF.Builder.CreateStructGEP(destField, 6, "x");
+
+    // src->x
+    llvm::Value *srcField = CGF.GetAddrOfLocalVar(&src);
+    srcField = CGF.Builder.CreateLoad(srcField);
+    srcField = CGF.Builder.CreateBitCast(srcField, byrefPtrType);
+    srcField = CGF.Builder.CreateStructGEP(srcField, 6, "x");
+
+    byrefInfo.emitCopy(CGF, destField, srcField);
+  }  
+
+  CGF.FinishFunction();
+
+  return llvm::ConstantExpr::getBitCast(Fn, CGF.Int8PtrTy);
 }
 
-llvm::Constant *
-CodeGenFunction::GeneratebyrefDestroyHelperFunction(const llvm::Type *T,
-                                                    BlockFieldFlags flags,
-                                                    const VarDecl *variable) {
-  QualType R = getContext().VoidTy;
+/// Build the copy helper for a __block variable.
+static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM,
+                                            const llvm::StructType &byrefType,
+                                            CodeGenModule::ByrefHelpers &info) {
+  CodeGenFunction CGF(CGM);
+  return generateByrefCopyHelper(CGF, byrefType, info);
+}
 
-  FunctionArgList Args;
-  // FIXME: This leaks
-  ImplicitParamDecl *Src =
-    ImplicitParamDecl::Create(getContext(), 0,
-                              SourceLocation(), 0,
-                              getContext().getPointerType(getContext().VoidTy));
+/// Generate code for a __block variable's dispose helper.
+static llvm::Constant *
+generateByrefDisposeHelper(CodeGenFunction &CGF,
+                           const llvm::StructType &byrefType,
+                           CodeGenModule::ByrefHelpers &byrefInfo) {
+  ASTContext &Context = CGF.getContext();
+  QualType R = Context.VoidTy;
 
-  Args.push_back(std::make_pair(Src, Src->getType()));
+  FunctionArgList args;
+  ImplicitParamDecl src(0, SourceLocation(), 0, Context.VoidPtrTy);
+  args.push_back(&src);
 
   const CGFunctionInfo &FI =
-      CGM.getTypes().getFunctionInfo(R, Args, FunctionType::ExtInfo());
+    CGF.CGM.getTypes().getFunctionInfo(R, args, FunctionType::ExtInfo());
 
-  CodeGenTypes &Types = CGM.getTypes();
+  CodeGenTypes &Types = CGF.CGM.getTypes();
   const llvm::FunctionType *LTy = Types.GetFunctionType(FI, false);
 
   // FIXME: We'd like to put these into a mergable by content, with
@@ -1329,81 +1380,255 @@ CodeGenFunction::GeneratebyrefDestroyHelperFunction(const llvm::Type *T,
   llvm::Function *Fn =
     llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
                            "__Block_byref_object_dispose_",
-                           &CGM.getModule());
+                           &CGF.CGM.getModule());
 
   IdentifierInfo *II
-    = &CGM.getContext().Idents.get("__Block_byref_object_dispose_");
+    = &Context.Idents.get("__Block_byref_object_dispose_");
 
-  FunctionDecl *FD = FunctionDecl::Create(getContext(),
-                                          getContext().getTranslationUnitDecl(),
+  FunctionDecl *FD = FunctionDecl::Create(Context,
+                                          Context.getTranslationUnitDecl(),
+                                          SourceLocation(),
                                           SourceLocation(), II, R, 0,
                                           SC_Static,
                                           SC_None,
                                           false, true);
-  StartFunction(FD, R, Fn, Args, SourceLocation());
+  CGF.StartFunction(FD, R, Fn, FI, args, SourceLocation());
+
+  if (byrefInfo.needsDispose()) {
+    llvm::Value *V = CGF.GetAddrOfLocalVar(&src);
+    V = CGF.Builder.CreateLoad(V);
+    V = CGF.Builder.CreateBitCast(V, byrefType.getPointerTo(0));
+    V = CGF.Builder.CreateStructGEP(V, 6, "x");
 
-  llvm::Value *V = GetAddrOfLocalVar(Src);
-  V = Builder.CreateBitCast(V, llvm::PointerType::get(T, 0));
-  V = Builder.CreateLoad(V);
-  V = Builder.CreateStructGEP(V, 6, "x");
+    byrefInfo.emitDispose(CGF, V);
+  }
 
-  // If it's not any kind of special object, it must have a destructor
-  // or something.
-  if (!flags.isSpecialPointer()) {
-    EHScopeStack::stable_iterator CleanupDepth = EHStack.stable_begin();
-    PushDestructorCleanup(variable->getType(), V);
-    PopCleanupBlocks(CleanupDepth);
+  CGF.FinishFunction();
 
-  // Otherwise, call _Block_object_dispose.
-  } else {
-    V = Builder.CreateBitCast(V, llvm::PointerType::get(Int8PtrTy, 0));
-    V = Builder.CreateLoad(V);
+  return llvm::ConstantExpr::getBitCast(Fn, CGF.Int8PtrTy);
+}
 
-    flags |= BLOCK_BYREF_CALLER;
-    BuildBlockRelease(V, flags);
+/// Build the dispose helper for a __block variable.
+static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM,
+                                              const llvm::StructType &byrefType,
+                                            CodeGenModule::ByrefHelpers &info) {
+  CodeGenFunction CGF(CGM);
+  return generateByrefDisposeHelper(CGF, byrefType, info);
+}
+
+/// 
+template <class T> static T *buildByrefHelpers(CodeGenModule &CGM,
+                                               const llvm::StructType &byrefTy,
+                                               T &byrefInfo) {
+  // Increase the field's alignment to be at least pointer alignment,
+  // since the layout of the byref struct will guarantee at least that.
+  byrefInfo.Alignment = std::max(byrefInfo.Alignment,
+                              CharUnits::fromQuantity(CGM.PointerAlignInBytes));
+
+  llvm::FoldingSetNodeID id;
+  byrefInfo.Profile(id);
+
+  void *insertPos;
+  CodeGenModule::ByrefHelpers *node
+    = CGM.ByrefHelpersCache.FindNodeOrInsertPos(id, insertPos);
+  if (node) return static_cast<T*>(node);
+
+  byrefInfo.CopyHelper = buildByrefCopyHelper(CGM, byrefTy, byrefInfo);
+  byrefInfo.DisposeHelper = buildByrefDisposeHelper(CGM, byrefTy, byrefInfo);
+
+  T *copy = new (CGM.getContext()) T(byrefInfo);
+  CGM.ByrefHelpersCache.InsertNode(copy, insertPos);
+  return copy;
+}
+
+CodeGenModule::ByrefHelpers *
+CodeGenFunction::buildByrefHelpers(const llvm::StructType &byrefType,
+                                   const AutoVarEmission &emission) {
+  const VarDecl &var = *emission.Variable;
+  QualType type = var.getType();
+
+  if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) {
+    const Expr *copyExpr = CGM.getContext().getBlockVarCopyInits(&var);
+    if (!copyExpr && record->hasTrivialDestructor()) return 0;
+
+    CXXByrefHelpers byrefInfo(emission.Alignment, type, copyExpr);
+    return ::buildByrefHelpers(CGM, byrefType, byrefInfo);
   }
 
-  FinishFunction();
+  BlockFieldFlags flags;
+  if (type->isBlockPointerType()) {
+    flags |= BLOCK_FIELD_IS_BLOCK;
+  } else if (CGM.getContext().isObjCNSObjectType(type) || 
+             type->isObjCObjectPointerType()) {
+    flags |= BLOCK_FIELD_IS_OBJECT;
+  } else {
+    return 0;
+  }
 
-  return llvm::ConstantExpr::getBitCast(Fn, Int8PtrTy);
+  if (type.isObjCGCWeak())
+    flags |= BLOCK_FIELD_IS_WEAK;
+
+  ObjectByrefHelpers byrefInfo(emission.Alignment, flags);
+  return ::buildByrefHelpers(CGM, byrefType, byrefInfo);
 }
 
-llvm::Constant *CodeGenModule::BuildbyrefCopyHelper(const llvm::Type *T,
-                                                    BlockFieldFlags flags,
-                                                    unsigned align,
-                                                    const VarDecl *var) {
-  // All alignments below pointer alignment are bumped up, as we
-  // always have at least that much alignment to begin with.
-  if (align < PointerAlignInBytes) align = PointerAlignInBytes;
+unsigned CodeGenFunction::getByRefValueLLVMField(const ValueDecl *VD) const {
+  assert(ByRefValueInfo.count(VD) && "Did not find value!");
   
-  // As an optimization, we only generate a single function of each kind we
-  // might need.  We need a different one for each alignment and for each
-  // setting of flags.  We mix Align and flag to get the kind.
-  uint64_t Kind = (uint64_t)align*BLOCK_BYREF_CURRENT_MAX + flags.getBitMask();
-  llvm::Constant *&Entry = AssignCache[Kind];
-  if (!Entry)
-    Entry = CodeGenFunction(*this).
-                GeneratebyrefCopyHelperFunction(T, flags, var);
-  return Entry;
+  return ByRefValueInfo.find(VD)->second.second;
+}
+
+llvm::Value *CodeGenFunction::BuildBlockByrefAddress(llvm::Value *BaseAddr,
+                                                     const VarDecl *V) {
+  llvm::Value *Loc = Builder.CreateStructGEP(BaseAddr, 1, "forwarding");
+  Loc = Builder.CreateLoad(Loc);
+  Loc = Builder.CreateStructGEP(Loc, getByRefValueLLVMField(V),
+                                V->getNameAsString());
+  return Loc;
 }
 
-llvm::Constant *CodeGenModule::BuildbyrefDestroyHelper(const llvm::Type *T,
-                                                       BlockFieldFlags flags,
-                                                       unsigned align,
-                                                       const VarDecl *var) {
-  // All alignments below pointer alignment are bumped up, as we
-  // always have at least that much alignment to begin with.
-  if (align < PointerAlignInBytes) align = PointerAlignInBytes;
+/// BuildByRefType - This routine changes a __block variable declared as T x
+///   into:
+///
+///      struct {
+///        void *__isa;
+///        void *__forwarding;
+///        int32_t __flags;
+///        int32_t __size;
+///        void *__copy_helper;       // only if needed
+///        void *__destroy_helper;    // only if needed
+///        char padding[X];           // only if needed
+///        T x;
+///      } x
+///
+const llvm::Type *CodeGenFunction::BuildByRefType(const VarDecl *D) {
+  std::pair<const llvm::Type *, unsigned> &Info = ByRefValueInfo[D];
+  if (Info.first)
+    return Info.first;
+  
+  QualType Ty = D->getType();
+
+  std::vector<const llvm::Type *> Types;
+  
+  llvm::PATypeHolder ByRefTypeHolder = llvm::OpaqueType::get(getLLVMContext());
+  
+  // void *__isa;
+  Types.push_back(Int8PtrTy);
+  
+  // void *__forwarding;
+  Types.push_back(llvm::PointerType::getUnqual(ByRefTypeHolder));
+  
+  // int32_t __flags;
+  Types.push_back(Int32Ty);
+    
+  // int32_t __size;
+  Types.push_back(Int32Ty);
+
+  bool HasCopyAndDispose = getContext().BlockRequiresCopying(Ty);
+  if (HasCopyAndDispose) {
+    /// void *__copy_helper;
+    Types.push_back(Int8PtrTy);
+    
+    /// void *__destroy_helper;
+    Types.push_back(Int8PtrTy);
+  }
+
+  bool Packed = false;
+  CharUnits Align = getContext().getDeclAlign(D);
+  if (Align > getContext().toCharUnitsFromBits(Target.getPointerAlign(0))) {
+    // We have to insert padding.
+    
+    // The struct above has 2 32-bit integers.
+    unsigned CurrentOffsetInBytes = 4 * 2;
+    
+    // And either 2 or 4 pointers.
+    CurrentOffsetInBytes += (HasCopyAndDispose ? 4 : 2) *
+      CGM.getTargetData().getTypeAllocSize(Int8PtrTy);
+    
+    // Align the offset.
+    unsigned AlignedOffsetInBytes = 
+      llvm::RoundUpToAlignment(CurrentOffsetInBytes, Align.getQuantity());
+    
+    unsigned NumPaddingBytes = AlignedOffsetInBytes - CurrentOffsetInBytes;
+    if (NumPaddingBytes > 0) {
+      const llvm::Type *Ty = llvm::Type::getInt8Ty(getLLVMContext());
+      // FIXME: We need a sema error for alignment larger than the minimum of
+      // the maximal stack alignmint and the alignment of malloc on the system.
+      if (NumPaddingBytes > 1)
+        Ty = llvm::ArrayType::get(Ty, NumPaddingBytes);
+    
+      Types.push_back(Ty);
+
+      // We want a packed struct.
+      Packed = true;
+    }
+  }
+
+  // T x;
+  Types.push_back(ConvertTypeForMem(Ty));
+  
+  const llvm::Type *T = llvm::StructType::get(getLLVMContext(), Types, Packed);
+  
+  cast<llvm::OpaqueType>(ByRefTypeHolder.get())->refineAbstractTypeTo(T);
+  CGM.getModule().addTypeName("struct.__block_byref_" + D->getNameAsString(), 
+                              ByRefTypeHolder.get());
   
-  // As an optimization, we only generate a single function of each kind we
-  // might need.  We need a different one for each alignment and for each
-  // setting of flags.  We mix Align and flag to get the kind.
-  uint64_t Kind = (uint64_t)align*BLOCK_BYREF_CURRENT_MAX + flags.getBitMask();
-  llvm::Constant *&Entry = DestroyCache[Kind];
-  if (!Entry)
-    Entry = CodeGenFunction(*this).
-                GeneratebyrefDestroyHelperFunction(T, flags, var);
-  return Entry;
+  Info.first = ByRefTypeHolder.get();
+  
+  Info.second = Types.size() - 1;
+  
+  return Info.first;
+}
+
+/// Initialize the structural components of a __block variable, i.e.
+/// everything but the actual object.
+void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
+  // Find the address of the local.
+  llvm::Value *addr = emission.Address;
+
+  // That's an alloca of the byref structure type.
+  const llvm::StructType *byrefType = cast<llvm::StructType>(
+                 cast<llvm::PointerType>(addr->getType())->getElementType());
+
+  // Build the byref helpers if necessary.  This is null if we don't need any.
+  CodeGenModule::ByrefHelpers *helpers =
+    buildByrefHelpers(*byrefType, emission);
+
+  const VarDecl &D = *emission.Variable;
+  QualType type = D.getType();
+
+  llvm::Value *V;
+
+  // Initialize the 'isa', which is just 0 or 1.
+  int isa = 0;
+  if (type.isObjCGCWeak())
+    isa = 1;
+  V = Builder.CreateIntToPtr(Builder.getInt32(isa), Int8PtrTy, "isa");
+  Builder.CreateStore(V, Builder.CreateStructGEP(addr, 0, "byref.isa"));
+
+  // Store the address of the variable into its own forwarding pointer.
+  Builder.CreateStore(addr,
+                      Builder.CreateStructGEP(addr, 1, "byref.forwarding"));
+
+  // Blocks ABI:
+  //   c) the flags field is set to either 0 if no helper functions are
+  //      needed or BLOCK_HAS_COPY_DISPOSE if they are,
+  BlockFlags flags;
+  if (helpers) flags |= BLOCK_HAS_COPY_DISPOSE;
+  Builder.CreateStore(llvm::ConstantInt::get(IntTy, flags.getBitMask()),
+                      Builder.CreateStructGEP(addr, 2, "byref.flags"));
+
+  CharUnits byrefSize = CGM.GetTargetTypeStoreSize(byrefType);
+  V = llvm::ConstantInt::get(IntTy, byrefSize.getQuantity());
+  Builder.CreateStore(V, Builder.CreateStructGEP(addr, 3, "byref.size"));
+
+  if (helpers) {
+    llvm::Value *copy_helper = Builder.CreateStructGEP(addr, 4);
+    Builder.CreateStore(helpers->CopyHelper, copy_helper);
+
+    llvm::Value *destroy_helper = Builder.CreateStructGEP(addr, 5);
+    Builder.CreateStore(helpers->DisposeHelper, destroy_helper);
+  }
 }
 
 void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags) {
@@ -1413,3 +1638,26 @@ void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags) {
   N = llvm::ConstantInt::get(Int32Ty, flags.getBitMask());
   Builder.CreateCall2(F, V, N);
 }
+
+namespace {
+  struct CallBlockRelease : EHScopeStack::Cleanup {
+    llvm::Value *Addr;
+    CallBlockRelease(llvm::Value *Addr) : Addr(Addr) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      CGF.BuildBlockRelease(Addr, BLOCK_FIELD_IS_BYREF);
+    }
+  };
+}
+
+/// Enter a cleanup to destroy a __block variable.  Note that this
+/// cleanup should be a no-op if the variable hasn't left the stack
+/// yet; if a cleanup is required for the variable itself, that needs
+/// to be done externally.
+void CodeGenFunction::enterByrefCleanup(const AutoVarEmission &emission) {
+  // We don't enter this cleanup if we're in pure-GC mode.
+  if (CGM.getLangOptions().getGCMode() == LangOptions::GCOnly)
+    return;
+
+  EHStack.pushCleanup<CallBlockRelease>(NormalAndEHCleanup, emission.Address);
+}