diff options
Diffstat (limited to 'contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp')
| -rw-r--r-- | contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp | 1149 |
1 files changed, 1149 insertions, 0 deletions
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp new file mode 100644 index 0000000..89bde2c --- /dev/null +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp @@ -0,0 +1,1149 @@ +//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains the code for emitting atomic operations. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenFunction.h" +#include "CGCall.h" +#include "CodeGenModule.h" +#include "clang/AST/ASTContext.h" +#include "clang/CodeGen/CGFunctionInfo.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Operator.h" + +using namespace clang; +using namespace CodeGen; + +namespace { + class AtomicInfo { + CodeGenFunction &CGF; + QualType AtomicTy; + QualType ValueTy; + uint64_t AtomicSizeInBits; + uint64_t ValueSizeInBits; + CharUnits AtomicAlign; + CharUnits ValueAlign; + CharUnits LValueAlign; + TypeEvaluationKind EvaluationKind; + bool UseLibcall; + public: + AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) { + assert(lvalue.isSimple()); + + AtomicTy = lvalue.getType(); + ValueTy = AtomicTy->castAs<AtomicType>()->getValueType(); + EvaluationKind = CGF.getEvaluationKind(ValueTy); + + ASTContext &C = CGF.getContext(); + + uint64_t valueAlignInBits; + std::tie(ValueSizeInBits, valueAlignInBits) = C.getTypeInfo(ValueTy); + + uint64_t atomicAlignInBits; + std::tie(AtomicSizeInBits, atomicAlignInBits) = C.getTypeInfo(AtomicTy); + + assert(ValueSizeInBits <= AtomicSizeInBits); + assert(valueAlignInBits <= atomicAlignInBits); + + AtomicAlign = C.toCharUnitsFromBits(atomicAlignInBits); + ValueAlign = C.toCharUnitsFromBits(valueAlignInBits); + if (lvalue.getAlignment().isZero()) + lvalue.setAlignment(AtomicAlign); + + UseLibcall = + (AtomicSizeInBits > uint64_t(C.toBits(lvalue.getAlignment())) || + AtomicSizeInBits > C.getTargetInfo().getMaxAtomicInlineWidth()); + } + + QualType getAtomicType() const { return AtomicTy; } + QualType getValueType() const { return ValueTy; } + CharUnits getAtomicAlignment() const { return AtomicAlign; } + CharUnits getValueAlignment() const { return ValueAlign; } + uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } + uint64_t getValueSizeInBits() const { return AtomicSizeInBits; } + TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } + bool shouldUseLibcall() const { return UseLibcall; } + + /// Is the atomic size larger than the underlying value type? + /// + /// Note that the absence of padding does not mean that atomic + /// objects are completely interchangeable with non-atomic + /// objects: we might have promoted the alignment of a type + /// without making it bigger. + bool hasPadding() const { + return (ValueSizeInBits != AtomicSizeInBits); + } + + bool emitMemSetZeroIfNecessary(LValue dest) const; + + llvm::Value *getAtomicSizeValue() const { + CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); + return CGF.CGM.getSize(size); + } + + /// Cast the given pointer to an integer pointer suitable for + /// atomic operations. + llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const; + + /// Turn an atomic-layout object into an r-value. + RValue convertTempToRValue(llvm::Value *addr, + AggValueSlot resultSlot, + SourceLocation loc) const; + + /// Copy an atomic r-value into atomic-layout memory. + void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const; + + /// Project an l-value down to the value field. + LValue projectValue(LValue lvalue) const { + llvm::Value *addr = lvalue.getAddress(); + if (hasPadding()) + addr = CGF.Builder.CreateStructGEP(addr, 0); + + return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(), + CGF.getContext(), lvalue.getTBAAInfo()); + } + + /// Materialize an atomic r-value in atomic-layout memory. + llvm::Value *materializeRValue(RValue rvalue) const; + + private: + bool requiresMemSetZero(llvm::Type *type) const; + }; +} + +static RValue emitAtomicLibcall(CodeGenFunction &CGF, + StringRef fnName, + QualType resultType, + CallArgList &args) { + const CGFunctionInfo &fnInfo = + CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args, + FunctionType::ExtInfo(), RequiredArgs::All); + llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); + llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); + return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args); +} + +/// Does a store of the given IR type modify the full expected width? +static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, + uint64_t expectedSize) { + return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); +} + +/// Does the atomic type require memsetting to zero before initialization? +/// +/// The IR type is provided as a way of making certain queries faster. +bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { + // If the atomic type has size padding, we definitely need a memset. + if (hasPadding()) return true; + + // Otherwise, do some simple heuristics to try to avoid it: + switch (getEvaluationKind()) { + // For scalars and complexes, check whether the store size of the + // type uses the full size. + case TEK_Scalar: + return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); + case TEK_Complex: + return !isFullSizeType(CGF.CGM, type->getStructElementType(0), + AtomicSizeInBits / 2); + + // Padding in structs has an undefined bit pattern. User beware. + case TEK_Aggregate: + return false; + } + llvm_unreachable("bad evaluation kind"); +} + +bool AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const { + llvm::Value *addr = dest.getAddress(); + if (!requiresMemSetZero(addr->getType()->getPointerElementType())) + return false; + + CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), + AtomicSizeInBits / 8, + dest.getAlignment().getQuantity()); + return true; +} + +static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, + llvm::Value *Dest, llvm::Value *Ptr, + llvm::Value *Val1, llvm::Value *Val2, + uint64_t Size, unsigned Align, + llvm::AtomicOrdering SuccessOrder, + llvm::AtomicOrdering FailureOrder) { + // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. + llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1); + Expected->setAlignment(Align); + llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2); + Desired->setAlignment(Align); + + llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( + Ptr, Expected, Desired, SuccessOrder, FailureOrder); + Pair->setVolatile(E->isVolatile()); + Pair->setWeak(IsWeak); + + // Cmp holds the result of the compare-exchange operation: true on success, + // false on failure. + llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); + llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); + + // This basic block is used to hold the store instruction if the operation + // failed. + llvm::BasicBlock *StoreExpectedBB = + CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); + + // This basic block is the exit point of the operation, we should end up + // here regardless of whether or not the operation succeeded. + llvm::BasicBlock *ContinueBB = + CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); + + // Update Expected if Expected isn't equal to Old, otherwise branch to the + // exit point. + CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); + + CGF.Builder.SetInsertPoint(StoreExpectedBB); + // Update the memory at Expected with Old's value. + llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1); + StoreExpected->setAlignment(Align); + // Finally, branch to the exit point. + CGF.Builder.CreateBr(ContinueBB); + + CGF.Builder.SetInsertPoint(ContinueBB); + // Update the memory at Dest with Cmp's value. + CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); + return; +} + +/// Given an ordering required on success, emit all possible cmpxchg +/// instructions to cope with the provided (but possibly only dynamically known) +/// FailureOrder. +static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, + bool IsWeak, llvm::Value *Dest, + llvm::Value *Ptr, llvm::Value *Val1, + llvm::Value *Val2, + llvm::Value *FailureOrderVal, + uint64_t Size, unsigned Align, + llvm::AtomicOrdering SuccessOrder) { + llvm::AtomicOrdering FailureOrder; + if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { + switch (FO->getSExtValue()) { + default: + FailureOrder = llvm::Monotonic; + break; + case AtomicExpr::AO_ABI_memory_order_consume: + case AtomicExpr::AO_ABI_memory_order_acquire: + FailureOrder = llvm::Acquire; + break; + case AtomicExpr::AO_ABI_memory_order_seq_cst: + FailureOrder = llvm::SequentiallyConsistent; + break; + } + if (FailureOrder >= SuccessOrder) { + // Don't assert on undefined behaviour. + FailureOrder = + llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder); + } + emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, Align, + SuccessOrder, FailureOrder); + return; + } + + // Create all the relevant BB's + llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, + *SeqCstBB = nullptr; + MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); + if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release) + AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); + if (SuccessOrder == llvm::SequentiallyConsistent) + SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); + + llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); + + llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); + + // Emit all the different atomics + + // MonotonicBB is arbitrarily chosen as the default case; in practice, this + // doesn't matter unless someone is crazy enough to use something that + // doesn't fold to a constant for the ordering. + CGF.Builder.SetInsertPoint(MonotonicBB); + emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, + Size, Align, SuccessOrder, llvm::Monotonic); + CGF.Builder.CreateBr(ContBB); + + if (AcquireBB) { + CGF.Builder.SetInsertPoint(AcquireBB); + emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, + Size, Align, SuccessOrder, llvm::Acquire); + CGF.Builder.CreateBr(ContBB); + SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume), + AcquireBB); + SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire), + AcquireBB); + } + if (SeqCstBB) { + CGF.Builder.SetInsertPoint(SeqCstBB); + emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, + Size, Align, SuccessOrder, llvm::SequentiallyConsistent); + CGF.Builder.CreateBr(ContBB); + SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst), + SeqCstBB); + } + + CGF.Builder.SetInsertPoint(ContBB); +} + +static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest, + llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2, + llvm::Value *IsWeak, llvm::Value *FailureOrder, + uint64_t Size, unsigned Align, + llvm::AtomicOrdering Order) { + llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; + llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; + + switch (E->getOp()) { + case AtomicExpr::AO__c11_atomic_init: + llvm_unreachable("Already handled!"); + + case AtomicExpr::AO__c11_atomic_compare_exchange_strong: + emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, + FailureOrder, Size, Align, Order); + return; + case AtomicExpr::AO__c11_atomic_compare_exchange_weak: + emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, + FailureOrder, Size, Align, Order); + return; + case AtomicExpr::AO__atomic_compare_exchange: + case AtomicExpr::AO__atomic_compare_exchange_n: { + if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { + emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, + Val1, Val2, FailureOrder, Size, Align, Order); + } else { + // Create all the relevant BB's + llvm::BasicBlock *StrongBB = + CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); + llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); + llvm::BasicBlock *ContBB = + CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); + + llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); + SI->addCase(CGF.Builder.getInt1(false), StrongBB); + + CGF.Builder.SetInsertPoint(StrongBB); + emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, + FailureOrder, Size, Align, Order); + CGF.Builder.CreateBr(ContBB); + + CGF.Builder.SetInsertPoint(WeakBB); + emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, + FailureOrder, Size, Align, Order); + CGF.Builder.CreateBr(ContBB); + + CGF.Builder.SetInsertPoint(ContBB); + } + return; + } + case AtomicExpr::AO__c11_atomic_load: + case AtomicExpr::AO__atomic_load_n: + case AtomicExpr::AO__atomic_load: { + llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); + Load->setAtomic(Order); + Load->setAlignment(Size); + Load->setVolatile(E->isVolatile()); + llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest); + StoreDest->setAlignment(Align); + return; + } + + case AtomicExpr::AO__c11_atomic_store: + case AtomicExpr::AO__atomic_store: + case AtomicExpr::AO__atomic_store_n: { + assert(!Dest && "Store does not return a value"); + llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); + LoadVal1->setAlignment(Align); + llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); + Store->setAtomic(Order); + Store->setAlignment(Size); + Store->setVolatile(E->isVolatile()); + return; + } + + case AtomicExpr::AO__c11_atomic_exchange: + case AtomicExpr::AO__atomic_exchange_n: + case AtomicExpr::AO__atomic_exchange: + Op = llvm::AtomicRMWInst::Xchg; + break; + + case AtomicExpr::AO__atomic_add_fetch: + PostOp = llvm::Instruction::Add; + // Fall through. + case AtomicExpr::AO__c11_atomic_fetch_add: + case AtomicExpr::AO__atomic_fetch_add: + Op = llvm::AtomicRMWInst::Add; + break; + + case AtomicExpr::AO__atomic_sub_fetch: + PostOp = llvm::Instruction::Sub; + // Fall through. + case AtomicExpr::AO__c11_atomic_fetch_sub: + case AtomicExpr::AO__atomic_fetch_sub: + Op = llvm::AtomicRMWInst::Sub; + break; + + case AtomicExpr::AO__atomic_and_fetch: + PostOp = llvm::Instruction::And; + // Fall through. + case AtomicExpr::AO__c11_atomic_fetch_and: + case AtomicExpr::AO__atomic_fetch_and: + Op = llvm::AtomicRMWInst::And; + break; + + case AtomicExpr::AO__atomic_or_fetch: + PostOp = llvm::Instruction::Or; + // Fall through. + case AtomicExpr::AO__c11_atomic_fetch_or: + case AtomicExpr::AO__atomic_fetch_or: + Op = llvm::AtomicRMWInst::Or; + break; + + case AtomicExpr::AO__atomic_xor_fetch: + PostOp = llvm::Instruction::Xor; + // Fall through. + case AtomicExpr::AO__c11_atomic_fetch_xor: + case AtomicExpr::AO__atomic_fetch_xor: + Op = llvm::AtomicRMWInst::Xor; + break; + + case AtomicExpr::AO__atomic_nand_fetch: + PostOp = llvm::Instruction::And; + // Fall through. + case AtomicExpr::AO__atomic_fetch_nand: + Op = llvm::AtomicRMWInst::Nand; + break; + } + + llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); + LoadVal1->setAlignment(Align); + llvm::AtomicRMWInst *RMWI = + CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order); + RMWI->setVolatile(E->isVolatile()); + + // For __atomic_*_fetch operations, perform the operation again to + // determine the value which was written. + llvm::Value *Result = RMWI; + if (PostOp) + Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); + if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) + Result = CGF.Builder.CreateNot(Result); + llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest); + StoreDest->setAlignment(Align); +} + +// This function emits any expression (scalar, complex, or aggregate) +// into a temporary alloca. +static llvm::Value * +EmitValToTemp(CodeGenFunction &CGF, Expr *E) { + llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); + CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), + /*Init*/ true); + return DeclPtr; +} + +static void +AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, + bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, + SourceLocation Loc) { + if (UseOptimizedLibcall) { + // Load value and pass it to the function directly. + unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity(); + Val = CGF.EmitLoadOfScalar(Val, false, Align, ValTy, Loc); + Args.add(RValue::get(Val), ValTy); + } else { + // Non-optimized functions always take a reference. + Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), + CGF.getContext().VoidPtrTy); + } +} + +RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { + QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); + QualType MemTy = AtomicTy; + if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) + MemTy = AT->getValueType(); + CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy); + uint64_t Size = sizeChars.getQuantity(); + CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy); + unsigned Align = alignChars.getQuantity(); + unsigned MaxInlineWidthInBits = + getTarget().getMaxAtomicInlineWidth(); + bool UseLibcall = (Size != Align || + getContext().toBits(sizeChars) > MaxInlineWidthInBits); + + llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr, + *Val2 = nullptr; + llvm::Value *Ptr = EmitScalarExpr(E->getPtr()); + + if (E->getOp() == AtomicExpr::AO__c11_atomic_init) { + assert(!Dest && "Init does not return a value"); + LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext()); + EmitAtomicInit(E->getVal1(), lvalue); + return RValue::get(nullptr); + } + + llvm::Value *Order = EmitScalarExpr(E->getOrder()); + + switch (E->getOp()) { + case AtomicExpr::AO__c11_atomic_init: + llvm_unreachable("Already handled!"); + + case AtomicExpr::AO__c11_atomic_load: + case AtomicExpr::AO__atomic_load_n: + break; + + case AtomicExpr::AO__atomic_load: + Dest = EmitScalarExpr(E->getVal1()); + break; + + case AtomicExpr::AO__atomic_store: + Val1 = EmitScalarExpr(E->getVal1()); + break; + + case AtomicExpr::AO__atomic_exchange: + Val1 = EmitScalarExpr(E->getVal1()); + Dest = EmitScalarExpr(E->getVal2()); + break; + + case AtomicExpr::AO__c11_atomic_compare_exchange_strong: + case AtomicExpr::AO__c11_atomic_compare_exchange_weak: + case AtomicExpr::AO__atomic_compare_exchange_n: + case AtomicExpr::AO__atomic_compare_exchange: + Val1 = EmitScalarExpr(E->getVal1()); + if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) + Val2 = EmitScalarExpr(E->getVal2()); + else + Val2 = EmitValToTemp(*this, E->getVal2()); + OrderFail = EmitScalarExpr(E->getOrderFail()); + if (E->getNumSubExprs() == 6) + IsWeak = EmitScalarExpr(E->getWeak()); + break; + + case AtomicExpr::AO__c11_atomic_fetch_add: + case AtomicExpr::AO__c11_atomic_fetch_sub: + if (MemTy->isPointerType()) { + // For pointer arithmetic, we're required to do a bit of math: + // adding 1 to an int* is not the same as adding 1 to a uintptr_t. + // ... but only for the C11 builtins. The GNU builtins expect the + // user to multiply by sizeof(T). + QualType Val1Ty = E->getVal1()->getType(); + llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); + CharUnits PointeeIncAmt = + getContext().getTypeSizeInChars(MemTy->getPointeeType()); + Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); + Val1 = CreateMemTemp(Val1Ty, ".atomictmp"); + EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty)); + break; + } + // Fall through. + case AtomicExpr::AO__atomic_fetch_add: + case AtomicExpr::AO__atomic_fetch_sub: + case AtomicExpr::AO__atomic_add_fetch: + case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__c11_atomic_store: + case AtomicExpr::AO__c11_atomic_exchange: + case AtomicExpr::AO__atomic_store_n: + case AtomicExpr::AO__atomic_exchange_n: + case AtomicExpr::AO__c11_atomic_fetch_and: + case AtomicExpr::AO__c11_atomic_fetch_or: + case AtomicExpr::AO__c11_atomic_fetch_xor: + case AtomicExpr::AO__atomic_fetch_and: + case AtomicExpr::AO__atomic_fetch_or: + case AtomicExpr::AO__atomic_fetch_xor: + case AtomicExpr::AO__atomic_fetch_nand: + case AtomicExpr::AO__atomic_and_fetch: + case AtomicExpr::AO__atomic_or_fetch: + case AtomicExpr::AO__atomic_xor_fetch: + case AtomicExpr::AO__atomic_nand_fetch: + Val1 = EmitValToTemp(*this, E->getVal1()); + break; + } + + if (!E->getType()->isVoidType() && !Dest) + Dest = CreateMemTemp(E->getType(), ".atomicdst"); + + // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . + if (UseLibcall) { + bool UseOptimizedLibcall = false; + switch (E->getOp()) { + case AtomicExpr::AO__c11_atomic_fetch_add: + case AtomicExpr::AO__atomic_fetch_add: + case AtomicExpr::AO__c11_atomic_fetch_and: + case AtomicExpr::AO__atomic_fetch_and: + case AtomicExpr::AO__c11_atomic_fetch_or: + case AtomicExpr::AO__atomic_fetch_or: + case AtomicExpr::AO__c11_atomic_fetch_sub: + case AtomicExpr::AO__atomic_fetch_sub: + case AtomicExpr::AO__c11_atomic_fetch_xor: + case AtomicExpr::AO__atomic_fetch_xor: + // For these, only library calls for certain sizes exist. + UseOptimizedLibcall = true; + break; + default: + // Only use optimized library calls for sizes for which they exist. + if (Size == 1 || Size == 2 || Size == 4 || Size == 8) + UseOptimizedLibcall = true; + break; + } + + CallArgList Args; + if (!UseOptimizedLibcall) { + // For non-optimized library calls, the size is the first parameter + Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), + getContext().getSizeType()); + } + // Atomic address is the first or second parameter + Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), getContext().VoidPtrTy); + + std::string LibCallName; + QualType LoweredMemTy = + MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; + QualType RetTy; + bool HaveRetTy = false; + switch (E->getOp()) { + // There is only one libcall for compare an exchange, because there is no + // optimisation benefit possible from a libcall version of a weak compare + // and exchange. + // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, + // void *desired, int success, int failure) + // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, + // int success, int failure) + case AtomicExpr::AO__c11_atomic_compare_exchange_weak: + case AtomicExpr::AO__c11_atomic_compare_exchange_strong: + case AtomicExpr::AO__atomic_compare_exchange: + case AtomicExpr::AO__atomic_compare_exchange_n: + LibCallName = "__atomic_compare_exchange"; + RetTy = getContext().BoolTy; + HaveRetTy = true; + Args.add(RValue::get(EmitCastToVoidPtr(Val1)), getContext().VoidPtrTy); + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy, + E->getExprLoc()); + Args.add(RValue::get(Order), getContext().IntTy); + Order = OrderFail; + break; + // void __atomic_exchange(size_t size, void *mem, void *val, void *return, + // int order) + // T __atomic_exchange_N(T *mem, T val, int order) + case AtomicExpr::AO__c11_atomic_exchange: + case AtomicExpr::AO__atomic_exchange_n: + case AtomicExpr::AO__atomic_exchange: + LibCallName = "__atomic_exchange"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, + E->getExprLoc()); + break; + // void __atomic_store(size_t size, void *mem, void *val, int order) + // void __atomic_store_N(T *mem, T val, int order) + case AtomicExpr::AO__c11_atomic_store: + case AtomicExpr::AO__atomic_store: + case AtomicExpr::AO__atomic_store_n: + LibCallName = "__atomic_store"; + RetTy = getContext().VoidTy; + HaveRetTy = true; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, + E->getExprLoc()); + break; + // void __atomic_load(size_t size, void *mem, void *return, int order) + // T __atomic_load_N(T *mem, int order) + case AtomicExpr::AO__c11_atomic_load: + case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__atomic_load_n: + LibCallName = "__atomic_load"; + break; + // T __atomic_fetch_add_N(T *mem, T val, int order) + case AtomicExpr::AO__c11_atomic_fetch_add: + case AtomicExpr::AO__atomic_fetch_add: + LibCallName = "__atomic_fetch_add"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy, + E->getExprLoc()); + break; + // T __atomic_fetch_and_N(T *mem, T val, int order) + case AtomicExpr::AO__c11_atomic_fetch_and: + case AtomicExpr::AO__atomic_fetch_and: + LibCallName = "__atomic_fetch_and"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, + E->getExprLoc()); + break; + // T __atomic_fetch_or_N(T *mem, T val, int order) + case AtomicExpr::AO__c11_atomic_fetch_or: + case AtomicExpr::AO__atomic_fetch_or: + LibCallName = "__atomic_fetch_or"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, + E->getExprLoc()); + break; + // T __atomic_fetch_sub_N(T *mem, T val, int order) + case AtomicExpr::AO__c11_atomic_fetch_sub: + case AtomicExpr::AO__atomic_fetch_sub: + LibCallName = "__atomic_fetch_sub"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy, + E->getExprLoc()); + break; + // T __atomic_fetch_xor_N(T *mem, T val, int order) + case AtomicExpr::AO__c11_atomic_fetch_xor: + case AtomicExpr::AO__atomic_fetch_xor: + LibCallName = "__atomic_fetch_xor"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, + E->getExprLoc()); + break; + default: return EmitUnsupportedRValue(E, "atomic library call"); + } + + // Optimized functions have the size in their name. + if (UseOptimizedLibcall) + LibCallName += "_" + llvm::utostr(Size); + // By default, assume we return a value of the atomic type. + if (!HaveRetTy) { + if (UseOptimizedLibcall) { + // Value is returned directly. + RetTy = MemTy; + } else { + // Value is returned through parameter before the order. + RetTy = getContext().VoidTy; + Args.add(RValue::get(EmitCastToVoidPtr(Dest)), + getContext().VoidPtrTy); + } + } + // order is always the last parameter + Args.add(RValue::get(Order), + getContext().IntTy); + + const CGFunctionInfo &FuncInfo = + CGM.getTypes().arrangeFreeFunctionCall(RetTy, Args, + FunctionType::ExtInfo(), RequiredArgs::All); + llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); + llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); + RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); + if (!RetTy->isVoidType()) + return Res; + if (E->getType()->isVoidType()) + return RValue::get(nullptr); + return convertTempToRValue(Dest, E->getType(), E->getExprLoc()); + } + + bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || + E->getOp() == AtomicExpr::AO__atomic_store || + E->getOp() == AtomicExpr::AO__atomic_store_n; + bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || + E->getOp() == AtomicExpr::AO__atomic_load || + E->getOp() == AtomicExpr::AO__atomic_load_n; + + llvm::Type *IPtrTy = + llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo(); + llvm::Value *OrigDest = Dest; + Ptr = Builder.CreateBitCast(Ptr, IPtrTy); + if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy); + if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy); + if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy); + + if (isa<llvm::ConstantInt>(Order)) { + int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); + switch (ord) { + case AtomicExpr::AO_ABI_memory_order_relaxed: + EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, Align, llvm::Monotonic); + break; + case AtomicExpr::AO_ABI_memory_order_consume: + case AtomicExpr::AO_ABI_memory_order_acquire: + if (IsStore) + break; // Avoid crashing on code with undefined behavior + EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, Align, llvm::Acquire); + break; + case AtomicExpr::AO_ABI_memory_order_release: + if (IsLoad) + break; // Avoid crashing on code with undefined behavior + EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, Align, llvm::Release); + break; + case AtomicExpr::AO_ABI_memory_order_acq_rel: + if (IsLoad || IsStore) + break; // Avoid crashing on code with undefined behavior + EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, Align, llvm::AcquireRelease); + break; + case AtomicExpr::AO_ABI_memory_order_seq_cst: + EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, Align, llvm::SequentiallyConsistent); + break; + default: // invalid order + // We should not ever get here normally, but it's hard to + // enforce that in general. + break; + } + if (E->getType()->isVoidType()) + return RValue::get(nullptr); + return convertTempToRValue(OrigDest, E->getType(), E->getExprLoc()); + } + + // Long case, when Order isn't obviously constant. + + // Create all the relevant BB's + llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, + *ReleaseBB = nullptr, *AcqRelBB = nullptr, + *SeqCstBB = nullptr; + MonotonicBB = createBasicBlock("monotonic", CurFn); + if (!IsStore) + AcquireBB = createBasicBlock("acquire", CurFn); + if (!IsLoad) + ReleaseBB = createBasicBlock("release", CurFn); + if (!IsLoad && !IsStore) + AcqRelBB = createBasicBlock("acqrel", CurFn); + SeqCstBB = createBasicBlock("seqcst", CurFn); + llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); + + // Create the switch for the split + // MonotonicBB is arbitrarily chosen as the default case; in practice, this + // doesn't matter unless someone is crazy enough to use something that + // doesn't fold to a constant for the ordering. + Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); + llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); + + // Emit all the different atomics + Builder.SetInsertPoint(MonotonicBB); + EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, Align, llvm::Monotonic); + Builder.CreateBr(ContBB); + if (!IsStore) { + Builder.SetInsertPoint(AcquireBB); + EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, Align, llvm::Acquire); + Builder.CreateBr(ContBB); + SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume), + AcquireBB); + SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire), + AcquireBB); + } + if (!IsLoad) { + Builder.SetInsertPoint(ReleaseBB); + EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, Align, llvm::Release); + Builder.CreateBr(ContBB); + SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release), + ReleaseBB); + } + if (!IsLoad && !IsStore) { + Builder.SetInsertPoint(AcqRelBB); + EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, Align, llvm::AcquireRelease); + Builder.CreateBr(ContBB); + SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel), + AcqRelBB); + } + Builder.SetInsertPoint(SeqCstBB); + EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, Align, llvm::SequentiallyConsistent); + Builder.CreateBr(ContBB); + SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst), + SeqCstBB); + + // Cleanup and return + Builder.SetInsertPoint(ContBB); + if (E->getType()->isVoidType()) + return RValue::get(nullptr); + return convertTempToRValue(OrigDest, E->getType(), E->getExprLoc()); +} + +llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const { + unsigned addrspace = + cast<llvm::PointerType>(addr->getType())->getAddressSpace(); + llvm::IntegerType *ty = + llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); + return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); +} + +RValue AtomicInfo::convertTempToRValue(llvm::Value *addr, + AggValueSlot resultSlot, + SourceLocation loc) const { + if (EvaluationKind == TEK_Aggregate) + return resultSlot.asRValue(); + + // Drill into the padding structure if we have one. + if (hasPadding()) + addr = CGF.Builder.CreateStructGEP(addr, 0); + + // Otherwise, just convert the temporary to an r-value using the + // normal conversion routine. + return CGF.convertTempToRValue(addr, getValueType(), loc); +} + +/// Emit a load from an l-value of atomic type. Note that the r-value +/// we produce is an r-value of the atomic *value* type. +RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, + AggValueSlot resultSlot) { + AtomicInfo atomics(*this, src); + + // Check whether we should use a library call. + if (atomics.shouldUseLibcall()) { + llvm::Value *tempAddr; + if (!resultSlot.isIgnored()) { + assert(atomics.getEvaluationKind() == TEK_Aggregate); + tempAddr = resultSlot.getAddr(); + } else { + tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp"); + } + + // void __atomic_load(size_t size, void *mem, void *return, int order); + CallArgList args; + args.add(RValue::get(atomics.getAtomicSizeValue()), + getContext().getSizeType()); + args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())), + getContext().VoidPtrTy); + args.add(RValue::get(EmitCastToVoidPtr(tempAddr)), + getContext().VoidPtrTy); + args.add(RValue::get(llvm::ConstantInt::get( + IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)), + getContext().IntTy); + emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args); + + // Produce the r-value. + return atomics.convertTempToRValue(tempAddr, resultSlot, loc); + } + + // Okay, we're doing this natively. + llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress()); + llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load"); + load->setAtomic(llvm::SequentiallyConsistent); + + // Other decoration. + load->setAlignment(src.getAlignment().getQuantity()); + if (src.isVolatileQualified()) + load->setVolatile(true); + if (src.getTBAAInfo()) + CGM.DecorateInstruction(load, src.getTBAAInfo()); + + // Okay, turn that back into the original value type. + QualType valueType = atomics.getValueType(); + llvm::Value *result = load; + + // If we're ignoring an aggregate return, don't do anything. + if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored()) + return RValue::getAggregate(nullptr, false); + + // The easiest way to do this this is to go through memory, but we + // try not to in some easy cases. + if (atomics.getEvaluationKind() == TEK_Scalar && !atomics.hasPadding()) { + llvm::Type *resultTy = CGM.getTypes().ConvertTypeForMem(valueType); + if (isa<llvm::IntegerType>(resultTy)) { + assert(result->getType() == resultTy); + result = EmitFromMemory(result, valueType); + } else if (isa<llvm::PointerType>(resultTy)) { + result = Builder.CreateIntToPtr(result, resultTy); + } else { + result = Builder.CreateBitCast(result, resultTy); + } + return RValue::get(result); + } + + // Create a temporary. This needs to be big enough to hold the + // atomic integer. + llvm::Value *temp; + bool tempIsVolatile = false; + CharUnits tempAlignment; + if (atomics.getEvaluationKind() == TEK_Aggregate) { + assert(!resultSlot.isIgnored()); + temp = resultSlot.getAddr(); + tempAlignment = atomics.getValueAlignment(); + tempIsVolatile = resultSlot.isVolatile(); + } else { + temp = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp"); + tempAlignment = atomics.getAtomicAlignment(); + } + + // Slam the integer into the temporary. + llvm::Value *castTemp = atomics.emitCastToAtomicIntPointer(temp); + Builder.CreateAlignedStore(result, castTemp, tempAlignment.getQuantity()) + ->setVolatile(tempIsVolatile); + + return atomics.convertTempToRValue(temp, resultSlot, loc); +} + + + +/// Copy an r-value into memory as part of storing to an atomic type. +/// This needs to create a bit-pattern suitable for atomic operations. +void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const { + // If we have an r-value, the rvalue should be of the atomic type, + // which means that the caller is responsible for having zeroed + // any padding. Just do an aggregate copy of that type. + if (rvalue.isAggregate()) { + CGF.EmitAggregateCopy(dest.getAddress(), + rvalue.getAggregateAddr(), + getAtomicType(), + (rvalue.isVolatileQualified() + || dest.isVolatileQualified()), + dest.getAlignment()); + return; + } + + // Okay, otherwise we're copying stuff. + + // Zero out the buffer if necessary. + emitMemSetZeroIfNecessary(dest); + + // Drill past the padding if present. + dest = projectValue(dest); + + // Okay, store the rvalue in. + if (rvalue.isScalar()) { + CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true); + } else { + CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true); + } +} + + +/// Materialize an r-value into memory for the purposes of storing it +/// to an atomic type. +llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const { + // Aggregate r-values are already in memory, and EmitAtomicStore + // requires them to be values of the atomic type. + if (rvalue.isAggregate()) + return rvalue.getAggregateAddr(); + + // Otherwise, make a temporary and materialize into it. + llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp"); + LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment()); + emitCopyIntoMemory(rvalue, tempLV); + return temp; +} + +/// Emit a store to an l-value of atomic type. +/// +/// Note that the r-value is expected to be an r-value *of the atomic +/// type*; this means that for aggregate r-values, it should include +/// storage for any padding that was necessary. +void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) { + // If this is an aggregate r-value, it should agree in type except + // maybe for address-space qualification. + assert(!rvalue.isAggregate() || + rvalue.getAggregateAddr()->getType()->getPointerElementType() + == dest.getAddress()->getType()->getPointerElementType()); + + AtomicInfo atomics(*this, dest); + + // If this is an initialization, just put the value there normally. + if (isInit) { + atomics.emitCopyIntoMemory(rvalue, dest); + return; + } + + // Check whether we should use a library call. + if (atomics.shouldUseLibcall()) { + // Produce a source address. + llvm::Value *srcAddr = atomics.materializeRValue(rvalue); + + // void __atomic_store(size_t size, void *mem, void *val, int order) + CallArgList args; + args.add(RValue::get(atomics.getAtomicSizeValue()), + getContext().getSizeType()); + args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())), + getContext().VoidPtrTy); + args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), + getContext().VoidPtrTy); + args.add(RValue::get(llvm::ConstantInt::get( + IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)), + getContext().IntTy); + emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); + return; + } + + // Okay, we're doing this natively. + llvm::Value *intValue; + + // If we've got a scalar value of the right size, try to avoid going + // through memory. + if (rvalue.isScalar() && !atomics.hasPadding()) { + llvm::Value *value = rvalue.getScalarVal(); + if (isa<llvm::IntegerType>(value->getType())) { + intValue = value; + } else { + llvm::IntegerType *inputIntTy = + llvm::IntegerType::get(getLLVMContext(), atomics.getValueSizeInBits()); + if (isa<llvm::PointerType>(value->getType())) { + intValue = Builder.CreatePtrToInt(value, inputIntTy); + } else { + intValue = Builder.CreateBitCast(value, inputIntTy); + } + } + + // Otherwise, we need to go through memory. + } else { + // Put the r-value in memory. + llvm::Value *addr = atomics.materializeRValue(rvalue); + + // Cast the temporary to the atomic int type and pull a value out. + addr = atomics.emitCastToAtomicIntPointer(addr); + intValue = Builder.CreateAlignedLoad(addr, + atomics.getAtomicAlignment().getQuantity()); + } + + // Do the atomic store. + llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress()); + llvm::StoreInst *store = Builder.CreateStore(intValue, addr); + + // Initializations don't need to be atomic. + if (!isInit) store->setAtomic(llvm::SequentiallyConsistent); + + // Other decoration. + store->setAlignment(dest.getAlignment().getQuantity()); + if (dest.isVolatileQualified()) + store->setVolatile(true); + if (dest.getTBAAInfo()) + CGM.DecorateInstruction(store, dest.getTBAAInfo()); +} + +void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { + AtomicInfo atomics(*this, dest); + + switch (atomics.getEvaluationKind()) { + case TEK_Scalar: { + llvm::Value *value = EmitScalarExpr(init); + atomics.emitCopyIntoMemory(RValue::get(value), dest); + return; + } + + case TEK_Complex: { + ComplexPairTy value = EmitComplexExpr(init); + atomics.emitCopyIntoMemory(RValue::getComplex(value), dest); + return; + } + + case TEK_Aggregate: { + // Fix up the destination if the initializer isn't an expression + // of atomic type. + bool Zeroed = false; + if (!init->getType()->isAtomicType()) { + Zeroed = atomics.emitMemSetZeroIfNecessary(dest); + dest = atomics.projectValue(dest); + } + + // Evaluate the expression directly into the destination. + AggValueSlot slot = AggValueSlot::forLValue(dest, + AggValueSlot::IsNotDestructed, + AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased, + Zeroed ? AggValueSlot::IsZeroed : + AggValueSlot::IsNotZeroed); + + EmitAggExpr(init, slot); + return; + } + } + llvm_unreachable("bad evaluation kind"); +} |
