diff options
Diffstat (limited to 'contrib/llvm/tools/clang/lib/CodeGen')
65 files changed, 13770 insertions, 8343 deletions
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h index cc8652e..a65f270 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h @@ -25,6 +25,8 @@ namespace clang { class TargetInfo; namespace CodeGen { + class ABIArgInfo; + class Address; class CGCXXABI; class CGFunctionInfo; class CodeGenFunction; @@ -79,8 +81,15 @@ namespace clang { // the ABI information any lower than CodeGen. Of course, for // VAArg handling it has to be at this level; there is no way to // abstract this out. - virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGen::CodeGenFunction &CGF) const = 0; + virtual CodeGen::Address EmitVAArg(CodeGen::CodeGenFunction &CGF, + CodeGen::Address VAListAddr, + QualType Ty) const = 0; + + /// Emit the target dependent code to load a value of + /// \arg Ty from the \c __builtin_ms_va_list pointed to by \arg VAListAddr. + virtual CodeGen::Address EmitMSVAArg(CodeGen::CodeGenFunction &CGF, + CodeGen::Address VAListAddr, + QualType Ty) const; virtual bool isHomogeneousAggregateBaseType(QualType Ty) const; @@ -92,6 +101,15 @@ namespace clang { bool isHomogeneousAggregate(QualType Ty, const Type *&Base, uint64_t &Members) const; + /// A convenience method to return an indirect ABIArgInfo with an + /// expected alignment equal to the ABI alignment of the given type. + CodeGen::ABIArgInfo + getNaturalAlignIndirect(QualType Ty, bool ByRef = true, + bool Realign = false, + llvm::Type *Padding = nullptr) const; + + CodeGen::ABIArgInfo + getNaturalAlignIndirectInReg(QualType Ty, bool Realign = false) const; }; } // end namespace clang diff --git a/contrib/llvm/tools/clang/lib/CodeGen/Address.h b/contrib/llvm/tools/clang/lib/CodeGen/Address.h new file mode 100644 index 0000000..9d145fa --- /dev/null +++ b/contrib/llvm/tools/clang/lib/CodeGen/Address.h @@ -0,0 +1,126 @@ +//===-- Address.h - An aligned address -------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This class provides a simple wrapper for a pair of a pointer and an +// alignment. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_CODEGEN_ADDRESS_H +#define LLVM_CLANG_LIB_CODEGEN_ADDRESS_H + +#include "llvm/IR/Constants.h" +#include "clang/AST/CharUnits.h" + +namespace clang { +namespace CodeGen { + +/// An aligned address. +class Address { + llvm::Value *Pointer; + CharUnits Alignment; +public: + Address(llvm::Value *pointer, CharUnits alignment) + : Pointer(pointer), Alignment(alignment) { + assert((!alignment.isZero() || pointer == nullptr) && + "creating valid address with invalid alignment"); + } + + static Address invalid() { return Address(nullptr, CharUnits()); } + bool isValid() const { return Pointer != nullptr; } + + llvm::Value *getPointer() const { + assert(isValid()); + return Pointer; + } + + /// Return the type of the pointer value. + llvm::PointerType *getType() const { + return llvm::cast<llvm::PointerType>(getPointer()->getType()); + } + + /// Return the type of the values stored in this address. + /// + /// When IR pointer types lose their element type, we should simply + /// store it in Address instead for the convenience of writing code. + llvm::Type *getElementType() const { + return getType()->getElementType(); + } + + /// Return the address space that this address resides in. + unsigned getAddressSpace() const { + return getType()->getAddressSpace(); + } + + /// Return the IR name of the pointer value. + llvm::StringRef getName() const { + return getPointer()->getName(); + } + + /// Return the alignment of this pointer. + CharUnits getAlignment() const { + assert(isValid()); + return Alignment; + } +}; + +/// A specialization of Address that requires the address to be an +/// LLVM Constant. +class ConstantAddress : public Address { +public: + ConstantAddress(llvm::Constant *pointer, CharUnits alignment) + : Address(pointer, alignment) {} + + static ConstantAddress invalid() { + return ConstantAddress(nullptr, CharUnits()); + } + + llvm::Constant *getPointer() const { + return llvm::cast<llvm::Constant>(Address::getPointer()); + } + + ConstantAddress getBitCast(llvm::Type *ty) const { + return ConstantAddress(llvm::ConstantExpr::getBitCast(getPointer(), ty), + getAlignment()); + } + + ConstantAddress getElementBitCast(llvm::Type *ty) const { + return getBitCast(ty->getPointerTo(getAddressSpace())); + } + + static bool isaImpl(Address addr) { + return llvm::isa<llvm::Constant>(addr.getPointer()); + } + static ConstantAddress castImpl(Address addr) { + return ConstantAddress(llvm::cast<llvm::Constant>(addr.getPointer()), + addr.getAlignment()); + } +}; + +} +} + +namespace llvm { + // Present a minimal LLVM-like casting interface. + template <class U> inline U cast(clang::CodeGen::Address addr) { + return U::castImpl(addr); + } + template <class U> inline bool isa(clang::CodeGen::Address addr) { + return U::isaImpl(addr); + } +} + +namespace clang { + // Make our custom isa and cast available in namespace clang, to mirror + // what we do for LLVM's versions in Basic/LLVM.h. + using llvm::isa; + using llvm::cast; +} + +#endif diff --git a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp index afcb9e5..82297e7 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp @@ -14,6 +14,7 @@ #include "clang/Frontend/CodeGenOptions.h" #include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Frontend/Utils.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" @@ -21,6 +22,7 @@ #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/FunctionInfo.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" @@ -52,6 +54,7 @@ class EmitAssemblyHelper { const clang::TargetOptions &TargetOpts; const LangOptions &LangOpts; Module *TheModule; + std::unique_ptr<FunctionInfoIndex> FunctionIndex; Timer CodeGenerationTime; @@ -112,15 +115,14 @@ private: bool AddEmitPasses(BackendAction Action, raw_pwrite_stream &OS); public: - EmitAssemblyHelper(DiagnosticsEngine &_Diags, - const CodeGenOptions &CGOpts, + EmitAssemblyHelper(DiagnosticsEngine &_Diags, const CodeGenOptions &CGOpts, const clang::TargetOptions &TOpts, - const LangOptions &LOpts, - Module *M) - : Diags(_Diags), CodeGenOpts(CGOpts), TargetOpts(TOpts), LangOpts(LOpts), - TheModule(M), CodeGenerationTime("Code Generation Time"), - CodeGenPasses(nullptr), PerModulePasses(nullptr), - PerFunctionPasses(nullptr) {} + const LangOptions &LOpts, Module *M, + std::unique_ptr<FunctionInfoIndex> Index) + : Diags(_Diags), CodeGenOpts(CGOpts), TargetOpts(TOpts), LangOpts(LOpts), + TheModule(M), FunctionIndex(std::move(Index)), + CodeGenerationTime("Code Generation Time"), CodeGenPasses(nullptr), + PerModulePasses(nullptr), PerFunctionPasses(nullptr) {} ~EmitAssemblyHelper() { delete CodeGenPasses; @@ -166,14 +168,6 @@ static void addObjCARCOptPass(const PassManagerBuilder &Builder, PassManagerBase PM.add(createObjCARCOptPass()); } -static void addSampleProfileLoaderPass(const PassManagerBuilder &Builder, - legacy::PassManagerBase &PM) { - const PassManagerBuilderWrapper &BuilderWrapper = - static_cast<const PassManagerBuilderWrapper &>(Builder); - const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); - PM.add(createSampleProfileLoaderPass(CGOpts.SampleProfileFile)); -} - static void addAddDiscriminatorsPass(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { PM.add(createAddDiscriminatorsPass()); @@ -201,14 +195,20 @@ static void addSanitizerCoveragePass(const PassManagerBuilder &Builder, static void addAddressSanitizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { - PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/false)); - PM.add(createAddressSanitizerModulePass(/*CompileKernel*/false)); + const PassManagerBuilderWrapper &BuilderWrapper = + static_cast<const PassManagerBuilderWrapper&>(Builder); + const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); + bool Recover = CGOpts.SanitizeRecover.has(SanitizerKind::Address); + PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/false, Recover)); + PM.add(createAddressSanitizerModulePass(/*CompileKernel*/false, Recover)); } static void addKernelAddressSanitizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM) { - PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/true)); - PM.add(createAddressSanitizerModulePass(/*CompileKernel*/true)); + PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/true, + /*Recover*/true)); + PM.add(createAddressSanitizerModulePass(/*CompileKernel*/true, + /*Recover*/true)); } static void addMemorySanitizerPass(const PassManagerBuilder &Builder, @@ -272,6 +272,9 @@ static void addSymbolRewriterPass(const CodeGenOptions &Opts, } void EmitAssemblyHelper::CreatePasses() { + if (CodeGenOpts.DisableLLVMPasses) + return; + unsigned OptLevel = CodeGenOpts.OptimizationLevel; CodeGenOptions::InliningMethod Inlining = CodeGenOpts.getInlining(); @@ -283,6 +286,29 @@ void EmitAssemblyHelper::CreatePasses() { } PassManagerBuilderWrapper PMBuilder(CodeGenOpts, LangOpts); + + // Figure out TargetLibraryInfo. + Triple TargetTriple(TheModule->getTargetTriple()); + PMBuilder.LibraryInfo = createTLII(TargetTriple, CodeGenOpts); + + switch (Inlining) { + case CodeGenOptions::NoInlining: + break; + case CodeGenOptions::NormalInlining: { + PMBuilder.Inliner = + createFunctionInliningPass(OptLevel, CodeGenOpts.OptimizeSize); + break; + } + case CodeGenOptions::OnlyAlwaysInlining: + // Respect always_inline. + if (OptLevel == 0) + // Do not insert lifetime intrinsics at -O0. + PMBuilder.Inliner = createAlwaysInlinerPass(false); + else + PMBuilder.Inliner = createAlwaysInlinerPass(); + break; + } + PMBuilder.OptLevel = OptLevel; PMBuilder.SizeLevel = CodeGenOpts.OptimizeSize; PMBuilder.BBVectorize = CodeGenOpts.VectorizeBB; @@ -295,13 +321,20 @@ void EmitAssemblyHelper::CreatePasses() { PMBuilder.PrepareForLTO = CodeGenOpts.PrepareForLTO; PMBuilder.RerollLoops = CodeGenOpts.RerollLoops; + legacy::PassManager *MPM = getPerModulePasses(); + + // If we are performing a ThinLTO importing compile, invoke the LTO + // pipeline and pass down the in-memory function index. + if (!CodeGenOpts.ThinLTOIndexFile.empty()) { + assert(FunctionIndex && "Expected non-empty function index"); + PMBuilder.FunctionIndex = FunctionIndex.get(); + PMBuilder.populateLTOPassManager(*MPM); + return; + } + PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible, addAddDiscriminatorsPass); - if (!CodeGenOpts.SampleProfileFile.empty()) - PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible, - addSampleProfileLoaderPass); - // In ObjC ARC mode, add the main ARC optimization passes. if (LangOpts.ObjCAutoRefCount) { PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible, @@ -363,27 +396,6 @@ void EmitAssemblyHelper::CreatePasses() { addDataFlowSanitizerPass); } - // Figure out TargetLibraryInfo. - Triple TargetTriple(TheModule->getTargetTriple()); - PMBuilder.LibraryInfo = createTLII(TargetTriple, CodeGenOpts); - - switch (Inlining) { - case CodeGenOptions::NoInlining: break; - case CodeGenOptions::NormalInlining: { - PMBuilder.Inliner = - createFunctionInliningPass(OptLevel, CodeGenOpts.OptimizeSize); - break; - } - case CodeGenOptions::OnlyAlwaysInlining: - // Respect always_inline. - if (OptLevel == 0) - // Do not insert lifetime intrinsics at -O0. - PMBuilder.Inliner = createAlwaysInlinerPass(false); - else - PMBuilder.Inliner = createAlwaysInlinerPass(); - break; - } - // Set up the per-function pass manager. legacy::FunctionPassManager *FPM = getPerFunctionPasses(); if (CodeGenOpts.VerifyModule) @@ -391,7 +403,6 @@ void EmitAssemblyHelper::CreatePasses() { PMBuilder.populateFunctionPassManager(*FPM); // Set up the per-module pass manager. - legacy::PassManager *MPM = getPerModulePasses(); if (!CodeGenOpts.RewriteMapFiles.empty()) addSymbolRewriterPass(CodeGenOpts, MPM); @@ -420,6 +431,9 @@ void EmitAssemblyHelper::CreatePasses() { MPM->add(createInstrProfilingPass(Options)); } + if (!CodeGenOpts.SampleProfileFile.empty()) + MPM->add(createSampleProfileLoaderPass(CodeGenOpts.SampleProfileFile)); + PMBuilder.populateModulePassManager(*MPM); } @@ -455,20 +469,16 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { BackendArgs.push_back("-limit-float-precision"); BackendArgs.push_back(CodeGenOpts.LimitFloatPrecision.c_str()); } - for (unsigned i = 0, e = CodeGenOpts.BackendOptions.size(); i != e; ++i) - BackendArgs.push_back(CodeGenOpts.BackendOptions[i].c_str()); + for (const std::string &BackendOption : CodeGenOpts.BackendOptions) + BackendArgs.push_back(BackendOption.c_str()); BackendArgs.push_back(nullptr); llvm::cl::ParseCommandLineOptions(BackendArgs.size() - 1, BackendArgs.data()); - std::string FeaturesStr; - if (!TargetOpts.Features.empty()) { - SubtargetFeatures Features; - for (const std::string &Feature : TargetOpts.Features) - Features.AddFeature(Feature); - FeaturesStr = Features.getString(); - } + std::string FeaturesStr = + llvm::join(TargetOpts.Features.begin(), TargetOpts.Features.end(), ","); + // Keep this synced with the equivalent code in tools/driver/cc1as_main.cpp. llvm::Reloc::Model RM = llvm::Reloc::Default; if (CodeGenOpts.RelocationModel == "static") { RM = llvm::Reloc::Static; @@ -497,24 +507,16 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { .Case("posix", llvm::ThreadModel::POSIX) .Case("single", llvm::ThreadModel::Single); - if (CodeGenOpts.DisableIntegratedAS) - Options.DisableIntegratedAS = true; - - if (CodeGenOpts.CompressDebugSections) - Options.CompressDebugSections = true; - - if (CodeGenOpts.UseInitArray) - Options.UseInitArray = true; - // Set float ABI type. - if (CodeGenOpts.FloatABI == "soft" || CodeGenOpts.FloatABI == "softfp") - Options.FloatABIType = llvm::FloatABI::Soft; - else if (CodeGenOpts.FloatABI == "hard") - Options.FloatABIType = llvm::FloatABI::Hard; - else { - assert(CodeGenOpts.FloatABI.empty() && "Invalid float abi!"); - Options.FloatABIType = llvm::FloatABI::Default; - } + assert((CodeGenOpts.FloatABI == "soft" || CodeGenOpts.FloatABI == "softfp" || + CodeGenOpts.FloatABI == "hard" || CodeGenOpts.FloatABI.empty()) && + "Invalid Floating Point ABI!"); + Options.FloatABIType = + llvm::StringSwitch<llvm::FloatABI::ABIType>(CodeGenOpts.FloatABI) + .Case("soft", llvm::FloatABI::Soft) + .Case("softfp", llvm::FloatABI::Soft) + .Case("hard", llvm::FloatABI::Hard) + .Default(llvm::FloatABI::Default); // Set FP fusion mode. switch (CodeGenOpts.getFPContractMode()) { @@ -529,6 +531,17 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { break; } + Options.UseInitArray = CodeGenOpts.UseInitArray; + Options.DisableIntegratedAS = CodeGenOpts.DisableIntegratedAS; + Options.CompressDebugSections = CodeGenOpts.CompressDebugSections; + + // Set EABI version. + Options.EABIVersion = llvm::StringSwitch<llvm::EABI>(CodeGenOpts.EABIVersion) + .Case("4", llvm::EABI::EABI4) + .Case("5", llvm::EABI::EABI5) + .Case("gnu", llvm::EABI::GNU) + .Default(llvm::EABI::Default); + Options.LessPreciseFPMADOption = CodeGenOpts.LessPreciseFPMAD; Options.NoInfsFPMath = CodeGenOpts.NoInfsFPMath; Options.NoNaNsFPMath = CodeGenOpts.NoNaNsFPMath; @@ -539,11 +552,27 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { Options.FunctionSections = CodeGenOpts.FunctionSections; Options.DataSections = CodeGenOpts.DataSections; Options.UniqueSectionNames = CodeGenOpts.UniqueSectionNames; + Options.EmulatedTLS = CodeGenOpts.EmulatedTLS; + switch (CodeGenOpts.getDebuggerTuning()) { + case CodeGenOptions::DebuggerKindGDB: + Options.DebuggerTuning = llvm::DebuggerKind::GDB; + break; + case CodeGenOptions::DebuggerKindLLDB: + Options.DebuggerTuning = llvm::DebuggerKind::LLDB; + break; + case CodeGenOptions::DebuggerKindSCE: + Options.DebuggerTuning = llvm::DebuggerKind::SCE; + break; + default: + break; + } Options.MCOptions.MCRelaxAll = CodeGenOpts.RelaxAll; Options.MCOptions.MCSaveTempLabels = CodeGenOpts.SaveTempLabels; Options.MCOptions.MCUseDwarfDirectory = !CodeGenOpts.NoDwarfDirectoryAsm; Options.MCOptions.MCNoExecStack = CodeGenOpts.NoExecStack; + Options.MCOptions.MCIncrementalLinkerCompatible = + CodeGenOpts.IncrementalLinkerCompatible; Options.MCOptions.MCFatalWarnings = CodeGenOpts.FatalWarnings; Options.MCOptions.AsmVerbose = CodeGenOpts.AsmVerbose; Options.MCOptions.ABIName = TargetOpts.ABI; @@ -605,7 +634,7 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, if (UsesCodeGen && !TM) return; if (TM) - TheModule->setDataLayout(*TM->getDataLayout()); + TheModule->setDataLayout(TM->createDataLayout()); CreatePasses(); switch (Action) { @@ -613,8 +642,8 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, break; case Backend_EmitBC: - getPerModulePasses()->add( - createBitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists)); + getPerModulePasses()->add(createBitcodeWriterPass( + *OS, CodeGenOpts.EmitLLVMUseLists, CodeGenOpts.EmitFunctionSummary)); break; case Backend_EmitLL: @@ -659,16 +688,17 @@ void clang::EmitBackendOutput(DiagnosticsEngine &Diags, const clang::TargetOptions &TOpts, const LangOptions &LOpts, StringRef TDesc, Module *M, BackendAction Action, - raw_pwrite_stream *OS) { - EmitAssemblyHelper AsmHelper(Diags, CGOpts, TOpts, LOpts, M); + raw_pwrite_stream *OS, + std::unique_ptr<FunctionInfoIndex> Index) { + EmitAssemblyHelper AsmHelper(Diags, CGOpts, TOpts, LOpts, M, + std::move(Index)); AsmHelper.EmitAssembly(Action, OS); // If an optional clang TargetInfo description string was passed in, use it to // verify the LLVM TargetMachine's DataLayout. if (AsmHelper.TM && !TDesc.empty()) { - std::string DLDesc = - AsmHelper.TM->getDataLayout()->getStringRepresentation(); + std::string DLDesc = M->getDataLayout().getStringRepresentation(); if (DLDesc != TDesc) { unsigned DiagID = Diags.getCustomDiagID( DiagnosticsEngine::Error, "backend data layout '%0' does not match " diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp index fc4b66b..24de30b 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp @@ -80,7 +80,7 @@ namespace { AtomicSizeInBits = C.toBits( C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) .RoundUpToAlignment(lvalue.getAlignment())); - auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldAddr()); + auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); auto OffsetInChars = (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * lvalue.getAlignment(); @@ -94,8 +94,9 @@ namespace { BFI.Offset = Offset; BFI.StorageSize = AtomicSizeInBits; BFI.StorageOffset += OffsetInChars; - LVal = LValue::MakeBitfield(Addr, BFI, lvalue.getType(), - lvalue.getAlignment()); + LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()), + BFI, lvalue.getType(), + lvalue.getAlignmentSource()); LVal.setTBAAInfo(lvalue.getTBAAInfo()); AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); if (AtomicTy.isNull()) { @@ -118,10 +119,8 @@ namespace { ValueTy = lvalue.getType(); ValueSizeInBits = C.getTypeSize(ValueTy); AtomicTy = ValueTy = CGF.getContext().getExtVectorType( - lvalue.getType(), lvalue.getExtVectorAddr() - ->getType() - ->getPointerElementType() - ->getVectorNumElements()); + lvalue.getType(), lvalue.getExtVectorAddress() + .getElementType()->getVectorNumElements()); AtomicSizeInBits = C.getTypeSize(AtomicTy); AtomicAlign = ValueAlign = lvalue.getAlignment(); LVal = lvalue; @@ -139,15 +138,22 @@ namespace { TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } bool shouldUseLibcall() const { return UseLibcall; } const LValue &getAtomicLValue() const { return LVal; } - llvm::Value *getAtomicAddress() const { + llvm::Value *getAtomicPointer() const { if (LVal.isSimple()) - return LVal.getAddress(); + return LVal.getPointer(); else if (LVal.isBitField()) - return LVal.getBitFieldAddr(); + return LVal.getBitFieldPointer(); else if (LVal.isVectorElt()) - return LVal.getVectorAddr(); + return LVal.getVectorPointer(); assert(LVal.isExtVectorElt()); - return LVal.getExtVectorAddr(); + return LVal.getExtVectorPointer(); + } + Address getAtomicAddress() const { + return Address(getAtomicPointer(), getAtomicAlignment()); + } + + Address getAtomicAddressAsAtomicIntPointer() const { + return emitCastToAtomicIntPointer(getAtomicAddress()); } /// Is the atomic size larger than the underlying value type? @@ -167,13 +173,18 @@ namespace { return CGF.CGM.getSize(size); } - /// Cast the given pointer to an integer pointer suitable for - /// atomic operations. - llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const; + /// Cast the given pointer to an integer pointer suitable for atomic + /// operations if the source. + Address emitCastToAtomicIntPointer(Address Addr) const; + + /// If Addr is compatible with the iN that will be used for an atomic + /// operation, bitcast it. Otherwise, create a temporary that is suitable + /// and copy the value across. + Address convertToAtomicIntPointer(Address Addr) const; /// Turn an atomic-layout object into an r-value. - RValue convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot, - SourceLocation loc, bool AsValue) const; + RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, + SourceLocation loc, bool AsValue) const; /// \brief Converts a rvalue to integer value. llvm::Value *convertRValueToInt(RValue RVal) const; @@ -188,12 +199,12 @@ namespace { /// Project an l-value down to the value field. LValue projectValue() const { assert(LVal.isSimple()); - llvm::Value *addr = getAtomicAddress(); + Address addr = getAtomicAddress(); if (hasPadding()) - addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0); + addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); - return LValue::MakeAddr(addr, getValueType(), LVal.getAlignment(), - CGF.getContext(), LVal.getTBAAInfo()); + return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), + LVal.getAlignmentSource(), LVal.getTBAAInfo()); } /// \brief Emits atomic load. @@ -228,18 +239,18 @@ namespace { bool IsVolatile); /// Materialize an atomic r-value in atomic-layout memory. - llvm::Value *materializeRValue(RValue rvalue) const; + Address materializeRValue(RValue rvalue) const; /// \brief Translates LLVM atomic ordering to GNU atomic ordering for /// libcalls. static AtomicExpr::AtomicOrderingKind translateAtomicOrdering(const llvm::AtomicOrdering AO); + /// \brief Creates temp alloca for intermediate operations on atomic value. + Address CreateTempAlloca() const; private: bool requiresMemSetZero(llvm::Type *type) const; - /// \brief Creates temp alloca for intermediate operations on atomic value. - llvm::Value *CreateTempAlloca() const; /// \brief Emits atomic load as a libcall. void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, @@ -294,16 +305,16 @@ AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) { llvm_unreachable("Unhandled AtomicOrdering"); } -llvm::Value *AtomicInfo::CreateTempAlloca() const { - auto *TempAlloca = CGF.CreateMemTemp( +Address AtomicInfo::CreateTempAlloca() const { + Address TempAlloca = CGF.CreateMemTemp( (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy : AtomicTy, + getAtomicAlignment(), "atomic-temp"); - TempAlloca->setAlignment(getAtomicAlignment().getQuantity()); // Cast to pointer to value type for bitfields. if (LVal.isBitField()) return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - TempAlloca, getAtomicAddress()->getType()); + TempAlloca, getAtomicAddress().getType()); return TempAlloca; } @@ -351,7 +362,7 @@ bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { bool AtomicInfo::emitMemSetZeroIfNecessary() const { assert(LVal.isSimple()); - llvm::Value *addr = LVal.getAddress(); + llvm::Value *addr = LVal.getPointer(); if (!requiresMemSetZero(addr->getType()->getPointerElementType())) return false; @@ -363,19 +374,17 @@ bool AtomicInfo::emitMemSetZeroIfNecessary() const { } static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, - llvm::Value *Dest, llvm::Value *Ptr, - llvm::Value *Val1, llvm::Value *Val2, - uint64_t Size, unsigned Align, + Address Dest, Address Ptr, + Address Val1, Address Val2, + uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::AtomicOrdering FailureOrder) { // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. - llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1); - Expected->setAlignment(Align); - llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2); - Desired->setAlignment(Align); + llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); + llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( - Ptr, Expected, Desired, SuccessOrder, FailureOrder); + Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder); Pair->setVolatile(E->isVolatile()); Pair->setWeak(IsWeak); @@ -400,26 +409,24 @@ static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, CGF.Builder.SetInsertPoint(StoreExpectedBB); // Update the memory at Expected with Old's value. - llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1); - StoreExpected->setAlignment(Align); + CGF.Builder.CreateStore(Old, Val1); // Finally, branch to the exit point. CGF.Builder.CreateBr(ContinueBB); CGF.Builder.SetInsertPoint(ContinueBB); // Update the memory at Dest with Cmp's value. CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); - return; } /// Given an ordering required on success, emit all possible cmpxchg /// instructions to cope with the provided (but possibly only dynamically known) /// FailureOrder. static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, - bool IsWeak, llvm::Value *Dest, - llvm::Value *Ptr, llvm::Value *Val1, - llvm::Value *Val2, + bool IsWeak, Address Dest, + Address Ptr, Address Val1, + Address Val2, llvm::Value *FailureOrderVal, - uint64_t Size, unsigned Align, + uint64_t Size, llvm::AtomicOrdering SuccessOrder) { llvm::AtomicOrdering FailureOrder; if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { @@ -440,7 +447,7 @@ static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, FailureOrder = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder); } - emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, Align, + emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, FailureOrder); return; } @@ -465,13 +472,13 @@ static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, // doesn't fold to a constant for the ordering. CGF.Builder.SetInsertPoint(MonotonicBB); emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, - Size, Align, SuccessOrder, llvm::Monotonic); + Size, SuccessOrder, llvm::Monotonic); CGF.Builder.CreateBr(ContBB); if (AcquireBB) { CGF.Builder.SetInsertPoint(AcquireBB); emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, - Size, Align, SuccessOrder, llvm::Acquire); + Size, SuccessOrder, llvm::Acquire); CGF.Builder.CreateBr(ContBB); SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume), AcquireBB); @@ -481,7 +488,7 @@ static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, if (SeqCstBB) { CGF.Builder.SetInsertPoint(SeqCstBB); emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, - Size, Align, SuccessOrder, llvm::SequentiallyConsistent); + Size, SuccessOrder, llvm::SequentiallyConsistent); CGF.Builder.CreateBr(ContBB); SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst), SeqCstBB); @@ -490,11 +497,10 @@ static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, CGF.Builder.SetInsertPoint(ContBB); } -static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest, - llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2, +static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, + Address Ptr, Address Val1, Address Val2, llvm::Value *IsWeak, llvm::Value *FailureOrder, - uint64_t Size, unsigned Align, - llvm::AtomicOrdering Order) { + uint64_t Size, llvm::AtomicOrdering Order) { llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; @@ -504,17 +510,17 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest, case AtomicExpr::AO__c11_atomic_compare_exchange_strong: emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, - FailureOrder, Size, Align, Order); + FailureOrder, Size, Order); return; case AtomicExpr::AO__c11_atomic_compare_exchange_weak: emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, - FailureOrder, Size, Align, Order); + FailureOrder, Size, Order); return; case AtomicExpr::AO__atomic_compare_exchange: case AtomicExpr::AO__atomic_compare_exchange_n: { if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, - Val1, Val2, FailureOrder, Size, Align, Order); + Val1, Val2, FailureOrder, Size, Order); } else { // Create all the relevant BB's llvm::BasicBlock *StrongBB = @@ -528,12 +534,12 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest, CGF.Builder.SetInsertPoint(StrongBB); emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, - FailureOrder, Size, Align, Order); + FailureOrder, Size, Order); CGF.Builder.CreateBr(ContBB); CGF.Builder.SetInsertPoint(WeakBB); emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, - FailureOrder, Size, Align, Order); + FailureOrder, Size, Order); CGF.Builder.CreateBr(ContBB); CGF.Builder.SetInsertPoint(ContBB); @@ -545,22 +551,17 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest, case AtomicExpr::AO__atomic_load: { llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); Load->setAtomic(Order); - Load->setAlignment(Size); Load->setVolatile(E->isVolatile()); - llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest); - StoreDest->setAlignment(Align); + CGF.Builder.CreateStore(Load, Dest); return; } case AtomicExpr::AO__c11_atomic_store: case AtomicExpr::AO__atomic_store: case AtomicExpr::AO__atomic_store_n: { - assert(!Dest && "Store does not return a value"); - llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); - LoadVal1->setAlignment(Align); + llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); Store->setAtomic(Order); - Store->setAlignment(Size); Store->setVolatile(E->isVolatile()); return; } @@ -612,17 +613,16 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest, break; case AtomicExpr::AO__atomic_nand_fetch: - PostOp = llvm::Instruction::And; - // Fall through. + PostOp = llvm::Instruction::And; // the NOT is special cased below + // Fall through. case AtomicExpr::AO__atomic_fetch_nand: Op = llvm::AtomicRMWInst::Nand; break; } - llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); - LoadVal1->setAlignment(Align); + llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); llvm::AtomicRMWInst *RMWI = - CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order); + CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order); RMWI->setVolatile(E->isVolatile()); // For __atomic_*_fetch operations, perform the operation again to @@ -632,15 +632,14 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest, Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) Result = CGF.Builder.CreateNot(Result); - llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest); - StoreDest->setAlignment(Align); + CGF.Builder.CreateStore(Result, Dest); } // This function emits any expression (scalar, complex, or aggregate) // into a temporary alloca. -static llvm::Value * +static Address EmitValToTemp(CodeGenFunction &CGF, Expr *E) { - llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); + Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), /*Init*/ true); return DeclPtr; @@ -652,14 +651,15 @@ AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, SourceLocation Loc, CharUnits SizeInChars) { if (UseOptimizedLibcall) { // Load value and pass it to the function directly. - unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity(); + CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); ValTy = CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits)->getPointerTo(); - Val = CGF.EmitLoadOfScalar(CGF.Builder.CreateBitCast(Val, IPtrTy), false, - Align, CGF.getContext().getPointerType(ValTy), + Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align); + Val = CGF.EmitLoadOfScalar(Ptr, false, + CGF.getContext().getPointerType(ValTy), Loc); // Coerce the value into an appropriately sized integer type. Args.add(RValue::get(Val), ValTy); @@ -670,27 +670,27 @@ AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, } } -RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { +RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); QualType MemTy = AtomicTy; if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) MemTy = AT->getValueType(); - CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy); + CharUnits sizeChars, alignChars; + std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy); uint64_t Size = sizeChars.getQuantity(); - CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy); - unsigned Align = alignChars.getQuantity(); - unsigned MaxInlineWidthInBits = - getTarget().getMaxAtomicInlineWidth(); - bool UseLibcall = (Size != Align || + unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); + bool UseLibcall = (sizeChars != alignChars || getContext().toBits(sizeChars) > MaxInlineWidthInBits); - llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr, - *Val2 = nullptr; - llvm::Value *Ptr = EmitScalarExpr(E->getPtr()); + llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; + + Address Val1 = Address::invalid(); + Address Val2 = Address::invalid(); + Address Dest = Address::invalid(); + Address Ptr(EmitScalarExpr(E->getPtr()), alignChars); if (E->getOp() == AtomicExpr::AO__c11_atomic_init) { - assert(!Dest && "Init does not return a value"); - LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext()); + LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); EmitAtomicInit(E->getVal1(), lvalue); return RValue::get(nullptr); } @@ -706,25 +706,25 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { break; case AtomicExpr::AO__atomic_load: - Dest = EmitScalarExpr(E->getVal1()); + Dest = EmitPointerWithAlignment(E->getVal1()); break; case AtomicExpr::AO__atomic_store: - Val1 = EmitScalarExpr(E->getVal1()); + Val1 = EmitPointerWithAlignment(E->getVal1()); break; case AtomicExpr::AO__atomic_exchange: - Val1 = EmitScalarExpr(E->getVal1()); - Dest = EmitScalarExpr(E->getVal2()); + Val1 = EmitPointerWithAlignment(E->getVal1()); + Dest = EmitPointerWithAlignment(E->getVal2()); break; case AtomicExpr::AO__c11_atomic_compare_exchange_strong: case AtomicExpr::AO__c11_atomic_compare_exchange_weak: case AtomicExpr::AO__atomic_compare_exchange_n: case AtomicExpr::AO__atomic_compare_exchange: - Val1 = EmitScalarExpr(E->getVal1()); + Val1 = EmitPointerWithAlignment(E->getVal1()); if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) - Val2 = EmitScalarExpr(E->getVal2()); + Val2 = EmitPointerWithAlignment(E->getVal2()); else Val2 = EmitValToTemp(*this, E->getVal2()); OrderFail = EmitScalarExpr(E->getOrderFail()); @@ -744,8 +744,9 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { CharUnits PointeeIncAmt = getContext().getTypeSizeInChars(MemTy->getPointeeType()); Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); - Val1 = CreateMemTemp(Val1Ty, ".atomictmp"); - EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty)); + auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); + Val1 = Temp; + EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); break; } // Fall through. @@ -774,12 +775,21 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { QualType RValTy = E->getType().getUnqualifiedType(); - auto GetDest = [&] { - if (!RValTy->isVoidType() && !Dest) { - Dest = CreateMemTemp(RValTy, ".atomicdst"); - } - return Dest; - }; + // The inlined atomics only function on iN types, where N is a power of 2. We + // need to make sure (via temporaries if necessary) that all incoming values + // are compatible. + LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); + AtomicInfo Atomics(*this, AtomicVal); + + Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); + if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1); + if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2); + if (Dest.isValid()) + Dest = Atomics.emitCastToAtomicIntPointer(Dest); + else if (E->isCmpXChg()) + Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); + else if (!RValTy->isVoidType()) + Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca()); // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . if (UseLibcall) { @@ -835,13 +845,15 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { getContext().getSizeType()); } // Atomic address is the first or second parameter - Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), getContext().VoidPtrTy); + Args.add(RValue::get(EmitCastToVoidPtr(Ptr.getPointer())), + getContext().VoidPtrTy); std::string LibCallName; QualType LoweredMemTy = MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; QualType RetTy; bool HaveRetTy = false; + llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; switch (E->getOp()) { case AtomicExpr::AO__c11_atomic_init: llvm_unreachable("Already handled!"); @@ -860,9 +872,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { LibCallName = "__atomic_compare_exchange"; RetTy = getContext().BoolTy; HaveRetTy = true; - Args.add(RValue::get(EmitCastToVoidPtr(Val1)), getContext().VoidPtrTy); - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy, - E->getExprLoc(), sizeChars); + Args.add(RValue::get(EmitCastToVoidPtr(Val1.getPointer())), + getContext().VoidPtrTy); + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), + MemTy, E->getExprLoc(), sizeChars); Args.add(RValue::get(Order), getContext().IntTy); Order = OrderFail; break; @@ -873,8 +886,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { case AtomicExpr::AO__atomic_exchange_n: case AtomicExpr::AO__atomic_exchange: LibCallName = "__atomic_exchange"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, - E->getExprLoc(), sizeChars); + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), + MemTy, E->getExprLoc(), sizeChars); break; // void __atomic_store(size_t size, void *mem, void *val, int order) // void __atomic_store_N(T *mem, T val, int order) @@ -884,8 +897,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { LibCallName = "__atomic_store"; RetTy = getContext().VoidTy; HaveRetTy = true; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, - E->getExprLoc(), sizeChars); + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), + MemTy, E->getExprLoc(), sizeChars); break; // void __atomic_load(size_t size, void *mem, void *return, int order) // T __atomic_load_N(T *mem, int order) @@ -894,83 +907,70 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { case AtomicExpr::AO__atomic_load_n: LibCallName = "__atomic_load"; break; + // T __atomic_add_fetch_N(T *mem, T val, int order) // T __atomic_fetch_add_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_add_fetch: + PostOp = llvm::Instruction::Add; + // Fall through. case AtomicExpr::AO__c11_atomic_fetch_add: case AtomicExpr::AO__atomic_fetch_add: LibCallName = "__atomic_fetch_add"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy, - E->getExprLoc(), sizeChars); + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), + LoweredMemTy, E->getExprLoc(), sizeChars); break; + // T __atomic_and_fetch_N(T *mem, T val, int order) // T __atomic_fetch_and_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_and_fetch: + PostOp = llvm::Instruction::And; + // Fall through. case AtomicExpr::AO__c11_atomic_fetch_and: case AtomicExpr::AO__atomic_fetch_and: LibCallName = "__atomic_fetch_and"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, - E->getExprLoc(), sizeChars); + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), + MemTy, E->getExprLoc(), sizeChars); break; + // T __atomic_or_fetch_N(T *mem, T val, int order) // T __atomic_fetch_or_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_or_fetch: + PostOp = llvm::Instruction::Or; + // Fall through. case AtomicExpr::AO__c11_atomic_fetch_or: case AtomicExpr::AO__atomic_fetch_or: LibCallName = "__atomic_fetch_or"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, - E->getExprLoc(), sizeChars); + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), + MemTy, E->getExprLoc(), sizeChars); break; + // T __atomic_sub_fetch_N(T *mem, T val, int order) // T __atomic_fetch_sub_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_sub_fetch: + PostOp = llvm::Instruction::Sub; + // Fall through. case AtomicExpr::AO__c11_atomic_fetch_sub: case AtomicExpr::AO__atomic_fetch_sub: LibCallName = "__atomic_fetch_sub"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy, - E->getExprLoc(), sizeChars); + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), + LoweredMemTy, E->getExprLoc(), sizeChars); break; + // T __atomic_xor_fetch_N(T *mem, T val, int order) // T __atomic_fetch_xor_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_xor_fetch: + PostOp = llvm::Instruction::Xor; + // Fall through. case AtomicExpr::AO__c11_atomic_fetch_xor: case AtomicExpr::AO__atomic_fetch_xor: LibCallName = "__atomic_fetch_xor"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, - E->getExprLoc(), sizeChars); + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), + MemTy, E->getExprLoc(), sizeChars); break; + // T __atomic_nand_fetch_N(T *mem, T val, int order) // T __atomic_fetch_nand_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_nand_fetch: + PostOp = llvm::Instruction::And; // the NOT is special cased below + // Fall through. case AtomicExpr::AO__atomic_fetch_nand: LibCallName = "__atomic_fetch_nand"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, - E->getExprLoc(), sizeChars); - break; - - // T __atomic_add_fetch_N(T *mem, T val, int order) - case AtomicExpr::AO__atomic_add_fetch: - LibCallName = "__atomic_add_fetch"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy, - E->getExprLoc(), sizeChars); - break; - // T __atomic_and_fetch_N(T *mem, T val, int order) - case AtomicExpr::AO__atomic_and_fetch: - LibCallName = "__atomic_and_fetch"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, - E->getExprLoc(), sizeChars); - break; - // T __atomic_or_fetch_N(T *mem, T val, int order) - case AtomicExpr::AO__atomic_or_fetch: - LibCallName = "__atomic_or_fetch"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, - E->getExprLoc(), sizeChars); - break; - // T __atomic_sub_fetch_N(T *mem, T val, int order) - case AtomicExpr::AO__atomic_sub_fetch: - LibCallName = "__atomic_sub_fetch"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy, - E->getExprLoc(), sizeChars); - break; - // T __atomic_xor_fetch_N(T *mem, T val, int order) - case AtomicExpr::AO__atomic_xor_fetch: - LibCallName = "__atomic_xor_fetch"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, - E->getExprLoc(), sizeChars); - break; - // T __atomic_nand_fetch_N(T *mem, T val, int order) - case AtomicExpr::AO__atomic_nand_fetch: - LibCallName = "__atomic_nand_fetch"; - AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, - E->getExprLoc(), sizeChars); + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), + MemTy, E->getExprLoc(), sizeChars); break; } @@ -987,30 +987,46 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { } else { // Value is returned through parameter before the order. RetTy = getContext().VoidTy; - Args.add(RValue::get(EmitCastToVoidPtr(Dest)), getContext().VoidPtrTy); + Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), + getContext().VoidPtrTy); } } // order is always the last parameter Args.add(RValue::get(Order), getContext().IntTy); + // PostOp is only needed for the atomic_*_fetch operations, and + // thus is only needed for and implemented in the + // UseOptimizedLibcall codepath. + assert(UseOptimizedLibcall || !PostOp); + RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); // The value is returned directly from the libcall. - if (HaveRetTy && !RetTy->isVoidType()) + if (E->isCmpXChg()) return Res; - // The value is returned via an explicit out param. - if (RetTy->isVoidType()) - return RValue::get(nullptr); - // The value is returned directly for optimized libcalls but the caller is - // expected an out-param. - if (UseOptimizedLibcall) { + + // The value is returned directly for optimized libcalls but the expr + // provided an out-param. + if (UseOptimizedLibcall && Res.getScalarVal()) { llvm::Value *ResVal = Res.getScalarVal(); - llvm::StoreInst *StoreDest = Builder.CreateStore( + if (PostOp) { + llvm::Value *LoadVal1 = Args[1].RV.getScalarVal(); + ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); + } + if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) + ResVal = Builder.CreateNot(ResVal); + + Builder.CreateStore( ResVal, - Builder.CreateBitCast(GetDest(), ResVal->getType()->getPointerTo())); - StoreDest->setAlignment(Align); + Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo())); } - return convertTempToRValue(Dest, RValTy, E->getExprLoc()); + + if (RValTy->isVoidType()) + return RValue::get(nullptr); + + return convertTempToRValue( + Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), + RValTy, E->getExprLoc()); } bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || @@ -1020,45 +1036,35 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { E->getOp() == AtomicExpr::AO__atomic_load || E->getOp() == AtomicExpr::AO__atomic_load_n; - llvm::Type *ITy = - llvm::IntegerType::get(getLLVMContext(), Size * 8); - llvm::Value *OrigDest = GetDest(); - Ptr = Builder.CreateBitCast( - Ptr, ITy->getPointerTo(Ptr->getType()->getPointerAddressSpace())); - if (Val1) Val1 = Builder.CreateBitCast(Val1, ITy->getPointerTo()); - if (Val2) Val2 = Builder.CreateBitCast(Val2, ITy->getPointerTo()); - if (Dest && !E->isCmpXChg()) - Dest = Builder.CreateBitCast(Dest, ITy->getPointerTo()); - if (isa<llvm::ConstantInt>(Order)) { int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); switch (ord) { case AtomicExpr::AO_ABI_memory_order_relaxed: EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, - Size, Align, llvm::Monotonic); + Size, llvm::Monotonic); break; case AtomicExpr::AO_ABI_memory_order_consume: case AtomicExpr::AO_ABI_memory_order_acquire: if (IsStore) break; // Avoid crashing on code with undefined behavior EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, - Size, Align, llvm::Acquire); + Size, llvm::Acquire); break; case AtomicExpr::AO_ABI_memory_order_release: if (IsLoad) break; // Avoid crashing on code with undefined behavior EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, - Size, Align, llvm::Release); + Size, llvm::Release); break; case AtomicExpr::AO_ABI_memory_order_acq_rel: if (IsLoad || IsStore) break; // Avoid crashing on code with undefined behavior EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, - Size, Align, llvm::AcquireRelease); + Size, llvm::AcquireRelease); break; case AtomicExpr::AO_ABI_memory_order_seq_cst: EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, - Size, Align, llvm::SequentiallyConsistent); + Size, llvm::SequentiallyConsistent); break; default: // invalid order // We should not ever get here normally, but it's hard to @@ -1067,7 +1073,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { } if (RValTy->isVoidType()) return RValue::get(nullptr); - return convertTempToRValue(OrigDest, RValTy, E->getExprLoc()); + + return convertTempToRValue( + Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), + RValTy, E->getExprLoc()); } // Long case, when Order isn't obviously constant. @@ -1096,12 +1105,12 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { // Emit all the different atomics Builder.SetInsertPoint(MonotonicBB); EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, - Size, Align, llvm::Monotonic); + Size, llvm::Monotonic); Builder.CreateBr(ContBB); if (!IsStore) { Builder.SetInsertPoint(AcquireBB); EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, - Size, Align, llvm::Acquire); + Size, llvm::Acquire); Builder.CreateBr(ContBB); SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume), AcquireBB); @@ -1111,7 +1120,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { if (!IsLoad) { Builder.SetInsertPoint(ReleaseBB); EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, - Size, Align, llvm::Release); + Size, llvm::Release); Builder.CreateBr(ContBB); SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release), ReleaseBB); @@ -1119,14 +1128,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { if (!IsLoad && !IsStore) { Builder.SetInsertPoint(AcqRelBB); EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, - Size, Align, llvm::AcquireRelease); + Size, llvm::AcquireRelease); Builder.CreateBr(ContBB); SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel), AcqRelBB); } Builder.SetInsertPoint(SeqCstBB); EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, - Size, Align, llvm::SequentiallyConsistent); + Size, llvm::SequentiallyConsistent); Builder.CreateBr(ContBB); SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst), SeqCstBB); @@ -1135,47 +1144,65 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { Builder.SetInsertPoint(ContBB); if (RValTy->isVoidType()) return RValue::get(nullptr); - return convertTempToRValue(OrigDest, RValTy, E->getExprLoc()); + + assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); + return convertTempToRValue( + Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), + RValTy, E->getExprLoc()); } -llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const { +Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { unsigned addrspace = - cast<llvm::PointerType>(addr->getType())->getAddressSpace(); + cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace(); llvm::IntegerType *ty = llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); } -RValue AtomicInfo::convertTempToRValue(llvm::Value *addr, - AggValueSlot resultSlot, - SourceLocation loc, bool AsValue) const { +Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { + llvm::Type *Ty = Addr.getElementType(); + uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); + if (SourceSizeInBits != AtomicSizeInBits) { + Address Tmp = CreateTempAlloca(); + CGF.Builder.CreateMemCpy(Tmp, Addr, + std::min(AtomicSizeInBits, SourceSizeInBits) / 8); + Addr = Tmp; + } + + return emitCastToAtomicIntPointer(Addr); +} + +RValue AtomicInfo::convertAtomicTempToRValue(Address addr, + AggValueSlot resultSlot, + SourceLocation loc, + bool asValue) const { if (LVal.isSimple()) { if (EvaluationKind == TEK_Aggregate) return resultSlot.asRValue(); // Drill into the padding structure if we have one. if (hasPadding()) - addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0); + addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); // Otherwise, just convert the temporary to an r-value using the // normal conversion routine. return CGF.convertTempToRValue(addr, getValueType(), loc); } - if (!AsValue) + if (!asValue) // Get RValue from temp memory as atomic for non-simple lvalues - return RValue::get( - CGF.Builder.CreateAlignedLoad(addr, AtomicAlign.getQuantity())); + return RValue::get(CGF.Builder.CreateLoad(addr)); if (LVal.isBitField()) - return CGF.EmitLoadOfBitfieldLValue(LValue::MakeBitfield( - addr, LVal.getBitFieldInfo(), LVal.getType(), LVal.getAlignment())); + return CGF.EmitLoadOfBitfieldLValue( + LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), + LVal.getAlignmentSource())); if (LVal.isVectorElt()) - return CGF.EmitLoadOfLValue(LValue::MakeVectorElt(addr, LVal.getVectorIdx(), - LVal.getType(), - LVal.getAlignment()), - loc); + return CGF.EmitLoadOfLValue( + LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), + LVal.getAlignmentSource()), loc); assert(LVal.isExtVectorElt()); return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( - addr, LVal.getExtVectorElts(), LVal.getType(), LVal.getAlignment())); + addr, LVal.getExtVectorElts(), LVal.getType(), + LVal.getAlignmentSource())); } RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, @@ -1191,7 +1218,7 @@ RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, !AsValue)) { auto *ValTy = AsValue ? CGF.ConvertTypeForMem(ValueTy) - : getAtomicAddress()->getType()->getPointerElementType(); + : getAtomicAddress().getType()->getPointerElementType(); if (ValTy->isIntegerTy()) { assert(IntVal->getType() == ValTy && "Different integer types."); return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); @@ -1203,25 +1230,22 @@ RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, // Create a temporary. This needs to be big enough to hold the // atomic integer. - llvm::Value *Temp; + Address Temp = Address::invalid(); bool TempIsVolatile = false; - CharUnits TempAlignment; if (AsValue && getEvaluationKind() == TEK_Aggregate) { assert(!ResultSlot.isIgnored()); - Temp = ResultSlot.getAddr(); - TempAlignment = getValueAlignment(); + Temp = ResultSlot.getAddress(); TempIsVolatile = ResultSlot.isVolatile(); } else { Temp = CreateTempAlloca(); - TempAlignment = getAtomicAlignment(); } // Slam the integer into the temporary. - llvm::Value *CastTemp = emitCastToAtomicIntPointer(Temp); - CGF.Builder.CreateAlignedStore(IntVal, CastTemp, TempAlignment.getQuantity()) + Address CastTemp = emitCastToAtomicIntPointer(Temp); + CGF.Builder.CreateStore(IntVal, CastTemp) ->setVolatile(TempIsVolatile); - return convertTempToRValue(Temp, ResultSlot, Loc, AsValue); + return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); } void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, @@ -1229,7 +1253,7 @@ void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, // void __atomic_load(size_t size, void *mem, void *return, int order); CallArgList Args; Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); - Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())), + Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), CGF.getContext().VoidPtrTy); Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), CGF.getContext().VoidPtrTy); @@ -1242,16 +1266,15 @@ void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile) { // Okay, we're doing this natively. - llvm::Value *Addr = emitCastToAtomicIntPointer(getAtomicAddress()); + Address Addr = getAtomicAddressAsAtomicIntPointer(); llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); Load->setAtomic(AO); // Other decoration. - Load->setAlignment(getAtomicAlignment().getQuantity()); if (IsVolatile) Load->setVolatile(true); if (LVal.getTBAAInfo()) - CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo()); + CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); return Load; } @@ -1259,11 +1282,12 @@ llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, /// we are operating under /volatile:ms *and* the LValue itself is volatile and /// performing such an operation can be performed without a libcall. bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { + if (!CGM.getCodeGenOpts().MSVolatile) return false; AtomicInfo AI(*this, LV); bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); // An atomic is inline if we don't need to use a libcall. bool AtomicIsInline = !AI.shouldUseLibcall(); - return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline; + return IsVolatile && AtomicIsInline; } /// An type is a candidate for having its loads and stores be made atomic if @@ -1295,18 +1319,18 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, bool IsVolatile) { // Check whether we should use a library call. if (shouldUseLibcall()) { - llvm::Value *TempAddr; + Address TempAddr = Address::invalid(); if (LVal.isSimple() && !ResultSlot.isIgnored()) { assert(getEvaluationKind() == TEK_Aggregate); - TempAddr = ResultSlot.getAddr(); + TempAddr = ResultSlot.getAddress(); } else TempAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(TempAddr, AO, IsVolatile); + EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); // Okay, turn that back into the original value or whole atomic (for // non-simple lvalues) type. - return convertTempToRValue(TempAddr, ResultSlot, Loc, AsValue); + return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); } // Okay, we're doing this natively. @@ -1314,7 +1338,7 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, // If we're ignoring an aggregate return, don't do anything. if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) - return RValue::getAggregate(nullptr, false); + return RValue::getAggregate(Address::invalid(), false); // Okay, turn that back into the original value or atomic (for non-simple // lvalues) type. @@ -1340,11 +1364,10 @@ void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { // any padding. Just do an aggregate copy of that type. if (rvalue.isAggregate()) { CGF.EmitAggregateCopy(getAtomicAddress(), - rvalue.getAggregateAddr(), + rvalue.getAggregateAddress(), getAtomicType(), (rvalue.isVolatileQualified() - || LVal.isVolatileQualified()), - LVal.getAlignment()); + || LVal.isVolatileQualified())); return; } @@ -1367,15 +1390,14 @@ void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { /// Materialize an r-value into memory for the purposes of storing it /// to an atomic type. -llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const { +Address AtomicInfo::materializeRValue(RValue rvalue) const { // Aggregate r-values are already in memory, and EmitAtomicStore // requires them to be values of the atomic type. if (rvalue.isAggregate()) - return rvalue.getAggregateAddr(); + return rvalue.getAggregateAddress(); // Otherwise, make a temporary and materialize into it. - LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType(), - getAtomicAlignment()); + LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); AtomicInfo Atomics(CGF, TempLV); Atomics.emitCopyIntoMemory(rvalue); return TempLV.getAddress(); @@ -1400,20 +1422,20 @@ llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { } // Otherwise, we need to go through memory. // Put the r-value in memory. - llvm::Value *Addr = materializeRValue(RVal); + Address Addr = materializeRValue(RVal); // Cast the temporary to the atomic int type and pull a value out. Addr = emitCastToAtomicIntPointer(Addr); - return CGF.Builder.CreateAlignedLoad(Addr, - getAtomicAlignment().getQuantity()); + return CGF.Builder.CreateLoad(Addr); } std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( llvm::Value *ExpectedVal, llvm::Value *DesiredVal, llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { // Do the atomic store. - auto *Addr = emitCastToAtomicIntPointer(getAtomicAddress()); - auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal, + Address Addr = getAtomicAddressAsAtomicIntPointer(); + auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), + ExpectedVal, DesiredVal, Success, Failure); // Other decoration. Inst->setVolatile(LVal.isVolatileQualified()); @@ -1434,7 +1456,7 @@ AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, // void *desired, int success, int failure); CallArgList Args; Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); - Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())), + Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), CGF.getContext().VoidPtrTy); Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), CGF.getContext().VoidPtrTy); @@ -1462,13 +1484,14 @@ std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( // Check whether we should use a library call. if (shouldUseLibcall()) { // Produce a source address. - auto *ExpectedAddr = materializeRValue(Expected); - auto *DesiredAddr = materializeRValue(Desired); - auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, + Address ExpectedAddr = materializeRValue(Expected); + Address DesiredAddr = materializeRValue(Desired); + auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), + DesiredAddr.getPointer(), Success, Failure); return std::make_pair( - convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(), - SourceLocation(), /*AsValue=*/false), + convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false), Res); } @@ -1487,42 +1510,41 @@ std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, const llvm::function_ref<RValue(RValue)> &UpdateOp, - llvm::Value *DesiredAddr) { - llvm::Value *Ptr = nullptr; - LValue UpdateLVal; + Address DesiredAddr) { RValue UpRVal; LValue AtomicLVal = Atomics.getAtomicLValue(); LValue DesiredLVal; if (AtomicLVal.isSimple()) { UpRVal = OldRVal; - DesiredLVal = - LValue::MakeAddr(DesiredAddr, AtomicLVal.getType(), - AtomicLVal.getAlignment(), CGF.CGM.getContext()); + DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); } else { // Build new lvalue for temp address - Ptr = Atomics.materializeRValue(OldRVal); + Address Ptr = Atomics.materializeRValue(OldRVal); + LValue UpdateLVal; if (AtomicLVal.isBitField()) { UpdateLVal = LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), - AtomicLVal.getType(), AtomicLVal.getAlignment()); + AtomicLVal.getType(), + AtomicLVal.getAlignmentSource()); DesiredLVal = LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), - AtomicLVal.getType(), AtomicLVal.getAlignment()); + AtomicLVal.getType(), + AtomicLVal.getAlignmentSource()); } else if (AtomicLVal.isVectorElt()) { UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), - AtomicLVal.getAlignment()); + AtomicLVal.getAlignmentSource()); DesiredLVal = LValue::MakeVectorElt( DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), - AtomicLVal.getAlignment()); + AtomicLVal.getAlignmentSource()); } else { assert(AtomicLVal.isExtVectorElt()); UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), - AtomicLVal.getAlignment()); + AtomicLVal.getAlignmentSource()); DesiredLVal = LValue::MakeExtVectorElt( DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), - AtomicLVal.getAlignment()); + AtomicLVal.getAlignmentSource()); } UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); @@ -1544,26 +1566,26 @@ void AtomicInfo::EmitAtomicUpdateLibcall( bool IsVolatile) { auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); - llvm::Value *ExpectedAddr = CreateTempAlloca(); + Address ExpectedAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile); + EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); CGF.EmitBlock(ContBB); - auto *DesiredAddr = CreateTempAlloca(); + Address DesiredAddr = CreateTempAlloca(); if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || - requiresMemSetZero( - getAtomicAddress()->getType()->getPointerElementType())) { - auto *OldVal = CGF.Builder.CreateAlignedLoad( - ExpectedAddr, getAtomicAlignment().getQuantity()); - CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr, - getAtomicAlignment().getQuantity()); + requiresMemSetZero(getAtomicAddress().getElementType())) { + auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); + CGF.Builder.CreateStore(OldVal, DesiredAddr); } - auto OldRVal = convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(), - SourceLocation(), /*AsValue=*/false); + auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, + AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false); EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); auto *Res = - EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure); + EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), + DesiredAddr.getPointer(), + AO, Failure); CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -1583,19 +1605,16 @@ void AtomicInfo::EmitAtomicUpdateOp( llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), /*NumReservedValues=*/2); PHI->addIncoming(OldVal, CurBB); - auto *NewAtomicAddr = CreateTempAlloca(); - auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); + Address NewAtomicAddr = CreateTempAlloca(); + Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || - requiresMemSetZero( - getAtomicAddress()->getType()->getPointerElementType())) { - CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr, - getAtomicAlignment().getQuantity()); + requiresMemSetZero(getAtomicAddress().getElementType())) { + CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); } auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), SourceLocation(), /*AsValue=*/false); EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); - auto *DesiredVal = CGF.Builder.CreateAlignedLoad( - NewAtomicIntAddr, getAtomicAlignment().getQuantity()); + auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); // Try to write new value using cmpxchg operation auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); @@ -1604,23 +1623,25 @@ void AtomicInfo::EmitAtomicUpdateOp( } static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, - RValue UpdateRVal, llvm::Value *DesiredAddr) { + RValue UpdateRVal, Address DesiredAddr) { LValue AtomicLVal = Atomics.getAtomicLValue(); LValue DesiredLVal; // Build new lvalue for temp address if (AtomicLVal.isBitField()) { DesiredLVal = LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), - AtomicLVal.getType(), AtomicLVal.getAlignment()); + AtomicLVal.getType(), + AtomicLVal.getAlignmentSource()); } else if (AtomicLVal.isVectorElt()) { DesiredLVal = LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), - AtomicLVal.getType(), AtomicLVal.getAlignment()); + AtomicLVal.getType(), + AtomicLVal.getAlignmentSource()); } else { assert(AtomicLVal.isExtVectorElt()); DesiredLVal = LValue::MakeExtVectorElt( DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), - AtomicLVal.getAlignment()); + AtomicLVal.getAlignmentSource()); } DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); // Store new value in the corresponding memory area @@ -1632,24 +1653,23 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, bool IsVolatile) { auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); - llvm::Value *ExpectedAddr = CreateTempAlloca(); + Address ExpectedAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile); + EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); CGF.EmitBlock(ContBB); - auto *DesiredAddr = CreateTempAlloca(); + Address DesiredAddr = CreateTempAlloca(); if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || - requiresMemSetZero( - getAtomicAddress()->getType()->getPointerElementType())) { - auto *OldVal = CGF.Builder.CreateAlignedLoad( - ExpectedAddr, getAtomicAlignment().getQuantity()); - CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr, - getAtomicAlignment().getQuantity()); + requiresMemSetZero(getAtomicAddress().getElementType())) { + auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); + CGF.Builder.CreateStore(OldVal, DesiredAddr); } EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); auto *Res = - EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure); + EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), + DesiredAddr.getPointer(), + AO, Failure); CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -1668,17 +1688,14 @@ void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), /*NumReservedValues=*/2); PHI->addIncoming(OldVal, CurBB); - auto *NewAtomicAddr = CreateTempAlloca(); - auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); + Address NewAtomicAddr = CreateTempAlloca(); + Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || - requiresMemSetZero( - getAtomicAddress()->getType()->getPointerElementType())) { - CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr, - getAtomicAlignment().getQuantity()); + requiresMemSetZero(getAtomicAddress().getElementType())) { + CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); } EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); - auto *DesiredVal = CGF.Builder.CreateAlignedLoad( - NewAtomicIntAddr, getAtomicAlignment().getQuantity()); + auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); // Try to write new value using cmpxchg operation auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); @@ -1729,8 +1746,8 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, // If this is an aggregate r-value, it should agree in type except // maybe for address-space qualification. assert(!rvalue.isAggregate() || - rvalue.getAggregateAddr()->getType()->getPointerElementType() - == dest.getAddress()->getType()->getPointerElementType()); + rvalue.getAggregateAddress().getElementType() + == dest.getAddress().getElementType()); AtomicInfo atomics(*this, dest); LValue LVal = atomics.getAtomicLValue(); @@ -1745,15 +1762,16 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, // Check whether we should use a library call. if (atomics.shouldUseLibcall()) { // Produce a source address. - llvm::Value *srcAddr = atomics.materializeRValue(rvalue); + Address srcAddr = atomics.materializeRValue(rvalue); // void __atomic_store(size_t size, void *mem, void *val, int order) CallArgList args; args.add(RValue::get(atomics.getAtomicSizeValue()), getContext().getSizeType()); - args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicAddress())), + args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), + getContext().VoidPtrTy); + args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), getContext().VoidPtrTy); - args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), getContext().VoidPtrTy); args.add(RValue::get(llvm::ConstantInt::get( IntTy, AtomicInfo::translateAtomicOrdering(AO))), getContext().IntTy); @@ -1765,10 +1783,10 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, llvm::Value *intValue = atomics.convertRValueToInt(rvalue); // Do the atomic store. - llvm::Value *addr = + Address addr = atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); intValue = Builder.CreateIntCast( - intValue, addr->getType()->getPointerElementType(), /*isSigned=*/false); + intValue, addr.getElementType(), /*isSigned=*/false); llvm::StoreInst *store = Builder.CreateStore(intValue, addr); // Initializations don't need to be atomic. @@ -1776,11 +1794,10 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, store->setAtomic(AO); // Other decoration. - store->setAlignment(dest.getAlignment().getQuantity()); if (IsVolatile) store->setVolatile(true); if (dest.getTBAAInfo()) - CGM.DecorateInstruction(store, dest.getTBAAInfo()); + CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); return; } @@ -1797,11 +1814,11 @@ std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( // If this is an aggregate r-value, it should agree in type except // maybe for address-space qualification. assert(!Expected.isAggregate() || - Expected.getAggregateAddr()->getType()->getPointerElementType() == - Obj.getAddress()->getType()->getPointerElementType()); + Expected.getAggregateAddress().getElementType() == + Obj.getAddress().getElementType()); assert(!Desired.isAggregate() || - Desired.getAggregateAddr()->getType()->getPointerElementType() == - Obj.getAddress()->getType()->getPointerElementType()); + Desired.getAggregateAddress().getElementType() == + Obj.getAddress().getElementType()); AtomicInfo Atomics(*this, Obj); return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp index 3fd344c..ba2941e 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp @@ -1,4 +1,4 @@ -//===--- CGBlocks.cpp - Emit LLVM Code for declarations -------------------===// +//===--- CGBlocks.cpp - Emit LLVM Code for declarations ---------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -30,7 +30,7 @@ using namespace CodeGen; CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name) : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false), HasCXXObject(false), UsesStret(false), HasCapturedVariableLayout(false), - StructureType(nullptr), Block(block), + LocalAddress(Address::invalid()), StructureType(nullptr), Block(block), DominatingIP(nullptr) { // Skip asm prefix, if any. 'name' is usually taken directly from @@ -40,7 +40,7 @@ CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name) } // Anchor the vtable to this translation unit. -CodeGenModule::ByrefHelpers::~ByrefHelpers() {} +BlockByrefHelpers::~BlockByrefHelpers() {} /// Build the given block as a global block. static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM, @@ -78,7 +78,7 @@ static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM, ASTContext &C = CGM.getContext(); llvm::Type *ulong = CGM.getTypes().ConvertType(C.UnsignedLongTy); - llvm::Type *i8p = NULL; + llvm::Type *i8p = nullptr; if (CGM.getLangOpts().OpenCL) i8p = llvm::Type::getInt8PtrTy( @@ -111,7 +111,7 @@ static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM, std::string typeAtEncoding = CGM.getContext().getObjCEncodingForBlock(blockInfo.getBlockExpr()); elements.push_back(llvm::ConstantExpr::getBitCast( - CGM.GetAddrOfConstantCString(typeAtEncoding), i8p)); + CGM.GetAddrOfConstantCString(typeAtEncoding).getPointer(), i8p)); // GC layout. if (C.getLangOpts().ObjC1) { @@ -203,46 +203,36 @@ namespace { Capture(capture), Type(type) {} /// Tell the block info that this chunk has the given field index. - void setIndex(CGBlockInfo &info, unsigned index) { - if (!Capture) + void setIndex(CGBlockInfo &info, unsigned index, CharUnits offset) { + if (!Capture) { info.CXXThisIndex = index; - else - info.Captures[Capture->getVariable()] - = CGBlockInfo::Capture::makeIndex(index); + info.CXXThisOffset = offset; + } else { + info.Captures.insert({Capture->getVariable(), + CGBlockInfo::Capture::makeIndex(index, offset)}); + } } }; /// Order by 1) all __strong together 2) next, all byfref together 3) next, /// all __weak together. Preserve descending alignment in all situations. bool operator<(const BlockLayoutChunk &left, const BlockLayoutChunk &right) { - CharUnits LeftValue, RightValue; - bool LeftByref = left.Capture ? left.Capture->isByRef() : false; - bool RightByref = right.Capture ? right.Capture->isByRef() : false; - - if (left.Lifetime == Qualifiers::OCL_Strong && - left.Alignment >= right.Alignment) - LeftValue = CharUnits::fromQuantity(64); - else if (LeftByref && left.Alignment >= right.Alignment) - LeftValue = CharUnits::fromQuantity(32); - else if (left.Lifetime == Qualifiers::OCL_Weak && - left.Alignment >= right.Alignment) - LeftValue = CharUnits::fromQuantity(16); - else - LeftValue = left.Alignment; - if (right.Lifetime == Qualifiers::OCL_Strong && - right.Alignment >= left.Alignment) - RightValue = CharUnits::fromQuantity(64); - else if (RightByref && right.Alignment >= left.Alignment) - RightValue = CharUnits::fromQuantity(32); - else if (right.Lifetime == Qualifiers::OCL_Weak && - right.Alignment >= left.Alignment) - RightValue = CharUnits::fromQuantity(16); - else - RightValue = right.Alignment; - - return LeftValue > RightValue; + if (left.Alignment != right.Alignment) + return left.Alignment > right.Alignment; + + auto getPrefOrder = [](const BlockLayoutChunk &chunk) { + if (chunk.Capture && chunk.Capture->isByRef()) + return 1; + if (chunk.Lifetime == Qualifiers::OCL_Strong) + return 0; + if (chunk.Lifetime == Qualifiers::OCL_Weak) + return 2; + return 3; + }; + + return getPrefOrder(left) < getPrefOrder(right); } -} +} // end anonymous namespace /// Determines if the given type is safe for constant capture in C++. static bool isSafeForCXXConstantCapture(QualType type) { @@ -302,31 +292,20 @@ static CharUnits getLowBit(CharUnits v) { static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info, SmallVectorImpl<llvm::Type*> &elementTypes) { - ASTContext &C = CGM.getContext(); - - // The header is basically a 'struct { void *; int; int; void *; void *; }'. - CharUnits ptrSize, ptrAlign, intSize, intAlign; - std::tie(ptrSize, ptrAlign) = C.getTypeInfoInChars(C.VoidPtrTy); - std::tie(intSize, intAlign) = C.getTypeInfoInChars(C.IntTy); - - // Are there crazy embedded platforms where this isn't true? - assert(intSize <= ptrSize && "layout assumptions horribly violated"); + // The header is basically 'struct { void *; int; int; void *; void *; }'. + // Assert that that struct is packed. + assert(CGM.getIntSize() <= CGM.getPointerSize()); + assert(CGM.getIntAlign() <= CGM.getPointerAlign()); + assert((2 * CGM.getIntSize()).isMultipleOf(CGM.getPointerAlign())); - CharUnits headerSize = ptrSize; - if (2 * intSize < ptrAlign) headerSize += ptrSize; - else headerSize += 2 * intSize; - headerSize += 2 * ptrSize; - - info.BlockAlign = ptrAlign; - info.BlockSize = headerSize; + info.BlockAlign = CGM.getPointerAlign(); + info.BlockSize = 3 * CGM.getPointerSize() + 2 * CGM.getIntSize(); assert(elementTypes.empty()); - llvm::Type *i8p = CGM.getTypes().ConvertType(C.VoidPtrTy); - llvm::Type *intTy = CGM.getTypes().ConvertType(C.IntTy); - elementTypes.push_back(i8p); - elementTypes.push_back(intTy); - elementTypes.push_back(intTy); - elementTypes.push_back(i8p); + elementTypes.push_back(CGM.VoidPtrTy); + elementTypes.push_back(CGM.IntTy); + elementTypes.push_back(CGM.IntTy); + elementTypes.push_back(CGM.VoidPtrTy); elementTypes.push_back(CGM.getBlockDescriptorType()); assert(elementTypes.size() == BlockHeaderSize); @@ -365,6 +344,8 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, "Can't capture 'this' outside a method"); QualType thisType = cast<CXXMethodDecl>(CGF->CurFuncDecl)->getThisType(C); + // Theoretically, this could be in a different address space, so + // don't assume standard pointer size/align. llvm::Type *llvmType = CGM.getTypes().ConvertType(thisType); std::pair<CharUnits,CharUnits> tinfo = CGM.getContext().getTypeInfoInChars(thisType); @@ -384,15 +365,12 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, info.NeedsCopyDispose = true; // Just use void* instead of a pointer to the byref type. - QualType byRefPtrTy = C.VoidPtrTy; - - llvm::Type *llvmType = CGM.getTypes().ConvertType(byRefPtrTy); - std::pair<CharUnits,CharUnits> tinfo - = CGM.getContext().getTypeInfoInChars(byRefPtrTy); - maxFieldAlign = std::max(maxFieldAlign, tinfo.second); + CharUnits align = CGM.getPointerAlign(); + maxFieldAlign = std::max(maxFieldAlign, align); - layout.push_back(BlockLayoutChunk(tinfo.second, tinfo.first, - Qualifiers::OCL_None, &CI, llvmType)); + layout.push_back(BlockLayoutChunk(align, CGM.getPointerSize(), + Qualifiers::OCL_None, &CI, + CGM.VoidPtrTy)); continue; } @@ -421,9 +399,15 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, // Block pointers require copy/dispose. So do Objective-C pointers. } else if (variable->getType()->isObjCRetainableType()) { - info.NeedsCopyDispose = true; - // used for mrr below. - lifetime = Qualifiers::OCL_Strong; + // But honor the inert __unsafe_unretained qualifier, which doesn't + // actually make it into the type system. + if (variable->getType()->isObjCInertUnsafeUnretainedType()) { + lifetime = Qualifiers::OCL_ExplicitNone; + } else { + info.NeedsCopyDispose = true; + // used for mrr below. + lifetime = Qualifiers::OCL_Strong; + } // So do types that require non-trivial copy construction. } else if (CI.hasCopyExpr()) { @@ -504,18 +488,13 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, for (; li != le; ++li) { assert(endAlign >= li->Alignment); - li->setIndex(info, elementTypes.size()); + li->setIndex(info, elementTypes.size(), blockSize); elementTypes.push_back(li->Type); blockSize += li->Size; endAlign = getLowBit(blockSize); // ...until we get to the alignment of the maximum field. if (endAlign >= maxFieldAlign) { - if (li == first) { - // No user field was appended. So, a gap was added. - // Save total gap size for use in block layout bit map. - info.BlockHeaderForcedGapSize = li->Size; - } break; } } @@ -532,6 +511,12 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, CharUnits newBlockSize = blockSize.RoundUpToAlignment(maxFieldAlign); CharUnits padding = newBlockSize - blockSize; + // If we haven't yet added any fields, remember that there was an + // initial gap; this need to go into the block layout bit map. + if (blockSize == info.BlockHeaderForcedGapOffset) { + info.BlockHeaderForcedGapSize = padding; + } + elementTypes.push_back(llvm::ArrayType::get(CGM.Int8Ty, padding.getQuantity())); blockSize = newBlockSize; @@ -556,7 +541,7 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF, endAlign = getLowBit(blockSize); } assert(endAlign >= li->Alignment); - li->setIndex(info, elementTypes.size()); + li->setIndex(info, elementTypes.size(), blockSize); elementTypes.push_back(li->Type); blockSize += li->Size; endAlign = getLowBit(blockSize); @@ -586,9 +571,8 @@ static void enterBlockScope(CodeGenFunction &CGF, BlockDecl *block) { if (blockInfo.CanBeGlobal) return; // Make the allocation for the block. - blockInfo.Address = - CGF.CreateTempAlloca(blockInfo.StructureType, "block"); - blockInfo.Address->setAlignment(blockInfo.BlockAlign.getQuantity()); + blockInfo.LocalAddress = CGF.CreateTempAlloca(blockInfo.StructureType, + blockInfo.BlockAlign, "block"); // If there are cleanups to emit, enter them (but inactive). if (!blockInfo.NeedsCopyDispose) return; @@ -621,12 +605,13 @@ static void enterBlockScope(CodeGenFunction &CGF, BlockDecl *block) { } // GEP down to the address. - llvm::Value *addr = CGF.Builder.CreateStructGEP( - blockInfo.StructureType, blockInfo.Address, capture.getIndex()); + Address addr = CGF.Builder.CreateStructGEP(blockInfo.LocalAddress, + capture.getIndex(), + capture.getOffset()); // We can use that GEP as the dominating IP. if (!blockInfo.DominatingIP) - blockInfo.DominatingIP = cast<llvm::Instruction>(addr); + blockInfo.DominatingIP = cast<llvm::Instruction>(addr.getPointer()); CleanupKind cleanupKind = InactiveNormalCleanup; bool useArrayEHCleanup = CGF.needsEHCleanup(dtorKind); @@ -721,9 +706,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // Build the block descriptor. llvm::Constant *descriptor = buildBlockDescriptor(CGM, blockInfo); - llvm::Type *blockTy = blockInfo.StructureType; - llvm::AllocaInst *blockAddr = blockInfo.Address; - assert(blockAddr && "block has no address!"); + Address blockAddr = blockInfo.LocalAddress; + assert(blockAddr.isValid() && "block has no address!"); // Compute the initial on-stack block flags. BlockFlags flags = BLOCK_HAS_SIGNATURE; @@ -732,27 +716,44 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { if (blockInfo.HasCXXObject) flags |= BLOCK_HAS_CXX_OBJ; if (blockInfo.UsesStret) flags |= BLOCK_USE_STRET; - // Initialize the block literal. - Builder.CreateStore( - isa, Builder.CreateStructGEP(blockTy, blockAddr, 0, "block.isa")); - Builder.CreateStore( - llvm::ConstantInt::get(IntTy, flags.getBitMask()), - Builder.CreateStructGEP(blockTy, blockAddr, 1, "block.flags")); - Builder.CreateStore( - llvm::ConstantInt::get(IntTy, 0), - Builder.CreateStructGEP(blockTy, blockAddr, 2, "block.reserved")); - Builder.CreateStore( - blockFn, Builder.CreateStructGEP(blockTy, blockAddr, 3, "block.invoke")); - Builder.CreateStore(descriptor, Builder.CreateStructGEP(blockTy, blockAddr, 4, - "block.descriptor")); + auto projectField = + [&](unsigned index, CharUnits offset, const Twine &name) -> Address { + return Builder.CreateStructGEP(blockAddr, index, offset, name); + }; + auto storeField = + [&](llvm::Value *value, unsigned index, CharUnits offset, + const Twine &name) { + Builder.CreateStore(value, projectField(index, offset, name)); + }; + + // Initialize the block header. + { + // We assume all the header fields are densely packed. + unsigned index = 0; + CharUnits offset; + auto addHeaderField = + [&](llvm::Value *value, CharUnits size, const Twine &name) { + storeField(value, index, offset, name); + offset += size; + index++; + }; + + addHeaderField(isa, getPointerSize(), "block.isa"); + addHeaderField(llvm::ConstantInt::get(IntTy, flags.getBitMask()), + getIntSize(), "block.flags"); + addHeaderField(llvm::ConstantInt::get(IntTy, 0), + getIntSize(), "block.reserved"); + addHeaderField(blockFn, getPointerSize(), "block.invoke"); + addHeaderField(descriptor, getPointerSize(), "block.descriptor"); + } // Finally, capture all the values into the block. const BlockDecl *blockDecl = blockInfo.getBlockDecl(); // First, 'this'. if (blockDecl->capturesCXXThis()) { - llvm::Value *addr = Builder.CreateStructGEP( - blockTy, blockAddr, blockInfo.CXXThisIndex, "block.captured-this.addr"); + Address addr = projectField(blockInfo.CXXThisIndex, blockInfo.CXXThisOffset, + "block.captured-this.addr"); Builder.CreateStore(LoadCXXThis(), addr); } @@ -765,35 +766,37 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { if (capture.isConstant()) continue; QualType type = variable->getType(); - CharUnits align = getContext().getDeclAlign(variable); // This will be a [[type]]*, except that a byref entry will just be // an i8**. - llvm::Value *blockField = Builder.CreateStructGEP( - blockTy, blockAddr, capture.getIndex(), "block.captured"); + Address blockField = + projectField(capture.getIndex(), capture.getOffset(), "block.captured"); // Compute the address of the thing we're going to move into the // block literal. - llvm::Value *src; + Address src = Address::invalid(); if (BlockInfo && CI.isNested()) { // We need to use the capture from the enclosing block. const CGBlockInfo::Capture &enclosingCapture = BlockInfo->getCapture(variable); // This is a [[type]]*, except that a byref entry wil just be an i8**. - src = Builder.CreateStructGEP(BlockInfo->StructureType, LoadBlockStruct(), + src = Builder.CreateStructGEP(LoadBlockStruct(), enclosingCapture.getIndex(), + enclosingCapture.getOffset(), "block.capture.addr"); } else if (blockDecl->isConversionFromLambda()) { // The lambda capture in a lambda's conversion-to-block-pointer is // special; we'll simply emit it directly. - src = nullptr; + src = Address::invalid(); } else { // Just look it up in the locals map, which will give us back a // [[type]]*. If that doesn't work, do the more elaborate DRE // emission. - src = LocalDeclMap.lookup(variable); - if (!src) { + auto it = LocalDeclMap.find(variable); + if (it != LocalDeclMap.end()) { + src = it->second; + } else { DeclRefExpr declRef( const_cast<VarDecl *>(variable), /*RefersToEnclosingVariableOrCapture*/ CI.isNested(), type, @@ -808,14 +811,14 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // live a shorter life than the stack byref anyway. if (CI.isByRef()) { // Get a void* that points to the byref struct. + llvm::Value *byrefPointer; if (CI.isNested()) - src = Builder.CreateAlignedLoad(src, align.getQuantity(), - "byref.capture"); + byrefPointer = Builder.CreateLoad(src, "byref.capture"); else - src = Builder.CreateBitCast(src, VoidPtrTy); + byrefPointer = Builder.CreateBitCast(src.getPointer(), VoidPtrTy); // Write that void* into the capture field. - Builder.CreateAlignedStore(src, blockField, align.getQuantity()); + Builder.CreateStore(byrefPointer, blockField); // If we have a copy constructor, evaluate that into the block field. } else if (const Expr *copyExpr = CI.getCopyExpr()) { @@ -823,7 +826,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // If we have a lambda conversion, emit the expression // directly into the block instead. AggValueSlot Slot = - AggValueSlot::forAddr(blockField, align, Qualifiers(), + AggValueSlot::forAddr(blockField, Qualifiers(), AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased); @@ -834,9 +837,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // If it's a reference variable, copy the reference into the block field. } else if (type->isReferenceType()) { - llvm::Value *ref = - Builder.CreateAlignedLoad(src, align.getQuantity(), "ref.val"); - Builder.CreateAlignedStore(ref, blockField, align.getQuantity()); + llvm::Value *ref = Builder.CreateLoad(src, "ref.val"); + Builder.CreateStore(ref, blockField); // If this is an ARC __strong block-pointer variable, don't do a // block copy. @@ -848,13 +850,11 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { } else if (type.getObjCLifetime() == Qualifiers::OCL_Strong && type->isBlockPointerType()) { // Load the block and do a simple retain. - LValue srcLV = MakeAddrLValue(src, type, align); - llvm::Value *value = EmitLoadOfScalar(srcLV, SourceLocation()); + llvm::Value *value = Builder.CreateLoad(src, "block.captured_block"); value = EmitARCRetainNonBlock(value); // Do a primitive store to the block field. - LValue destLV = MakeAddrLValue(blockField, type, align); - EmitStoreOfScalar(value, destLV, /*init*/ true); + Builder.CreateStore(value, blockField); // Otherwise, fake up a POD copy into the block field. } else { @@ -876,7 +876,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // attributed to a reasonable location - otherwise it may be attributed to // locations of subexpressions in the initialization. EmitExprAsInit(&l2r, &blockFieldPseudoVar, - MakeAddrLValue(blockField, type, align), + MakeAddrLValue(blockField, type, AlignmentSource::Decl), /*captured by init*/ false); } @@ -891,7 +891,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // Cast to the converted block-pointer type, which happens (somewhat // unfortunately) to be a pointer to function type. llvm::Value *result = - Builder.CreateBitCast(blockAddr, + Builder.CreateBitCast(blockAddr.getPointer(), ConvertType(blockInfo.getBlockExpr()->getType())); return result; @@ -949,7 +949,6 @@ llvm::Type *CodeGenModule::getGenericBlockLiteralType() { return GenericBlockLiteralType; } - RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue) { const BlockPointerType *BPT = @@ -966,8 +965,8 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, Builder.CreateBitCast(Callee, BlockLiteralTy, "block.literal"); // Get the function pointer from the literal. - llvm::Value *FuncPtr = Builder.CreateStructGEP( - CGM.getGenericBlockLiteralType(), BlockLiteral, 3); + llvm::Value *FuncPtr = + Builder.CreateStructGEP(CGM.getGenericBlockLiteralType(), BlockLiteral, 3); BlockLiteral = Builder.CreateBitCast(BlockLiteral, VoidPtrTy); @@ -978,11 +977,10 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, QualType FnType = BPT->getPointeeType(); // And the rest of the arguments. - EmitCallArgs(Args, FnType->getAs<FunctionProtoType>(), - E->arg_begin(), E->arg_end()); + EmitCallArgs(Args, FnType->getAs<FunctionProtoType>(), E->arguments()); // Load the function. - llvm::Value *Func = Builder.CreateLoad(FuncPtr); + llvm::Value *Func = Builder.CreateAlignedLoad(FuncPtr, getPointerAlign()); const FunctionType *FuncTy = FnType->castAs<FunctionType>(); const CGFunctionInfo &FnInfo = @@ -998,41 +996,35 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, return EmitCall(FnInfo, Func, ReturnValue, Args); } -llvm::Value *CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable, - bool isByRef) { +Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable, + bool isByRef) { assert(BlockInfo && "evaluating block ref without block information?"); const CGBlockInfo::Capture &capture = BlockInfo->getCapture(variable); // Handle constant captures. - if (capture.isConstant()) return LocalDeclMap[variable]; + if (capture.isConstant()) return LocalDeclMap.find(variable)->second; - llvm::Value *addr = - Builder.CreateStructGEP(BlockInfo->StructureType, LoadBlockStruct(), - capture.getIndex(), "block.capture.addr"); + Address addr = + Builder.CreateStructGEP(LoadBlockStruct(), capture.getIndex(), + capture.getOffset(), "block.capture.addr"); if (isByRef) { // addr should be a void** right now. Load, then cast the result // to byref*. - addr = Builder.CreateLoad(addr); - auto *byrefType = BuildByRefType(variable); - llvm::PointerType *byrefPointerType = llvm::PointerType::get(byrefType, 0); - addr = Builder.CreateBitCast(addr, byrefPointerType, - "byref.addr"); - - // Follow the forwarding pointer. - addr = Builder.CreateStructGEP(byrefType, addr, 1, "byref.forwarding"); - addr = Builder.CreateLoad(addr, "byref.addr.forwarded"); - - // Cast back to byref* and GEP over to the actual object. - addr = Builder.CreateBitCast(addr, byrefPointerType); - addr = Builder.CreateStructGEP(byrefType, addr, - getByRefValueLLVMField(variable).second, - variable->getNameAsString()); + auto &byrefInfo = getBlockByrefInfo(variable); + addr = Address(Builder.CreateLoad(addr), byrefInfo.ByrefAlignment); + + auto byrefPointerType = llvm::PointerType::get(byrefInfo.Type, 0); + addr = Builder.CreateBitCast(addr, byrefPointerType, "byref.addr"); + + addr = emitBlockByrefAddress(addr, byrefInfo, /*follow*/ true, + variable->getName()); } - if (variable->getType()->isReferenceType()) - addr = Builder.CreateLoad(addr, "ref.tmp"); + if (auto refType = variable->getType()->getAs<ReferenceType>()) { + addr = EmitLoadOfReference(addr, refType); + } return addr; } @@ -1049,7 +1041,7 @@ CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *blockExpr, // Using that metadata, generate the actual block function. llvm::Constant *blockFn; { - llvm::DenseMap<const Decl*, llvm::Value*> LocalDeclMap; + CodeGenFunction::DeclMapTy LocalDeclMap; blockFn = CodeGenFunction(*this).GenerateBlockFunction(GlobalDecl(), blockInfo, LocalDeclMap, @@ -1103,6 +1095,44 @@ static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM, return llvm::ConstantExpr::getBitCast(literal, requiredType); } +void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D, + unsigned argNum, + llvm::Value *arg) { + assert(BlockInfo && "not emitting prologue of block invocation function?!"); + + llvm::Value *localAddr = nullptr; + if (CGM.getCodeGenOpts().OptimizationLevel == 0) { + // Allocate a stack slot to let the debug info survive the RA. + Address alloc = CreateMemTemp(D->getType(), D->getName() + ".addr"); + Builder.CreateStore(arg, alloc); + localAddr = Builder.CreateLoad(alloc); + } + + if (CGDebugInfo *DI = getDebugInfo()) { + if (CGM.getCodeGenOpts().getDebugInfo() + >= CodeGenOptions::LimitedDebugInfo) { + DI->setLocation(D->getLocation()); + DI->EmitDeclareOfBlockLiteralArgVariable(*BlockInfo, arg, argNum, + localAddr, Builder); + } + } + + SourceLocation StartLoc = BlockInfo->getBlockExpr()->getBody()->getLocStart(); + ApplyDebugLocation Scope(*this, StartLoc); + + // Instead of messing around with LocalDeclMap, just set the value + // directly as BlockPointer. + BlockPointer = Builder.CreateBitCast(arg, + BlockInfo->StructureType->getPointerTo(), + "block"); +} + +Address CodeGenFunction::LoadBlockStruct() { + assert(BlockInfo && "not in a block invocation function!"); + assert(BlockPointer && "no block pointer set!"); + return Address(BlockPointer, BlockInfo->BlockAlign); +} + llvm::Function * CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo &blockInfo, @@ -1122,7 +1152,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, for (DeclMapTy::const_iterator i = ldm.begin(), e = ldm.end(); i != e; ++i) { const auto *var = dyn_cast<VarDecl>(i->first); if (var && !var->hasLocalStorage()) - LocalDeclMap[var] = i->second; + setAddrOfLocalVar(var, i->second); } // Begin building the function declaration. @@ -1163,35 +1193,28 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, blockInfo.getBlockExpr()->getBody()->getLocStart()); // Okay. Undo some of what StartFunction did. - - // Pull the 'self' reference out of the local decl map. - llvm::Value *blockAddr = LocalDeclMap[&selfDecl]; - LocalDeclMap.erase(&selfDecl); - BlockPointer = Builder.CreateBitCast(blockAddr, - blockInfo.StructureType->getPointerTo(), - "block"); + // At -O0 we generate an explicit alloca for the BlockPointer, so the RA // won't delete the dbg.declare intrinsics for captured variables. llvm::Value *BlockPointerDbgLoc = BlockPointer; if (CGM.getCodeGenOpts().OptimizationLevel == 0) { // Allocate a stack slot for it, so we can point the debugger to it - llvm::AllocaInst *Alloca = CreateTempAlloca(BlockPointer->getType(), - "block.addr"); - unsigned Align = getContext().getDeclAlign(&selfDecl).getQuantity(); - Alloca->setAlignment(Align); + Address Alloca = CreateTempAlloca(BlockPointer->getType(), + getPointerAlign(), + "block.addr"); // Set the DebugLocation to empty, so the store is recognized as a // frame setup instruction by llvm::DwarfDebug::beginFunction(). auto NL = ApplyDebugLocation::CreateEmpty(*this); - Builder.CreateAlignedStore(BlockPointer, Alloca, Align); - BlockPointerDbgLoc = Alloca; + Builder.CreateStore(BlockPointer, Alloca); + BlockPointerDbgLoc = Alloca.getPointer(); } // If we have a C++ 'this' reference, go ahead and force it into // existence now. if (blockDecl->capturesCXXThis()) { - llvm::Value *addr = - Builder.CreateStructGEP(blockInfo.StructureType, BlockPointer, - blockInfo.CXXThisIndex, "block.captured-this"); + Address addr = + Builder.CreateStructGEP(LoadBlockStruct(), blockInfo.CXXThisIndex, + blockInfo.CXXThisOffset, "block.captured-this"); CXXThisValue = Builder.CreateLoad(addr, "this"); } @@ -1201,15 +1224,13 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable); if (!capture.isConstant()) continue; - unsigned align = getContext().getDeclAlign(variable).getQuantity(); - - llvm::AllocaInst *alloca = - CreateMemTemp(variable->getType(), "block.captured-const"); - alloca->setAlignment(align); + CharUnits align = getContext().getDeclAlign(variable); + Address alloca = + CreateMemTemp(variable->getType(), align, "block.captured-const"); - Builder.CreateAlignedStore(capture.getConstant(), alloca, align); + Builder.CreateStore(capture.getConstant(), alloca); - LocalDeclMap[variable] = alloca; + setAddrOfLocalVar(variable, alloca); } // Save a spot to insert the debug information for all the DeclRefExprs. @@ -1220,7 +1241,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, if (IsLambdaConversionToBlock) EmitLambdaBlockInvokeBody(); else { - PGO.assignRegionCounters(blockDecl, fn); + PGO.assignRegionCounters(GlobalDecl(blockDecl), fn); incrementProfileCounter(blockDecl->getBody()); EmitStmt(blockDecl->getBody()); } @@ -1243,15 +1264,15 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, >= CodeGenOptions::LimitedDebugInfo) { const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable); if (capture.isConstant()) { - DI->EmitDeclareOfAutoVariable(variable, LocalDeclMap[variable], + auto addr = LocalDeclMap.find(variable)->second; + DI->EmitDeclareOfAutoVariable(variable, addr.getPointer(), Builder); continue; } - DI->EmitDeclareOfBlockDeclRefVariable(variable, BlockPointerDbgLoc, - Builder, blockInfo, - entry_ptr == entry->end() - ? nullptr : entry_ptr); + DI->EmitDeclareOfBlockDeclRefVariable( + variable, BlockPointerDbgLoc, Builder, blockInfo, + entry_ptr == entry->end() ? nullptr : &*entry_ptr); } } // Recover location if it was changed in the above loop. @@ -1288,7 +1309,6 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, } */ - /// Generate the copy-helper function for a block closure object: /// static void block_copy_helper(block_t *dst, block_t *src); /// The runtime will have previously initialized 'dst' by doing a @@ -1330,18 +1350,21 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { nullptr, SC_Static, false, false); + + CGM.SetInternalFunctionAttributes(nullptr, Fn, FI); + auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(FD, C.VoidTy, Fn, FI, args); // Create a scope with an artificial location for the body of this function. auto AL = ApplyDebugLocation::CreateArtificial(*this); llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo(); - llvm::Value *src = GetAddrOfLocalVar(&srcDecl); - src = Builder.CreateLoad(src); + Address src = GetAddrOfLocalVar(&srcDecl); + src = Address(Builder.CreateLoad(src), blockInfo.BlockAlign); src = Builder.CreateBitCast(src, structPtrTy, "block.source"); - llvm::Value *dst = GetAddrOfLocalVar(&dstDecl); - dst = Builder.CreateLoad(dst); + Address dst = GetAddrOfLocalVar(&dstDecl); + dst = Address(Builder.CreateLoad(dst), blockInfo.BlockAlign); dst = Builder.CreateBitCast(dst, structPtrTy, "block.dest"); const BlockDecl *blockDecl = blockInfo.getBlockDecl(); @@ -1375,40 +1398,38 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { flags = BLOCK_FIELD_IS_BLOCK; // Special rules for ARC captures: - if (getLangOpts().ObjCAutoRefCount) { - Qualifiers qs = type.getQualifiers(); - - // We need to register __weak direct captures with the runtime. - if (qs.getObjCLifetime() == Qualifiers::OCL_Weak) { - useARCWeakCopy = true; - - // We need to retain the copied value for __strong direct captures. - } else if (qs.getObjCLifetime() == Qualifiers::OCL_Strong) { - // If it's a block pointer, we have to copy the block and - // assign that to the destination pointer, so we might as - // well use _Block_object_assign. Otherwise we can avoid that. - if (!isBlockPointer) - useARCStrongCopy = true; - - // Otherwise the memcpy is fine. - } else { - continue; - } + Qualifiers qs = type.getQualifiers(); + + // We need to register __weak direct captures with the runtime. + if (qs.getObjCLifetime() == Qualifiers::OCL_Weak) { + useARCWeakCopy = true; + + // We need to retain the copied value for __strong direct captures. + } else if (qs.getObjCLifetime() == Qualifiers::OCL_Strong) { + // If it's a block pointer, we have to copy the block and + // assign that to the destination pointer, so we might as + // well use _Block_object_assign. Otherwise we can avoid that. + if (!isBlockPointer) + useARCStrongCopy = true; // Non-ARC captures of retainable pointers are strong and // therefore require a call to _Block_object_assign. - } else { + } else if (!qs.getObjCLifetime() && !getLangOpts().ObjCAutoRefCount) { // fall through + + // Otherwise the memcpy is fine. + } else { + continue; } + + // For all other types, the memcpy is fine. } else { continue; } unsigned index = capture.getIndex(); - llvm::Value *srcField = - Builder.CreateStructGEP(blockInfo.StructureType, src, index); - llvm::Value *dstField = - Builder.CreateStructGEP(blockInfo.StructureType, dst, index); + Address srcField = Builder.CreateStructGEP(src, index, capture.getOffset()); + Address dstField = Builder.CreateStructGEP(dst, index, capture.getOffset()); // If there's an explicit copy expression, we do that. if (copyExpr) { @@ -1435,11 +1456,12 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { // We don't need this anymore, so kill it. It's not quite // worth the annoyance to avoid creating it in the first place. - cast<llvm::Instruction>(dstField)->eraseFromParent(); + cast<llvm::Instruction>(dstField.getPointer())->eraseFromParent(); } } else { srcValue = Builder.CreateBitCast(srcValue, VoidPtrTy); - llvm::Value *dstAddr = Builder.CreateBitCast(dstField, VoidPtrTy); + llvm::Value *dstAddr = + Builder.CreateBitCast(dstField.getPointer(), VoidPtrTy); llvm::Value *args[] = { dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask()) }; @@ -1502,6 +1524,9 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) { SourceLocation(), II, C.VoidTy, nullptr, SC_Static, false, false); + + CGM.SetInternalFunctionAttributes(nullptr, Fn, FI); + // Create a scope with an artificial location for the body of this function. auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(FD, C.VoidTy, Fn, FI, args); @@ -1509,8 +1534,8 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) { llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo(); - llvm::Value *src = GetAddrOfLocalVar(&srcDecl); - src = Builder.CreateLoad(src); + Address src = GetAddrOfLocalVar(&srcDecl); + src = Address(Builder.CreateLoad(src), blockInfo.BlockAlign); src = Builder.CreateBitCast(src, structPtrTy, "block"); const BlockDecl *blockDecl = blockInfo.getBlockDecl(); @@ -1544,29 +1569,31 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) { flags = BLOCK_FIELD_IS_BLOCK; // Special rules for ARC captures. - if (getLangOpts().ObjCAutoRefCount) { - Qualifiers qs = type.getQualifiers(); + Qualifiers qs = type.getQualifiers(); - // Don't generate special dispose logic for a captured object - // unless it's __strong or __weak. - if (!qs.hasStrongOrWeakObjCLifetime()) - continue; + // Use objc_storeStrong for __strong direct captures; the + // dynamic tools really like it when we do this. + if (qs.getObjCLifetime() == Qualifiers::OCL_Strong) { + useARCStrongDestroy = true; + + // Support __weak direct captures. + } else if (qs.getObjCLifetime() == Qualifiers::OCL_Weak) { + useARCWeakDestroy = true; - // Support __weak direct captures. - if (qs.getObjCLifetime() == Qualifiers::OCL_Weak) - useARCWeakDestroy = true; + // Non-ARC captures are strong, and we need to use _Block_object_dispose. + } else if (!qs.hasObjCLifetime() && !getLangOpts().ObjCAutoRefCount) { + // fall through - // Tools really want us to use objc_storeStrong here. - else - useARCStrongDestroy = true; + // Otherwise, we have nothing to do. + } else { + continue; } } else { continue; } - unsigned index = capture.getIndex(); - llvm::Value *srcField = - Builder.CreateStructGEP(blockInfo.StructureType, src, index); + Address srcField = + Builder.CreateStructGEP(src, capture.getIndex(), capture.getOffset()); // If there's an explicit copy expression, we do that. if (dtor) { @@ -1600,15 +1627,15 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) { namespace { /// Emits the copy/dispose helper functions for a __block object of id type. -class ObjectByrefHelpers : public CodeGenModule::ByrefHelpers { +class ObjectByrefHelpers final : public BlockByrefHelpers { BlockFieldFlags Flags; public: ObjectByrefHelpers(CharUnits alignment, BlockFieldFlags flags) - : ByrefHelpers(alignment), Flags(flags) {} + : BlockByrefHelpers(alignment), Flags(flags) {} - void emitCopy(CodeGenFunction &CGF, llvm::Value *destField, - llvm::Value *srcField) override { + void emitCopy(CodeGenFunction &CGF, Address destField, + Address srcField) override { destField = CGF.Builder.CreateBitCast(destField, CGF.VoidPtrTy); srcField = CGF.Builder.CreateBitCast(srcField, CGF.VoidPtrPtrTy); @@ -1619,11 +1646,11 @@ public: llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags); llvm::Value *fn = CGF.CGM.getBlockObjectAssign(); - llvm::Value *args[] = { destField, srcValue, flagsVal }; + llvm::Value *args[] = { destField.getPointer(), srcValue, flagsVal }; CGF.EmitNounwindRuntimeCall(fn, args); } - void emitDispose(CodeGenFunction &CGF, llvm::Value *field) override { + void emitDispose(CodeGenFunction &CGF, Address field) override { field = CGF.Builder.CreateBitCast(field, CGF.Int8PtrTy->getPointerTo(0)); llvm::Value *value = CGF.Builder.CreateLoad(field); @@ -1636,16 +1663,16 @@ public: }; /// Emits the copy/dispose helpers for an ARC __block __weak variable. -class ARCWeakByrefHelpers : public CodeGenModule::ByrefHelpers { +class ARCWeakByrefHelpers final : public BlockByrefHelpers { public: - ARCWeakByrefHelpers(CharUnits alignment) : ByrefHelpers(alignment) {} + ARCWeakByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {} - void emitCopy(CodeGenFunction &CGF, llvm::Value *destField, - llvm::Value *srcField) override { + void emitCopy(CodeGenFunction &CGF, Address destField, + Address srcField) override { CGF.EmitARCMoveWeak(destField, srcField); } - void emitDispose(CodeGenFunction &CGF, llvm::Value *field) override { + void emitDispose(CodeGenFunction &CGF, Address field) override { CGF.EmitARCDestroyWeak(field); } @@ -1657,36 +1684,31 @@ public: /// Emits the copy/dispose helpers for an ARC __block __strong variable /// that's not of block-pointer type. -class ARCStrongByrefHelpers : public CodeGenModule::ByrefHelpers { +class ARCStrongByrefHelpers final : public BlockByrefHelpers { public: - ARCStrongByrefHelpers(CharUnits alignment) : ByrefHelpers(alignment) {} + ARCStrongByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {} - void emitCopy(CodeGenFunction &CGF, llvm::Value *destField, - llvm::Value *srcField) override { + void emitCopy(CodeGenFunction &CGF, Address destField, + Address srcField) override { // Do a "move" by copying the value and then zeroing out the old // variable. - llvm::LoadInst *value = CGF.Builder.CreateLoad(srcField); - value->setAlignment(Alignment.getQuantity()); + llvm::Value *value = CGF.Builder.CreateLoad(srcField); llvm::Value *null = llvm::ConstantPointerNull::get(cast<llvm::PointerType>(value->getType())); if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) { - llvm::StoreInst *store = CGF.Builder.CreateStore(null, destField); - store->setAlignment(Alignment.getQuantity()); + CGF.Builder.CreateStore(null, destField); CGF.EmitARCStoreStrongCall(destField, value, /*ignored*/ true); CGF.EmitARCStoreStrongCall(srcField, null, /*ignored*/ true); return; } - llvm::StoreInst *store = CGF.Builder.CreateStore(value, destField); - store->setAlignment(Alignment.getQuantity()); - - store = CGF.Builder.CreateStore(null, srcField); - store->setAlignment(Alignment.getQuantity()); + CGF.Builder.CreateStore(value, destField); + CGF.Builder.CreateStore(null, srcField); } - void emitDispose(CodeGenFunction &CGF, llvm::Value *field) override { + void emitDispose(CodeGenFunction &CGF, Address field) override { CGF.EmitARCDestroyStrong(field, ARCImpreciseLifetime); } @@ -1698,25 +1720,22 @@ public: /// Emits the copy/dispose helpers for an ARC __block __strong /// variable that's of block-pointer type. -class ARCStrongBlockByrefHelpers : public CodeGenModule::ByrefHelpers { +class ARCStrongBlockByrefHelpers final : public BlockByrefHelpers { public: - ARCStrongBlockByrefHelpers(CharUnits alignment) : ByrefHelpers(alignment) {} + ARCStrongBlockByrefHelpers(CharUnits alignment) + : BlockByrefHelpers(alignment) {} - void emitCopy(CodeGenFunction &CGF, llvm::Value *destField, - llvm::Value *srcField) override { + void emitCopy(CodeGenFunction &CGF, Address destField, + Address srcField) override { // Do the copy with objc_retainBlock; that's all that // _Block_object_assign would do anyway, and we'd have to pass the // right arguments to make sure it doesn't get no-op'ed. - llvm::LoadInst *oldValue = CGF.Builder.CreateLoad(srcField); - oldValue->setAlignment(Alignment.getQuantity()); - + llvm::Value *oldValue = CGF.Builder.CreateLoad(srcField); llvm::Value *copy = CGF.EmitARCRetainBlock(oldValue, /*mandatory*/ true); - - llvm::StoreInst *store = CGF.Builder.CreateStore(copy, destField); - store->setAlignment(Alignment.getQuantity()); + CGF.Builder.CreateStore(copy, destField); } - void emitDispose(CodeGenFunction &CGF, llvm::Value *field) override { + void emitDispose(CodeGenFunction &CGF, Address field) override { CGF.EmitARCDestroyStrong(field, ARCImpreciseLifetime); } @@ -1728,23 +1747,23 @@ public: /// Emits the copy/dispose helpers for a __block variable with a /// nontrivial copy constructor or destructor. -class CXXByrefHelpers : public CodeGenModule::ByrefHelpers { +class CXXByrefHelpers final : public BlockByrefHelpers { QualType VarType; const Expr *CopyExpr; public: CXXByrefHelpers(CharUnits alignment, QualType type, const Expr *copyExpr) - : ByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {} + : BlockByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {} bool needsCopy() const override { return CopyExpr != nullptr; } - void emitCopy(CodeGenFunction &CGF, llvm::Value *destField, - llvm::Value *srcField) override { + void emitCopy(CodeGenFunction &CGF, Address destField, + Address srcField) override { if (!CopyExpr) return; CGF.EmitSynthesizedCXXCopyCtor(destField, srcField, CopyExpr); } - void emitDispose(CodeGenFunction &CGF, llvm::Value *field) override { + void emitDispose(CodeGenFunction &CGF, Address field) override { EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin(); CGF.PushDestructorCleanup(VarType, field); CGF.PopCleanupBlocks(cleanupDepth); @@ -1757,10 +1776,8 @@ public: } // end anonymous namespace static llvm::Constant * -generateByrefCopyHelper(CodeGenFunction &CGF, - llvm::StructType &byrefType, - unsigned valueFieldIndex, - CodeGenModule::ByrefHelpers &byrefInfo) { +generateByrefCopyHelper(CodeGenFunction &CGF, const BlockByrefInfo &byrefInfo, + BlockByrefHelpers &generator) { ASTContext &Context = CGF.getContext(); QualType R = Context.VoidTy; @@ -1777,8 +1794,7 @@ generateByrefCopyHelper(CodeGenFunction &CGF, const CGFunctionInfo &FI = CGF.CGM.getTypes().arrangeFreeFunctionDeclaration( R, args, FunctionType::ExtInfo(), /*variadic=*/false); - CodeGenTypes &Types = CGF.CGM.getTypes(); - llvm::FunctionType *LTy = Types.GetFunctionType(FI); + llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(FI); // FIXME: We'd like to put these into a mergable by content, with // internal linkage. @@ -1796,26 +1812,30 @@ generateByrefCopyHelper(CodeGenFunction &CGF, SC_Static, false, false); + CGF.CGM.SetInternalFunctionAttributes(nullptr, Fn, FI); + CGF.StartFunction(FD, R, Fn, FI, args); - if (byrefInfo.needsCopy()) { - llvm::Type *byrefPtrType = byrefType.getPointerTo(0); + if (generator.needsCopy()) { + llvm::Type *byrefPtrType = byrefInfo.Type->getPointerTo(0); // dst->x - llvm::Value *destField = CGF.GetAddrOfLocalVar(&dst); - destField = CGF.Builder.CreateLoad(destField); + Address destField = CGF.GetAddrOfLocalVar(&dst); + destField = Address(CGF.Builder.CreateLoad(destField), + byrefInfo.ByrefAlignment); destField = CGF.Builder.CreateBitCast(destField, byrefPtrType); - destField = CGF.Builder.CreateStructGEP(&byrefType, destField, - valueFieldIndex, "x"); + destField = CGF.emitBlockByrefAddress(destField, byrefInfo, false, + "dest-object"); // src->x - llvm::Value *srcField = CGF.GetAddrOfLocalVar(&src); - srcField = CGF.Builder.CreateLoad(srcField); + Address srcField = CGF.GetAddrOfLocalVar(&src); + srcField = Address(CGF.Builder.CreateLoad(srcField), + byrefInfo.ByrefAlignment); srcField = CGF.Builder.CreateBitCast(srcField, byrefPtrType); - srcField = - CGF.Builder.CreateStructGEP(&byrefType, srcField, valueFieldIndex, "x"); + srcField = CGF.emitBlockByrefAddress(srcField, byrefInfo, false, + "src-object"); - byrefInfo.emitCopy(CGF, destField, srcField); + generator.emitCopy(CGF, destField, srcField); } CGF.FinishFunction(); @@ -1825,19 +1845,17 @@ generateByrefCopyHelper(CodeGenFunction &CGF, /// Build the copy helper for a __block variable. static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM, - llvm::StructType &byrefType, - unsigned byrefValueIndex, - CodeGenModule::ByrefHelpers &info) { + const BlockByrefInfo &byrefInfo, + BlockByrefHelpers &generator) { CodeGenFunction CGF(CGM); - return generateByrefCopyHelper(CGF, byrefType, byrefValueIndex, info); + return generateByrefCopyHelper(CGF, byrefInfo, generator); } /// Generate code for a __block variable's dispose helper. static llvm::Constant * generateByrefDisposeHelper(CodeGenFunction &CGF, - llvm::StructType &byrefType, - unsigned byrefValueIndex, - CodeGenModule::ByrefHelpers &byrefInfo) { + const BlockByrefInfo &byrefInfo, + BlockByrefHelpers &generator) { ASTContext &Context = CGF.getContext(); QualType R = Context.VoidTy; @@ -1849,8 +1867,7 @@ generateByrefDisposeHelper(CodeGenFunction &CGF, const CGFunctionInfo &FI = CGF.CGM.getTypes().arrangeFreeFunctionDeclaration( R, args, FunctionType::ExtInfo(), /*variadic=*/false); - CodeGenTypes &Types = CGF.CGM.getTypes(); - llvm::FunctionType *LTy = Types.GetFunctionType(FI); + llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(FI); // FIXME: We'd like to put these into a mergable by content, with // internal linkage. @@ -1868,15 +1885,19 @@ generateByrefDisposeHelper(CodeGenFunction &CGF, SourceLocation(), II, R, nullptr, SC_Static, false, false); + + CGF.CGM.SetInternalFunctionAttributes(nullptr, Fn, FI); + CGF.StartFunction(FD, R, Fn, FI, args); - if (byrefInfo.needsDispose()) { - llvm::Value *V = CGF.GetAddrOfLocalVar(&src); - V = CGF.Builder.CreateLoad(V); - V = CGF.Builder.CreateBitCast(V, byrefType.getPointerTo(0)); - V = CGF.Builder.CreateStructGEP(&byrefType, V, byrefValueIndex, "x"); + if (generator.needsDispose()) { + Address addr = CGF.GetAddrOfLocalVar(&src); + addr = Address(CGF.Builder.CreateLoad(addr), byrefInfo.ByrefAlignment); + auto byrefPtrType = byrefInfo.Type->getPointerTo(0); + addr = CGF.Builder.CreateBitCast(addr, byrefPtrType); + addr = CGF.emitBlockByrefAddress(addr, byrefInfo, false, "object"); - byrefInfo.emitDispose(CGF, V); + generator.emitDispose(CGF, addr); } CGF.FinishFunction(); @@ -1886,38 +1907,29 @@ generateByrefDisposeHelper(CodeGenFunction &CGF, /// Build the dispose helper for a __block variable. static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM, - llvm::StructType &byrefType, - unsigned byrefValueIndex, - CodeGenModule::ByrefHelpers &info) { + const BlockByrefInfo &byrefInfo, + BlockByrefHelpers &generator) { CodeGenFunction CGF(CGM); - return generateByrefDisposeHelper(CGF, byrefType, byrefValueIndex, info); + return generateByrefDisposeHelper(CGF, byrefInfo, generator); } /// Lazily build the copy and dispose helpers for a __block variable /// with the given information. -template <class T> static T *buildByrefHelpers(CodeGenModule &CGM, - llvm::StructType &byrefTy, - unsigned byrefValueIndex, - T &byrefInfo) { - // Increase the field's alignment to be at least pointer alignment, - // since the layout of the byref struct will guarantee at least that. - byrefInfo.Alignment = std::max(byrefInfo.Alignment, - CharUnits::fromQuantity(CGM.PointerAlignInBytes)); - +template <class T> +static T *buildByrefHelpers(CodeGenModule &CGM, const BlockByrefInfo &byrefInfo, + T &&generator) { llvm::FoldingSetNodeID id; - byrefInfo.Profile(id); + generator.Profile(id); void *insertPos; - CodeGenModule::ByrefHelpers *node + BlockByrefHelpers *node = CGM.ByrefHelpersCache.FindNodeOrInsertPos(id, insertPos); if (node) return static_cast<T*>(node); - byrefInfo.CopyHelper = - buildByrefCopyHelper(CGM, byrefTy, byrefValueIndex, byrefInfo); - byrefInfo.DisposeHelper = - buildByrefDisposeHelper(CGM, byrefTy, byrefValueIndex,byrefInfo); + generator.CopyHelper = buildByrefCopyHelper(CGM, byrefInfo, generator); + generator.DisposeHelper = buildByrefDisposeHelper(CGM, byrefInfo, generator); - T *copy = new (CGM.getContext()) T(byrefInfo); + T *copy = new (CGM.getContext()) T(std::move(generator)); CGM.ByrefHelpersCache.InsertNode(copy, insertPos); return copy; } @@ -1925,20 +1937,25 @@ template <class T> static T *buildByrefHelpers(CodeGenModule &CGM, /// Build the copy and dispose helpers for the given __block variable /// emission. Places the helpers in the global cache. Returns null /// if no helpers are required. -CodeGenModule::ByrefHelpers * +BlockByrefHelpers * CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType, const AutoVarEmission &emission) { const VarDecl &var = *emission.Variable; QualType type = var.getType(); - unsigned byrefValueIndex = getByRefValueLLVMField(&var).second; + auto &byrefInfo = getBlockByrefInfo(&var); + + // The alignment we care about for the purposes of uniquing byref + // helpers is the alignment of the actual byref value field. + CharUnits valueAlignment = + byrefInfo.ByrefAlignment.alignmentAtOffset(byrefInfo.FieldOffset); if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) { const Expr *copyExpr = CGM.getContext().getBlockVarCopyInits(&var); if (!copyExpr && record->hasTrivialDestructor()) return nullptr; - CXXByrefHelpers byrefInfo(emission.Alignment, type, copyExpr); - return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo); + return ::buildByrefHelpers( + CGM, byrefInfo, CXXByrefHelpers(valueAlignment, type, copyExpr)); } // Otherwise, if we don't have a retainable type, there's nothing to do. @@ -1949,8 +1966,6 @@ CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType, // If we have lifetime, that dominates. if (Qualifiers::ObjCLifetime lifetime = qs.getObjCLifetime()) { - assert(getLangOpts().ObjCAutoRefCount); - switch (lifetime) { case Qualifiers::OCL_None: llvm_unreachable("impossible"); @@ -1961,24 +1976,23 @@ CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType, // Tell the runtime that this is ARC __weak, called by the // byref routines. - case Qualifiers::OCL_Weak: { - ARCWeakByrefHelpers byrefInfo(emission.Alignment); - return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo); - } + case Qualifiers::OCL_Weak: + return ::buildByrefHelpers(CGM, byrefInfo, + ARCWeakByrefHelpers(valueAlignment)); // ARC __strong __block variables need to be retained. case Qualifiers::OCL_Strong: // Block pointers need to be copied, and there's no direct // transfer possible. if (type->isBlockPointerType()) { - ARCStrongBlockByrefHelpers byrefInfo(emission.Alignment); - return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo); + return ::buildByrefHelpers(CGM, byrefInfo, + ARCStrongBlockByrefHelpers(valueAlignment)); // Otherwise, we transfer ownership of the retain from the stack // to the heap. } else { - ARCStrongByrefHelpers byrefInfo(emission.Alignment); - return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo); + return ::buildByrefHelpers(CGM, byrefInfo, + ARCStrongByrefHelpers(valueAlignment)); } } llvm_unreachable("fell out of lifetime switch!"); @@ -1997,28 +2011,33 @@ CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType, if (type.isObjCGCWeak()) flags |= BLOCK_FIELD_IS_WEAK; - ObjectByrefHelpers byrefInfo(emission.Alignment, flags); - return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo); + return ::buildByrefHelpers(CGM, byrefInfo, + ObjectByrefHelpers(valueAlignment, flags)); } -std::pair<llvm::Type *, unsigned> -CodeGenFunction::getByRefValueLLVMField(const ValueDecl *VD) const { - assert(ByRefValueInfo.count(VD) && "Did not find value!"); - - return ByRefValueInfo.find(VD)->second; +Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr, + const VarDecl *var, + bool followForward) { + auto &info = getBlockByrefInfo(var); + return emitBlockByrefAddress(baseAddr, info, followForward, var->getName()); } -llvm::Value *CodeGenFunction::BuildBlockByrefAddress(llvm::Value *BaseAddr, - const VarDecl *V) { - auto P = getByRefValueLLVMField(V); - llvm::Value *Loc = - Builder.CreateStructGEP(P.first, BaseAddr, 1, "forwarding"); - Loc = Builder.CreateLoad(Loc); - Loc = Builder.CreateStructGEP(P.first, Loc, P.second, V->getNameAsString()); - return Loc; +Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr, + const BlockByrefInfo &info, + bool followForward, + const llvm::Twine &name) { + // Chase the forwarding address if requested. + if (followForward) { + Address forwardingAddr = + Builder.CreateStructGEP(baseAddr, 1, getPointerSize(), "forwarding"); + baseAddr = Address(Builder.CreateLoad(forwardingAddr), info.ByrefAlignment); + } + + return Builder.CreateStructGEP(baseAddr, info.FieldIndex, + info.FieldOffset, name); } -/// BuildByRefType - This routine changes a __block variable declared as T x +/// BuildByrefInfo - This routine changes a __block variable declared as T x /// into: /// /// struct { @@ -2033,108 +2052,116 @@ llvm::Value *CodeGenFunction::BuildBlockByrefAddress(llvm::Value *BaseAddr, /// T x; /// } x /// -llvm::Type *CodeGenFunction::BuildByRefType(const VarDecl *D) { - std::pair<llvm::Type *, unsigned> &Info = ByRefValueInfo[D]; - if (Info.first) - return Info.first; +const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) { + auto it = BlockByrefInfos.find(D); + if (it != BlockByrefInfos.end()) + return it->second; + + llvm::StructType *byrefType = + llvm::StructType::create(getLLVMContext(), + "struct.__block_byref_" + D->getNameAsString()); QualType Ty = D->getType(); + CharUnits size; SmallVector<llvm::Type *, 8> types; - llvm::StructType *ByRefType = - llvm::StructType::create(getLLVMContext(), - "struct.__block_byref_" + D->getNameAsString()); - // void *__isa; types.push_back(Int8PtrTy); + size += getPointerSize(); // void *__forwarding; - types.push_back(llvm::PointerType::getUnqual(ByRefType)); + types.push_back(llvm::PointerType::getUnqual(byrefType)); + size += getPointerSize(); // int32_t __flags; types.push_back(Int32Ty); + size += CharUnits::fromQuantity(4); // int32_t __size; types.push_back(Int32Ty); + size += CharUnits::fromQuantity(4); + // Note that this must match *exactly* the logic in buildByrefHelpers. - bool HasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D); - if (HasCopyAndDispose) { + bool hasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D); + if (hasCopyAndDispose) { /// void *__copy_helper; types.push_back(Int8PtrTy); + size += getPointerSize(); /// void *__destroy_helper; types.push_back(Int8PtrTy); + size += getPointerSize(); } + bool HasByrefExtendedLayout = false; Qualifiers::ObjCLifetime Lifetime; if (getContext().getByrefLifetime(Ty, Lifetime, HasByrefExtendedLayout) && - HasByrefExtendedLayout) + HasByrefExtendedLayout) { /// void *__byref_variable_layout; types.push_back(Int8PtrTy); + size += CharUnits::fromQuantity(PointerSizeInBytes); + } - bool Packed = false; - CharUnits Align = getContext().getDeclAlign(D); - if (Align > - getContext().toCharUnitsFromBits(getTarget().getPointerAlign(0))) { - // We have to insert padding. - - // The struct above has 2 32-bit integers. - unsigned CurrentOffsetInBytes = 4 * 2; - - // And either 2, 3, 4 or 5 pointers. - unsigned noPointers = 2; - if (HasCopyAndDispose) - noPointers += 2; - if (HasByrefExtendedLayout) - noPointers += 1; - - CurrentOffsetInBytes += noPointers * CGM.getDataLayout().getTypeAllocSize(Int8PtrTy); - - // Align the offset. - unsigned AlignedOffsetInBytes = - llvm::RoundUpToAlignment(CurrentOffsetInBytes, Align.getQuantity()); - - unsigned NumPaddingBytes = AlignedOffsetInBytes - CurrentOffsetInBytes; - if (NumPaddingBytes > 0) { - llvm::Type *Ty = Int8Ty; - // FIXME: We need a sema error for alignment larger than the minimum of - // the maximal stack alignment and the alignment of malloc on the system. - if (NumPaddingBytes > 1) - Ty = llvm::ArrayType::get(Ty, NumPaddingBytes); - - types.push_back(Ty); + // T x; + llvm::Type *varTy = ConvertTypeForMem(Ty); - // We want a packed struct. - Packed = true; - } + bool packed = false; + CharUnits varAlign = getContext().getDeclAlign(D); + CharUnits varOffset = size.RoundUpToAlignment(varAlign); + + // We may have to insert padding. + if (varOffset != size) { + llvm::Type *paddingTy = + llvm::ArrayType::get(Int8Ty, (varOffset - size).getQuantity()); + + types.push_back(paddingTy); + size = varOffset; + + // Conversely, we might have to prevent LLVM from inserting padding. + } else if (CGM.getDataLayout().getABITypeAlignment(varTy) + > varAlign.getQuantity()) { + packed = true; } + types.push_back(varTy); - // T x; - types.push_back(ConvertTypeForMem(Ty)); - - ByRefType->setBody(types, Packed); - - Info.first = ByRefType; - - Info.second = types.size() - 1; - - return Info.first; + byrefType->setBody(types, packed); + + BlockByrefInfo info; + info.Type = byrefType; + info.FieldIndex = types.size() - 1; + info.FieldOffset = varOffset; + info.ByrefAlignment = std::max(varAlign, getPointerAlign()); + + auto pair = BlockByrefInfos.insert({D, info}); + assert(pair.second && "info was inserted recursively?"); + return pair.first->second; } /// Initialize the structural components of a __block variable, i.e. /// everything but the actual object. void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { // Find the address of the local. - llvm::Value *addr = emission.Address; + Address addr = emission.Addr; // That's an alloca of the byref structure type. llvm::StructType *byrefType = cast<llvm::StructType>( - cast<llvm::PointerType>(addr->getType())->getElementType()); + cast<llvm::PointerType>(addr.getPointer()->getType())->getElementType()); + + unsigned nextHeaderIndex = 0; + CharUnits nextHeaderOffset; + auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize, + const Twine &name) { + auto fieldAddr = Builder.CreateStructGEP(addr, nextHeaderIndex, + nextHeaderOffset, name); + Builder.CreateStore(value, fieldAddr); + + nextHeaderIndex++; + nextHeaderOffset += fieldSize; + }; // Build the byref helpers if necessary. This is null if we don't need any. - CodeGenModule::ByrefHelpers *helpers = - buildByrefHelpers(*byrefType, emission); + BlockByrefHelpers *helpers = buildByrefHelpers(*byrefType, emission); const VarDecl &D = *emission.Variable; QualType type = D.getType(); @@ -2143,7 +2170,7 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { Qualifiers::ObjCLifetime ByrefLifetime; bool ByRefHasLifetime = getContext().getByrefLifetime(type, ByrefLifetime, HasByrefExtendedLayout); - + llvm::Value *V; // Initialize the 'isa', which is just 0 or 1. @@ -2151,12 +2178,10 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { if (type.isObjCGCWeak()) isa = 1; V = Builder.CreateIntToPtr(Builder.getInt32(isa), Int8PtrTy, "isa"); - Builder.CreateStore(V, - Builder.CreateStructGEP(nullptr, addr, 0, "byref.isa")); + storeHeaderField(V, getPointerSize(), "byref.isa"); // Store the address of the variable into its own forwarding pointer. - Builder.CreateStore( - addr, Builder.CreateStructGEP(nullptr, addr, 1, "byref.forwarding")); + storeHeaderField(addr.getPointer(), getPointerSize(), "byref.forwarding"); // Blocks ABI: // c) the flags field is set to either 0 if no helper functions are @@ -2202,31 +2227,23 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { printf("\n"); } } - - Builder.CreateStore(llvm::ConstantInt::get(IntTy, flags.getBitMask()), - Builder.CreateStructGEP(nullptr, addr, 2, "byref.flags")); + storeHeaderField(llvm::ConstantInt::get(IntTy, flags.getBitMask()), + getIntSize(), "byref.flags"); CharUnits byrefSize = CGM.GetTargetTypeStoreSize(byrefType); V = llvm::ConstantInt::get(IntTy, byrefSize.getQuantity()); - Builder.CreateStore(V, - Builder.CreateStructGEP(nullptr, addr, 3, "byref.size")); + storeHeaderField(V, getIntSize(), "byref.size"); if (helpers) { - llvm::Value *copy_helper = Builder.CreateStructGEP(nullptr, addr, 4); - Builder.CreateStore(helpers->CopyHelper, copy_helper); - - llvm::Value *destroy_helper = Builder.CreateStructGEP(nullptr, addr, 5); - Builder.CreateStore(helpers->DisposeHelper, destroy_helper); + storeHeaderField(helpers->CopyHelper, getPointerSize(), + "byref.copyHelper"); + storeHeaderField(helpers->DisposeHelper, getPointerSize(), + "byref.disposeHelper"); } + if (ByRefHasLifetime && HasByrefExtendedLayout) { - llvm::Constant* ByrefLayoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, type); - llvm::Value *ByrefInfoAddr = - Builder.CreateStructGEP(nullptr, addr, helpers ? 6 : 4, "byref.layout"); - // cast destination to pointer to source type. - llvm::Type *DesTy = ByrefLayoutInfo->getType(); - DesTy = DesTy->getPointerTo(); - llvm::Value *BC = Builder.CreatePointerCast(ByrefInfoAddr, DesTy); - Builder.CreateStore(ByrefLayoutInfo, BC); + auto layoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, type); + storeHeaderField(layoutInfo, getPointerSize(), "byref.layout"); } } @@ -2240,7 +2257,8 @@ void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags) { } namespace { - struct CallBlockRelease : EHScopeStack::Cleanup { + /// Release a __block variable. + struct CallBlockRelease final : EHScopeStack::Cleanup { llvm::Value *Addr; CallBlockRelease(llvm::Value *Addr) : Addr(Addr) {} @@ -2249,7 +2267,7 @@ namespace { CGF.BuildBlockRelease(Addr, BLOCK_FIELD_IS_BYREF); } }; -} +} // end anonymous namespace /// Enter a cleanup to destroy a __block variable. Note that this /// cleanup should be a no-op if the variable hasn't left the stack @@ -2260,7 +2278,8 @@ void CodeGenFunction::enterByrefCleanup(const AutoVarEmission &emission) { if (CGM.getLangOpts().getGC() == LangOptions::GCOnly) return; - EHStack.pushCleanup<CallBlockRelease>(NormalAndEHCleanup, emission.Address); + EHStack.pushCleanup<CallBlockRelease>(NormalAndEHCleanup, + emission.Addr.getPointer()); } /// Adjust the declaration of something from the blocks API. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h index c4eed0d..1edabef 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.h @@ -140,6 +140,15 @@ inline BlockFieldFlags operator|(BlockFieldFlag_t l, BlockFieldFlag_t r) { return BlockFieldFlags(l) | BlockFieldFlags(r); } +/// Information about the layout of a __block variable. +class BlockByrefInfo { +public: + llvm::StructType *Type; + unsigned FieldIndex; + CharUnits ByrefAlignment; + CharUnits FieldOffset; +}; + /// CGBlockInfo - Information to generate a block literal. class CGBlockInfo { public: @@ -152,14 +161,19 @@ public: class Capture { uintptr_t Data; EHScopeStack::stable_iterator Cleanup; + CharUnits::QuantityType Offset; public: bool isIndex() const { return (Data & 1) != 0; } bool isConstant() const { return !isIndex(); } - unsigned getIndex() const { assert(isIndex()); return Data >> 1; } - llvm::Value *getConstant() const { - assert(isConstant()); - return reinterpret_cast<llvm::Value*>(Data); + + unsigned getIndex() const { + assert(isIndex()); + return Data >> 1; + } + CharUnits getOffset() const { + assert(isIndex()); + return CharUnits::fromQuantity(Offset); } EHScopeStack::stable_iterator getCleanup() const { assert(isIndex()); @@ -170,9 +184,15 @@ public: Cleanup = cleanup; } - static Capture makeIndex(unsigned index) { + llvm::Value *getConstant() const { + assert(isConstant()); + return reinterpret_cast<llvm::Value*>(Data); + } + + static Capture makeIndex(unsigned index, CharUnits offset) { Capture v; v.Data = (index << 1) | 1; + v.Offset = offset.getQuantity(); return v; } @@ -205,12 +225,13 @@ public: /// The mapping of allocated indexes within the block. llvm::DenseMap<const VarDecl*, Capture> Captures; - llvm::AllocaInst *Address; + Address LocalAddress; llvm::StructType *StructureType; const BlockDecl *Block; const BlockExpr *BlockExpression; CharUnits BlockSize; CharUnits BlockAlign; + CharUnits CXXThisOffset; // Offset of the gap caused by block header having a smaller // alignment than the alignment of the block descriptor. This diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h b/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h index 6610659..489f341 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h @@ -11,6 +11,8 @@ #define LLVM_CLANG_LIB_CODEGEN_CGBUILDER_H #include "llvm/IR/IRBuilder.h" +#include "Address.h" +#include "CodeGenTypeCache.h" namespace clang { namespace CodeGen { @@ -22,9 +24,9 @@ class CodeGenFunction; /// instructions. template <bool PreserveNames> class CGBuilderInserter - : protected llvm::IRBuilderDefaultInserter<PreserveNames> { + : protected llvm::IRBuilderDefaultInserter<PreserveNames> { public: - CGBuilderInserter() : CGF(nullptr) {} + CGBuilderInserter() = default; explicit CGBuilderInserter(CodeGenFunction *CGF) : CGF(CGF) {} protected: @@ -33,9 +35,7 @@ protected: llvm::BasicBlock *BB, llvm::BasicBlock::iterator InsertPt) const; private: - void operator=(const CGBuilderInserter &) = delete; - - CodeGenFunction *CGF; + CodeGenFunction *CGF = nullptr; }; // Don't preserve names on values in an optimized build. @@ -44,9 +44,260 @@ private: #else #define PreserveNames true #endif + typedef CGBuilderInserter<PreserveNames> CGBuilderInserterTy; + typedef llvm::IRBuilder<PreserveNames, llvm::ConstantFolder, - CGBuilderInserterTy> CGBuilderTy; + CGBuilderInserterTy> CGBuilderBaseTy; + +class CGBuilderTy : public CGBuilderBaseTy { + /// Storing a reference to the type cache here makes it a lot easier + /// to build natural-feeling, target-specific IR. + const CodeGenTypeCache &TypeCache; +public: + CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::LLVMContext &C) + : CGBuilderBaseTy(C), TypeCache(TypeCache) {} + CGBuilderTy(const CodeGenTypeCache &TypeCache, + llvm::LLVMContext &C, const llvm::ConstantFolder &F, + const CGBuilderInserterTy &Inserter) + : CGBuilderBaseTy(C, F, Inserter), TypeCache(TypeCache) {} + CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::Instruction *I) + : CGBuilderBaseTy(I), TypeCache(TypeCache) {} + CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::BasicBlock *BB) + : CGBuilderBaseTy(BB), TypeCache(TypeCache) {} + + llvm::ConstantInt *getSize(CharUnits N) { + return llvm::ConstantInt::get(TypeCache.SizeTy, N.getQuantity()); + } + llvm::ConstantInt *getSize(uint64_t N) { + return llvm::ConstantInt::get(TypeCache.SizeTy, N); + } + + // Note that we intentionally hide the CreateLoad APIs that don't + // take an alignment. + llvm::LoadInst *CreateLoad(Address Addr, const llvm::Twine &Name = "") { + return CreateAlignedLoad(Addr.getPointer(), + Addr.getAlignment().getQuantity(), + Name); + } + llvm::LoadInst *CreateLoad(Address Addr, const char *Name) { + // This overload is required to prevent string literals from + // ending up in the IsVolatile overload. + return CreateAlignedLoad(Addr.getPointer(), + Addr.getAlignment().getQuantity(), + Name); + } + llvm::LoadInst *CreateLoad(Address Addr, bool IsVolatile, + const llvm::Twine &Name = "") { + return CreateAlignedLoad(Addr.getPointer(), + Addr.getAlignment().getQuantity(), + IsVolatile, + Name); + } + + using CGBuilderBaseTy::CreateAlignedLoad; + llvm::LoadInst *CreateAlignedLoad(llvm::Value *Addr, CharUnits Align, + const llvm::Twine &Name = "") { + return CreateAlignedLoad(Addr, Align.getQuantity(), Name); + } + llvm::LoadInst *CreateAlignedLoad(llvm::Value *Addr, CharUnits Align, + const char *Name) { + return CreateAlignedLoad(Addr, Align.getQuantity(), Name); + } + llvm::LoadInst *CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, + CharUnits Align, + const llvm::Twine &Name = "") { + assert(Addr->getType()->getPointerElementType() == Ty); + return CreateAlignedLoad(Addr, Align.getQuantity(), Name); + } + llvm::LoadInst *CreateAlignedLoad(llvm::Value *Addr, CharUnits Align, + bool IsVolatile, + const llvm::Twine &Name = "") { + return CreateAlignedLoad(Addr, Align.getQuantity(), IsVolatile, Name); + } + + // Note that we intentionally hide the CreateStore APIs that don't + // take an alignment. + llvm::StoreInst *CreateStore(llvm::Value *Val, Address Addr, + bool IsVolatile = false) { + return CreateAlignedStore(Val, Addr.getPointer(), + Addr.getAlignment().getQuantity(), IsVolatile); + } + + using CGBuilderBaseTy::CreateAlignedStore; + llvm::StoreInst *CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, + CharUnits Align, bool IsVolatile = false) { + return CreateAlignedStore(Val, Addr, Align.getQuantity(), IsVolatile); + } + + // FIXME: these "default-aligned" APIs should be removed, + // but I don't feel like fixing all the builtin code right now. + llvm::LoadInst *CreateDefaultAlignedLoad(llvm::Value *Addr, + const llvm::Twine &Name = "") { + return CGBuilderBaseTy::CreateLoad(Addr, false, Name); + } + llvm::LoadInst *CreateDefaultAlignedLoad(llvm::Value *Addr, + const char *Name) { + return CGBuilderBaseTy::CreateLoad(Addr, false, Name); + } + llvm::LoadInst *CreateDefaultAlignedLoad(llvm::Value *Addr, bool IsVolatile, + const llvm::Twine &Name = "") { + return CGBuilderBaseTy::CreateLoad(Addr, IsVolatile, Name); + } + + llvm::StoreInst *CreateDefaultAlignedStore(llvm::Value *Val, + llvm::Value *Addr, + bool IsVolatile = false) { + return CGBuilderBaseTy::CreateStore(Val, Addr, IsVolatile); + } + + /// Emit a load from an i1 flag variable. + llvm::LoadInst *CreateFlagLoad(llvm::Value *Addr, + const llvm::Twine &Name = "") { + assert(Addr->getType()->getPointerElementType() == getInt1Ty()); + return CreateAlignedLoad(getInt1Ty(), Addr, CharUnits::One(), Name); + } + + /// Emit a store to an i1 flag variable. + llvm::StoreInst *CreateFlagStore(bool Value, llvm::Value *Addr) { + assert(Addr->getType()->getPointerElementType() == getInt1Ty()); + return CreateAlignedStore(getInt1(Value), Addr, CharUnits::One()); + } + + using CGBuilderBaseTy::CreateBitCast; + Address CreateBitCast(Address Addr, llvm::Type *Ty, + const llvm::Twine &Name = "") { + return Address(CreateBitCast(Addr.getPointer(), Ty, Name), + Addr.getAlignment()); + } + + /// Cast the element type of the given address to a different type, + /// preserving information like the alignment and address space. + Address CreateElementBitCast(Address Addr, llvm::Type *Ty, + const llvm::Twine &Name = "") { + auto PtrTy = Ty->getPointerTo(Addr.getAddressSpace()); + return CreateBitCast(Addr, PtrTy, Name); + } + + using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast; + Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, + const llvm::Twine &Name = "") { + llvm::Value *Ptr = + CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name); + return Address(Ptr, Addr.getAlignment()); + } + + using CGBuilderBaseTy::CreateStructGEP; + Address CreateStructGEP(Address Addr, unsigned Index, CharUnits Offset, + const llvm::Twine &Name = "") { + return Address(CreateStructGEP(Addr.getElementType(), + Addr.getPointer(), Index, Name), + Addr.getAlignment().alignmentAtOffset(Offset)); + } + + /// Given + /// %addr = [n x T]* ... + /// produce + /// %name = getelementptr inbounds %addr, i64 0, i64 index + /// where i64 is actually the target word size. + /// + /// This API assumes that drilling into an array like this is always + /// an inbounds operation. + /// + /// \param EltSize - the size of the type T in bytes + Address CreateConstArrayGEP(Address Addr, uint64_t Index, CharUnits EltSize, + const llvm::Twine &Name = "") { + return Address(CreateInBoundsGEP(Addr.getPointer(), + {getSize(CharUnits::Zero()), + getSize(Index)}, + Name), + Addr.getAlignment().alignmentAtOffset(Index * EltSize)); + } + + /// Given + /// %addr = T* ... + /// produce + /// %name = getelementptr inbounds %addr, i64 index + /// where i64 is actually the target word size. + /// + /// \param EltSize - the size of the type T in bytes + Address CreateConstInBoundsGEP(Address Addr, uint64_t Index, + CharUnits EltSize, + const llvm::Twine &Name = "") { + return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), + getSize(Index), Name), + Addr.getAlignment().alignmentAtOffset(Index * EltSize)); + } + + /// Given + /// %addr = T* ... + /// produce + /// %name = getelementptr inbounds %addr, i64 index + /// where i64 is actually the target word size. + /// + /// \param EltSize - the size of the type T in bytes + Address CreateConstGEP(Address Addr, uint64_t Index, CharUnits EltSize, + const llvm::Twine &Name = "") { + return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), + getSize(Index), Name), + Addr.getAlignment().alignmentAtOffset(Index * EltSize)); + } + + /// Given a pointer to i8, adjust it by a given constant offset. + Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset, + const llvm::Twine &Name = "") { + assert(Addr.getElementType() == TypeCache.Int8Ty); + return Address(CreateInBoundsGEP(Addr.getPointer(), getSize(Offset), Name), + Addr.getAlignment().alignmentAtOffset(Offset)); + } + Address CreateConstByteGEP(Address Addr, CharUnits Offset, + const llvm::Twine &Name = "") { + assert(Addr.getElementType() == TypeCache.Int8Ty); + return Address(CreateGEP(Addr.getPointer(), getSize(Offset), Name), + Addr.getAlignment().alignmentAtOffset(Offset)); + } + + llvm::Value *CreateConstInBoundsByteGEP(llvm::Value *Ptr, CharUnits Offset, + const llvm::Twine &Name = "") { + assert(Ptr->getType()->getPointerElementType() == TypeCache.Int8Ty); + return CreateInBoundsGEP(Ptr, getSize(Offset), Name); + } + llvm::Value *CreateConstByteGEP(llvm::Value *Ptr, CharUnits Offset, + const llvm::Twine &Name = "") { + assert(Ptr->getType()->getPointerElementType() == TypeCache.Int8Ty); + return CreateGEP(Ptr, getSize(Offset), Name); + } + + using CGBuilderBaseTy::CreateMemCpy; + llvm::CallInst *CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, + bool IsVolatile = false) { + auto Align = std::min(Dest.getAlignment(), Src.getAlignment()); + return CreateMemCpy(Dest.getPointer(), Src.getPointer(), Size, + Align.getQuantity(), IsVolatile); + } + llvm::CallInst *CreateMemCpy(Address Dest, Address Src, uint64_t Size, + bool IsVolatile = false) { + auto Align = std::min(Dest.getAlignment(), Src.getAlignment()); + return CreateMemCpy(Dest.getPointer(), Src.getPointer(), Size, + Align.getQuantity(), IsVolatile); + } + + using CGBuilderBaseTy::CreateMemMove; + llvm::CallInst *CreateMemMove(Address Dest, Address Src, llvm::Value *Size, + bool IsVolatile = false) { + auto Align = std::min(Dest.getAlignment(), Src.getAlignment()); + return CreateMemMove(Dest.getPointer(), Src.getPointer(), Size, + Align.getQuantity(), IsVolatile); + } + + using CGBuilderBaseTy::CreateMemSet; + llvm::CallInst *CreateMemSet(Address Dest, llvm::Value *Value, + llvm::Value *Size, bool IsVolatile = false) { + return CreateMemSet(Dest.getPointer(), Value, Size, + Dest.getAlignment().getQuantity(), IsVolatile); + } +}; + #undef PreserveNames } // end namespace CodeGen diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp index 9b8694f..787ac53 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp @@ -48,7 +48,7 @@ llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, if (FD->hasAttr<AsmLabelAttr>()) Name = getMangledName(D); else - Name = Context.BuiltinInfo.GetName(BuiltinID) + 10; + Name = Context.BuiltinInfo.getName(BuiltinID) + 10; llvm::FunctionType *Ty = cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType())); @@ -111,6 +111,28 @@ static Value *MakeBinaryAtomicValue(CodeGenFunction &CGF, return EmitFromInt(CGF, Result, T, ValueType); } +static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) { + Value *Val = CGF.EmitScalarExpr(E->getArg(0)); + Value *Address = CGF.EmitScalarExpr(E->getArg(1)); + + // Convert the type of the pointer to a pointer to the stored type. + Val = CGF.EmitToMemory(Val, E->getArg(0)->getType()); + Value *BC = CGF.Builder.CreateBitCast( + Address, llvm::PointerType::getUnqual(Val->getType()), "cast"); + LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType()); + LV.setNontemporal(true); + CGF.EmitStoreOfScalar(Val, LV, false); + return nullptr; +} + +static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) { + Value *Address = CGF.EmitScalarExpr(E->getArg(0)); + + LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType()); + LV.setNontemporal(true); + return CGF.EmitLoadOfScalar(LV, E->getExprLoc()); +} + static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E) { @@ -215,10 +237,20 @@ static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) { llvm::Type *IntTy = llvm::IntegerType::get(C, Width); V = CGF.Builder.CreateBitCast(V, IntTy); if (Ty->isPPC_FP128Ty()) { - // The higher-order double comes first, and so we need to truncate the - // pair to extract the overall sign. The order of the pair is the same - // in both little- and big-Endian modes. + // We want the sign bit of the higher-order double. The bitcast we just + // did works as if the double-double was stored to memory and then + // read as an i128. The "store" will put the higher-order double in the + // lower address in both little- and big-Endian modes, but the "load" + // will treat those bits as a different part of the i128: the low bits in + // little-Endian, the high bits in big-Endian. Therefore, on big-Endian + // we need to shift the high bits down to the low before truncating. Width >>= 1; + if (CGF.getTarget().isBigEndian()) { + Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width); + V = CGF.Builder.CreateLShr(V, ShiftCst); + } + // We are truncating value in order to extract the higher-order + // double, which we will be using to extract the sign from. IntTy = llvm::IntegerType::get(C, Width); V = CGF.Builder.CreateTrunc(V, IntTy); } @@ -256,6 +288,125 @@ static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, return CGF.Builder.CreateExtractValue(Tmp, 0); } +namespace { + struct WidthAndSignedness { + unsigned Width; + bool Signed; + }; +} + +static WidthAndSignedness +getIntegerWidthAndSignedness(const clang::ASTContext &context, + const clang::QualType Type) { + assert(Type->isIntegerType() && "Given type is not an integer."); + unsigned Width = Type->isBooleanType() ? 1 : context.getTypeInfo(Type).Width; + bool Signed = Type->isSignedIntegerType(); + return {Width, Signed}; +} + +// Given one or more integer types, this function produces an integer type that +// encompasses them: any value in one of the given types could be expressed in +// the encompassing type. +static struct WidthAndSignedness +EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) { + assert(Types.size() > 0 && "Empty list of types."); + + // If any of the given types is signed, we must return a signed type. + bool Signed = false; + for (const auto &Type : Types) { + Signed |= Type.Signed; + } + + // The encompassing type must have a width greater than or equal to the width + // of the specified types. Aditionally, if the encompassing type is signed, + // its width must be strictly greater than the width of any unsigned types + // given. + unsigned Width = 0; + for (const auto &Type : Types) { + unsigned MinWidth = Type.Width + (Signed && !Type.Signed); + if (Width < MinWidth) { + Width = MinWidth; + } + } + + return {Width, Signed}; +} + +Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) { + llvm::Type *DestType = Int8PtrTy; + if (ArgValue->getType() != DestType) + ArgValue = + Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data()); + + Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend; + return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue); +} + +/// Checks if using the result of __builtin_object_size(p, @p From) in place of +/// __builtin_object_size(p, @p To) is correct +static bool areBOSTypesCompatible(int From, int To) { + // Note: Our __builtin_object_size implementation currently treats Type=0 and + // Type=2 identically. Encoding this implementation detail here may make + // improving __builtin_object_size difficult in the future, so it's omitted. + return From == To || (From == 0 && To == 1) || (From == 3 && To == 2); +} + +static llvm::Value * +getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) { + return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true); +} + +llvm::Value * +CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type, + llvm::IntegerType *ResType) { + uint64_t ObjectSize; + if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type)) + return emitBuiltinObjectSize(E, Type, ResType); + return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true); +} + +/// Returns a Value corresponding to the size of the given expression. +/// This Value may be either of the following: +/// - A llvm::Argument (if E is a param with the pass_object_size attribute on +/// it) +/// - A call to the @llvm.objectsize intrinsic +llvm::Value * +CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type, + llvm::IntegerType *ResType) { + // We need to reference an argument if the pointer is a parameter with the + // pass_object_size attribute. + if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) { + auto *Param = dyn_cast<ParmVarDecl>(D->getDecl()); + auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>(); + if (Param != nullptr && PS != nullptr && + areBOSTypesCompatible(PS->getType(), Type)) { + auto Iter = SizeArguments.find(Param); + assert(Iter != SizeArguments.end()); + + const ImplicitParamDecl *D = Iter->second; + auto DIter = LocalDeclMap.find(D); + assert(DIter != LocalDeclMap.end()); + + return EmitLoadOfScalar(DIter->second, /*volatile=*/false, + getContext().getSizeType(), E->getLocStart()); + } + } + + // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't + // evaluate E for side-effects. In either case, we shouldn't lower to + // @llvm.objectsize. + if (Type == 3 || E->HasSideEffects(getContext())) + return getDefaultBuiltinObjectSizeResult(Type, ResType); + + // LLVM only supports 0 and 2, make sure that we pass along that + // as a boolean. + auto *CI = ConstantInt::get(Builder.getInt1Ty(), (Type & 2) >> 1); + // FIXME: Get right address space. + llvm::Type *Tys[] = {ResType, Builder.getInt8PtrTy(0)}; + Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); + return Builder.CreateCall(F, {EmitScalarExpr(E), CI}); +} + RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue) { @@ -279,22 +430,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, case Builtin::BI__builtin_stdarg_start: case Builtin::BI__builtin_va_start: case Builtin::BI__va_start: - case Builtin::BI__builtin_va_end: { - Value *ArgValue = (BuiltinID == Builtin::BI__va_start) - ? EmitScalarExpr(E->getArg(0)) - : EmitVAListRef(E->getArg(0)); - llvm::Type *DestType = Int8PtrTy; - if (ArgValue->getType() != DestType) - ArgValue = Builder.CreateBitCast(ArgValue, DestType, - ArgValue->getName().data()); - - Intrinsic::ID inst = (BuiltinID == Builtin::BI__builtin_va_end) ? - Intrinsic::vaend : Intrinsic::vastart; - return RValue::get(Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue)); - } + case Builtin::BI__builtin_va_end: + return RValue::get( + EmitVAStartEnd(BuiltinID == Builtin::BI__va_start + ? EmitScalarExpr(E->getArg(0)) + : EmitVAListRef(E->getArg(0)).getPointer(), + BuiltinID != Builtin::BI__builtin_va_end)); case Builtin::BI__builtin_va_copy: { - Value *DstPtr = EmitVAListRef(E->getArg(0)); - Value *SrcPtr = EmitVAListRef(E->getArg(1)); + Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer(); + Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer(); llvm::Type *Type = Int8PtrTy; @@ -455,6 +599,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, "cast"); return RValue::get(Result); } + case Builtin::BI__builtin_unpredictable: { + // Always return the argument of __builtin_unpredictable. LLVM does not + // handle this builtin. Metadata for this builtin should be added directly + // to instructions such as branches or switches that use it. + return RValue::get(EmitScalarExpr(E->getArg(0))); + } case Builtin::BI__builtin_expect: { Value *ArgValue = EmitScalarExpr(E->getArg(0)); llvm::Type *ArgType = ArgValue->getType(); @@ -501,26 +651,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, return RValue::get(Builder.CreateCall(F, ArgValue)); } case Builtin::BI__builtin_object_size: { - // We rely on constant folding to deal with expressions with side effects. - assert(!E->getArg(0)->HasSideEffects(getContext()) && - "should have been constant folded"); - - // We pass this builtin onto the optimizer so that it can - // figure out the object size in more complex cases. - llvm::Type *ResType = ConvertType(E->getType()); - - // LLVM only supports 0 and 2, make sure that we pass along that - // as a boolean. - Value *Ty = EmitScalarExpr(E->getArg(1)); - ConstantInt *CI = dyn_cast<ConstantInt>(Ty); - assert(CI); - uint64_t val = CI->getZExtValue(); - CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1); - // FIXME: Get right address space. - llvm::Type *Tys[] = { ResType, Builder.getInt8PtrTy(0) }; - Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); - return RValue::get( - Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0)), CI})); + unsigned Type = + E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue(); + auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType())); + + // We pass this builtin onto the optimizer so that it can figure out the + // object size in more complex cases. + return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType)); } case Builtin::BI__builtin_prefetch: { Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0)); @@ -737,29 +874,24 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, } case Builtin::BIbzero: case Builtin::BI__builtin_bzero: { - std::pair<llvm::Value*, unsigned> Dest = - EmitPointerWithAlignment(E->getArg(0)); + Address Dest = EmitPointerWithAlignment(E->getArg(0)); Value *SizeVal = EmitScalarExpr(E->getArg(1)); - EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(), + EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); - Builder.CreateMemSet(Dest.first, Builder.getInt8(0), SizeVal, - Dest.second, false); - return RValue::get(Dest.first); + Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false); + return RValue::get(Dest.getPointer()); } case Builtin::BImemcpy: case Builtin::BI__builtin_memcpy: { - std::pair<llvm::Value*, unsigned> Dest = - EmitPointerWithAlignment(E->getArg(0)); - std::pair<llvm::Value*, unsigned> Src = - EmitPointerWithAlignment(E->getArg(1)); + Address Dest = EmitPointerWithAlignment(E->getArg(0)); + Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = EmitScalarExpr(E->getArg(2)); - unsigned Align = std::min(Dest.second, Src.second); - EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(), + EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); - EmitNonNullArgCheck(RValue::get(Src.first), E->getArg(1)->getType(), + EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD, 1); - Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false); - return RValue::get(Dest.first); + Builder.CreateMemCpy(Dest, Src, SizeVal, false); + return RValue::get(Dest.getPointer()); } case Builtin::BI__builtin___memcpy_chk: { @@ -770,23 +902,20 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, break; if (Size.ugt(DstSize)) break; - std::pair<llvm::Value*, unsigned> Dest = - EmitPointerWithAlignment(E->getArg(0)); - std::pair<llvm::Value*, unsigned> Src = - EmitPointerWithAlignment(E->getArg(1)); + Address Dest = EmitPointerWithAlignment(E->getArg(0)); + Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); - unsigned Align = std::min(Dest.second, Src.second); - Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false); - return RValue::get(Dest.first); + Builder.CreateMemCpy(Dest, Src, SizeVal, false); + return RValue::get(Dest.getPointer()); } case Builtin::BI__builtin_objc_memmove_collectable: { - Value *Address = EmitScalarExpr(E->getArg(0)); - Value *SrcAddr = EmitScalarExpr(E->getArg(1)); + Address DestAddr = EmitPointerWithAlignment(E->getArg(0)); + Address SrcAddr = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = EmitScalarExpr(E->getArg(2)); CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, - Address, SrcAddr, SizeVal); - return RValue::get(Address); + DestAddr, SrcAddr, SizeVal); + return RValue::get(DestAddr.getPointer()); } case Builtin::BI__builtin___memmove_chk: { @@ -797,42 +926,35 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, break; if (Size.ugt(DstSize)) break; - std::pair<llvm::Value*, unsigned> Dest = - EmitPointerWithAlignment(E->getArg(0)); - std::pair<llvm::Value*, unsigned> Src = - EmitPointerWithAlignment(E->getArg(1)); + Address Dest = EmitPointerWithAlignment(E->getArg(0)); + Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); - unsigned Align = std::min(Dest.second, Src.second); - Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false); - return RValue::get(Dest.first); + Builder.CreateMemMove(Dest, Src, SizeVal, false); + return RValue::get(Dest.getPointer()); } case Builtin::BImemmove: case Builtin::BI__builtin_memmove: { - std::pair<llvm::Value*, unsigned> Dest = - EmitPointerWithAlignment(E->getArg(0)); - std::pair<llvm::Value*, unsigned> Src = - EmitPointerWithAlignment(E->getArg(1)); + Address Dest = EmitPointerWithAlignment(E->getArg(0)); + Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = EmitScalarExpr(E->getArg(2)); - unsigned Align = std::min(Dest.second, Src.second); - EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(), + EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); - EmitNonNullArgCheck(RValue::get(Src.first), E->getArg(1)->getType(), + EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD, 1); - Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false); - return RValue::get(Dest.first); + Builder.CreateMemMove(Dest, Src, SizeVal, false); + return RValue::get(Dest.getPointer()); } case Builtin::BImemset: case Builtin::BI__builtin_memset: { - std::pair<llvm::Value*, unsigned> Dest = - EmitPointerWithAlignment(E->getArg(0)); + Address Dest = EmitPointerWithAlignment(E->getArg(0)); Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty()); Value *SizeVal = EmitScalarExpr(E->getArg(2)); - EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(), + EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); - Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false); - return RValue::get(Dest.first); + Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); + return RValue::get(Dest.getPointer()); } case Builtin::BI__builtin___memset_chk: { // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2. @@ -842,13 +964,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, break; if (Size.ugt(DstSize)) break; - std::pair<llvm::Value*, unsigned> Dest = - EmitPointerWithAlignment(E->getArg(0)); + Address Dest = EmitPointerWithAlignment(E->getArg(0)); Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty()); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); - Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false); - return RValue::get(Dest.first); + Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); + return RValue::get(Dest.getPointer()); } case Builtin::BI__builtin_dwarf_cfa: { // The offset in bytes from the first argument to the CFA. @@ -952,7 +1073,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, } case Builtin::BI__builtin_setjmp: { // Buffer is a void**. - Value *Buf = EmitScalarExpr(E->getArg(0)); + Address Buf = EmitPointerWithAlignment(E->getArg(0)); // Store the frame pointer to the setjmp buffer. Value *FrameAddr = @@ -963,14 +1084,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, // Store the stack pointer to the setjmp buffer. Value *StackAddr = Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); - Value *StackSaveSlot = - Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2)); + Address StackSaveSlot = + Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize()); Builder.CreateStore(StackAddr, StackSaveSlot); // Call LLVM's EH setjmp, which is lightweight. Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); Buf = Builder.CreateBitCast(Buf, Int8PtrTy); - return RValue::get(Builder.CreateCall(F, Buf)); + return RValue::get(Builder.CreateCall(F, Buf.getPointer())); } case Builtin::BI__builtin_longjmp: { Value *Buf = EmitScalarExpr(E->getArg(0)); @@ -1135,8 +1256,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, StoreSize.getQuantity() * 8); Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo()); llvm::StoreInst *Store = - Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr); - Store->setAlignment(StoreSize.getQuantity()); + Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr, + StoreSize); Store->setAtomic(llvm::Release); return RValue::get(nullptr); } @@ -1153,6 +1274,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, return RValue::get(nullptr); } + case Builtin::BI__builtin_nontemporal_load: + return RValue::get(EmitNontemporalLoad(*this, E)); + case Builtin::BI__builtin_nontemporal_store: + return RValue::get(EmitNontemporalStore(*this, E)); case Builtin::BI__c11_atomic_is_lock_free: case Builtin::BI__atomic_is_lock_free: { // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the @@ -1270,15 +1395,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, bool Volatile = PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); - Value *Ptr = EmitScalarExpr(E->getArg(0)); - unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); + Address Ptr = EmitPointerWithAlignment(E->getArg(0)); + unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace(); Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); Value *NewVal = Builder.getInt8(0); Value *Order = EmitScalarExpr(E->getArg(1)); if (isa<llvm::ConstantInt>(Order)) { int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); - Store->setAlignment(1); switch (ord) { case 0: // memory_order_relaxed default: // invalid order @@ -1311,7 +1435,6 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, for (unsigned i = 0; i < 3; ++i) { Builder.SetInsertPoint(BBs[i]); StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); - Store->setAlignment(1); Store->setOrdering(Orders[i]); Builder.CreateBr(ContBB); } @@ -1493,8 +1616,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::Value *X = EmitScalarExpr(E->getArg(0)); llvm::Value *Y = EmitScalarExpr(E->getArg(1)); llvm::Value *Carryin = EmitScalarExpr(E->getArg(2)); - std::pair<llvm::Value*, unsigned> CarryOutPtr = - EmitPointerWithAlignment(E->getArg(3)); + Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3)); // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow. llvm::Intrinsic::ID IntrinsicId; @@ -1525,11 +1647,91 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Sum1, Carryin, Carry2); llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2), X->getType()); - llvm::StoreInst *CarryOutStore = Builder.CreateStore(CarryOut, - CarryOutPtr.first); - CarryOutStore->setAlignment(CarryOutPtr.second); + Builder.CreateStore(CarryOut, CarryOutPtr); return RValue::get(Sum2); } + + case Builtin::BI__builtin_add_overflow: + case Builtin::BI__builtin_sub_overflow: + case Builtin::BI__builtin_mul_overflow: { + const clang::Expr *LeftArg = E->getArg(0); + const clang::Expr *RightArg = E->getArg(1); + const clang::Expr *ResultArg = E->getArg(2); + + clang::QualType ResultQTy = + ResultArg->getType()->castAs<PointerType>()->getPointeeType(); + + WidthAndSignedness LeftInfo = + getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType()); + WidthAndSignedness RightInfo = + getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType()); + WidthAndSignedness ResultInfo = + getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy); + WidthAndSignedness EncompassingInfo = + EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo}); + + llvm::Type *EncompassingLLVMTy = + llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width); + + llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy); + + llvm::Intrinsic::ID IntrinsicId; + switch (BuiltinID) { + default: + llvm_unreachable("Unknown overflow builtin id."); + case Builtin::BI__builtin_add_overflow: + IntrinsicId = EncompassingInfo.Signed + ? llvm::Intrinsic::sadd_with_overflow + : llvm::Intrinsic::uadd_with_overflow; + break; + case Builtin::BI__builtin_sub_overflow: + IntrinsicId = EncompassingInfo.Signed + ? llvm::Intrinsic::ssub_with_overflow + : llvm::Intrinsic::usub_with_overflow; + break; + case Builtin::BI__builtin_mul_overflow: + IntrinsicId = EncompassingInfo.Signed + ? llvm::Intrinsic::smul_with_overflow + : llvm::Intrinsic::umul_with_overflow; + break; + } + + llvm::Value *Left = EmitScalarExpr(LeftArg); + llvm::Value *Right = EmitScalarExpr(RightArg); + Address ResultPtr = EmitPointerWithAlignment(ResultArg); + + // Extend each operand to the encompassing type. + Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed); + Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed); + + // Perform the operation on the extended values. + llvm::Value *Overflow, *Result; + Result = EmitOverflowIntrinsic(*this, IntrinsicId, Left, Right, Overflow); + + if (EncompassingInfo.Width > ResultInfo.Width) { + // The encompassing type is wider than the result type, so we need to + // truncate it. + llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy); + + // To see if the truncation caused an overflow, we will extend + // the result and then compare it to the original result. + llvm::Value *ResultTruncExt = Builder.CreateIntCast( + ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed); + llvm::Value *TruncationOverflow = + Builder.CreateICmpNE(Result, ResultTruncExt); + + Overflow = Builder.CreateOr(Overflow, TruncationOverflow); + Result = ResultTrunc; + } + + // Finally, store the result using the pointer. + bool isVolatile = + ResultArg->getType()->getPointeeType().isVolatileQualified(); + Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile); + + return RValue::get(Overflow); + } + case Builtin::BI__builtin_uadd_overflow: case Builtin::BI__builtin_uaddl_overflow: case Builtin::BI__builtin_uaddll_overflow: @@ -1554,13 +1756,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, // Scalarize our inputs. llvm::Value *X = EmitScalarExpr(E->getArg(0)); llvm::Value *Y = EmitScalarExpr(E->getArg(1)); - std::pair<llvm::Value *, unsigned> SumOutPtr = - EmitPointerWithAlignment(E->getArg(2)); + Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2)); // Decide which of the overflow intrinsics we are lowering to: llvm::Intrinsic::ID IntrinsicId; switch (BuiltinID) { - default: llvm_unreachable("Unknown security overflow builtin id."); + default: llvm_unreachable("Unknown overflow builtin id."); case Builtin::BI__builtin_uadd_overflow: case Builtin::BI__builtin_uaddl_overflow: case Builtin::BI__builtin_uaddll_overflow: @@ -1596,13 +1797,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::Value *Carry; llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry); - llvm::StoreInst *SumOutStore = Builder.CreateStore(Sum, SumOutPtr.first); - SumOutStore->setAlignment(SumOutPtr.second); + Builder.CreateStore(Sum, SumOutPtr); return RValue::get(Carry); } case Builtin::BI__builtin_addressof: - return RValue::get(EmitLValue(E->getArg(0)).getAddress()); + return RValue::get(EmitLValue(E->getArg(0)).getPointer()); case Builtin::BI__builtin_operator_new: return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), E->getArg(0), false); @@ -1777,8 +1977,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee())); + // Check that a call to a target specific builtin has the correct target + // features. + // This is down here to avoid non-target specific builtins, however, if + // generic builtins start to require generic target features then we + // can move this up to the beginning of the function. + checkTargetFeatures(E, FD); + // See if we have a target specific intrinsic. - const char *Name = getContext().BuiltinInfo.GetName(BuiltinID); + const char *Name = getContext().BuiltinInfo.getName(BuiltinID); Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic; if (const char *Prefix = llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) { @@ -1856,37 +2063,54 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, return GetUndefRValue(E->getType()); } -Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, - const CallExpr *E) { - switch (getTarget().getTriple().getArch()) { +static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, + unsigned BuiltinID, const CallExpr *E, + llvm::Triple::ArchType Arch) { + switch (Arch) { case llvm::Triple::arm: case llvm::Triple::armeb: case llvm::Triple::thumb: case llvm::Triple::thumbeb: - return EmitARMBuiltinExpr(BuiltinID, E); + return CGF->EmitARMBuiltinExpr(BuiltinID, E); case llvm::Triple::aarch64: case llvm::Triple::aarch64_be: - return EmitAArch64BuiltinExpr(BuiltinID, E); + return CGF->EmitAArch64BuiltinExpr(BuiltinID, E); case llvm::Triple::x86: case llvm::Triple::x86_64: - return EmitX86BuiltinExpr(BuiltinID, E); + return CGF->EmitX86BuiltinExpr(BuiltinID, E); case llvm::Triple::ppc: case llvm::Triple::ppc64: case llvm::Triple::ppc64le: - return EmitPPCBuiltinExpr(BuiltinID, E); + return CGF->EmitPPCBuiltinExpr(BuiltinID, E); case llvm::Triple::r600: case llvm::Triple::amdgcn: - return EmitAMDGPUBuiltinExpr(BuiltinID, E); + return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E); case llvm::Triple::systemz: - return EmitSystemZBuiltinExpr(BuiltinID, E); + return CGF->EmitSystemZBuiltinExpr(BuiltinID, E); case llvm::Triple::nvptx: case llvm::Triple::nvptx64: - return EmitNVPTXBuiltinExpr(BuiltinID, E); + return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E); + case llvm::Triple::wasm32: + case llvm::Triple::wasm64: + return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E); default: return nullptr; } } +Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, + const CallExpr *E) { + if (getContext().BuiltinInfo.isAuxBuiltinID(BuiltinID)) { + assert(getContext().getAuxTargetInfo() && "Missing aux target info"); + return EmitTargetArchBuiltinExpr( + this, getContext().BuiltinInfo.getAuxBuiltinID(BuiltinID), E, + getContext().getAuxTargetInfo()->getTriple().getArch()); + } + + return EmitTargetArchBuiltinExpr(this, BuiltinID, E, + getTarget().getTriple().getArch()); +} + static llvm::VectorType *GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool V1Ty=false) { @@ -1917,6 +2141,19 @@ static llvm::VectorType *GetNeonType(CodeGenFunction *CGF, llvm_unreachable("Unknown vector element type!"); } +static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF, + NeonTypeFlags IntTypeFlags) { + int IsQuad = IntTypeFlags.isQuad(); + switch (IntTypeFlags.getEltType()) { + case NeonTypeFlags::Int32: + return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad)); + case NeonTypeFlags::Int64: + return llvm::VectorType::get(CGF->DoubleTy, (1 << IsQuad)); + default: + llvm_unreachable("Type can't be converted to floating-point!"); + } +} + Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) { unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements(); Value* SV = llvm::ConstantVector::getSplat(nElts, C); @@ -1940,10 +2177,7 @@ Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty, bool neg) { int SV = cast<ConstantInt>(V)->getSExtValue(); - - llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); - llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV); - return llvm::ConstantVector::getSplat(VTy->getNumElements(), C); + return ConstantInt::get(Ty, neg ? -SV : SV); } // \brief Right-shift a vector by a constant. @@ -1962,8 +2196,7 @@ Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift, if (ShiftAmt == EltSize) { if (usgn) { // Right-shifting an unsigned value by its size yields 0. - llvm::Constant *Zero = ConstantInt::get(VTy->getElementType(), 0); - return llvm::ConstantVector::getSplat(VTy->getNumElements(), Zero); + return llvm::ConstantAggregateZero::get(VTy); } else { // Right-shifting a signed value by its size is equivalent // to a shift of size-1. @@ -1979,61 +2212,6 @@ Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift, return Builder.CreateAShr(Vec, Shift, name); } -/// GetPointeeAlignment - Given an expression with a pointer type, find the -/// alignment of the type referenced by the pointer. Skip over implicit -/// casts. -std::pair<llvm::Value*, unsigned> -CodeGenFunction::EmitPointerWithAlignment(const Expr *Addr) { - assert(Addr->getType()->isPointerType()); - Addr = Addr->IgnoreParens(); - if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Addr)) { - if ((ICE->getCastKind() == CK_BitCast || ICE->getCastKind() == CK_NoOp) && - ICE->getSubExpr()->getType()->isPointerType()) { - std::pair<llvm::Value*, unsigned> Ptr = - EmitPointerWithAlignment(ICE->getSubExpr()); - Ptr.first = Builder.CreateBitCast(Ptr.first, - ConvertType(Addr->getType())); - return Ptr; - } else if (ICE->getCastKind() == CK_ArrayToPointerDecay) { - LValue LV = EmitLValue(ICE->getSubExpr()); - unsigned Align = LV.getAlignment().getQuantity(); - if (!Align) { - // FIXME: Once LValues are fixed to always set alignment, - // zap this code. - QualType PtTy = ICE->getSubExpr()->getType(); - if (!PtTy->isIncompleteType()) - Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); - else - Align = 1; - } - return std::make_pair(LV.getAddress(), Align); - } - } - if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Addr)) { - if (UO->getOpcode() == UO_AddrOf) { - LValue LV = EmitLValue(UO->getSubExpr()); - unsigned Align = LV.getAlignment().getQuantity(); - if (!Align) { - // FIXME: Once LValues are fixed to always set alignment, - // zap this code. - QualType PtTy = UO->getSubExpr()->getType(); - if (!PtTy->isIncompleteType()) - Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); - else - Align = 1; - } - return std::make_pair(LV.getAddress(), Align); - } - } - - unsigned Align = 1; - QualType PtTy = Addr->getType()->getPointeeType(); - if (!PtTy->isIncompleteType()) - Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); - - return std::make_pair(EmitScalarExpr(Addr), Align); -} - enum { AddRetType = (1 << 0), Add1ArgType = (1 << 1), @@ -2056,31 +2234,36 @@ enum { AddRetType | VectorizeRetType | Add1ArgType | InventFloatType }; - struct NeonIntrinsicInfo { +namespace { +struct NeonIntrinsicInfo { + const char *NameHint; unsigned BuiltinID; unsigned LLVMIntrinsic; unsigned AltLLVMIntrinsic; - const char *NameHint; unsigned TypeModifier; bool operator<(unsigned RHSBuiltinID) const { return BuiltinID < RHSBuiltinID; } + bool operator<(const NeonIntrinsicInfo &TE) const { + return BuiltinID < TE.BuiltinID; + } }; +} // end anonymous namespace #define NEONMAP0(NameBase) \ - { NEON::BI__builtin_neon_ ## NameBase, 0, 0, #NameBase, 0 } + { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 } #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ - { NEON:: BI__builtin_neon_ ## NameBase, \ - Intrinsic::LLVMIntrinsic, 0, #NameBase, TypeModifier } + { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ + Intrinsic::LLVMIntrinsic, 0, TypeModifier } #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \ - { NEON:: BI__builtin_neon_ ## NameBase, \ + { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ - #NameBase, TypeModifier } + TypeModifier } -static NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { +static const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), NEONMAP1(vabs_v, arm_neon_vabs, 0), @@ -2106,7 +2289,7 @@ static NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { NEONMAP1(vclzq_v, ctlz, Add1ArgType), NEONMAP1(vcnt_v, ctpop, Add1ArgType), NEONMAP1(vcntq_v, ctpop, Add1ArgType), - NEONMAP1(vcvt_f16_v, arm_neon_vcvtfp2hf, 0), + NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0), NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0), NEONMAP0(vcvt_f32_v), NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), @@ -2297,7 +2480,7 @@ static NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { NEONMAP0(vzipq_v) }; -static NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { +static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { NEONMAP1(vabs_v, aarch64_neon_abs, 0), NEONMAP1(vabsq_v, aarch64_neon_abs, 0), NEONMAP0(vaddhn_v), @@ -2319,7 +2502,7 @@ static NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { NEONMAP1(vclzq_v, ctlz, Add1ArgType), NEONMAP1(vcnt_v, ctpop, Add1ArgType), NEONMAP1(vcntq_v, ctpop, Add1ArgType), - NEONMAP1(vcvt_f16_v, aarch64_neon_vcvtfp2hf, 0), + NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0), NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0), NEONMAP0(vcvt_f32_v), NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), @@ -2412,7 +2595,7 @@ static NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { NEONMAP0(vtstq_v), }; -static NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = { +static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = { NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType), NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType), NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType), @@ -2623,9 +2806,7 @@ findNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap, #ifndef NDEBUG if (!MapProvenSorted) { - // FIXME: use std::is_sorted once C++11 is allowed - for (unsigned i = 0; i < IntrinsicMap.size() - 1; ++i) - assert(IntrinsicMap[i].BuiltinID <= IntrinsicMap[i + 1].BuiltinID); + assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap))); MapProvenSorted = true; } #endif @@ -2744,7 +2925,7 @@ static Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, - SmallVectorImpl<llvm::Value *> &Ops, llvm::Value *Align) { + SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1) { // Get the last argument, which specifies the vector type. llvm::APSInt NeonTypeConst; const Expr *Arg = E->getArg(E->getNumArgs() - 1); @@ -2761,6 +2942,10 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( if (!Ty) return nullptr; + auto getAlignmentValue32 = [&](Address addr) -> Value* { + return Builder.getInt32(addr.getAlignment().getQuantity()); + }; + unsigned Int = LLVMIntrinsic; if ((Modifier & UnsignedAlts) && !Usgn) Int = AltLLVMIntrinsic; @@ -2782,9 +2967,8 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> - Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(), - SrcTy->getScalarSizeInBits() / 2); - ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt); + Constant *ShiftAmt = + ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2); Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); // %res = trunc <4 x i32> %high to <4 x i16> @@ -2822,13 +3006,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( case NEON::BI__builtin_neon_vcvt_n_f64_v: case NEON::BI__builtin_neon_vcvtq_n_f32_v: case NEON::BI__builtin_neon_vcvtq_n_f64_v: { - bool Double = - (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); - llvm::Type *FloatTy = - GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 - : NeonTypeFlags::Float32, - false, Quad)); - llvm::Type *Tys[2] = { FloatTy, Ty }; + llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty }; Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic; Function *F = CGM.getIntrinsic(Int, Tys); return EmitNeonCall(F, Ops, "vcvt_n"); @@ -2841,13 +3019,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( case NEON::BI__builtin_neon_vcvtq_n_u32_v: case NEON::BI__builtin_neon_vcvtq_n_s64_v: case NEON::BI__builtin_neon_vcvtq_n_u64_v: { - bool Double = - (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); - llvm::Type *FloatTy = - GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 - : NeonTypeFlags::Float32, - false, Quad)); - llvm::Type *Tys[2] = { Ty, FloatTy }; + llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); return EmitNeonCall(F, Ops, "vcvt_n"); } @@ -2859,13 +3031,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( case NEON::BI__builtin_neon_vcvtq_u32_v: case NEON::BI__builtin_neon_vcvtq_s64_v: case NEON::BI__builtin_neon_vcvtq_u64_v: { - bool Double = - (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); - llvm::Type *FloatTy = - GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 - : NeonTypeFlags::Float32, - false, Quad)); - Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); + Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); } @@ -2901,13 +3067,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( case NEON::BI__builtin_neon_vcvtmq_s64_v: case NEON::BI__builtin_neon_vcvtmq_u32_v: case NEON::BI__builtin_neon_vcvtmq_u64_v: { - bool Double = - (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); - llvm::Type *InTy = - GetNeonType(this, - NeonTypeFlags(Double ? NeonTypeFlags::Float64 - : NeonTypeFlags::Float32, false, Quad)); - llvm::Type *Tys[2] = { Ty, InTy }; + llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); } case NEON::BI__builtin_neon_vext_v: @@ -2933,28 +3093,31 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); } case NEON::BI__builtin_neon_vld1_v: - case NEON::BI__builtin_neon_vld1q_v: - Ops.push_back(Align); - return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vld1"); + case NEON::BI__builtin_neon_vld1q_v: { + llvm::Type *Tys[] = {Ty, Int8PtrTy}; + Ops.push_back(getAlignmentValue32(PtrOp0)); + return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); + } case NEON::BI__builtin_neon_vld2_v: case NEON::BI__builtin_neon_vld2q_v: case NEON::BI__builtin_neon_vld3_v: case NEON::BI__builtin_neon_vld3q_v: case NEON::BI__builtin_neon_vld4_v: case NEON::BI__builtin_neon_vld4q_v: { - Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); + llvm::Type *Tys[] = {Ty, Int8PtrTy}; + Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); + Value *Align = getAlignmentValue32(PtrOp1); Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vld1_dup_v: case NEON::BI__builtin_neon_vld1q_dup_v: { Value *V = UndefValue::get(Ty); Ty = llvm::PointerType::getUnqual(VTy->getElementType()); - Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - LoadInst *Ld = Builder.CreateLoad(Ops[0]); - Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); + PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty); + LoadInst *Ld = Builder.CreateLoad(PtrOp0); llvm::Constant *CI = ConstantInt::get(SizeTy, 0); Ops[0] = Builder.CreateInsertElement(V, Ld, CI); return EmitNeonSplat(Ops[0], CI); @@ -2965,14 +3128,15 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( case NEON::BI__builtin_neon_vld3q_lane_v: case NEON::BI__builtin_neon_vld4_lane_v: case NEON::BI__builtin_neon_vld4q_lane_v: { - Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); + llvm::Type *Tys[] = {Ty, Int8PtrTy}; + Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); for (unsigned I = 2; I < Ops.size() - 1; ++I) Ops[I] = Builder.CreateBitCast(Ops[I], Ty); - Ops.push_back(Align); + Ops.push_back(getAlignmentValue32(PtrOp1)); Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vmovl_v: { llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy); @@ -3019,14 +3183,10 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( case NEON::BI__builtin_neon_vqdmlal_v: case NEON::BI__builtin_neon_vqdmlsl_v: { SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); - Value *Mul = EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), - MulOps, "vqdmlal"); - - SmallVector<Value *, 2> AccumOps; - AccumOps.push_back(Ops[0]); - AccumOps.push_back(Mul); - return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), - AccumOps, NameHint); + Ops[1] = + EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal"); + Ops.resize(2); + return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); } case NEON::BI__builtin_neon_vqshl_n_v: case NEON::BI__builtin_neon_vqshlq_n_v: @@ -3088,9 +3248,11 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( case NEON::BI__builtin_neon_vst3_lane_v: case NEON::BI__builtin_neon_vst3q_lane_v: case NEON::BI__builtin_neon_vst4_lane_v: - case NEON::BI__builtin_neon_vst4q_lane_v: - Ops.push_back(Align); - return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, ""); + case NEON::BI__builtin_neon_vst4q_lane_v: { + llvm::Type *Tys[] = {Int8PtrTy, Ty}; + Ops.push_back(getAlignmentValue32(PtrOp0)); + return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); + } case NEON::BI__builtin_neon_vsubhn_v: { llvm::VectorType *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy); @@ -3101,9 +3263,8 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> - Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(), - SrcTy->getScalarSizeInBits() / 2); - ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt); + Constant *ShiftAmt = + ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2); Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); // %res = trunc <4 x i32> %high to <4 x i16> @@ -3125,7 +3286,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); - SV = Builder.CreateStore(SV, Addr); + SV = Builder.CreateDefaultAlignedStore(SV, Addr); } return SV; } @@ -3153,7 +3314,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); - SV = Builder.CreateStore(SV, Addr); + SV = Builder.CreateDefaultAlignedStore(SV, Addr); } return SV; } @@ -3173,7 +3334,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); - SV = Builder.CreateStore(SV, Addr); + SV = Builder.CreateDefaultAlignedStore(SV, Addr); } return SV; } @@ -3252,33 +3413,37 @@ static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, } Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) { + unsigned Value; switch (BuiltinID) { default: return nullptr; case ARM::BI__builtin_arm_nop: - return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint), - llvm::ConstantInt::get(Int32Ty, 0)); + Value = 0; + break; case ARM::BI__builtin_arm_yield: case ARM::BI__yield: - return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint), - llvm::ConstantInt::get(Int32Ty, 1)); + Value = 1; + break; case ARM::BI__builtin_arm_wfe: case ARM::BI__wfe: - return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint), - llvm::ConstantInt::get(Int32Ty, 2)); + Value = 2; + break; case ARM::BI__builtin_arm_wfi: case ARM::BI__wfi: - return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint), - llvm::ConstantInt::get(Int32Ty, 3)); + Value = 3; + break; case ARM::BI__builtin_arm_sev: case ARM::BI__sev: - return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint), - llvm::ConstantInt::get(Int32Ty, 4)); + Value = 4; + break; case ARM::BI__builtin_arm_sevl: case ARM::BI__sevl: - return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint), - llvm::ConstantInt::get(Int32Ty, 5)); + Value = 5; + break; } + + return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint), + llvm::ConstantInt::get(Int32Ty, Value)); } // Generates the IR for the read/write special register builtin, @@ -3428,9 +3593,9 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, if (BuiltinID == ARM::BI__clear_cache) { assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); const FunctionDecl *FD = E->getDirectCallee(); - SmallVector<Value*, 2> Ops; + Value *Ops[2]; for (unsigned i = 0; i < 2; i++) - Ops.push_back(EmitScalarExpr(E->getArg(i))); + Ops[i] = EmitScalarExpr(E->getArg(i)); llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); StringRef Name = FD->getName(); @@ -3504,11 +3669,11 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, : Intrinsic::arm_strexd); llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, nullptr); - Value *Tmp = CreateMemTemp(E->getArg(0)->getType()); + Address Tmp = CreateMemTemp(E->getArg(0)->getType()); Value *Val = EmitScalarExpr(E->getArg(0)); Builder.CreateStore(Val, Tmp); - Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); + Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); Val = Builder.CreateLoad(LdPtr); Value *Arg0 = Builder.CreateExtractValue(Val, 0); @@ -3627,8 +3792,13 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); assert(Error == ASTContext::GE_None && "Should not codegen an error"); + auto getAlignmentValue32 = [&](Address addr) -> Value* { + return Builder.getInt32(addr.getAlignment().getQuantity()); + }; + + Address PtrOp0 = Address::invalid(); + Address PtrOp1 = Address::invalid(); SmallVector<Value*, 4> Ops; - llvm::Value *Align = nullptr; bool HasExtraArg = HasExtraNeonArgument(BuiltinID); unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0); for (unsigned i = 0, e = NumArgs; i != e; i++) { @@ -3658,10 +3828,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vst4q_lane_v: // Get the alignment for the argument in addition to the value; // we'll use it later. - std::pair<llvm::Value*, unsigned> Src = - EmitPointerWithAlignment(E->getArg(0)); - Ops.push_back(Src.first); - Align = Builder.getInt32(Src.second); + PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); + Ops.push_back(PtrOp0.getPointer()); continue; } } @@ -3684,10 +3852,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vld4_dup_v: // Get the alignment for the argument in addition to the value; // we'll use it later. - std::pair<llvm::Value*, unsigned> Src = - EmitPointerWithAlignment(E->getArg(1)); - Ops.push_back(Src.first); - Align = Builder.getInt32(Src.second); + PtrOp1 = EmitPointerWithAlignment(E->getArg(1)); + Ops.push_back(PtrOp1.getPointer()); continue; } } @@ -3798,7 +3964,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, if (Builtin) return EmitCommonNeonBuiltinExpr( Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, - Builtin->NameHint, Builtin->TypeModifier, E, Ops, Align); + Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1); unsigned Int; switch (BuiltinID) { @@ -3809,27 +3975,25 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, if (VTy->getElementType()->isIntegerTy(64)) { // Extract the other lane. Ops[1] = Builder.CreateBitCast(Ops[1], Ty); - int Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); + uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane)); Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); // Load the value as a one-element vector. Ty = llvm::VectorType::get(VTy->getElementType(), 1); - Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty); + llvm::Type *Tys[] = {Ty, Int8PtrTy}; + Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys); + Value *Align = getAlignmentValue32(PtrOp0); Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); // Combine them. - SmallVector<Constant*, 2> Indices; - Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane)); - Indices.push_back(ConstantInt::get(Int32Ty, Lane)); - SV = llvm::ConstantVector::get(Indices); + uint32_t Indices[] = {1 - Lane, Lane}; + SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); } // fall through case NEON::BI__builtin_neon_vld1_lane_v: { Ops[1] = Builder.CreateBitCast(Ops[1], Ty); - Ty = llvm::PointerType::getUnqual(VTy->getElementType()); - Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - LoadInst *Ld = Builder.CreateLoad(Ops[0]); - Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); + PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType()); + Value *Ld = Builder.CreateLoad(PtrOp0); return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); } case NEON::BI__builtin_neon_vld2_dup_v: @@ -3849,11 +4013,13 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, break; default: llvm_unreachable("unknown vld_dup intrinsic?"); } - Function *F = CGM.getIntrinsic(Int, Ty); + llvm::Type *Tys[] = {Ty, Int8PtrTy}; + Function *F = CGM.getIntrinsic(Int, Tys); + llvm::Value *Align = getAlignmentValue32(PtrOp1); Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup"); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } switch (BuiltinID) { case NEON::BI__builtin_neon_vld2_dup_v: @@ -3867,7 +4033,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, break; default: llvm_unreachable("unknown vld_dup intrinsic?"); } - Function *F = CGM.getIntrinsic(Int, Ty); + llvm::Type *Tys[] = {Ty, Int8PtrTy}; + Function *F = CGM.getIntrinsic(Int, Tys); llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType()); SmallVector<Value*, 6> Args; @@ -3876,7 +4043,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); Args.push_back(CI); - Args.push_back(Align); + Args.push_back(getAlignmentValue32(PtrOp1)); Ops[1] = Builder.CreateCall(F, Args, "vld_dup"); // splat lane 0 to all elts in each vector of the result. @@ -3889,7 +4056,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, } Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vqrshrn_n_v: Int = @@ -3941,18 +4108,17 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateBitCast(Ops[1], Ty); Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); - Ops[2] = Align; + Ops[2] = getAlignmentValue32(PtrOp0); + llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, - Ops[1]->getType()), Ops); + Tys), Ops); } // fall through case NEON::BI__builtin_neon_vst1_lane_v: { Ops[1] = Builder.CreateBitCast(Ops[1], Ty); Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); - StoreInst *St = Builder.CreateStore(Ops[1], - Builder.CreateBitCast(Ops[0], Ty)); - St->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); + auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty)); return St; } case NEON::BI__builtin_neon_vtbl1_v: @@ -4029,52 +4195,41 @@ static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID // Determine the type of this overloaded NEON intrinsic. NeonTypeFlags Type(Result.getZExtValue()); - llvm::VectorType *VTy = GetNeonType(&CGF, Type); - llvm::Type *Ty = VTy; + llvm::VectorType *Ty = GetNeonType(&CGF, Type); if (!Ty) return nullptr; - unsigned nElts = VTy->getNumElements(); - CodeGen::CGBuilderTy &Builder = CGF.Builder; // AArch64 scalar builtins are not overloaded, they do not have an extra // argument that specifies the vector type, need to handle each case. - SmallVector<Value *, 2> TblOps; switch (BuiltinID) { case NEON::BI__builtin_neon_vtbl1_v: { - TblOps.push_back(Ops[0]); - return packTBLDVectorList(CGF, TblOps, nullptr, Ops[1], Ty, - Intrinsic::aarch64_neon_tbl1, "vtbl1"); + return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr, + Ops[1], Ty, Intrinsic::aarch64_neon_tbl1, + "vtbl1"); } case NEON::BI__builtin_neon_vtbl2_v: { - TblOps.push_back(Ops[0]); - TblOps.push_back(Ops[1]); - return packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty, - Intrinsic::aarch64_neon_tbl1, "vtbl1"); + return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr, + Ops[2], Ty, Intrinsic::aarch64_neon_tbl1, + "vtbl1"); } case NEON::BI__builtin_neon_vtbl3_v: { - TblOps.push_back(Ops[0]); - TblOps.push_back(Ops[1]); - TblOps.push_back(Ops[2]); - return packTBLDVectorList(CGF, TblOps, nullptr, Ops[3], Ty, - Intrinsic::aarch64_neon_tbl2, "vtbl2"); + return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr, + Ops[3], Ty, Intrinsic::aarch64_neon_tbl2, + "vtbl2"); } case NEON::BI__builtin_neon_vtbl4_v: { - TblOps.push_back(Ops[0]); - TblOps.push_back(Ops[1]); - TblOps.push_back(Ops[2]); - TblOps.push_back(Ops[3]); - return packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty, - Intrinsic::aarch64_neon_tbl2, "vtbl2"); + return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr, + Ops[4], Ty, Intrinsic::aarch64_neon_tbl2, + "vtbl2"); } case NEON::BI__builtin_neon_vtbx1_v: { - TblOps.push_back(Ops[1]); - Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty, - Intrinsic::aarch64_neon_tbl1, "vtbl1"); + Value *TblRes = + packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2], + Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1"); - llvm::Constant *Eight = ConstantInt::get(VTy->getElementType(), 8); - Value* EightV = llvm::ConstantVector::getSplat(nElts, Eight); + llvm::Constant *EightV = ConstantInt::get(Ty, 8); Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); CmpRes = Builder.CreateSExt(CmpRes, Ty); @@ -4083,20 +4238,16 @@ static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); } case NEON::BI__builtin_neon_vtbx2_v: { - TblOps.push_back(Ops[1]); - TblOps.push_back(Ops[2]); - return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[3], Ty, - Intrinsic::aarch64_neon_tbx1, "vtbx1"); + return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0], + Ops[3], Ty, Intrinsic::aarch64_neon_tbx1, + "vtbx1"); } case NEON::BI__builtin_neon_vtbx3_v: { - TblOps.push_back(Ops[1]); - TblOps.push_back(Ops[2]); - TblOps.push_back(Ops[3]); - Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty, - Intrinsic::aarch64_neon_tbl2, "vtbl2"); - - llvm::Constant *TwentyFour = ConstantInt::get(VTy->getElementType(), 24); - Value* TwentyFourV = llvm::ConstantVector::getSplat(nElts, TwentyFour); + Value *TblRes = + packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4], + Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2"); + + llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24); Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], TwentyFourV); CmpRes = Builder.CreateSExt(CmpRes, Ty); @@ -4106,12 +4257,9 @@ static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); } case NEON::BI__builtin_neon_vtbx4_v: { - TblOps.push_back(Ops[1]); - TblOps.push_back(Ops[2]); - TblOps.push_back(Ops[3]); - TblOps.push_back(Ops[4]); - return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[5], Ty, - Intrinsic::aarch64_neon_tbx2, "vtbx2"); + return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0], + Ops[5], Ty, Intrinsic::aarch64_neon_tbx2, + "vtbx2"); } case NEON::BI__builtin_neon_vqtbl1_v: case NEON::BI__builtin_neon_vqtbl1q_v: @@ -4156,15 +4304,6 @@ Value *CodeGenFunction::vectorWrapScalar16(Value *Op) { return Op; } -Value *CodeGenFunction::vectorWrapScalar8(Value *Op) { - llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8); - Op = Builder.CreateBitCast(Op, Int8Ty); - Value *V = UndefValue::get(VTy); - llvm::Constant *CI = ConstantInt::get(SizeTy, 0); - Op = Builder.CreateInsertElement(V, Op, CI); - return Op; -} - Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E) { unsigned HintID = static_cast<unsigned>(-1); @@ -4236,9 +4375,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, if (BuiltinID == AArch64::BI__clear_cache) { assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); const FunctionDecl *FD = E->getDirectCallee(); - SmallVector<Value*, 2> Ops; + Value *Ops[2]; for (unsigned i = 0; i < 2; i++) - Ops.push_back(EmitScalarExpr(E->getArg(i))); + Ops[i] = EmitScalarExpr(E->getArg(i)); llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); StringRef Name = FD->getName(); @@ -4297,14 +4436,11 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, : Intrinsic::aarch64_stxp); llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, nullptr); - Value *One = llvm::ConstantInt::get(Int32Ty, 1); - Value *Tmp = Builder.CreateAlloca(ConvertType(E->getArg(0)->getType()), - One); - Value *Val = EmitScalarExpr(E->getArg(0)); - Builder.CreateStore(Val, Tmp); + Address Tmp = CreateMemTemp(E->getArg(0)->getType()); + EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true); - Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); - Val = Builder.CreateLoad(LdPtr); + Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy)); + llvm::Value *Val = Builder.CreateLoad(Tmp); Value *Arg0 = Builder.CreateExtractValue(Val, 0); Value *Arg1 = Builder.CreateExtractValue(Val, 1); @@ -4342,6 +4478,11 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, return Builder.CreateCall(F); } + if (BuiltinID == AArch64::BI__builtin_thread_pointer) { + Function *F = CGM.getIntrinsic(Intrinsic::aarch64_thread_pointer); + return Builder.CreateCall(F); + } + // CRC32 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; switch (BuiltinID) { @@ -4453,12 +4594,12 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vldrq_p128: { llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy); - return Builder.CreateLoad(Ptr); + return Builder.CreateDefaultAlignedLoad(Ptr); } case NEON::BI__builtin_neon_vstrq_p128: { llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); - return Builder.CreateStore(EmitScalarExpr(E->getArg(1)), Ptr); + return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr); } case NEON::BI__builtin_neon_vcvts_u32_f32: case NEON::BI__builtin_neon_vcvtd_u64_f64: @@ -4491,8 +4632,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, return Builder.CreateSIToFP(Ops[0], FTy); } case NEON::BI__builtin_neon_vpaddd_s64: { - llvm::Type *Ty = - llvm::VectorType::get(llvm::Type::getInt64Ty(getLLVMContext()), 2); + llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2); Value *Vec = EmitScalarExpr(E->getArg(0)); // The vector is v2f64, so make sure it's bitcast to that. Vec = Builder.CreateBitCast(Vec, Ty, "v2i64"); @@ -4505,7 +4645,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, } case NEON::BI__builtin_neon_vpaddd_f64: { llvm::Type *Ty = - llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2); + llvm::VectorType::get(DoubleTy, 2); Value *Vec = EmitScalarExpr(E->getArg(0)); // The vector is v2f64, so make sure it's bitcast to that. Vec = Builder.CreateBitCast(Vec, Ty, "v2f64"); @@ -4518,7 +4658,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, } case NEON::BI__builtin_neon_vpadds_f32: { llvm::Type *Ty = - llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2); + llvm::VectorType::get(FloatTy, 2); Value *Vec = EmitScalarExpr(E->getArg(0)); // The vector is v2f32, so make sure it's bitcast to that. Vec = Builder.CreateBitCast(Vec, Ty, "v2f32"); @@ -4566,12 +4706,11 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz"); case NEON::BI__builtin_neon_vceqzd_u64: { - llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); Ops.push_back(EmitScalarExpr(E->getArg(0))); - Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - Ops[0] = Builder.CreateICmp(llvm::ICmpInst::ICMP_EQ, Ops[0], - llvm::Constant::getNullValue(Ty)); - return Builder.CreateSExt(Ops[0], Ty, "vceqzd"); + Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); + Ops[0] = + Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); + return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); } case NEON::BI__builtin_neon_vceqd_f64: case NEON::BI__builtin_neon_vcled_f64: @@ -4645,14 +4784,13 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, } case NEON::BI__builtin_neon_vtstd_s64: case NEON::BI__builtin_neon_vtstd_u64: { - llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); Ops.push_back(EmitScalarExpr(E->getArg(1))); - Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - Ops[1] = Builder.CreateBitCast(Ops[1], Ty); + Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); + Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], - llvm::Constant::getNullValue(Ty)); - return Builder.CreateSExt(Ops[0], Ty, "vtstd"); + llvm::Constant::getNullValue(Int64Ty)); + return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); } case NEON::BI__builtin_neon_vset_lane_i8: case NEON::BI__builtin_neon_vset_lane_i16: @@ -4675,89 +4813,80 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vsetq_lane_f64: // The vector type needs a cast for the v2f64 variant. Ops[1] = Builder.CreateBitCast(Ops[1], - llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2)); + llvm::VectorType::get(DoubleTy, 2)); Ops.push_back(EmitScalarExpr(E->getArg(2))); return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); case NEON::BI__builtin_neon_vget_lane_i8: case NEON::BI__builtin_neon_vdupb_lane_i8: - Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8)); + Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vget_lane"); case NEON::BI__builtin_neon_vgetq_lane_i8: case NEON::BI__builtin_neon_vdupb_laneq_i8: - Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16)); + Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vgetq_lane"); case NEON::BI__builtin_neon_vget_lane_i16: case NEON::BI__builtin_neon_vduph_lane_i16: - Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4)); + Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vget_lane"); case NEON::BI__builtin_neon_vgetq_lane_i16: case NEON::BI__builtin_neon_vduph_laneq_i16: - Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8)); + Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vgetq_lane"); case NEON::BI__builtin_neon_vget_lane_i32: case NEON::BI__builtin_neon_vdups_lane_i32: - Ops[0] = Builder.CreateBitCast( - Ops[0], - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 2)); + Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vget_lane"); case NEON::BI__builtin_neon_vdups_lane_f32: Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2)); + llvm::VectorType::get(FloatTy, 2)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vdups_lane"); case NEON::BI__builtin_neon_vgetq_lane_i32: case NEON::BI__builtin_neon_vdups_laneq_i32: - Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 4)); + Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vgetq_lane"); case NEON::BI__builtin_neon_vget_lane_i64: case NEON::BI__builtin_neon_vdupd_lane_i64: - Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 1)); + Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vget_lane"); case NEON::BI__builtin_neon_vdupd_lane_f64: Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1)); + llvm::VectorType::get(DoubleTy, 1)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vdupd_lane"); case NEON::BI__builtin_neon_vgetq_lane_i64: case NEON::BI__builtin_neon_vdupd_laneq_i64: - Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 2)); + Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vgetq_lane"); case NEON::BI__builtin_neon_vget_lane_f32: Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2)); + llvm::VectorType::get(FloatTy, 2)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vget_lane"); case NEON::BI__builtin_neon_vget_lane_f64: Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1)); + llvm::VectorType::get(DoubleTy, 1)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vget_lane"); case NEON::BI__builtin_neon_vgetq_lane_f32: case NEON::BI__builtin_neon_vdups_laneq_f32: Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 4)); + llvm::VectorType::get(FloatTy, 4)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vgetq_lane"); case NEON::BI__builtin_neon_vgetq_lane_f64: case NEON::BI__builtin_neon_vdupd_laneq_f64: Ops[0] = Builder.CreateBitCast(Ops[0], - llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2)); + llvm::VectorType::get(DoubleTy, 2)); return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), "vgetq_lane"); case NEON::BI__builtin_neon_vaddd_s64: @@ -4930,7 +5059,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, if (Builtin) return EmitCommonNeonBuiltinExpr( Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, - Builtin->NameHint, Builtin->TypeModifier, E, Ops, nullptr); + Builtin->NameHint, Builtin->TypeModifier, E, Ops, + /*never use addresses*/ Address::invalid(), Address::invalid()); if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops)) return V; @@ -5096,15 +5226,13 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Int = Intrinsic::aarch64_neon_fmaxnm; return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); case NEON::BI__builtin_neon_vrecpss_f32: { - llvm::Type *f32Type = llvm::Type::getFloatTy(getLLVMContext()); Ops.push_back(EmitScalarExpr(E->getArg(1))); - return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f32Type), + return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy), Ops, "vrecps"); } case NEON::BI__builtin_neon_vrecpsd_f64: { - llvm::Type *f64Type = llvm::Type::getDoubleTy(getLLVMContext()); Ops.push_back(EmitScalarExpr(E->getArg(1))); - return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f64Type), + return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy), Ops, "vrecps"); } case NEON::BI__builtin_neon_vqshrun_n_v: @@ -5207,13 +5335,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vcvtq_u32_v: case NEON::BI__builtin_neon_vcvtq_s64_v: case NEON::BI__builtin_neon_vcvtq_u64_v: { - bool Double = - (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); - llvm::Type *InTy = - GetNeonType(this, - NeonTypeFlags(Double ? NeonTypeFlags::Float64 - : NeonTypeFlags::Float32, false, quad)); - Ops[0] = Builder.CreateBitCast(Ops[0], InTy); + Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); if (usgn) return Builder.CreateFPToUI(Ops[0], Ty); return Builder.CreateFPToSI(Ops[0], Ty); @@ -5227,13 +5349,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vcvta_u64_v: case NEON::BI__builtin_neon_vcvtaq_u64_v: { Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas; - bool Double = - (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); - llvm::Type *InTy = - GetNeonType(this, - NeonTypeFlags(Double ? NeonTypeFlags::Float64 - : NeonTypeFlags::Float32, false, quad)); - llvm::Type *Tys[2] = { Ty, InTy }; + llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); } case NEON::BI__builtin_neon_vcvtm_s32_v: @@ -5245,13 +5361,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vcvtm_u64_v: case NEON::BI__builtin_neon_vcvtmq_u64_v: { Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms; - bool Double = - (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); - llvm::Type *InTy = - GetNeonType(this, - NeonTypeFlags(Double ? NeonTypeFlags::Float64 - : NeonTypeFlags::Float32, false, quad)); - llvm::Type *Tys[2] = { Ty, InTy }; + llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); } case NEON::BI__builtin_neon_vcvtn_s32_v: @@ -5263,13 +5373,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vcvtn_u64_v: case NEON::BI__builtin_neon_vcvtnq_u64_v: { Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns; - bool Double = - (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); - llvm::Type *InTy = - GetNeonType(this, - NeonTypeFlags(Double ? NeonTypeFlags::Float64 - : NeonTypeFlags::Float32, false, quad)); - llvm::Type *Tys[2] = { Ty, InTy }; + llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); } case NEON::BI__builtin_neon_vcvtp_s32_v: @@ -5281,13 +5385,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vcvtp_u64_v: case NEON::BI__builtin_neon_vcvtpq_u64_v: { Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps; - bool Double = - (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); - llvm::Type *InTy = - GetNeonType(this, - NeonTypeFlags(Double ? NeonTypeFlags::Float64 - : NeonTypeFlags::Float32, false, quad)); - llvm::Type *Tys[2] = { Ty, InTy }; + llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); } case NEON::BI__builtin_neon_vmulx_v: @@ -5338,232 +5436,192 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, // FALLTHROUGH case NEON::BI__builtin_neon_vaddv_s8: { Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 8)); + return Builder.CreateTrunc(Ops[0], Int8Ty); } case NEON::BI__builtin_neon_vaddv_u16: usgn = true; // FALLTHROUGH case NEON::BI__builtin_neon_vaddv_s16: { Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 4); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vaddvq_u8: usgn = true; // FALLTHROUGH case NEON::BI__builtin_neon_vaddvq_s8: { Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 16); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 8)); + return Builder.CreateTrunc(Ops[0], Int8Ty); } case NEON::BI__builtin_neon_vaddvq_u16: usgn = true; // FALLTHROUGH case NEON::BI__builtin_neon_vaddvq_s16: { Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vmaxv_u8: { Int = Intrinsic::aarch64_neon_umaxv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 8)); + return Builder.CreateTrunc(Ops[0], Int8Ty); } case NEON::BI__builtin_neon_vmaxv_u16: { Int = Intrinsic::aarch64_neon_umaxv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 4); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vmaxvq_u8: { Int = Intrinsic::aarch64_neon_umaxv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 16); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 8)); + return Builder.CreateTrunc(Ops[0], Int8Ty); } case NEON::BI__builtin_neon_vmaxvq_u16: { Int = Intrinsic::aarch64_neon_umaxv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vmaxv_s8: { Int = Intrinsic::aarch64_neon_smaxv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 8)); + return Builder.CreateTrunc(Ops[0], Int8Ty); } case NEON::BI__builtin_neon_vmaxv_s16: { Int = Intrinsic::aarch64_neon_smaxv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 4); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vmaxvq_s8: { Int = Intrinsic::aarch64_neon_smaxv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 16); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 8)); + return Builder.CreateTrunc(Ops[0], Int8Ty); } case NEON::BI__builtin_neon_vmaxvq_s16: { Int = Intrinsic::aarch64_neon_smaxv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vminv_u8: { Int = Intrinsic::aarch64_neon_uminv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 8)); + return Builder.CreateTrunc(Ops[0], Int8Ty); } case NEON::BI__builtin_neon_vminv_u16: { Int = Intrinsic::aarch64_neon_uminv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 4); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vminvq_u8: { Int = Intrinsic::aarch64_neon_uminv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 16); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 8)); + return Builder.CreateTrunc(Ops[0], Int8Ty); } case NEON::BI__builtin_neon_vminvq_u16: { Int = Intrinsic::aarch64_neon_uminv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vminv_s8: { Int = Intrinsic::aarch64_neon_sminv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 8)); + return Builder.CreateTrunc(Ops[0], Int8Ty); } case NEON::BI__builtin_neon_vminv_s16: { Int = Intrinsic::aarch64_neon_sminv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 4); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vminvq_s8: { Int = Intrinsic::aarch64_neon_sminv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 16); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 8)); + return Builder.CreateTrunc(Ops[0], Int8Ty); } case NEON::BI__builtin_neon_vminvq_s16: { Int = Intrinsic::aarch64_neon_sminv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vmul_n_f64: { Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); @@ -5572,80 +5630,68 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, } case NEON::BI__builtin_neon_vaddlv_u8: { Int = Intrinsic::aarch64_neon_uaddlv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vaddlv_u16: { Int = Intrinsic::aarch64_neon_uaddlv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 4); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); } case NEON::BI__builtin_neon_vaddlvq_u8: { Int = Intrinsic::aarch64_neon_uaddlv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 16); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vaddlvq_u16: { Int = Intrinsic::aarch64_neon_uaddlv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); } case NEON::BI__builtin_neon_vaddlv_s8: { Int = Intrinsic::aarch64_neon_saddlv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vaddlv_s16: { Int = Intrinsic::aarch64_neon_saddlv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 4); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); } case NEON::BI__builtin_neon_vaddlvq_s8: { Int = Intrinsic::aarch64_neon_saddlv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int8Ty, 16); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); - return Builder.CreateTrunc(Ops[0], - llvm::IntegerType::get(getLLVMContext(), 16)); + return Builder.CreateTrunc(Ops[0], Int16Ty); } case NEON::BI__builtin_neon_vaddlvq_s16: { Int = Intrinsic::aarch64_neon_saddlv; - Ty = llvm::IntegerType::get(getLLVMContext(), 32); - VTy = - llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); + Ty = Int32Ty; + VTy = llvm::VectorType::get(Int16Ty, 8); llvm::Type *Tys[2] = { Ty, VTy }; Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); @@ -5708,7 +5754,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vst1_x2_v: case NEON::BI__builtin_neon_vst1q_x2_v: @@ -5733,32 +5779,31 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Int = Intrinsic::aarch64_neon_st1x4; break; } - SmallVector<Value *, 4> IntOps(Ops.begin()+1, Ops.end()); - IntOps.push_back(Ops[0]); - return EmitNeonCall(CGM.getIntrinsic(Int, Tys), IntOps, ""); + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); + return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); } case NEON::BI__builtin_neon_vld1_v: case NEON::BI__builtin_neon_vld1q_v: Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); - return Builder.CreateLoad(Ops[0]); + return Builder.CreateDefaultAlignedLoad(Ops[0]); case NEON::BI__builtin_neon_vst1_v: case NEON::BI__builtin_neon_vst1q_v: Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); Ops[1] = Builder.CreateBitCast(Ops[1], VTy); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); case NEON::BI__builtin_neon_vld1_lane_v: case NEON::BI__builtin_neon_vld1q_lane_v: Ops[1] = Builder.CreateBitCast(Ops[1], Ty); Ty = llvm::PointerType::getUnqual(VTy->getElementType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - Ops[0] = Builder.CreateLoad(Ops[0]); + Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); case NEON::BI__builtin_neon_vld1_dup_v: case NEON::BI__builtin_neon_vld1q_dup_v: { Value *V = UndefValue::get(Ty); Ty = llvm::PointerType::getUnqual(VTy->getElementType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - Ops[0] = Builder.CreateLoad(Ops[0]); + Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); return EmitNeonSplat(Ops[0], CI); @@ -5768,7 +5813,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateBitCast(Ops[1], Ty); Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); - return Builder.CreateStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty)); + return Builder.CreateDefaultAlignedStore(Ops[1], + Builder.CreateBitCast(Ops[0], Ty)); case NEON::BI__builtin_neon_vld2_v: case NEON::BI__builtin_neon_vld2q_v: { llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); @@ -5778,7 +5824,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ops[1]->getType())); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vld3_v: case NEON::BI__builtin_neon_vld3q_v: { @@ -5789,7 +5835,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ops[1]->getType())); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vld4_v: case NEON::BI__builtin_neon_vld4q_v: { @@ -5800,7 +5846,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ops[1]->getType())); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vld2_dup_v: case NEON::BI__builtin_neon_vld2q_dup_v: { @@ -5812,7 +5858,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ops[1]->getType())); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vld3_dup_v: case NEON::BI__builtin_neon_vld3q_dup_v: { @@ -5824,7 +5870,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ops[1]->getType())); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vld4_dup_v: case NEON::BI__builtin_neon_vld4q_dup_v: { @@ -5836,7 +5882,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ops[1]->getType())); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vld2_lane_v: case NEON::BI__builtin_neon_vld2q_lane_v: { @@ -5846,12 +5892,11 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops.erase(Ops.begin()+1); Ops[1] = Builder.CreateBitCast(Ops[1], Ty); Ops[2] = Builder.CreateBitCast(Ops[2], Ty); - Ops[3] = Builder.CreateZExt(Ops[3], - llvm::IntegerType::get(getLLVMContext(), 64)); + Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vld3_lane_v: case NEON::BI__builtin_neon_vld3q_lane_v: { @@ -5862,12 +5907,11 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateBitCast(Ops[1], Ty); Ops[2] = Builder.CreateBitCast(Ops[2], Ty); Ops[3] = Builder.CreateBitCast(Ops[3], Ty); - Ops[4] = Builder.CreateZExt(Ops[4], - llvm::IntegerType::get(getLLVMContext(), 64)); + Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vld4_lane_v: case NEON::BI__builtin_neon_vld4q_lane_v: { @@ -5879,12 +5923,11 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[2] = Builder.CreateBitCast(Ops[2], Ty); Ops[3] = Builder.CreateBitCast(Ops[3], Ty); Ops[4] = Builder.CreateBitCast(Ops[4], Ty); - Ops[5] = Builder.CreateZExt(Ops[5], - llvm::IntegerType::get(getLLVMContext(), 64)); + Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case NEON::BI__builtin_neon_vst2_v: case NEON::BI__builtin_neon_vst2q_v: { @@ -5898,8 +5941,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vst2q_lane_v: { Ops.push_back(Ops[0]); Ops.erase(Ops.begin()); - Ops[2] = Builder.CreateZExt(Ops[2], - llvm::IntegerType::get(getLLVMContext(), 64)); + Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys), Ops, ""); @@ -5916,8 +5958,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vst3q_lane_v: { Ops.push_back(Ops[0]); Ops.erase(Ops.begin()); - Ops[3] = Builder.CreateZExt(Ops[3], - llvm::IntegerType::get(getLLVMContext(), 64)); + Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys), Ops, ""); @@ -5934,8 +5975,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vst4q_lane_v: { Ops.push_back(Ops[0]); Ops.erase(Ops.begin()); - Ops[4] = Builder.CreateZExt(Ops[4], - llvm::IntegerType::get(getLLVMContext(), 64)); + Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys), Ops, ""); @@ -5956,7 +5996,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); - SV = Builder.CreateStore(SV, Addr); + SV = Builder.CreateDefaultAlignedStore(SV, Addr); } return SV; } @@ -5975,7 +6015,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); - SV = Builder.CreateStore(SV, Addr); + SV = Builder.CreateDefaultAlignedStore(SV, Addr); } return SV; } @@ -5995,7 +6035,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); - SV = Builder.CreateStore(SV, Addr); + SV = Builder.CreateDefaultAlignedStore(SV, Addr); } return SV; } @@ -6072,6 +6112,31 @@ BuildVector(ArrayRef<llvm::Value*> Ops) { Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E) { + if (BuiltinID == X86::BI__builtin_ms_va_start || + BuiltinID == X86::BI__builtin_ms_va_end) + return EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(), + BuiltinID == X86::BI__builtin_ms_va_start); + if (BuiltinID == X86::BI__builtin_ms_va_copy) { + // Lower this manually. We can't reliably determine whether or not any + // given va_copy() is for a Win64 va_list from the calling convention + // alone, because it's legal to do this from a System V ABI function. + // With opaque pointer types, we won't have enough information in LLVM + // IR to determine this from the argument types, either. Best to do it + // now, while we have enough information. + Address DestAddr = EmitMSVAListRef(E->getArg(0)); + Address SrcAddr = EmitMSVAListRef(E->getArg(1)); + + llvm::Type *BPP = Int8PtrPtrTy; + + DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"), + DestAddr.getAlignment()); + SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"), + SrcAddr.getAlignment()); + + Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val"); + return Builder.CreateStore(ArgPtr, DestAddr); + } + SmallVector<Value*, 4> Ops; // Find out if any arguments are required to be integer constant expressions. @@ -6167,7 +6232,8 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, ConstantInt::get(Int32Ty, 0) }; Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs); - Value *Features = Builder.CreateLoad(CpuFeatures); + Value *Features = Builder.CreateAlignedLoad(CpuFeatures, + CharUnits::fromQuantity(4)); // Check the value of the bit corresponding to the feature requested. Value *Bitset = Builder.CreateAnd( @@ -6175,13 +6241,17 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0)); } case X86::BI_mm_prefetch: { - Value *Address = EmitScalarExpr(E->getArg(0)); + Value *Address = Ops[0]; Value *RW = ConstantInt::get(Int32Ty, 0); - Value *Locality = EmitScalarExpr(E->getArg(1)); + Value *Locality = Ops[1]; Value *Data = ConstantInt::get(Int32Ty, 1); Value *F = CGM.getIntrinsic(Intrinsic::prefetch); return Builder.CreateCall(F, {Address, RW, Locality, Data}); } + case X86::BI__builtin_ia32_undef128: + case X86::BI__builtin_ia32_undef256: + case X86::BI__builtin_ia32_undef512: + return UndefValue::get(ConvertType(E->getType())); case X86::BI__builtin_ia32_vec_init_v8qi: case X86::BI__builtin_ia32_vec_init_v4hi: case X86::BI__builtin_ia32_vec_init_v2si: @@ -6191,17 +6261,57 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, return Builder.CreateExtractElement(Ops[0], llvm::ConstantInt::get(Ops[1]->getType(), 0)); case X86::BI__builtin_ia32_ldmxcsr: { - Value *Tmp = CreateMemTemp(E->getArg(0)->getType()); + Address Tmp = CreateMemTemp(E->getArg(0)->getType()); Builder.CreateStore(Ops[0], Tmp); return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), - Builder.CreateBitCast(Tmp, Int8PtrTy)); + Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy)); } case X86::BI__builtin_ia32_stmxcsr: { - Value *Tmp = CreateMemTemp(E->getType()); + Address Tmp = CreateMemTemp(E->getType()); Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), - Builder.CreateBitCast(Tmp, Int8PtrTy)); + Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy)); return Builder.CreateLoad(Tmp, "stmxcsr"); } + case X86::BI__builtin_ia32_xsave: + case X86::BI__builtin_ia32_xsave64: + case X86::BI__builtin_ia32_xrstor: + case X86::BI__builtin_ia32_xrstor64: + case X86::BI__builtin_ia32_xsaveopt: + case X86::BI__builtin_ia32_xsaveopt64: + case X86::BI__builtin_ia32_xrstors: + case X86::BI__builtin_ia32_xrstors64: + case X86::BI__builtin_ia32_xsavec: + case X86::BI__builtin_ia32_xsavec64: + case X86::BI__builtin_ia32_xsaves: + case X86::BI__builtin_ia32_xsaves64: { + Intrinsic::ID ID; +#define INTRINSIC_X86_XSAVE_ID(NAME) \ + case X86::BI__builtin_ia32_##NAME: \ + ID = Intrinsic::x86_##NAME; \ + break + switch (BuiltinID) { + default: llvm_unreachable("Unsupported intrinsic!"); + INTRINSIC_X86_XSAVE_ID(xsave); + INTRINSIC_X86_XSAVE_ID(xsave64); + INTRINSIC_X86_XSAVE_ID(xrstor); + INTRINSIC_X86_XSAVE_ID(xrstor64); + INTRINSIC_X86_XSAVE_ID(xsaveopt); + INTRINSIC_X86_XSAVE_ID(xsaveopt64); + INTRINSIC_X86_XSAVE_ID(xrstors); + INTRINSIC_X86_XSAVE_ID(xrstors64); + INTRINSIC_X86_XSAVE_ID(xsavec); + INTRINSIC_X86_XSAVE_ID(xsavec64); + INTRINSIC_X86_XSAVE_ID(xsaves); + INTRINSIC_X86_XSAVE_ID(xsaves64); + } +#undef INTRINSIC_X86_XSAVE_ID + Value *Mhi = Builder.CreateTrunc( + Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); + Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); + Ops[1] = Mhi; + Ops.push_back(Mlo); + return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); + } case X86::BI__builtin_ia32_storehps: case X86::BI__builtin_ia32_storelps: { llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty); @@ -6217,7 +6327,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, // cast pointer to i64 & store Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); - return Builder.CreateStore(Ops[1], Ops[0]); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); } case X86::BI__builtin_ia32_palignr128: case X86::BI__builtin_ia32_palignr256: { @@ -6242,18 +6352,19 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); } - SmallVector<llvm::Constant*, 32> Indices; + uint32_t Indices[32]; // 256-bit palignr operates on 128-bit lanes so we need to handle that for (unsigned l = 0; l != NumElts; l += NumLaneElts) { for (unsigned i = 0; i != NumLaneElts; ++i) { unsigned Idx = ShiftVal + i; if (Idx >= NumLaneElts) Idx += NumElts - NumLaneElts; // End of lane, switch operand. - Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l)); + Indices[l + i] = Idx + l; } } - Value* SV = llvm::ConstantVector::get(Indices); + Value *SV = llvm::ConstantDataVector::get(getLLVMContext(), + makeArrayRef(Indices, NumElts)); return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); } case X86::BI__builtin_ia32_pslldqi256: { @@ -6264,13 +6375,13 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, if (shiftVal >= 16) return llvm::Constant::getNullValue(ConvertType(E->getType())); - SmallVector<llvm::Constant*, 32> Indices; + uint32_t Indices[32]; // 256-bit pslldq operates on 128-bit lanes so we need to handle that for (unsigned l = 0; l != 32; l += 16) { for (unsigned i = 0; i != 16; ++i) { unsigned Idx = 32 + i - shiftVal; if (Idx < 32) Idx -= 16; // end of lane, switch operand. - Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l)); + Indices[l + i] = Idx + l; } } @@ -6278,7 +6389,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); Value *Zero = llvm::Constant::getNullValue(VecTy); - Value *SV = llvm::ConstantVector::get(Indices); + Value *SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); SV = Builder.CreateShuffleVector(Zero, Ops[0], SV, "pslldq"); llvm::Type *ResultType = ConvertType(E->getType()); return Builder.CreateBitCast(SV, ResultType, "cast"); @@ -6291,13 +6402,13 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, if (shiftVal >= 16) return llvm::Constant::getNullValue(ConvertType(E->getType())); - SmallVector<llvm::Constant*, 32> Indices; + uint32_t Indices[32]; // 256-bit psrldq operates on 128-bit lanes so we need to handle that for (unsigned l = 0; l != 32; l += 16) { for (unsigned i = 0; i != 16; ++i) { unsigned Idx = i + shiftVal; if (Idx >= 16) Idx += 16; // end of lane, switch operand. - Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l)); + Indices[l + i] = Idx + l; } } @@ -6305,7 +6416,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); Value *Zero = llvm::Constant::getNullValue(VecTy); - Value *SV = llvm::ConstantVector::get(Indices); + Value *SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); SV = Builder.CreateShuffleVector(Ops[0], Zero, SV, "psrldq"); llvm::Type *ResultType = ConvertType(E->getType()); return Builder.CreateBitCast(SV, ResultType, "cast"); @@ -6325,7 +6436,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, Value *BC = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ops[1]->getType()), "cast"); - StoreInst *SI = Builder.CreateStore(Ops[1], BC); + StoreInst *SI = Builder.CreateDefaultAlignedStore(Ops[1], BC); SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); // If the operand is an integer, we can't assume alignment. Otherwise, @@ -6377,7 +6488,8 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, } Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID)); - Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]); + Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0), + Ops[0]); return Builder.CreateExtractValue(Call, 1); } // SSE comparison intrisics @@ -6544,6 +6656,11 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, switch (BuiltinID) { default: return nullptr; + // __builtin_ppc_get_timebase is GCC 4.8+'s PowerPC-specific name for what we + // call __builtin_readcyclecounter. + case PPC::BI__builtin_ppc_get_timebase: + return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter)); + // vec_ld, vec_lvsl, vec_lvsr case PPC::BI__builtin_altivec_lvx: case PPC::BI__builtin_altivec_lvxl: @@ -6775,8 +6892,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, // Translate from the intrinsics's struct return to the builtin's out // argument. - std::pair<llvm::Value *, unsigned> FlagOutPtr - = EmitPointerWithAlignment(E->getArg(3)); + Address FlagOutPtr = EmitPointerWithAlignment(E->getArg(3)); llvm::Value *X = EmitScalarExpr(E->getArg(0)); llvm::Value *Y = EmitScalarExpr(E->getArg(1)); @@ -6791,11 +6907,10 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1); llvm::Type *RealFlagType - = FlagOutPtr.first->getType()->getPointerElementType(); + = FlagOutPtr.getPointer()->getType()->getPointerElementType(); llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType); - llvm::StoreInst *FlagStore = Builder.CreateStore(FlagExt, FlagOutPtr.first); - FlagStore->setAlignment(FlagOutPtr.second); + Builder.CreateStore(FlagExt, FlagOutPtr); return Result; } case AMDGPU::BI__builtin_amdgpu_div_fmas: @@ -6846,7 +6961,7 @@ static Value *EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, SmallVector<Value *, 8> Args(NumArgs); for (unsigned I = 0; I < NumArgs; ++I) Args[I] = CGF.EmitScalarExpr(E->getArg(I)); - Value *CCPtr = CGF.EmitScalarExpr(E->getArg(NumArgs)); + Address CCPtr = CGF.EmitPointerWithAlignment(E->getArg(NumArgs)); Value *F = CGF.CGM.getIntrinsic(IntrinsicID); Value *Call = CGF.Builder.CreateCall(F, Args); Value *CC = CGF.Builder.CreateExtractValue(Call, 1); @@ -7115,23 +7230,29 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, case NVPTX::BI__nvvm_atom_max_gen_i: case NVPTX::BI__nvvm_atom_max_gen_l: case NVPTX::BI__nvvm_atom_max_gen_ll: + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E); + case NVPTX::BI__nvvm_atom_max_gen_ui: case NVPTX::BI__nvvm_atom_max_gen_ul: case NVPTX::BI__nvvm_atom_max_gen_ull: - return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E); + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMax, E); case NVPTX::BI__nvvm_atom_min_gen_i: case NVPTX::BI__nvvm_atom_min_gen_l: case NVPTX::BI__nvvm_atom_min_gen_ll: + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E); + case NVPTX::BI__nvvm_atom_min_gen_ui: case NVPTX::BI__nvvm_atom_min_gen_ul: case NVPTX::BI__nvvm_atom_min_gen_ull: - return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E); + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMin, E); case NVPTX::BI__nvvm_atom_cas_gen_i: case NVPTX::BI__nvvm_atom_cas_gen_l: case NVPTX::BI__nvvm_atom_cas_gen_ll: - return MakeAtomicCmpXchgValue(*this, E, true); + // __nvvm_atom_cas_gen_* should return the old value rather than the + // success flag. + return MakeAtomicCmpXchgValue(*this, E, /*ReturnBool=*/false); case NVPTX::BI__nvvm_atom_add_gen_f: { Value *Ptr = EmitScalarExpr(E->getArg(0)); @@ -7147,3 +7268,22 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, return nullptr; } } + +Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, + const CallExpr *E) { + switch (BuiltinID) { + case WebAssembly::BI__builtin_wasm_memory_size: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_size, ResultType); + return Builder.CreateCall(Callee); + } + case WebAssembly::BI__builtin_wasm_grow_memory: { + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_grow_memory, X->getType()); + return Builder.CreateCall(Callee, X); + } + + default: + return nullptr; + } +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp index 67d0ab7..045e19b 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp @@ -57,9 +57,9 @@ private: unsigned Alignment = 0) { llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0), llvm::ConstantInt::get(SizeTy, 0)}; - auto *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str()); - return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), - ConstStr, Zeros); + auto ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str()); + return llvm::ConstantExpr::getGetElementPtr(ConstStr.getElementType(), + ConstStr.getPointer(), Zeros); } void emitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args); @@ -121,7 +121,7 @@ void CGNVCUDARuntime::emitDeviceStubBody(CodeGenFunction &CGF, std::vector<llvm::Type *> ArgTypes; for (FunctionArgList::const_iterator I = Args.begin(), E = Args.end(); I != E; ++I) { - llvm::Value *V = CGF.GetAddrOfLocalVar(*I); + llvm::Value *V = CGF.GetAddrOfLocalVar(*I).getPointer(); ArgValues.push_back(V); assert(isa<llvm::PointerType>(V->getType()) && "Arg type not PointerType"); ArgTypes.push_back(cast<llvm::PointerType>(V->getType())->getElementType()); @@ -173,7 +173,7 @@ llvm::Function *CGNVCUDARuntime::makeRegisterKernelsFn() { llvm::GlobalValue::InternalLinkage, "__cuda_register_kernels", &TheModule); llvm::BasicBlock *EntryBB = llvm::BasicBlock::Create(Context, "entry", RegisterKernelsFunc); - CGBuilderTy Builder(Context); + CGBuilderTy Builder(CGM, Context); Builder.SetInsertPoint(EntryBB); // void __cudaRegisterFunction(void **, const char *, char *, const char *, @@ -230,7 +230,7 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { llvm::GlobalValue::InternalLinkage, "__cuda_module_ctor", &TheModule); llvm::BasicBlock *CtorEntryBB = llvm::BasicBlock::Create(Context, "entry", ModuleCtorFunc); - CGBuilderTy CtorBuilder(Context); + CGBuilderTy CtorBuilder(CGM, Context); CtorBuilder.SetInsertPoint(CtorEntryBB); @@ -267,7 +267,8 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { llvm::GlobalVariable *GpuBinaryHandle = new llvm::GlobalVariable( TheModule, VoidPtrPtrTy, false, llvm::GlobalValue::InternalLinkage, llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__cuda_gpubin_handle"); - CtorBuilder.CreateStore(RegisterFatbinCall, GpuBinaryHandle, false); + CtorBuilder.CreateAlignedStore(RegisterFatbinCall, GpuBinaryHandle, + CGM.getPointerAlign()); // Call __cuda_register_kernels(GpuBinaryHandle); CtorBuilder.CreateCall(RegisterKernelsFunc, RegisterFatbinCall); @@ -300,12 +301,13 @@ llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() { llvm::GlobalValue::InternalLinkage, "__cuda_module_dtor", &TheModule); llvm::BasicBlock *DtorEntryBB = llvm::BasicBlock::Create(Context, "entry", ModuleDtorFunc); - CGBuilderTy DtorBuilder(Context); + CGBuilderTy DtorBuilder(CGM, Context); DtorBuilder.SetInsertPoint(DtorEntryBB); for (llvm::GlobalVariable *GpuBinaryHandle : GpuBinaryHandles) { - DtorBuilder.CreateCall(UnregisterFatbinFunc, - DtorBuilder.CreateLoad(GpuBinaryHandle, false)); + auto HandleValue = + DtorBuilder.CreateAlignedLoad(GpuBinaryHandle, CGM.getPointerAlign()); + DtorBuilder.CreateCall(UnregisterFatbinFunc, HandleValue); } DtorBuilder.CreateRetVoid(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp index 7d7ed78..6847df9 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp @@ -28,6 +28,7 @@ using namespace clang; using namespace CodeGen; + /// Try to emit a base destructor as an alias to its primary /// base-class destructor. bool CodeGenModule::TryEmitBaseDestructorAsAlias(const CXXDestructorDecl *D) { @@ -39,6 +40,12 @@ bool CodeGenModule::TryEmitBaseDestructorAsAlias(const CXXDestructorDecl *D) { if (getCodeGenOpts().OptimizationLevel == 0) return true; + // If sanitizing memory to check for use-after-dtor, do not emit as + // an alias, unless this class owns no members. + if (getCodeGenOpts().SanitizeMemoryUseAfterDtor && + !D->getParent()->field_empty()) + return true; + // If the destructor doesn't have a trivial body, we have to emit it // separately. if (!D->hasTrivialBody()) @@ -124,11 +131,6 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl, if (!llvm::GlobalAlias::isValidLinkage(Linkage)) return true; - // Don't create a weak alias for a dllexport'd symbol. - if (AliasDecl.getDecl()->hasAttr<DLLExportAttr>() && - llvm::GlobalValue::isWeakForLinker(Linkage)) - return true; - llvm::GlobalValue::LinkageTypes TargetLinkage = getFunctionLinkage(TargetDecl); @@ -141,8 +143,8 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl, return false; // Derive the type for the alias. - llvm::PointerType *AliasType - = getTypes().GetFunctionType(AliasDecl)->getPointerTo(); + llvm::Type *AliasValueType = getTypes().GetFunctionType(AliasDecl); + llvm::PointerType *AliasType = AliasValueType->getPointerTo(); // Find the referent. Some aliases might require a bitcast, in // which case the caller is responsible for ensuring the soundness @@ -166,6 +168,16 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl, return false; } + // If we have a weak, non-discardable alias (weak, weak_odr), like an extern + // template instantiation or a dllexported class, avoid forming it on COFF. + // A COFF weak external alias cannot satisfy a normal undefined symbol + // reference from another TU. The other TU must also mark the referenced + // symbol as weak, which we cannot rely on. + if (llvm::GlobalValue::isWeakForLinker(Linkage) && + getTriple().isOSBinFormatCOFF()) { + return true; + } + if (!InEveryTU) { // If we don't have a definition for the destructor yet, don't // emit. We can't emit aliases to declarations; that's just not @@ -182,8 +194,8 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl, return true; // Create the alias with no name. - auto *Alias = - llvm::GlobalAlias::create(AliasType, Linkage, "", Aliasee, &getModule()); + auto *Alias = llvm::GlobalAlias::create(AliasValueType, 0, Linkage, "", + Aliasee, &getModule()); // Switch any previous uses to the alias. if (Entry) { @@ -207,7 +219,8 @@ llvm::Function *CodeGenModule::codegenCXXStructor(const CXXMethodDecl *MD, const CGFunctionInfo &FnInfo = getTypes().arrangeCXXStructorDeclaration(MD, Type); auto *Fn = cast<llvm::Function>( - getAddrOfCXXStructor(MD, Type, &FnInfo, nullptr, true)); + getAddrOfCXXStructor(MD, Type, &FnInfo, /*FnType=*/nullptr, + /*DontDefer=*/true, /*IsForDefinition=*/true)); GlobalDecl GD; if (const auto *DD = dyn_cast<CXXDestructorDecl>(MD)) { @@ -226,9 +239,9 @@ llvm::Function *CodeGenModule::codegenCXXStructor(const CXXMethodDecl *MD, return Fn; } -llvm::GlobalValue *CodeGenModule::getAddrOfCXXStructor( +llvm::Constant *CodeGenModule::getAddrOfCXXStructor( const CXXMethodDecl *MD, StructorType Type, const CGFunctionInfo *FnInfo, - llvm::FunctionType *FnType, bool DontDefer) { + llvm::FunctionType *FnType, bool DontDefer, bool IsForDefinition) { GlobalDecl GD; if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) { GD = GlobalDecl(CD, toCXXCtorType(Type)); @@ -236,19 +249,15 @@ llvm::GlobalValue *CodeGenModule::getAddrOfCXXStructor( GD = GlobalDecl(cast<CXXDestructorDecl>(MD), toCXXDtorType(Type)); } - StringRef Name = getMangledName(GD); - if (llvm::GlobalValue *Existing = GetGlobalValue(Name)) - return Existing; - if (!FnType) { if (!FnInfo) FnInfo = &getTypes().arrangeCXXStructorDeclaration(MD, Type); FnType = getTypes().GetFunctionType(*FnInfo); } - return cast<llvm::Function>(GetOrCreateLLVMFunction(Name, FnType, GD, - /*ForVTable=*/false, - DontDefer)); + return GetOrCreateLLVMFunction( + getMangledName(GD), FnType, GD, /*ForVTable=*/false, DontDefer, + /*isThunk=*/false, /*ExtraAttrs=*/llvm::AttributeSet(), IsForDefinition); } static llvm::Value *BuildAppleKextVirtualCall(CodeGenFunction &CGF, @@ -270,7 +279,7 @@ static llvm::Value *BuildAppleKextVirtualCall(CodeGenFunction &CGF, VTableIndex += AddressPoint; llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfnkxt"); - return CGF.Builder.CreateLoad(VFuncPtr); + return CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.PointerAlignInBytes); } /// BuildAppleKextVirtualCall - This routine is to support gcc's kext ABI making diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp index dc16616..e4da447 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "CGCXXABI.h" +#include "CGCleanup.h" using namespace clang; using namespace CodeGen; @@ -73,25 +74,28 @@ CGCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) { } llvm::Value *CGCXXABI::EmitLoadOfMemberFunctionPointer( - CodeGenFunction &CGF, const Expr *E, llvm::Value *&This, + CodeGenFunction &CGF, const Expr *E, Address This, + llvm::Value *&ThisPtrForCall, llvm::Value *MemPtr, const MemberPointerType *MPT) { ErrorUnsupportedABI(CGF, "calls through member pointers"); + ThisPtrForCall = This.getPointer(); const FunctionProtoType *FPT = MPT->getPointeeType()->getAs<FunctionProtoType>(); const CXXRecordDecl *RD = cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl()); llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType( - CGM.getTypes().arrangeCXXMethodType(RD, FPT)); + CGM.getTypes().arrangeCXXMethodType(RD, FPT, /*FD=*/nullptr)); return llvm::Constant::getNullValue(FTy->getPointerTo()); } llvm::Value * CGCXXABI::EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E, - llvm::Value *Base, llvm::Value *MemPtr, + Address Base, llvm::Value *MemPtr, const MemberPointerType *MPT) { ErrorUnsupportedABI(CGF, "loads of member pointers"); - llvm::Type *Ty = CGF.ConvertType(MPT->getPointeeType())->getPointerTo(); + llvm::Type *Ty = CGF.ConvertType(MPT->getPointeeType()) + ->getPointerTo(Base.getAddressSpace()); return llvm::Constant::getNullValue(Ty); } @@ -159,13 +163,24 @@ void CGCXXABI::buildThisParam(CodeGenFunction &CGF, FunctionArgList ¶ms) { &CGM.getContext().Idents.get("this"), MD->getThisType(CGM.getContext())); params.push_back(ThisDecl); - getThisDecl(CGF) = ThisDecl; + CGF.CXXABIThisDecl = ThisDecl; + + // Compute the presumed alignment of 'this', which basically comes + // down to whether we know it's a complete object or not. + auto &Layout = CGF.getContext().getASTRecordLayout(MD->getParent()); + if (MD->getParent()->getNumVBases() == 0 || // avoid vcall in common case + MD->getParent()->hasAttr<FinalAttr>() || + !isThisCompleteObject(CGF.CurGD)) { + CGF.CXXABIThisAlignment = Layout.getAlignment(); + } else { + CGF.CXXABIThisAlignment = Layout.getNonVirtualAlignment(); + } } void CGCXXABI::EmitThisParam(CodeGenFunction &CGF) { /// Initialize the 'this' slot. assert(getThisDecl(CGF) && "no 'this' variable for function"); - getThisValue(CGF) + CGF.CXXABIThisValue = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(getThisDecl(CGF)), "this"); } @@ -186,14 +201,14 @@ CharUnits CGCXXABI::getArrayCookieSizeImpl(QualType elementType) { return CharUnits::Zero(); } -llvm::Value *CGCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, - llvm::Value *NewPtr, - llvm::Value *NumElements, - const CXXNewExpr *expr, - QualType ElementType) { +Address CGCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, + Address NewPtr, + llvm::Value *NumElements, + const CXXNewExpr *expr, + QualType ElementType) { // Should never be called. ErrorUnsupportedABI(CGF, "array cookie initialization"); - return nullptr; + return Address::invalid(); } bool CGCXXABI::requiresArrayCookie(const CXXDeleteExpr *expr, @@ -215,31 +230,30 @@ bool CGCXXABI::requiresArrayCookie(const CXXNewExpr *expr) { return expr->getAllocatedType().isDestructedType(); } -void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *ptr, +void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, Address ptr, const CXXDeleteExpr *expr, QualType eltTy, llvm::Value *&numElements, llvm::Value *&allocPtr, CharUnits &cookieSize) { // Derive a char* in the same address space as the pointer. - unsigned AS = ptr->getType()->getPointerAddressSpace(); - llvm::Type *charPtrTy = CGF.Int8Ty->getPointerTo(AS); - ptr = CGF.Builder.CreateBitCast(ptr, charPtrTy); + ptr = CGF.Builder.CreateElementBitCast(ptr, CGF.Int8Ty); // If we don't need an array cookie, bail out early. if (!requiresArrayCookie(expr, eltTy)) { - allocPtr = ptr; + allocPtr = ptr.getPointer(); numElements = nullptr; cookieSize = CharUnits::Zero(); return; } cookieSize = getArrayCookieSizeImpl(eltTy); - allocPtr = CGF.Builder.CreateConstInBoundsGEP1_64(ptr, - -cookieSize.getQuantity()); - numElements = readArrayCookieImpl(CGF, allocPtr, cookieSize); + Address allocAddr = + CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize); + allocPtr = allocAddr.getPointer(); + numElements = readArrayCookieImpl(CGF, allocAddr, cookieSize); } llvm::Value *CGCXXABI::readArrayCookieImpl(CodeGenFunction &CGF, - llvm::Value *ptr, + Address ptr, CharUnits cookieSize) { ErrorUnsupportedABI(CGF, "reading a new[] cookie"); return llvm::ConstantInt::get(CGF.SizeTy, 0); @@ -308,3 +322,11 @@ CGCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF, // Just call std::terminate and ignore the violating exception. return CGF.EmitNounwindRuntimeCall(CGF.CGM.getTerminateFn()); } + +CatchTypeInfo CGCXXABI::getCatchAllTypeInfo() { + return CatchTypeInfo{nullptr, 0}; +} + +std::vector<CharUnits> CGCXXABI::getVBPtrOffsets(const CXXRecordDecl *RD) { + return std::vector<CharUnits>(); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h index 436b96a..3f240b1 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h @@ -37,6 +37,7 @@ class MangleContext; namespace CodeGen { class CodeGenFunction; class CodeGenModule; +struct CatchTypeInfo; /// \brief Implements C++ ABI-specific code generation functions. class CGCXXABI { @@ -48,12 +49,15 @@ protected: : CGM(CGM), MangleCtx(CGM.getContext().createMangleContext()) {} protected: - ImplicitParamDecl *&getThisDecl(CodeGenFunction &CGF) { + ImplicitParamDecl *getThisDecl(CodeGenFunction &CGF) { return CGF.CXXABIThisDecl; } - llvm::Value *&getThisValue(CodeGenFunction &CGF) { + llvm::Value *getThisValue(CodeGenFunction &CGF) { return CGF.CXXABIThisValue; } + Address getThisAddress(CodeGenFunction &CGF) { + return Address(CGF.CXXABIThisValue, CGF.CXXABIThisAlignment); + } /// Issue a diagnostic about unsupported features in the ABI. void ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S); @@ -77,6 +81,12 @@ protected: virtual bool requiresArrayCookie(const CXXDeleteExpr *E, QualType eltType); virtual bool requiresArrayCookie(const CXXNewExpr *E); + /// Determine whether there's something special about the rules of + /// the ABI tell us that 'this' is a complete object within the + /// given function. Obvious common logic like being defined on a + /// final class will have been taken care of by the caller. + virtual bool isThisCompleteObject(GlobalDecl GD) const = 0; + public: virtual ~CGCXXABI(); @@ -135,13 +145,14 @@ public: /// pointer. Apply the this-adjustment and set 'This' to the /// adjusted value. virtual llvm::Value *EmitLoadOfMemberFunctionPointer( - CodeGenFunction &CGF, const Expr *E, llvm::Value *&This, - llvm::Value *MemPtr, const MemberPointerType *MPT); + CodeGenFunction &CGF, const Expr *E, Address This, + llvm::Value *&ThisPtrForCall, llvm::Value *MemPtr, + const MemberPointerType *MPT); /// Calculate an l-value from an object and a data member pointer. virtual llvm::Value * EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E, - llvm::Value *Base, llvm::Value *MemPtr, + Address Base, llvm::Value *MemPtr, const MemberPointerType *MPT); /// Perform a derived-to-base, base-to-derived, or bitcast member @@ -164,10 +175,6 @@ public: return true; } - virtual bool isTypeInfoCalculable(QualType Ty) const { - return !Ty->isIncompleteType(); - } - /// Create a null member pointer of the given type. virtual llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT); @@ -212,12 +219,17 @@ protected: public: virtual void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE, - llvm::Value *Ptr, QualType ElementType, + Address Ptr, QualType ElementType, const CXXDestructorDecl *Dtor) = 0; virtual void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) = 0; virtual void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) = 0; virtual llvm::GlobalVariable *getThrowInfo(QualType T) { return nullptr; } + /// \brief Determine whether it's possible to emit a vtable for \p RD, even + /// though we do not know that the vtable has been marked as used by semantic + /// analysis. + virtual bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const = 0; + virtual void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) = 0; virtual llvm::CallInst * @@ -225,33 +237,34 @@ public: llvm::Value *Exn); virtual llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) = 0; - virtual llvm::Constant * + virtual CatchTypeInfo getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) = 0; + virtual CatchTypeInfo getCatchAllTypeInfo(); virtual bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) = 0; virtual void EmitBadTypeidCall(CodeGenFunction &CGF) = 0; virtual llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy, - llvm::Value *ThisPtr, + Address ThisPtr, llvm::Type *StdTypeInfoPtrTy) = 0; virtual bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr, QualType SrcRecordTy) = 0; virtual llvm::Value * - EmitDynamicCastCall(CodeGenFunction &CGF, llvm::Value *Value, + EmitDynamicCastCall(CodeGenFunction &CGF, Address Value, QualType SrcRecordTy, QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) = 0; virtual llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, - llvm::Value *Value, + Address Value, QualType SrcRecordTy, QualType DestTy) = 0; virtual bool EmitBadCastCall(CodeGenFunction &CGF) = 0; virtual llvm::Value *GetVirtualBaseClassOffset(CodeGenFunction &CGF, - llvm::Value *This, + Address This, const CXXRecordDecl *ClassDecl, const CXXRecordDecl *BaseClassDecl) = 0; @@ -294,10 +307,9 @@ public: /// Perform ABI-specific "this" argument adjustment required prior to /// a call of a virtual function. /// The "VirtualCall" argument is true iff the call itself is virtual. - virtual llvm::Value * + virtual Address adjustThisArgumentForVirtualFunctionCall(CodeGenFunction &CGF, GlobalDecl GD, - llvm::Value *This, - bool VirtualCall) { + Address This, bool VirtualCall) { return This; } @@ -337,19 +349,31 @@ public: virtual void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type, bool ForVirtualBase, bool Delegating, - llvm::Value *This) = 0; + Address This) = 0; /// Emits the VTable definitions required for the given record type. virtual void emitVTableDefinitions(CodeGenVTables &CGVT, const CXXRecordDecl *RD) = 0; + /// Checks if ABI requires extra virtual offset for vtable field. + virtual bool + isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF, + CodeGenFunction::VPtr Vptr) = 0; + + /// Checks if ABI requires to initilize vptrs for given dynamic class. + virtual bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) = 0; + + /// Get the address point of the vtable for the given base subobject. + virtual llvm::Constant * + getVTableAddressPoint(BaseSubobject Base, + const CXXRecordDecl *VTableClass) = 0; + /// Get the address point of the vtable for the given base subobject while - /// building a constructor or a destructor. On return, NeedsVirtualOffset - /// tells if a virtual base adjustment is needed in order to get the offset - /// of the base subobject. - virtual llvm::Value *getVTableAddressPointInStructor( - CodeGenFunction &CGF, const CXXRecordDecl *RD, BaseSubobject Base, - const CXXRecordDecl *NearestVBase, bool &NeedsVirtualOffset) = 0; + /// building a constructor or a destructor. + virtual llvm::Value * + getVTableAddressPointInStructor(CodeGenFunction &CGF, const CXXRecordDecl *RD, + BaseSubobject Base, + const CXXRecordDecl *NearestVBase) = 0; /// Get the address point of the vtable for the given base subobject while /// building a constexpr. @@ -365,14 +389,14 @@ public: /// Build a virtual function pointer in the ABI-specific way. virtual llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, - llvm::Value *This, + Address This, llvm::Type *Ty, SourceLocation Loc) = 0; /// Emit the ABI-specific virtual destructor call. virtual llvm::Value * EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, - CXXDtorType DtorType, llvm::Value *This, + CXXDtorType DtorType, Address This, const CXXMemberCallExpr *CE) = 0; virtual void adjustCallArgsForDestructorThunk(CodeGenFunction &CGF, @@ -388,11 +412,11 @@ public: GlobalDecl GD, bool ReturnAdjustment) = 0; virtual llvm::Value *performThisAdjustment(CodeGenFunction &CGF, - llvm::Value *This, + Address This, const ThisAdjustment &TA) = 0; virtual llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, - llvm::Value *Ret, + Address Ret, const ReturnAdjustment &RA) = 0; virtual void EmitReturnFromThunk(CodeGenFunction &CGF, @@ -401,6 +425,9 @@ public: virtual size_t getSrcArgforCopyCtor(const CXXConstructorDecl *, FunctionArgList &Args) const = 0; + /// Gets the offsets of all the virtual base pointers in a given class. + virtual std::vector<CharUnits> getVBPtrOffsets(const CXXRecordDecl *RD); + /// Gets the pure virtual member call function. virtual StringRef GetPureVirtualCallName() = 0; @@ -429,11 +456,11 @@ public: /// always a size_t /// \param ElementType - the base element allocated type, /// i.e. the allocated type after stripping all array types - virtual llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF, - llvm::Value *NewPtr, - llvm::Value *NumElements, - const CXXNewExpr *expr, - QualType ElementType); + virtual Address InitializeArrayCookie(CodeGenFunction &CGF, + Address NewPtr, + llvm::Value *NumElements, + const CXXNewExpr *expr, + QualType ElementType); /// Reads the array cookie associated with the given pointer, /// if it has one. @@ -448,7 +475,7 @@ public: /// function /// \param CookieSize - an out parameter which will be initialized /// with the size of the cookie, or zero if there is no cookie - virtual void ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *Ptr, + virtual void ReadArrayCookie(CodeGenFunction &CGF, Address Ptr, const CXXDeleteExpr *expr, QualType ElementType, llvm::Value *&NumElements, llvm::Value *&AllocPtr, CharUnits &CookieSize); @@ -471,8 +498,7 @@ protected: /// Other parameters are as above. /// /// \return a size_t - virtual llvm::Value *readArrayCookieImpl(CodeGenFunction &IGF, - llvm::Value *ptr, + virtual llvm::Value *readArrayCookieImpl(CodeGenFunction &IGF, Address ptr, CharUnits cookieSize); public: @@ -512,11 +538,9 @@ public: /// thread_local variables, a list of functions to perform the /// initialization. virtual void EmitThreadLocalInitFuncs( - CodeGenModule &CGM, - ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>> - CXXThreadLocals, + CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals, ArrayRef<llvm::Function *> CXXThreadLocalInits, - ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) = 0; + ArrayRef<const VarDecl *> CXXThreadLocalInitVars) = 0; // Determine if references to thread_local global variables can be made // directly or require access through a thread wrapper function. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp index 0bcf59b..49b5df0 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp @@ -15,12 +15,14 @@ #include "CGCall.h" #include "ABIInfo.h" #include "CGCXXABI.h" +#include "CGCleanup.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" #include "TargetInfo.h" #include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclObjC.h" +#include "clang/Basic/TargetBuiltins.h" #include "clang/Basic/TargetInfo.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/CodeGenOptions.h" @@ -91,15 +93,41 @@ CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) { FTNP->getExtInfo(), RequiredArgs(0)); } +/// Adds the formal paramaters in FPT to the given prefix. If any parameter in +/// FPT has pass_object_size attrs, then we'll add parameters for those, too. +static void appendParameterTypes(const CodeGenTypes &CGT, + SmallVectorImpl<CanQualType> &prefix, + const CanQual<FunctionProtoType> &FPT, + const FunctionDecl *FD) { + // Fast path: unknown target. + if (FD == nullptr) { + prefix.append(FPT->param_type_begin(), FPT->param_type_end()); + return; + } + + // In the vast majority cases, we'll have precisely FPT->getNumParams() + // parameters; the only thing that can change this is the presence of + // pass_object_size. So, we preallocate for the common case. + prefix.reserve(prefix.size() + FPT->getNumParams()); + + assert(FD->getNumParams() == FPT->getNumParams()); + for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) { + prefix.push_back(FPT->getParamType(I)); + if (FD->getParamDecl(I)->hasAttr<PassObjectSizeAttr>()) + prefix.push_back(CGT.getContext().getSizeType()); + } +} + /// Arrange the LLVM function layout for a value of the given function /// type, on top of any implicit parameters already stored. static const CGFunctionInfo & arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod, SmallVectorImpl<CanQualType> &prefix, - CanQual<FunctionProtoType> FTP) { + CanQual<FunctionProtoType> FTP, + const FunctionDecl *FD) { RequiredArgs required = RequiredArgs::forPrototypePlus(FTP, prefix.size()); // FIXME: Kill copy. - prefix.append(FTP->param_type_begin(), FTP->param_type_end()); + appendParameterTypes(CGT, prefix, FTP, FD); CanQualType resultType = FTP->getReturnType().getUnqualifiedType(); return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod, /*chainCall=*/false, prefix, @@ -109,10 +137,11 @@ arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod, /// Arrange the argument and result information for a value of the /// given freestanding function type. const CGFunctionInfo & -CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) { +CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP, + const FunctionDecl *FD) { SmallVector<CanQualType, 16> argTypes; return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes, - FTP); + FTP, FD); } static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) { @@ -155,7 +184,8 @@ static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) { /// constructor or destructor. const CGFunctionInfo & CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD, - const FunctionProtoType *FTP) { + const FunctionProtoType *FTP, + const CXXMethodDecl *MD) { SmallVector<CanQualType, 16> argTypes; // Add the 'this' pointer. @@ -166,7 +196,7 @@ CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD, return ::arrangeLLVMFunctionInfo( *this, true, argTypes, - FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>()); + FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>(), MD); } /// Arrange the argument and result information for a declaration or @@ -183,10 +213,10 @@ CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) { if (MD->isInstance()) { // The abstract case is perfectly fine. const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD); - return arrangeCXXMethodType(ThisType, prototype.getTypePtr()); + return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD); } - return arrangeFreeFunctionType(prototype); + return arrangeFreeFunctionType(prototype, MD); } const CGFunctionInfo & @@ -207,7 +237,7 @@ CodeGenTypes::arrangeCXXStructorDeclaration(const CXXMethodDecl *MD, CanQual<FunctionProtoType> FTP = GetFormalType(MD); // Add the formal parameters. - argTypes.append(FTP->param_type_begin(), FTP->param_type_end()); + appendParameterTypes(*this, argTypes, FTP, MD); TheCXXABI.buildStructorSignature(MD, Type, argTypes); @@ -273,7 +303,7 @@ CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) { } assert(isa<FunctionProtoType>(FTy)); - return arrangeFreeFunctionType(FTy.getAs<FunctionProtoType>()); + return arrangeFreeFunctionType(FTy.getAs<FunctionProtoType>(), FD); } /// Arrange the argument and result information for the declaration or @@ -553,6 +583,7 @@ CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, FI->HasRegParm = info.getHasRegParm(); FI->RegParm = info.getRegParm(); FI->ArgStruct = nullptr; + FI->ArgStructAlign = 0; FI->NumArgs = argTypes.size(); FI->getArgsBuffer()[0].type = resultType; for (unsigned i = 0, e = argTypes.size(); i != e; ++i) @@ -726,6 +757,21 @@ CodeGenTypes::getExpandedTypes(QualType Ty, } } +static void forConstantArrayExpansion(CodeGenFunction &CGF, + ConstantArrayExpansion *CAE, + Address BaseAddr, + llvm::function_ref<void(Address)> Fn) { + CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy); + CharUnits EltAlign = + BaseAddr.getAlignment().alignmentOfArrayElement(EltSize); + + for (int i = 0, n = CAE->NumElts; i < n; i++) { + llvm::Value *EltAddr = + CGF.Builder.CreateConstGEP2_32(nullptr, BaseAddr.getPointer(), 0, i); + Fn(Address(EltAddr, EltAlign)); + } +} + void CodeGenFunction::ExpandTypeFromArgs( QualType Ty, LValue LV, SmallVectorImpl<llvm::Argument *>::iterator &AI) { assert(LV.isSimple() && @@ -733,17 +779,16 @@ void CodeGenFunction::ExpandTypeFromArgs( auto Exp = getTypeExpansion(Ty, getContext()); if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { - for (int i = 0, n = CAExp->NumElts; i < n; i++) { - llvm::Value *EltAddr = - Builder.CreateConstGEP2_32(nullptr, LV.getAddress(), 0, i); + forConstantArrayExpansion(*this, CAExp, LV.getAddress(), + [&](Address EltAddr) { LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy); ExpandTypeFromArgs(CAExp->EltTy, LV, AI); - } + }); } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { - llvm::Value *This = LV.getAddress(); + Address This = LV.getAddress(); for (const CXXBaseSpecifier *BS : RExp->Bases) { // Perform a single step derived-to-base conversion. - llvm::Value *Base = + Address Base = GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1, /*NullCheckValue=*/false, SourceLocation()); LValue SubLV = MakeAddrLValue(Base, BS->getType()); @@ -756,15 +801,10 @@ void CodeGenFunction::ExpandTypeFromArgs( LValue SubLV = EmitLValueForField(LV, FD); ExpandTypeFromArgs(FD->getType(), SubLV, AI); } - } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) { - llvm::Value *RealAddr = - Builder.CreateStructGEP(nullptr, LV.getAddress(), 0, "real"); - EmitStoreThroughLValue(RValue::get(*AI++), - MakeAddrLValue(RealAddr, CExp->EltTy)); - llvm::Value *ImagAddr = - Builder.CreateStructGEP(nullptr, LV.getAddress(), 1, "imag"); - EmitStoreThroughLValue(RValue::get(*AI++), - MakeAddrLValue(ImagAddr, CExp->EltTy)); + } else if (isa<ComplexExpansion>(Exp.get())) { + auto realValue = *AI++; + auto imagValue = *AI++; + EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true); } else { assert(isa<NoExpansion>(Exp.get())); EmitStoreThroughLValue(RValue::get(*AI++), LV); @@ -776,18 +816,17 @@ void CodeGenFunction::ExpandTypeToArgs( SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) { auto Exp = getTypeExpansion(Ty, getContext()); if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { - llvm::Value *Addr = RV.getAggregateAddr(); - for (int i = 0, n = CAExp->NumElts; i < n; i++) { - llvm::Value *EltAddr = Builder.CreateConstGEP2_32(nullptr, Addr, 0, i); + forConstantArrayExpansion(*this, CAExp, RV.getAggregateAddress(), + [&](Address EltAddr) { RValue EltRV = convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()); ExpandTypeToArgs(CAExp->EltTy, EltRV, IRFuncTy, IRCallArgs, IRCallArgPos); - } + }); } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { - llvm::Value *This = RV.getAggregateAddr(); + Address This = RV.getAggregateAddress(); for (const CXXBaseSpecifier *BS : RExp->Bases) { // Perform a single step derived-to-base conversion. - llvm::Value *Base = + Address Base = GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1, /*NullCheckValue=*/false, SourceLocation()); RValue BaseRV = RValue::getAggregate(Base); @@ -822,12 +861,22 @@ void CodeGenFunction::ExpandTypeToArgs( } } +/// Create a temporary allocation for the purposes of coercion. +static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty, + CharUnits MinAlign) { + // Don't use an alignment that's worse than what LLVM would prefer. + auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty); + CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign)); + + return CGF.CreateTempAlloca(Ty, Align); +} + /// EnterStructPointerForCoercedAccess - Given a struct pointer that we are /// accessing some number of bytes out of it, try to gep into the struct to get /// at its inner goodness. Dive as deep as possible without entering an element /// with an in-memory size smaller than DstSize. -static llvm::Value * -EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr, +static Address +EnterStructPointerForCoercedAccess(Address SrcPtr, llvm::StructType *SrcSTy, uint64_t DstSize, CodeGenFunction &CGF) { // We can't dive into a zero-element struct. @@ -846,11 +895,10 @@ EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr, return SrcPtr; // GEP into the first element. - SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcSTy, SrcPtr, 0, 0, "coerce.dive"); + SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, CharUnits(), "coerce.dive"); // If the first element is a struct, recurse. - llvm::Type *SrcTy = - cast<llvm::PointerType>(SrcPtr->getType())->getElementType(); + llvm::Type *SrcTy = SrcPtr.getElementType(); if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF); @@ -918,21 +966,19 @@ static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, /// This safely handles the case when the src type is smaller than the /// destination type; in this situation the values of bits which not /// present in the src are undefined. -static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr, - llvm::Type *Ty, CharUnits SrcAlign, +static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, CodeGenFunction &CGF) { - llvm::Type *SrcTy = - cast<llvm::PointerType>(SrcPtr->getType())->getElementType(); + llvm::Type *SrcTy = Src.getElementType(); // If SrcTy and Ty are the same, just do a load. if (SrcTy == Ty) - return CGF.Builder.CreateAlignedLoad(SrcPtr, SrcAlign.getQuantity()); + return CGF.Builder.CreateLoad(Src); uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty); if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) { - SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF); - SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType(); + Src = EnterStructPointerForCoercedAccess(Src, SrcSTy, DstSize, CGF); + SrcTy = Src.getType()->getElementType(); } uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy); @@ -941,8 +987,7 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr, // extension or truncation to the desired type. if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) && (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) { - llvm::LoadInst *Load = - CGF.Builder.CreateAlignedLoad(SrcPtr, SrcAlign.getQuantity()); + llvm::Value *Load = CGF.Builder.CreateLoad(Src); return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF); } @@ -954,22 +999,18 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr, // // FIXME: Assert that we aren't truncating non-padding bits when have access // to that information. - llvm::Value *Casted = - CGF.Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(Ty)); - return CGF.Builder.CreateAlignedLoad(Casted, SrcAlign.getQuantity()); - } - - // Otherwise do coercion through memory. This is stupid, but - // simple. - llvm::AllocaInst *Tmp = CGF.CreateTempAlloca(Ty); - Tmp->setAlignment(SrcAlign.getQuantity()); - llvm::Type *I8PtrTy = CGF.Builder.getInt8PtrTy(); - llvm::Value *Casted = CGF.Builder.CreateBitCast(Tmp, I8PtrTy); - llvm::Value *SrcCasted = CGF.Builder.CreateBitCast(SrcPtr, I8PtrTy); + Src = CGF.Builder.CreateBitCast(Src, llvm::PointerType::getUnqual(Ty)); + return CGF.Builder.CreateLoad(Src); + } + + // Otherwise do coercion through memory. This is stupid, but simple. + Address Tmp = CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment()); + Address Casted = CGF.Builder.CreateBitCast(Tmp, CGF.Int8PtrTy); + Address SrcCasted = CGF.Builder.CreateBitCast(Src, CGF.Int8PtrTy); CGF.Builder.CreateMemCpy(Casted, SrcCasted, llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize), - SrcAlign.getQuantity(), false); - return CGF.Builder.CreateAlignedLoad(Tmp, SrcAlign.getQuantity()); + false); + return CGF.Builder.CreateLoad(Tmp); } // Function to store a first-class aggregate into memory. We prefer to @@ -977,8 +1018,7 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr, // fast-isel. // FIXME: Do we need to recurse here? static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val, - llvm::Value *DestPtr, bool DestIsVolatile, - CharUnits DestAlign) { + Address Dest, bool DestIsVolatile) { // Prefer scalar stores to first-class aggregate stores. if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) { @@ -986,17 +1026,13 @@ static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val, CGF.CGM.getDataLayout().getStructLayout(STy); for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { - llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(STy, DestPtr, 0, i); + auto EltOffset = CharUnits::fromQuantity(Layout->getElementOffset(i)); + Address EltPtr = CGF.Builder.CreateStructGEP(Dest, i, EltOffset); llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i); - uint64_t EltOffset = Layout->getElementOffset(i); - CharUnits EltAlign = - DestAlign.alignmentAtOffset(CharUnits::fromQuantity(EltOffset)); - CGF.Builder.CreateAlignedStore(Elt, EltPtr, EltAlign.getQuantity(), - DestIsVolatile); + CGF.Builder.CreateStore(Elt, EltPtr, DestIsVolatile); } } else { - CGF.Builder.CreateAlignedStore(Val, DestPtr, DestAlign.getQuantity(), - DestIsVolatile); + CGF.Builder.CreateStore(Val, Dest, DestIsVolatile); } } @@ -1007,24 +1043,21 @@ static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val, /// This safely handles the case when the src type is larger than the /// destination type; the upper bits of the src will be lost. static void CreateCoercedStore(llvm::Value *Src, - llvm::Value *DstPtr, + Address Dst, bool DstIsVolatile, - CharUnits DstAlign, CodeGenFunction &CGF) { llvm::Type *SrcTy = Src->getType(); - llvm::Type *DstTy = - cast<llvm::PointerType>(DstPtr->getType())->getElementType(); + llvm::Type *DstTy = Dst.getType()->getElementType(); if (SrcTy == DstTy) { - CGF.Builder.CreateAlignedStore(Src, DstPtr, DstAlign.getQuantity(), - DstIsVolatile); + CGF.Builder.CreateStore(Src, Dst, DstIsVolatile); return; } uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy); if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) { - DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF); - DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType(); + Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy, SrcSize, CGF); + DstTy = Dst.getType()->getElementType(); } // If the source and destination are integer or pointer types, just do an @@ -1032,8 +1065,7 @@ static void CreateCoercedStore(llvm::Value *Src, if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) && (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) { Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF); - CGF.Builder.CreateAlignedStore(Src, DstPtr, DstAlign.getQuantity(), - DstIsVolatile); + CGF.Builder.CreateStore(Src, Dst, DstIsVolatile); return; } @@ -1041,9 +1073,8 @@ static void CreateCoercedStore(llvm::Value *Src, // If store is legal, just bitcast the src pointer. if (SrcSize <= DstSize) { - llvm::Value *Casted = - CGF.Builder.CreateBitCast(DstPtr, llvm::PointerType::getUnqual(SrcTy)); - BuildAggStore(CGF, Src, Casted, DstIsVolatile, DstAlign); + Dst = CGF.Builder.CreateBitCast(Dst, llvm::PointerType::getUnqual(SrcTy)); + BuildAggStore(CGF, Src, Dst, DstIsVolatile); } else { // Otherwise do coercion through memory. This is stupid, but // simple. @@ -1054,16 +1085,25 @@ static void CreateCoercedStore(llvm::Value *Src, // // FIXME: Assert that we aren't truncating non-padding bits when have access // to that information. - llvm::AllocaInst *Tmp = CGF.CreateTempAlloca(SrcTy); - Tmp->setAlignment(DstAlign.getQuantity()); - CGF.Builder.CreateAlignedStore(Src, Tmp, DstAlign.getQuantity()); - llvm::Type *I8PtrTy = CGF.Builder.getInt8PtrTy(); - llvm::Value *Casted = CGF.Builder.CreateBitCast(Tmp, I8PtrTy); - llvm::Value *DstCasted = CGF.Builder.CreateBitCast(DstPtr, I8PtrTy); + Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment()); + CGF.Builder.CreateStore(Src, Tmp); + Address Casted = CGF.Builder.CreateBitCast(Tmp, CGF.Int8PtrTy); + Address DstCasted = CGF.Builder.CreateBitCast(Dst, CGF.Int8PtrTy); CGF.Builder.CreateMemCpy(DstCasted, Casted, llvm::ConstantInt::get(CGF.IntPtrTy, DstSize), - DstAlign.getQuantity(), false); + false); + } +} + +static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr, + const ABIArgInfo &info) { + if (unsigned offset = info.getDirectOffset()) { + addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty); + addr = CGF.Builder.CreateConstInBoundsByteGEP(addr, + CharUnits::fromQuantity(offset)); + addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType()); } + return addr; } namespace { @@ -1380,8 +1420,19 @@ llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) { return GetFunctionType(*Info); } +static void AddAttributesFromFunctionProtoType(ASTContext &Ctx, + llvm::AttrBuilder &FuncAttrs, + const FunctionProtoType *FPT) { + if (!FPT) + return; + + if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) && + FPT->isNothrow(Ctx)) + FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); +} + void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, - const Decl *TargetDecl, + CGCalleeInfo CalleeInfo, AttributeListType &PAL, unsigned &CallingConv, bool AttrOnCallSite) { @@ -1394,6 +1445,13 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, if (FI.isNoReturn()) FuncAttrs.addAttribute(llvm::Attribute::NoReturn); + // If we have information about the function prototype, we can learn + // attributes form there. + AddAttributesFromFunctionProtoType(getContext(), FuncAttrs, + CalleeInfo.getCalleeFunctionProtoType()); + + const Decl *TargetDecl = CalleeInfo.getCalleeDecl(); + // FIXME: handle sseregparm someday... if (TargetDecl) { if (TargetDecl->hasAttr<ReturnsTwiceAttr>()) @@ -1406,9 +1464,8 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate); if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) { - const FunctionProtoType *FPT = Fn->getType()->getAs<FunctionProtoType>(); - if (FPT && FPT->isNothrow(getContext())) - FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); + AddAttributesFromFunctionProtoType( + getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>()); // Don't use [[noreturn]] or _Noreturn for a call to a virtual function. // These attributes are not inherited by overloads. const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn); @@ -1416,13 +1473,16 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, FuncAttrs.addAttribute(llvm::Attribute::NoReturn); } - // 'const' and 'pure' attribute functions are also nounwind. + // 'const', 'pure' and 'noalias' attributed functions are also nounwind. if (TargetDecl->hasAttr<ConstAttr>()) { FuncAttrs.addAttribute(llvm::Attribute::ReadNone); FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); } else if (TargetDecl->hasAttr<PureAttr>()) { FuncAttrs.addAttribute(llvm::Attribute::ReadOnly); FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); + } else if (TargetDecl->hasAttr<NoAliasAttr>()) { + FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly); + FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); } if (TargetDecl->hasAttr<RestrictAttr>()) RetAttrs.addAttribute(llvm::Attribute::NoAlias); @@ -1466,8 +1526,12 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, FuncAttrs.addAttribute("no-frame-pointer-elim-non-leaf"); } + bool DisableTailCalls = + CodeGenOpts.DisableTailCalls || + (TargetDecl && TargetDecl->hasAttr<DisableTailCallsAttr>()); FuncAttrs.addAttribute("disable-tail-calls", - llvm::toStringRef(CodeGenOpts.DisableTailCalls)); + llvm::toStringRef(DisableTailCalls)); + FuncAttrs.addAttribute("less-precise-fpmad", llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD)); FuncAttrs.addAttribute("no-infs-fp-math", @@ -1481,77 +1545,53 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, FuncAttrs.addAttribute("stack-protector-buffer-size", llvm::utostr(CodeGenOpts.SSPBufferSize)); - if (!CodeGenOpts.StackRealignment) - FuncAttrs.addAttribute("no-realign-stack"); + if (CodeGenOpts.StackRealignment) + FuncAttrs.addAttribute("stackrealign"); // Add target-cpu and target-features attributes to functions. If // we have a decl for the function and it has a target attribute then // parse that and add it to the feature set. StringRef TargetCPU = getTarget().getTargetOpts().CPU; - - // TODO: Features gets us the features on the command line including - // feature dependencies. For canonicalization purposes we might want to - // avoid putting features in the target-features set if we know it'll be - // one of the default features in the backend, e.g. corei7-avx and +avx or - // figure out non-explicit dependencies. - // Canonicalize the existing features in a new feature map. - // TODO: Migrate the existing backends to keep the map around rather than - // the vector. - llvm::StringMap<bool> FeatureMap; - for (auto F : getTarget().getTargetOpts().Features) { - const char *Name = F.c_str(); - bool Enabled = Name[0] == '+'; - getTarget().setFeatureEnabled(FeatureMap, Name + 1, Enabled); - } - const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl); - if (FD) { - if (const auto *TD = FD->getAttr<TargetAttr>()) { - StringRef FeaturesStr = TD->getFeatures(); - SmallVector<StringRef, 1> AttrFeatures; - FeaturesStr.split(AttrFeatures, ","); - - // Grab the various features and prepend a "+" to turn on the feature to - // the backend and add them to our existing set of features. - for (auto &Feature : AttrFeatures) { - // Go ahead and trim whitespace rather than either erroring or - // accepting it weirdly. - Feature = Feature.trim(); - - // While we're here iterating check for a different target cpu. - if (Feature.startswith("arch=")) - TargetCPU = Feature.split("=").second.trim(); - else if (Feature.startswith("tune=")) - // We don't support cpu tuning this way currently. - ; - else if (Feature.startswith("fpmath=")) - // TODO: Support the fpmath option this way. It will require checking - // overall feature validity for the function with the rest of the - // attributes on the function. - ; - else if (Feature.startswith("mno-")) - getTarget().setFeatureEnabled(FeatureMap, Feature.split("-").second, - false); - else - getTarget().setFeatureEnabled(FeatureMap, Feature, true); - } + if (FD && FD->hasAttr<TargetAttr>()) { + llvm::StringMap<bool> FeatureMap; + getFunctionFeatureMap(FeatureMap, FD); + + // Produce the canonical string for this set of features. + std::vector<std::string> Features; + for (llvm::StringMap<bool>::const_iterator it = FeatureMap.begin(), + ie = FeatureMap.end(); + it != ie; ++it) + Features.push_back((it->second ? "+" : "-") + it->first().str()); + + // Now add the target-cpu and target-features to the function. + // While we populated the feature map above, we still need to + // get and parse the target attribute so we can get the cpu for + // the function. + const auto *TD = FD->getAttr<TargetAttr>(); + TargetAttr::ParsedTargetAttr ParsedAttr = TD->parse(); + if (ParsedAttr.second != "") + TargetCPU = ParsedAttr.second; + if (TargetCPU != "") + FuncAttrs.addAttribute("target-cpu", TargetCPU); + if (!Features.empty()) { + std::sort(Features.begin(), Features.end()); + FuncAttrs.addAttribute( + "target-features", + llvm::join(Features.begin(), Features.end(), ",")); + } + } else { + // Otherwise just add the existing target cpu and target features to the + // function. + std::vector<std::string> &Features = getTarget().getTargetOpts().Features; + if (TargetCPU != "") + FuncAttrs.addAttribute("target-cpu", TargetCPU); + if (!Features.empty()) { + std::sort(Features.begin(), Features.end()); + FuncAttrs.addAttribute( + "target-features", + llvm::join(Features.begin(), Features.end(), ",")); } - } - - // Produce the canonical string for this set of features. - std::vector<std::string> Features; - for (llvm::StringMap<bool>::const_iterator it = FeatureMap.begin(), - ie = FeatureMap.end(); - it != ie; ++it) - Features.push_back((it->second ? "+" : "-") + it->first().str()); - - // Now add the target-cpu and target-features to the function. - if (TargetCPU != "") - FuncAttrs.addAttribute("target-cpu", TargetCPU); - if (!Features.empty()) { - std::sort(Features.begin(), Features.end()); - FuncAttrs.addAttribute("target-features", - llvm::join(Features.begin(), Features.end(), ",")); } } @@ -1655,20 +1695,37 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, Attrs.addAttribute(llvm::Attribute::InReg); break; - case ABIArgInfo::Indirect: + case ABIArgInfo::Indirect: { if (AI.getInReg()) Attrs.addAttribute(llvm::Attribute::InReg); if (AI.getIndirectByVal()) Attrs.addAttribute(llvm::Attribute::ByVal); - Attrs.addAlignmentAttr(AI.getIndirectAlign()); + CharUnits Align = AI.getIndirectAlign(); + + // In a byval argument, it is important that the required + // alignment of the type is honored, as LLVM might be creating a + // *new* stack object, and needs to know what alignment to give + // it. (Sometimes it can deduce a sensible alignment on its own, + // but not if clang decides it must emit a packed struct, or the + // user specifies increased alignment requirements.) + // + // This is different from indirect *not* byval, where the object + // exists already, and the align attribute is purely + // informative. + assert(!Align.isZero()); + + // For now, only add this when we have a byval argument. + // TODO: be less lazy about updating test cases. + if (AI.getIndirectByVal()) + Attrs.addAlignmentAttr(Align.getQuantity()); // byval disables readnone and readonly. FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly) .removeAttribute(llvm::Attribute::ReadNone); break; - + } case ABIArgInfo::Ignore: case ABIArgInfo::Expand: continue; @@ -1788,10 +1845,14 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, // If we're using inalloca, all the memory arguments are GEPs off of the last // parameter, which is a pointer to the complete memory area. - llvm::Value *ArgStruct = nullptr; + Address ArgStruct = Address::invalid(); + const llvm::StructLayout *ArgStructLayout = nullptr; if (IRFunctionArgs.hasInallocaArg()) { - ArgStruct = FnArgs[IRFunctionArgs.getInallocaArgNo()]; - assert(ArgStruct->getType() == FI.getArgStruct()->getPointerTo()); + ArgStructLayout = CGM.getDataLayout().getStructLayout(FI.getArgStruct()); + ArgStruct = Address(FnArgs[IRFunctionArgs.getInallocaArgNo()], + FI.getArgStructAlignment()); + + assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo()); } // Name the struct return parameter. @@ -1805,9 +1866,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, // Track if we received the parameter as a pointer (indirect, byval, or // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it // into a local alloca for us. - enum ValOrPointer { HaveValue = 0, HavePointer = 1 }; - typedef llvm::PointerIntPair<llvm::Value *, 1> ValueAndIsPtr; - SmallVector<ValueAndIsPtr, 16> ArgVals; + SmallVector<ParamValue, 16> ArgVals; ArgVals.reserve(Args.size()); // Create a pointer value for every parameter declaration. This usually @@ -1833,49 +1892,47 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, switch (ArgI.getKind()) { case ABIArgInfo::InAlloca: { assert(NumIRArgs == 0); - llvm::Value *V = - Builder.CreateStructGEP(FI.getArgStruct(), ArgStruct, - ArgI.getInAllocaFieldIndex(), Arg->getName()); - ArgVals.push_back(ValueAndIsPtr(V, HavePointer)); + auto FieldIndex = ArgI.getInAllocaFieldIndex(); + CharUnits FieldOffset = + CharUnits::fromQuantity(ArgStructLayout->getElementOffset(FieldIndex)); + Address V = Builder.CreateStructGEP(ArgStruct, FieldIndex, FieldOffset, + Arg->getName()); + ArgVals.push_back(ParamValue::forIndirect(V)); break; } case ABIArgInfo::Indirect: { assert(NumIRArgs == 1); - llvm::Value *V = FnArgs[FirstIRArg]; + Address ParamAddr = Address(FnArgs[FirstIRArg], ArgI.getIndirectAlign()); if (!hasScalarEvaluationKind(Ty)) { // Aggregates and complex variables are accessed by reference. All we - // need to do is realign the value, if requested + // need to do is realign the value, if requested. + Address V = ParamAddr; if (ArgI.getIndirectRealign()) { - llvm::Value *AlignedTemp = CreateMemTemp(Ty, "coerce"); + Address AlignedTemp = CreateMemTemp(Ty, "coerce"); // Copy from the incoming argument pointer to the temporary with the // appropriate alignment. // // FIXME: We should have a common utility for generating an aggregate // copy. - llvm::Type *I8PtrTy = Builder.getInt8PtrTy(); CharUnits Size = getContext().getTypeSizeInChars(Ty); - llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy); - llvm::Value *Src = Builder.CreateBitCast(V, I8PtrTy); - Builder.CreateMemCpy(Dst, - Src, - llvm::ConstantInt::get(IntPtrTy, - Size.getQuantity()), - ArgI.getIndirectAlign(), - false); + auto SizeVal = llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()); + Address Dst = Builder.CreateBitCast(AlignedTemp, Int8PtrTy); + Address Src = Builder.CreateBitCast(ParamAddr, Int8PtrTy); + Builder.CreateMemCpy(Dst, Src, SizeVal, false); V = AlignedTemp; } - ArgVals.push_back(ValueAndIsPtr(V, HavePointer)); + ArgVals.push_back(ParamValue::forIndirect(V)); } else { // Load scalar value from indirect argument. - V = EmitLoadOfScalar(V, false, ArgI.getIndirectAlign(), Ty, - Arg->getLocStart()); + llvm::Value *V = + EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getLocStart()); if (isPromoted) V = emitArgumentDemotion(*this, Arg, V); - ArgVals.push_back(ValueAndIsPtr(V, HaveValue)); + ArgVals.push_back(ParamValue::forDirect(V)); } break; } @@ -1980,87 +2037,66 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, if (V->getType() != LTy) V = Builder.CreateBitCast(V, LTy); - ArgVals.push_back(ValueAndIsPtr(V, HaveValue)); + ArgVals.push_back(ParamValue::forDirect(V)); break; } - llvm::AllocaInst *Alloca = CreateMemTemp(Ty, Arg->getName()); - - // The alignment we need to use is the max of the requested alignment for - // the argument plus the alignment required by our access code below. - unsigned AlignmentToUse = - CGM.getDataLayout().getABITypeAlignment(ArgI.getCoerceToType()); - AlignmentToUse = std::max(AlignmentToUse, - (unsigned)getContext().getDeclAlign(Arg).getQuantity()); + Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg), + Arg->getName()); - Alloca->setAlignment(AlignmentToUse); - llvm::Value *V = Alloca; - llvm::Value *Ptr = V; // Pointer to store into. - CharUnits PtrAlign = CharUnits::fromQuantity(AlignmentToUse); - - // If the value is offset in memory, apply the offset now. - if (unsigned Offs = ArgI.getDirectOffset()) { - Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy()); - Ptr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), Ptr, Offs); - Ptr = Builder.CreateBitCast(Ptr, - llvm::PointerType::getUnqual(ArgI.getCoerceToType())); - PtrAlign = PtrAlign.alignmentAtOffset(CharUnits::fromQuantity(Offs)); - } + // Pointer to store into. + Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI); // Fast-isel and the optimizer generally like scalar values better than // FCAs, so we flatten them if this is safe to do for this argument. llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType()); if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy && STy->getNumElements() > 1) { + auto SrcLayout = CGM.getDataLayout().getStructLayout(STy); uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy); - llvm::Type *DstTy = - cast<llvm::PointerType>(Ptr->getType())->getElementType(); + llvm::Type *DstTy = Ptr.getElementType(); uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy); + Address AddrToStoreInto = Address::invalid(); if (SrcSize <= DstSize) { - Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy)); - - assert(STy->getNumElements() == NumIRArgs); - for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { - auto AI = FnArgs[FirstIRArg + i]; - AI->setName(Arg->getName() + ".coerce" + Twine(i)); - llvm::Value *EltPtr = Builder.CreateConstGEP2_32(STy, Ptr, 0, i); - Builder.CreateStore(AI, EltPtr); - } + AddrToStoreInto = + Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy)); } else { - llvm::AllocaInst *TempAlloca = - CreateTempAlloca(ArgI.getCoerceToType(), "coerce"); - TempAlloca->setAlignment(AlignmentToUse); - llvm::Value *TempV = TempAlloca; - - assert(STy->getNumElements() == NumIRArgs); - for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { - auto AI = FnArgs[FirstIRArg + i]; - AI->setName(Arg->getName() + ".coerce" + Twine(i)); - llvm::Value *EltPtr = - Builder.CreateConstGEP2_32(ArgI.getCoerceToType(), TempV, 0, i); - Builder.CreateStore(AI, EltPtr); - } + AddrToStoreInto = + CreateTempAlloca(STy, Alloca.getAlignment(), "coerce"); + } - Builder.CreateMemCpy(Ptr, TempV, DstSize, AlignmentToUse); + assert(STy->getNumElements() == NumIRArgs); + for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { + auto AI = FnArgs[FirstIRArg + i]; + AI->setName(Arg->getName() + ".coerce" + Twine(i)); + auto Offset = CharUnits::fromQuantity(SrcLayout->getElementOffset(i)); + Address EltPtr = + Builder.CreateStructGEP(AddrToStoreInto, i, Offset); + Builder.CreateStore(AI, EltPtr); + } + + if (SrcSize > DstSize) { + Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize); } + } else { // Simple case, just do a coerced store of the argument into the alloca. assert(NumIRArgs == 1); auto AI = FnArgs[FirstIRArg]; AI->setName(Arg->getName() + ".coerce"); - CreateCoercedStore(AI, Ptr, /*DestIsVolatile=*/false, PtrAlign, *this); + CreateCoercedStore(AI, Ptr, /*DestIsVolatile=*/false, *this); } - // Match to what EmitParmDecl is expecting for this type. if (CodeGenFunction::hasScalarEvaluationKind(Ty)) { - V = EmitLoadOfScalar(V, false, AlignmentToUse, Ty, Arg->getLocStart()); + llvm::Value *V = + EmitLoadOfScalar(Alloca, false, Ty, Arg->getLocStart()); if (isPromoted) V = emitArgumentDemotion(*this, Arg, V); - ArgVals.push_back(ValueAndIsPtr(V, HaveValue)); + ArgVals.push_back(ParamValue::forDirect(V)); } else { - ArgVals.push_back(ValueAndIsPtr(V, HavePointer)); + ArgVals.push_back(ParamValue::forIndirect(Alloca)); } break; } @@ -2069,11 +2105,9 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, // If this structure was expanded into multiple arguments then // we need to create a temporary and reconstruct it from the // arguments. - llvm::AllocaInst *Alloca = CreateMemTemp(Ty); - CharUnits Align = getContext().getDeclAlign(Arg); - Alloca->setAlignment(Align.getQuantity()); - LValue LV = MakeAddrLValue(Alloca, Ty, Align); - ArgVals.push_back(ValueAndIsPtr(Alloca, HavePointer)); + Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg)); + LValue LV = MakeAddrLValue(Alloca, Ty); + ArgVals.push_back(ParamValue::forIndirect(Alloca)); auto FnArgIter = FnArgs.begin() + FirstIRArg; ExpandTypeFromArgs(Ty, LV, FnArgIter); @@ -2089,10 +2123,10 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, assert(NumIRArgs == 0); // Initialize the local variable appropriately. if (!hasScalarEvaluationKind(Ty)) { - ArgVals.push_back(ValueAndIsPtr(CreateMemTemp(Ty), HavePointer)); + ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty))); } else { llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType())); - ArgVals.push_back(ValueAndIsPtr(U, HaveValue)); + ArgVals.push_back(ParamValue::forDirect(U)); } break; } @@ -2100,12 +2134,10 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) { for (int I = Args.size() - 1; I >= 0; --I) - EmitParmDecl(*Args[I], ArgVals[I].getPointer(), ArgVals[I].getInt(), - I + 1); + EmitParmDecl(*Args[I], ArgVals[I], I + 1); } else { for (unsigned I = 0, E = Args.size(); I != E; ++I) - EmitParmDecl(*Args[I], ArgVals[I].getPointer(), ArgVals[I].getInt(), - I + 1); + EmitParmDecl(*Args[I], ArgVals[I], I + 1); } } @@ -2158,9 +2190,9 @@ static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF, bool doRetainAutorelease; - if (call->getCalledValue() == CGF.CGM.getARCEntrypoints().objc_retain) { + if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints().objc_retain) { doRetainAutorelease = true; - } else if (call->getCalledValue() == CGF.CGM.getARCEntrypoints() + } else if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints() .objc_retainAutoreleasedReturnValue) { doRetainAutorelease = false; @@ -2169,7 +2201,7 @@ static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF, // for that call. If we can't find it, we can't do this // optimization. But it should always be the immediately previous // instruction, unless we needed bitcasts around the call. - if (CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker) { + if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) { llvm::Instruction *prev = call->getPrevNode(); assert(prev); if (isa<llvm::BitCastInst>(prev)) { @@ -2178,7 +2210,7 @@ static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF, } assert(isa<llvm::CallInst>(prev)); assert(cast<llvm::CallInst>(prev)->getCalledValue() == - CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker); + CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker); insnsToKill.push_back(prev); } } else { @@ -2223,7 +2255,7 @@ static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF, llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(result->stripPointerCasts()); if (!retainCall || - retainCall->getCalledValue() != CGF.CGM.getARCEntrypoints().objc_retain) + retainCall->getCalledValue() != CGF.CGM.getObjCEntrypoints().objc_retain) return nullptr; // Look for an ordinary load of 'self'. @@ -2231,7 +2263,7 @@ static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF, llvm::LoadInst *load = dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts()); if (!load || load->isAtomic() || load->isVolatile() || - load->getPointerOperand() != CGF.GetAddrOfLocalVar(self)) + load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer()) return nullptr; // Okay! Burn it all down. This relies for correctness on the @@ -2268,11 +2300,23 @@ static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF, /// Heuristically search for a dominating store to the return-value slot. static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { + // Check if a User is a store which pointerOperand is the ReturnValue. + // We are looking for stores to the ReturnValue, not for stores of the + // ReturnValue to some other location. + auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * { + auto *SI = dyn_cast<llvm::StoreInst>(U); + if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer()) + return nullptr; + // These aren't actually possible for non-coerced returns, and we + // only care about non-coerced returns on this code path. + assert(!SI->isAtomic() && !SI->isVolatile()); + return SI; + }; // If there are multiple uses of the return-value slot, just check // for something immediately preceding the IP. Sometimes this can // happen with how we generate implicit-returns; it can also happen // with noreturn cleanups. - if (!CGF.ReturnValue->hasOneUse()) { + if (!CGF.ReturnValue.getPointer()->hasOneUse()) { llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock(); if (IP->empty()) return nullptr; llvm::Instruction *I = &IP->back(); @@ -2296,21 +2340,13 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { break; } - llvm::StoreInst *store = dyn_cast<llvm::StoreInst>(I); - if (!store) return nullptr; - if (store->getPointerOperand() != CGF.ReturnValue) return nullptr; - assert(!store->isAtomic() && !store->isVolatile()); // see below - return store; + return GetStoreIfValid(I); } llvm::StoreInst *store = - dyn_cast<llvm::StoreInst>(CGF.ReturnValue->user_back()); + GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back()); if (!store) return nullptr; - // These aren't actually possible for non-coerced returns, and we - // only care about non-coerced returns on this code path. - assert(!store->isAtomic() && !store->isVolatile()); - // Now do a first-and-dirty dominance check: just walk up the // single-predecessors chain from the current insertion point. llvm::BasicBlock *StoreBB = store->getParent(); @@ -2335,7 +2371,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, } // Functions with no result always return void. - if (!ReturnValue) { + if (!ReturnValue.isValid()) { Builder.CreateRetVoid(); return; } @@ -2353,10 +2389,10 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, if (RetAI.getInAllocaSRet()) { llvm::Function::arg_iterator EI = CurFn->arg_end(); --EI; - llvm::Value *ArgStruct = EI; + llvm::Value *ArgStruct = &*EI; llvm::Value *SRet = Builder.CreateStructGEP( nullptr, ArgStruct, RetAI.getInAllocaFieldIndex()); - RV = Builder.CreateLoad(SRet, "sret"); + RV = Builder.CreateAlignedLoad(SRet, getPointerAlign(), "sret"); } break; @@ -2367,9 +2403,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, switch (getEvaluationKind(RetTy)) { case TEK_Complex: { ComplexPairTy RT = - EmitLoadOfComplex(MakeNaturalAlignAddrLValue(ReturnValue, RetTy), - EndLoc); - EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(AI, RetTy), + EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc); + EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy), /*isInit*/ true); break; } @@ -2378,7 +2413,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, break; case TEK_Scalar: EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), - MakeNaturalAlignAddrLValue(AI, RetTy), + MakeNaturalAlignAddrLValue(&*AI, RetTy), /*isInit*/ true); break; } @@ -2406,9 +2441,12 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, SI->eraseFromParent(); // If that was the only use of the return value, nuke it as well now. - if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) { - cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent(); - ReturnValue = nullptr; + auto returnValueInst = ReturnValue.getPointer(); + if (returnValueInst->use_empty()) { + if (auto alloca = dyn_cast<llvm::AllocaInst>(returnValueInst)) { + alloca->eraseFromParent(); + ReturnValue = Address::invalid(); + } } // Otherwise, we have to do a simple load. @@ -2416,18 +2454,10 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, RV = Builder.CreateLoad(ReturnValue); } } else { - llvm::Value *V = ReturnValue; - CharUnits Align = getContext().getTypeAlignInChars(RetTy); // If the value is offset in memory, apply the offset now. - if (unsigned Offs = RetAI.getDirectOffset()) { - V = Builder.CreateBitCast(V, Builder.getInt8PtrTy()); - V = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), V, Offs); - V = Builder.CreateBitCast(V, - llvm::PointerType::getUnqual(RetAI.getCoerceToType())); - Align = Align.alignmentAtOffset(CharUnits::fromQuantity(Offs)); - } + Address V = emitAddressAtOffset(*this, ReturnValue, RetAI); - RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), Align, *this); + RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this); } // In ARC, end functions that return a retainable type with a call @@ -2450,8 +2480,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, llvm::Instruction *Ret; if (RV) { - if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute)) { - if (auto RetNNAttr = CurGD.getDecl()->getAttr<ReturnsNonNullAttr>()) { + if (CurCodeDecl && SanOpts.has(SanitizerKind::ReturnsNonnullAttribute)) { + if (auto RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>()) { SanitizerScope SanScope(this); llvm::Value *Cond = Builder.CreateICmpNE( RV, llvm::Constant::getNullValue(RV->getType())); @@ -2477,14 +2507,20 @@ static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) { return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory; } -static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) { +static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, + QualType Ty) { // FIXME: Generate IR in one pass, rather than going back and fixing up these // placeholders. llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty); llvm::Value *Placeholder = - llvm::UndefValue::get(IRTy->getPointerTo()->getPointerTo()); - Placeholder = CGF.Builder.CreateLoad(Placeholder); - return AggValueSlot::forAddr(Placeholder, CharUnits::Zero(), + llvm::UndefValue::get(IRTy->getPointerTo()->getPointerTo()); + Placeholder = CGF.Builder.CreateDefaultAlignedLoad(Placeholder); + + // FIXME: When we generate this IR in one pass, we shouldn't need + // this win32-specific alignment hack. + CharUnits Align = CharUnits::fromQuantity(4); + + return AggValueSlot::forAddr(Address(Placeholder, Align), Ty.getQualifiers(), AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers, @@ -2497,7 +2533,7 @@ void CodeGenFunction::EmitDelegateCallArg(CallArgList &args, // StartFunction converted the ABI-lowered parameter(s) into a // local alloca. We need to turn that into an r-value suitable // for EmitCall. - llvm::Value *local = GetAddrOfLocalVar(param); + Address local = GetAddrOfLocalVar(param); QualType type = param->getType(); @@ -2532,20 +2568,21 @@ static bool isProvablyNonNull(llvm::Value *addr) { static void emitWriteback(CodeGenFunction &CGF, const CallArgList::Writeback &writeback) { const LValue &srcLV = writeback.Source; - llvm::Value *srcAddr = srcLV.getAddress(); - assert(!isProvablyNull(srcAddr) && + Address srcAddr = srcLV.getAddress(); + assert(!isProvablyNull(srcAddr.getPointer()) && "shouldn't have writeback for provably null argument"); llvm::BasicBlock *contBB = nullptr; // If the argument wasn't provably non-null, we need to null check // before doing the store. - bool provablyNonNull = isProvablyNonNull(srcAddr); + bool provablyNonNull = isProvablyNonNull(srcAddr.getPointer()); if (!provablyNonNull) { llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback"); contBB = CGF.createBasicBlock("icr.done"); - llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull"); + llvm::Value *isNull = + CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); CGF.Builder.CreateCondBr(isNull, contBB, writebackBB); CGF.EmitBlock(writebackBB); } @@ -2554,9 +2591,8 @@ static void emitWriteback(CodeGenFunction &CGF, llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary); // Cast it back, in case we're writing an id to a Foo* or something. - value = CGF.Builder.CreateBitCast(value, - cast<llvm::PointerType>(srcAddr->getType())->getElementType(), - "icr.writeback-cast"); + value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(), + "icr.writeback-cast"); // Perform the writeback. @@ -2606,10 +2642,9 @@ static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF, ArrayRef<CallArgList::CallArgCleanup> Cleanups = CallArgs.getCleanupsToDeactivate(); // Iterate in reverse to increase the likelihood of popping the cleanup. - for (ArrayRef<CallArgList::CallArgCleanup>::reverse_iterator - I = Cleanups.rbegin(), E = Cleanups.rend(); I != E; ++I) { - CGF.DeactivateCleanupBlock(I->Cleanup, I->IsActiveIP); - I->IsActiveIP->eraseFromParent(); + for (const auto &I : llvm::reverse(Cleanups)) { + CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP); + I.IsActiveIP->eraseFromParent(); } } @@ -2621,7 +2656,9 @@ static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) { } /// Emit an argument that's being passed call-by-writeback. That is, -/// we are passing the address of +/// we are passing the address of an __autoreleased temporary; it +/// might be copy-initialized with the current value of the given +/// address, but it will definitely be copied out of after the call. static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, const ObjCIndirectCopyRestoreExpr *CRE) { LValue srcLV; @@ -2633,13 +2670,13 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, // Otherwise, just emit it as a scalar. } else { - llvm::Value *srcAddr = CGF.EmitScalarExpr(CRE->getSubExpr()); + Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr()); QualType srcAddrType = CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType(); - srcLV = CGF.MakeNaturalAlignAddrLValue(srcAddr, srcAddrType); + srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType); } - llvm::Value *srcAddr = srcLV.getAddress(); + Address srcAddr = srcLV.getAddress(); // The dest and src types don't necessarily match in LLVM terms // because of the crazy ObjC compatibility rules. @@ -2648,15 +2685,16 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, cast<llvm::PointerType>(CGF.ConvertType(CRE->getType())); // If the address is a constant null, just pass the appropriate null. - if (isProvablyNull(srcAddr)) { + if (isProvablyNull(srcAddr.getPointer())) { args.add(RValue::get(llvm::ConstantPointerNull::get(destType)), CRE->getType()); return; } // Create the temporary. - llvm::Value *temp = CGF.CreateTempAlloca(destType->getElementType(), - "icr.temp"); + Address temp = CGF.CreateTempAlloca(destType->getElementType(), + CGF.getPointerAlign(), + "icr.temp"); // Loading an l-value can introduce a cleanup if the l-value is __weak, // and that cleanup will be conditional if we can't prove that the l-value // isn't null, so we need to register a dominating point so that the cleanups @@ -2678,15 +2716,16 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, // If the address is *not* known to be non-null, we need to switch. llvm::Value *finalArgument; - bool provablyNonNull = isProvablyNonNull(srcAddr); + bool provablyNonNull = isProvablyNonNull(srcAddr.getPointer()); if (provablyNonNull) { - finalArgument = temp; + finalArgument = temp.getPointer(); } else { - llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull"); + llvm::Value *isNull = + CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); finalArgument = CGF.Builder.CreateSelect(isNull, llvm::ConstantPointerNull::get(destType), - temp, "icr.argument"); + temp.getPointer(), "icr.argument"); // If we need to copy, then the load has to be conditional, which // means we need control flow. @@ -2753,24 +2792,12 @@ void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) { // Save the stack. llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave); StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save"); - - // Control gets really tied up in landing pads, so we have to spill the - // stacksave to an alloca to avoid violating SSA form. - // TODO: This is dead if we never emit the cleanup. We should create the - // alloca and store lazily on the first cleanup emission. - StackBaseMem = CGF.CreateTempAlloca(CGF.Int8PtrTy, "inalloca.spmem"); - CGF.Builder.CreateStore(StackBase, StackBaseMem); - CGF.pushStackRestore(EHCleanup, StackBaseMem); - StackCleanup = CGF.EHStack.getInnermostEHScope(); - assert(StackCleanup.isValid()); } void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const { if (StackBase) { - CGF.DeactivateCleanupBlock(StackCleanup, StackBase); + // Restore the stack after the call. llvm::Value *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore); - // We could load StackBase from StackBaseMem, but in the non-exceptional - // case we can skip it. CGF.Builder.CreateCall(F, StackBase); } } @@ -2800,12 +2827,26 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType, "nonnull_arg", StaticData, None); } -void CodeGenFunction::EmitCallArgs(CallArgList &Args, - ArrayRef<QualType> ArgTypes, - CallExpr::const_arg_iterator ArgBeg, - CallExpr::const_arg_iterator ArgEnd, - const FunctionDecl *CalleeDecl, - unsigned ParamsToSkip) { +void CodeGenFunction::EmitCallArgs( + CallArgList &Args, ArrayRef<QualType> ArgTypes, + llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange, + const FunctionDecl *CalleeDecl, unsigned ParamsToSkip) { + assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin())); + + auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg) { + if (CalleeDecl == nullptr || I >= CalleeDecl->getNumParams()) + return; + auto *PS = CalleeDecl->getParamDecl(I)->getAttr<PassObjectSizeAttr>(); + if (PS == nullptr) + return; + + const auto &Context = getContext(); + auto SizeTy = Context.getSizeType(); + auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy)); + llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T); + Args.add(RValue::get(V), SizeTy); + }; + // We *have* to evaluate arguments from right to left in the MS C++ ABI, // because arguments are destroyed left to right in the callee. if (CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) { @@ -2822,10 +2863,11 @@ void CodeGenFunction::EmitCallArgs(CallArgList &Args, // Evaluate each argument. size_t CallArgsStart = Args.size(); for (int I = ArgTypes.size() - 1; I >= 0; --I) { - CallExpr::const_arg_iterator Arg = ArgBeg + I; + CallExpr::const_arg_iterator Arg = ArgRange.begin() + I; EmitCallArg(Args, *Arg, ArgTypes[I]); - EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], Arg->getExprLoc(), + EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], (*Arg)->getExprLoc(), CalleeDecl, ParamsToSkip + I); + MaybeEmitImplicitObjectSize(I, *Arg); } // Un-reverse the arguments we just evaluated so they match up with the LLVM @@ -2835,21 +2877,22 @@ void CodeGenFunction::EmitCallArgs(CallArgList &Args, } for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) { - CallExpr::const_arg_iterator Arg = ArgBeg + I; - assert(Arg != ArgEnd); + CallExpr::const_arg_iterator Arg = ArgRange.begin() + I; + assert(Arg != ArgRange.end()); EmitCallArg(Args, *Arg, ArgTypes[I]); - EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], Arg->getExprLoc(), + EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], (*Arg)->getExprLoc(), CalleeDecl, ParamsToSkip + I); + MaybeEmitImplicitObjectSize(I, *Arg); } } namespace { -struct DestroyUnpassedArg : EHScopeStack::Cleanup { - DestroyUnpassedArg(llvm::Value *Addr, QualType Ty) +struct DestroyUnpassedArg final : EHScopeStack::Cleanup { + DestroyUnpassedArg(Address Addr, QualType Ty) : Addr(Addr), Ty(Ty) {} - llvm::Value *Addr; + Address Addr; QualType Ty; void Emit(CodeGenFunction &CGF, Flags flags) override { @@ -2860,8 +2903,6 @@ struct DestroyUnpassedArg : EHScopeStack::Cleanup { } }; -} - struct DisableDebugLocationUpdates { CodeGenFunction &CGF; bool disabledDebugInfo; @@ -2875,6 +2916,8 @@ struct DisableDebugLocationUpdates { } }; +} // end anonymous namespace + void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, QualType type) { DisableDebugLocationUpdates Dis(*this, E); @@ -2923,7 +2966,8 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, // Create a no-op GEP between the placeholder and the cleanup so we can // RAUW it successfully. It also serves as a marker of the first // instruction where the cleanup is active. - pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddr(), type); + pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(), + type); // This unreachable is a temporary marker which will be removed later. llvm::Instruction *IsActive = Builder.CreateUnreachable(); args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive); @@ -2940,9 +2984,8 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, } else { // We can't represent a misaligned lvalue in the CallArgList, so copy // to an aligned temporary now. - llvm::Value *tmp = CreateMemTemp(type); - EmitAggregateCopy(tmp, L.getAddress(), type, L.isVolatile(), - L.getAlignment()); + Address tmp = CreateMemTemp(type); + EmitAggregateCopy(tmp, L.getAddress(), type, L.isVolatile()); args.add(RValue::getAggregate(tmp), type); } return; @@ -3015,19 +3058,41 @@ CodeGenFunction::EmitRuntimeCall(llvm::Value *callee, return call; } +// Calls which may throw must have operand bundles indicating which funclet +// they are nested within. +static void +getBundlesForFunclet(llvm::Value *Callee, + llvm::Instruction *CurrentFuncletPad, + SmallVectorImpl<llvm::OperandBundleDef> &BundleList) { + // There is no need for a funclet operand bundle if we aren't inside a funclet. + if (!CurrentFuncletPad) + return; + + // Skip intrinsics which cannot throw. + auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts()); + if (CalleeFn && CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) + return; + + BundleList.emplace_back("funclet", CurrentFuncletPad); +} + /// Emits a call or invoke to the given noreturn runtime function. void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(llvm::Value *callee, ArrayRef<llvm::Value*> args) { + SmallVector<llvm::OperandBundleDef, 1> BundleList; + getBundlesForFunclet(callee, CurrentFuncletPad, BundleList); + if (getInvokeDest()) { llvm::InvokeInst *invoke = Builder.CreateInvoke(callee, getUnreachableBlock(), getInvokeDest(), - args); + args, + BundleList); invoke->setDoesNotReturn(); invoke->setCallingConv(getRuntimeCC()); } else { - llvm::CallInst *call = Builder.CreateCall(callee, args); + llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList); call->setDoesNotReturn(); call->setCallingConv(getRuntimeCC()); Builder.CreateUnreachable(); @@ -3052,12 +3117,6 @@ CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::Value *callee, return callSite; } -llvm::CallSite -CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee, - const Twine &Name) { - return EmitCallOrInvoke(Callee, None, Name); -} - /// Emits a call or invoke instruction to the given function, depending /// on the current state of the EH stack. llvm::CallSite @@ -3102,7 +3161,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm::Value *Callee, ReturnValueSlot ReturnValue, const CallArgList &CallArgs, - const Decl *TargetDecl, + CGCalleeInfo CalleeInfo, llvm::Instruction **callOrInvoke) { // FIXME: We no longer need the types from CallArgs; lift up and simplify. @@ -3117,8 +3176,10 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If we're using inalloca, insert the allocation after the stack save. // FIXME: Do this earlier rather than hacking it in here! - llvm::AllocaInst *ArgMemory = nullptr; + Address ArgMemory = Address::invalid(); + const llvm::StructLayout *ArgMemoryLayout = nullptr; if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) { + ArgMemoryLayout = CGM.getDataLayout().getStructLayout(ArgStruct); llvm::Instruction *IP = CallArgs.getStackBase(); llvm::AllocaInst *AI; if (IP) { @@ -3127,36 +3188,44 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } else { AI = CreateTempAlloca(ArgStruct, "argmem"); } + auto Align = CallInfo.getArgStructAlignment(); + AI->setAlignment(Align.getQuantity()); AI->setUsedWithInAlloca(true); assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca()); - ArgMemory = AI; + ArgMemory = Address(AI, Align); } + // Helper function to drill into the inalloca allocation. + auto createInAllocaStructGEP = [&](unsigned FieldIndex) -> Address { + auto FieldOffset = + CharUnits::fromQuantity(ArgMemoryLayout->getElementOffset(FieldIndex)); + return Builder.CreateStructGEP(ArgMemory, FieldIndex, FieldOffset); + }; + ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo); SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs()); // If the call returns a temporary with struct return, create a temporary // alloca to hold the result, unless one is given to us. - llvm::Value *SRetPtr = nullptr; + Address SRetPtr = Address::invalid(); size_t UnusedReturnSize = 0; if (RetAI.isIndirect() || RetAI.isInAlloca()) { - SRetPtr = ReturnValue.getValue(); - if (!SRetPtr) { + if (!ReturnValue.isNull()) { + SRetPtr = ReturnValue.getValue(); + } else { SRetPtr = CreateMemTemp(RetTy); if (HaveInsertPoint() && ReturnValue.isUnused()) { uint64_t size = CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy)); - if (EmitLifetimeStart(size, SRetPtr)) + if (EmitLifetimeStart(size, SRetPtr.getPointer())) UnusedReturnSize = size; } } if (IRFunctionArgs.hasSRetArg()) { - IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr; + IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer(); } else { - llvm::Value *Addr = - Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory, - RetAI.getInAllocaFieldIndex()); - Builder.CreateStore(SRetPtr, Addr); + Address Addr = createInAllocaStructGEP(RetAI.getInAllocaFieldIndex()); + Builder.CreateStore(SRetPtr.getPointer(), Addr); } } @@ -3169,8 +3238,6 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, const ABIArgInfo &ArgInfo = info_it->info; RValue RV = I->RV; - CharUnits TypeAlign = getContext().getTypeAlignInChars(I->Ty); - // Insert a padding argument to ensure proper alignment. if (IRFunctionArgs.hasPaddingArg(ArgNo)) IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] = @@ -3186,27 +3253,23 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, if (RV.isAggregate()) { // Replace the placeholder with the appropriate argument slot GEP. llvm::Instruction *Placeholder = - cast<llvm::Instruction>(RV.getAggregateAddr()); + cast<llvm::Instruction>(RV.getAggregatePointer()); CGBuilderTy::InsertPoint IP = Builder.saveIP(); Builder.SetInsertPoint(Placeholder); - llvm::Value *Addr = - Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory, - ArgInfo.getInAllocaFieldIndex()); + Address Addr = createInAllocaStructGEP(ArgInfo.getInAllocaFieldIndex()); Builder.restoreIP(IP); - deferPlaceholderReplacement(Placeholder, Addr); + deferPlaceholderReplacement(Placeholder, Addr.getPointer()); } else { // Store the RValue into the argument struct. - llvm::Value *Addr = - Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory, - ArgInfo.getInAllocaFieldIndex()); - unsigned AS = Addr->getType()->getPointerAddressSpace(); + Address Addr = createInAllocaStructGEP(ArgInfo.getInAllocaFieldIndex()); + unsigned AS = Addr.getType()->getPointerAddressSpace(); llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS); // There are some cases where a trivial bitcast is not avoidable. The // definition of a type later in a translation unit may change it's type // from {}* to (%struct.foo*)*. - if (Addr->getType() != MemType) + if (Addr.getType() != MemType) Addr = Builder.CreateBitCast(Addr, MemType); - LValue argLV = MakeAddrLValue(Addr, I->Ty, TypeAlign); + LValue argLV = MakeAddrLValue(Addr, I->Ty); EmitInitStoreOfNonAggregate(*this, RV, argLV); } break; @@ -3216,12 +3279,10 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(NumIRArgs == 1); if (RV.isScalar() || RV.isComplex()) { // Make a temporary alloca to pass the argument. - llvm::AllocaInst *AI = CreateMemTemp(I->Ty); - if (ArgInfo.getIndirectAlign() > AI->getAlignment()) - AI->setAlignment(ArgInfo.getIndirectAlign()); - IRCallArgs[FirstIRArg] = AI; + Address Addr = CreateMemTemp(I->Ty, ArgInfo.getIndirectAlign()); + IRCallArgs[FirstIRArg] = Addr.getPointer(); - LValue argLV = MakeAddrLValue(AI, I->Ty, TypeAlign); + LValue argLV = MakeAddrLValue(Addr, I->Ty); EmitInitStoreOfNonAggregate(*this, RV, argLV); } else { // We want to avoid creating an unnecessary temporary+copy here; @@ -3232,27 +3293,27 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // we cannot force it to be sufficiently aligned. // 3. If the argument is byval, but RV is located in an address space // different than that of the argument (0). - llvm::Value *Addr = RV.getAggregateAddr(); - unsigned Align = ArgInfo.getIndirectAlign(); + Address Addr = RV.getAggregateAddress(); + CharUnits Align = ArgInfo.getIndirectAlign(); const llvm::DataLayout *TD = &CGM.getDataLayout(); - const unsigned RVAddrSpace = Addr->getType()->getPointerAddressSpace(); + const unsigned RVAddrSpace = Addr.getType()->getAddressSpace(); const unsigned ArgAddrSpace = (FirstIRArg < IRFuncTy->getNumParams() ? IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() : 0); if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) || - (ArgInfo.getIndirectByVal() && TypeAlign.getQuantity() < Align && - llvm::getOrEnforceKnownAlignment(Addr, Align, *TD) < Align) || + (ArgInfo.getIndirectByVal() && Addr.getAlignment() < Align && + llvm::getOrEnforceKnownAlignment(Addr.getPointer(), + Align.getQuantity(), *TD) + < Align.getQuantity()) || (ArgInfo.getIndirectByVal() && (RVAddrSpace != ArgAddrSpace))) { // Create an aligned temporary, and copy to it. - llvm::AllocaInst *AI = CreateMemTemp(I->Ty); - if (Align > AI->getAlignment()) - AI->setAlignment(Align); - IRCallArgs[FirstIRArg] = AI; + Address AI = CreateMemTemp(I->Ty, ArgInfo.getIndirectAlign()); + IRCallArgs[FirstIRArg] = AI.getPointer(); EmitAggregateCopy(AI, Addr, I->Ty, RV.isVolatileQualified()); } else { // Skip the extra memcpy call. - IRCallArgs[FirstIRArg] = Addr; + IRCallArgs[FirstIRArg] = Addr.getPointer(); } } break; @@ -3272,7 +3333,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, if (RV.isScalar()) V = RV.getScalarVal(); else - V = Builder.CreateLoad(RV.getAggregateAddr()); + V = Builder.CreateLoad(RV.getAggregateAddress()); // We might have to widen integers, but we should never truncate. if (ArgInfo.getCoerceToType() != V->getType() && @@ -3289,35 +3350,24 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } // FIXME: Avoid the conversion through memory if possible. - llvm::Value *SrcPtr; - CharUnits SrcAlign; + Address Src = Address::invalid(); if (RV.isScalar() || RV.isComplex()) { - SrcPtr = CreateMemTemp(I->Ty, "coerce"); - SrcAlign = TypeAlign; - LValue SrcLV = MakeAddrLValue(SrcPtr, I->Ty, TypeAlign); + Src = CreateMemTemp(I->Ty, "coerce"); + LValue SrcLV = MakeAddrLValue(Src, I->Ty); EmitInitStoreOfNonAggregate(*this, RV, SrcLV); } else { - SrcPtr = RV.getAggregateAddr(); - // This alignment is guaranteed by EmitCallArg. - SrcAlign = TypeAlign; + Src = RV.getAggregateAddress(); } // If the value is offset in memory, apply the offset now. - if (unsigned Offs = ArgInfo.getDirectOffset()) { - SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy()); - SrcPtr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), SrcPtr, Offs); - SrcPtr = Builder.CreateBitCast(SrcPtr, - llvm::PointerType::getUnqual(ArgInfo.getCoerceToType())); - SrcAlign = SrcAlign.alignmentAtOffset(CharUnits::fromQuantity(Offs)); - } + Src = emitAddressAtOffset(*this, Src, ArgInfo); // Fast-isel and the optimizer generally like scalar values better than // FCAs, so we flatten them if this is safe to do for this argument. llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType()); if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) { - llvm::Type *SrcTy = - cast<llvm::PointerType>(SrcPtr->getType())->getElementType(); + llvm::Type *SrcTy = Src.getType()->getElementType(); uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy); uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy); @@ -3326,29 +3376,28 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // of the destination type to allow loading all of it. The bits past // the source value are left undef. if (SrcSize < DstSize) { - llvm::AllocaInst *TempAlloca - = CreateTempAlloca(STy, SrcPtr->getName() + ".coerce"); - Builder.CreateMemCpy(TempAlloca, SrcPtr, SrcSize, 0); - SrcPtr = TempAlloca; + Address TempAlloca + = CreateTempAlloca(STy, Src.getAlignment(), + Src.getName() + ".coerce"); + Builder.CreateMemCpy(TempAlloca, Src, SrcSize); + Src = TempAlloca; } else { - SrcPtr = Builder.CreateBitCast(SrcPtr, - llvm::PointerType::getUnqual(STy)); + Src = Builder.CreateBitCast(Src, llvm::PointerType::getUnqual(STy)); } + auto SrcLayout = CGM.getDataLayout().getStructLayout(STy); assert(NumIRArgs == STy->getNumElements()); for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { - llvm::Value *EltPtr = Builder.CreateConstGEP2_32(STy, SrcPtr, 0, i); - llvm::LoadInst *LI = Builder.CreateLoad(EltPtr); - // We don't know what we're loading from. - LI->setAlignment(1); + auto Offset = CharUnits::fromQuantity(SrcLayout->getElementOffset(i)); + Address EltPtr = Builder.CreateStructGEP(Src, i, Offset); + llvm::Value *LI = Builder.CreateLoad(EltPtr); IRCallArgs[FirstIRArg + i] = LI; } } else { // In the simple case, just pass the coerced loaded value. assert(NumIRArgs == 1); IRCallArgs[FirstIRArg] = - CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(), - SrcAlign, *this); + CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this); } break; @@ -3362,8 +3411,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } } - if (ArgMemory) { - llvm::Value *Arg = ArgMemory; + if (ArgMemory.isValid()) { + llvm::Value *Arg = ArgMemory.getPointer(); if (CallInfo.isVariadic()) { // When passing non-POD arguments by value to variadic functions, we will // end up with a variadic prototype and an inalloca call site. In such @@ -3441,23 +3490,37 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, unsigned CallingConv; CodeGen::AttributeListType AttributeList; - CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList, - CallingConv, true); + CGM.ConstructAttributeList(CallInfo, CalleeInfo, AttributeList, CallingConv, + true); llvm::AttributeSet Attrs = llvm::AttributeSet::get(getLLVMContext(), AttributeList); - llvm::BasicBlock *InvokeDest = nullptr; - if (!Attrs.hasAttribute(llvm::AttributeSet::FunctionIndex, - llvm::Attribute::NoUnwind) || - currentFunctionUsesSEHTry()) - InvokeDest = getInvokeDest(); + bool CannotThrow; + if (currentFunctionUsesSEHTry()) { + // SEH cares about asynchronous exceptions, everything can "throw." + CannotThrow = false; + } else if (isCleanupPadScope() && + EHPersonality::get(*this).isMSVCXXPersonality()) { + // The MSVC++ personality will implicitly terminate the program if an + // exception is thrown. An unwind edge cannot be reached. + CannotThrow = true; + } else { + // Otherwise, nowunind callsites will never throw. + CannotThrow = Attrs.hasAttribute(llvm::AttributeSet::FunctionIndex, + llvm::Attribute::NoUnwind); + } + llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest(); + + SmallVector<llvm::OperandBundleDef, 1> BundleList; + getBundlesForFunclet(Callee, CurrentFuncletPad, BundleList); llvm::CallSite CS; if (!InvokeDest) { - CS = Builder.CreateCall(Callee, IRCallArgs); + CS = Builder.CreateCall(Callee, IRCallArgs, BundleList); } else { llvm::BasicBlock *Cont = createBasicBlock("invoke.cont"); - CS = Builder.CreateInvoke(Callee, Cont, InvokeDest, IRCallArgs); + CS = Builder.CreateInvoke(Callee, Cont, InvokeDest, IRCallArgs, + BundleList); EmitBlock(Cont); } if (callOrInvoke) @@ -3489,7 +3552,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, if (CS.doesNotReturn()) { if (UnusedReturnSize) EmitLifetimeEnd(llvm::ConstantInt::get(Int64Ty, UnusedReturnSize), - SRetPtr); + SRetPtr.getPointer()); Builder.CreateUnreachable(); Builder.ClearInsertionPoint(); @@ -3516,6 +3579,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // lexical order, so deactivate it and run it manually here. CallArgs.freeArgumentMemory(*this); + if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) { + const Decl *TargetDecl = CalleeInfo.getCalleeDecl(); + if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>()) + Call->setTailCallKind(llvm::CallInst::TCK_NoTail); + } + RValue Ret = [&] { switch (RetAI.getKind()) { case ABIArgInfo::InAlloca: @@ -3523,7 +3592,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation()); if (UnusedReturnSize) EmitLifetimeEnd(llvm::ConstantInt::get(Int64Ty, UnusedReturnSize), - SRetPtr); + SRetPtr.getPointer()); return ret; } @@ -3543,15 +3612,14 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, return RValue::getComplex(std::make_pair(Real, Imag)); } case TEK_Aggregate: { - llvm::Value *DestPtr = ReturnValue.getValue(); + Address DestPtr = ReturnValue.getValue(); bool DestIsVolatile = ReturnValue.isVolatile(); - CharUnits DestAlign = getContext().getTypeAlignInChars(RetTy); - if (!DestPtr) { + if (!DestPtr.isValid()) { DestPtr = CreateMemTemp(RetTy, "agg.tmp"); DestIsVolatile = false; } - BuildAggStore(*this, CI, DestPtr, DestIsVolatile, DestAlign); + BuildAggStore(*this, CI, DestPtr, DestIsVolatile); return RValue::getAggregate(DestPtr); } case TEK_Scalar: { @@ -3566,28 +3634,17 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm_unreachable("bad evaluation kind"); } - llvm::Value *DestPtr = ReturnValue.getValue(); + Address DestPtr = ReturnValue.getValue(); bool DestIsVolatile = ReturnValue.isVolatile(); - CharUnits DestAlign = getContext().getTypeAlignInChars(RetTy); - if (!DestPtr) { + if (!DestPtr.isValid()) { DestPtr = CreateMemTemp(RetTy, "coerce"); DestIsVolatile = false; } // If the value is offset in memory, apply the offset now. - llvm::Value *StorePtr = DestPtr; - CharUnits StoreAlign = DestAlign; - if (unsigned Offs = RetAI.getDirectOffset()) { - StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy()); - StorePtr = - Builder.CreateConstGEP1_32(Builder.getInt8Ty(), StorePtr, Offs); - StorePtr = Builder.CreateBitCast(StorePtr, - llvm::PointerType::getUnqual(RetAI.getCoerceToType())); - StoreAlign = - StoreAlign.alignmentAtOffset(CharUnits::fromQuantity(Offs)); - } - CreateCoercedStore(CI, StorePtr, DestIsVolatile, StoreAlign, *this); + Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI); + CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this); return convertTempToRValue(DestPtr, RetTy, SourceLocation()); } @@ -3599,6 +3656,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm_unreachable("Unhandled ABIArgInfo::Kind"); } (); + const Decl *TargetDecl = CalleeInfo.getCalleeDecl(); + if (Ret.isScalar() && TargetDecl) { if (const auto *AA = TargetDecl->getAttr<AssumeAlignedAttr>()) { llvm::Value *OffsetValue = nullptr; @@ -3617,6 +3676,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, /* VarArg handling */ -llvm::Value *CodeGenFunction::EmitVAArg(llvm::Value *VAListAddr, QualType Ty) { - return CGM.getTypes().getABIInfo().EmitVAArg(VAListAddr, Ty, *this); +Address CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr) { + VAListAddr = VE->isMicrosoftABI() + ? EmitMSVAListRef(VE->getSubExpr()) + : EmitVAListRef(VE->getSubExpr()); + QualType Ty = VE->getType(); + if (VE->isMicrosoftABI()) + return CGM.getTypes().getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty); + return CGM.getTypes().getABIInfo().EmitVAArg(*this, VAListAddr, Ty); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h index 7a4708e..2ebd09b 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h @@ -56,7 +56,7 @@ namespace CodeGen { class CallArgList : public SmallVector<CallArg, 16> { public: - CallArgList() : StackBase(nullptr), StackBaseMem(nullptr) {} + CallArgList() : StackBase(nullptr) {} struct Writeback { /// The original argument. Note that the argument l-value @@ -64,7 +64,7 @@ namespace CodeGen { LValue Source; /// The temporary alloca. - llvm::Value *Temporary; + Address Temporary; /// A value to "use" after the writeback, or null. llvm::Value *ToUse; @@ -88,12 +88,9 @@ namespace CodeGen { other.Writebacks.begin(), other.Writebacks.end()); } - void addWriteback(LValue srcLV, llvm::Value *temporary, + void addWriteback(LValue srcLV, Address temporary, llvm::Value *toUse) { - Writeback writeback; - writeback.Source = srcLV; - writeback.Temporary = temporary; - writeback.ToUse = toUse; + Writeback writeback = { srcLV, temporary, toUse }; Writebacks.push_back(writeback); } @@ -137,9 +134,6 @@ namespace CodeGen { /// The stacksave call. It dominates all of the argument evaluation. llvm::CallInst *StackBase; - /// The alloca holding the stackbase. We need it to maintain SSA form. - llvm::AllocaInst *StackBaseMem; - /// The iterator pointing to the stack restore cleanup. We manually run and /// deactivate this cleanup after the call in the unexceptional case because /// it doesn't run in the normal order. @@ -156,6 +150,7 @@ namespace CodeGen { /// function can be stored, and whether the address is volatile or not. class ReturnValueSlot { llvm::PointerIntPair<llvm::Value *, 2, unsigned int> Value; + CharUnits Alignment; // Return value slot flags enum Flags { @@ -165,14 +160,15 @@ namespace CodeGen { public: ReturnValueSlot() {} - ReturnValueSlot(llvm::Value *Value, bool IsVolatile, bool IsUnused = false) - : Value(Value, - (IsVolatile ? IS_VOLATILE : 0) | (IsUnused ? IS_UNUSED : 0)) {} + ReturnValueSlot(Address Addr, bool IsVolatile, bool IsUnused = false) + : Value(Addr.isValid() ? Addr.getPointer() : nullptr, + (IsVolatile ? IS_VOLATILE : 0) | (IsUnused ? IS_UNUSED : 0)), + Alignment(Addr.isValid() ? Addr.getAlignment() : CharUnits::Zero()) {} - bool isNull() const { return !getValue(); } + bool isNull() const { return !getValue().isValid(); } bool isVolatile() const { return Value.getInt() & IS_VOLATILE; } - llvm::Value *getValue() const { return Value.getPointer(); } + Address getValue() const { return Address(Value.getPointer(), Alignment); } bool isUnused() const { return Value.getInt() & IS_UNUSED; } }; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp index c49f182..2e566de 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp @@ -1,4 +1,4 @@ -//===--- CGClass.cpp - Emit LLVM Code for C++ classes ---------------------===// +//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -25,10 +25,124 @@ #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/CodeGenOptions.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Metadata.h" using namespace clang; using namespace CodeGen; +/// Return the best known alignment for an unknown pointer to a +/// particular class. +CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) { + if (!RD->isCompleteDefinition()) + return CharUnits::One(); // Hopefully won't be used anywhere. + + auto &layout = getContext().getASTRecordLayout(RD); + + // If the class is final, then we know that the pointer points to an + // object of that type and can use the full alignment. + if (RD->hasAttr<FinalAttr>()) { + return layout.getAlignment(); + + // Otherwise, we have to assume it could be a subclass. + } else { + return layout.getNonVirtualAlignment(); + } +} + +/// Return the best known alignment for a pointer to a virtual base, +/// given the alignment of a pointer to the derived class. +CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign, + const CXXRecordDecl *derivedClass, + const CXXRecordDecl *vbaseClass) { + // The basic idea here is that an underaligned derived pointer might + // indicate an underaligned base pointer. + + assert(vbaseClass->isCompleteDefinition()); + auto &baseLayout = getContext().getASTRecordLayout(vbaseClass); + CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment(); + + return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass, + expectedVBaseAlign); +} + +CharUnits +CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign, + const CXXRecordDecl *baseDecl, + CharUnits expectedTargetAlign) { + // If the base is an incomplete type (which is, alas, possible with + // member pointers), be pessimistic. + if (!baseDecl->isCompleteDefinition()) + return std::min(actualBaseAlign, expectedTargetAlign); + + auto &baseLayout = getContext().getASTRecordLayout(baseDecl); + CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment(); + + // If the class is properly aligned, assume the target offset is, too. + // + // This actually isn't necessarily the right thing to do --- if the + // class is a complete object, but it's only properly aligned for a + // base subobject, then the alignments of things relative to it are + // probably off as well. (Note that this requires the alignment of + // the target to be greater than the NV alignment of the derived + // class.) + // + // However, our approach to this kind of under-alignment can only + // ever be best effort; after all, we're never going to propagate + // alignments through variables or parameters. Note, in particular, + // that constructing a polymorphic type in an address that's less + // than pointer-aligned will generally trap in the constructor, + // unless we someday add some sort of attribute to change the + // assumed alignment of 'this'. So our goal here is pretty much + // just to allow the user to explicitly say that a pointer is + // under-aligned and then safely access its fields and v-tables. + if (actualBaseAlign >= expectedBaseAlign) { + return expectedTargetAlign; + } + + // Otherwise, we might be offset by an arbitrary multiple of the + // actual alignment. The correct adjustment is to take the min of + // the two alignments. + return std::min(actualBaseAlign, expectedTargetAlign); +} + +Address CodeGenFunction::LoadCXXThisAddress() { + assert(CurFuncDecl && "loading 'this' without a func declaration?"); + assert(isa<CXXMethodDecl>(CurFuncDecl)); + + // Lazily compute CXXThisAlignment. + if (CXXThisAlignment.isZero()) { + // Just use the best known alignment for the parent. + // TODO: if we're currently emitting a complete-object ctor/dtor, + // we can always use the complete-object alignment. + auto RD = cast<CXXMethodDecl>(CurFuncDecl)->getParent(); + CXXThisAlignment = CGM.getClassPointerAlignment(RD); + } + + return Address(LoadCXXThis(), CXXThisAlignment); +} + +/// Emit the address of a field using a member data pointer. +/// +/// \param E Only used for emergency diagnostics +Address +CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base, + llvm::Value *memberPtr, + const MemberPointerType *memberPtrType, + AlignmentSource *alignSource) { + // Ask the ABI to compute the actual address. + llvm::Value *ptr = + CGM.getCXXABI().EmitMemberDataPointerAddress(*this, E, base, + memberPtr, memberPtrType); + + QualType memberType = memberPtrType->getPointeeType(); + CharUnits memberAlign = getNaturalTypeAlignment(memberType, alignSource); + memberAlign = + CGM.getDynamicOffsetAlignment(base.getAlignment(), + memberPtrType->getClass()->getAsCXXRecordDecl(), + memberAlign); + return Address(ptr, memberAlign); +} + CharUnits CodeGenModule::computeNonVirtualBaseClassOffset( const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start, CastExpr::path_const_iterator End) { @@ -78,15 +192,13 @@ CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl, /// when the type is known to be complete (e.g. in complete destructors). /// /// The object pointed to by 'This' is assumed to be non-null. -llvm::Value * -CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(llvm::Value *This, +Address +CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This, const CXXRecordDecl *Derived, const CXXRecordDecl *Base, bool BaseIsVirtual) { // 'this' must be a pointer (in some address space) to Derived. - assert(This->getType()->isPointerTy() && - cast<llvm::PointerType>(This->getType())->getElementType() - == ConvertType(Derived)); + assert(This.getElementType() == ConvertType(Derived)); // Compute the offset of the virtual base. CharUnits Offset; @@ -98,20 +210,22 @@ CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(llvm::Value *This, // Shift and cast down to the base type. // TODO: for complete types, this should be possible with a GEP. - llvm::Value *V = This; - if (Offset.isPositive()) { - V = Builder.CreateBitCast(V, Int8PtrTy); - V = Builder.CreateConstInBoundsGEP1_64(V, Offset.getQuantity()); + Address V = This; + if (!Offset.isZero()) { + V = Builder.CreateElementBitCast(V, Int8Ty); + V = Builder.CreateConstInBoundsByteGEP(V, Offset); } - V = Builder.CreateBitCast(V, ConvertType(Base)->getPointerTo()); + V = Builder.CreateElementBitCast(V, ConvertType(Base)); return V; } -static llvm::Value * -ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, llvm::Value *ptr, +static Address +ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr, CharUnits nonVirtualOffset, - llvm::Value *virtualOffset) { + llvm::Value *virtualOffset, + const CXXRecordDecl *derivedClass, + const CXXRecordDecl *nearestVBase) { // Assert that we have something to do. assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr); @@ -128,13 +242,27 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, llvm::Value *ptr, } // Apply the base offset. + llvm::Value *ptr = addr.getPointer(); ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8PtrTy); ptr = CGF.Builder.CreateInBoundsGEP(ptr, baseOffset, "add.ptr"); - return ptr; + + // If we have a virtual component, the alignment of the result will + // be relative only to the known alignment of that vbase. + CharUnits alignment; + if (virtualOffset) { + assert(nearestVBase && "virtual offset without vbase?"); + alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(), + derivedClass, nearestVBase); + } else { + alignment = addr.getAlignment(); + } + alignment = alignment.alignmentAtOffset(nonVirtualOffset); + + return Address(ptr, alignment); } -llvm::Value *CodeGenFunction::GetAddressOfBaseClass( - llvm::Value *Value, const CXXRecordDecl *Derived, +Address CodeGenFunction::GetAddressOfBaseClass( + Address Value, const CXXRecordDecl *Derived, CastExpr::path_const_iterator PathBegin, CastExpr::path_const_iterator PathEnd, bool NullCheckValue, SourceLocation Loc) { @@ -174,14 +302,14 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( ConvertType((PathEnd[-1])->getType())->getPointerTo(); QualType DerivedTy = getContext().getRecordType(Derived); - CharUnits DerivedAlign = getContext().getTypeAlignInChars(DerivedTy); + CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived); // If the static offset is zero and we don't have a virtual step, // just do a bitcast; null checks are unnecessary. if (NonVirtualOffset.isZero() && !VBase) { if (sanitizePerformTypeCheck()) { - EmitTypeCheck(TCK_Upcast, Loc, Value, DerivedTy, DerivedAlign, - !NullCheckValue); + EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(), + DerivedTy, DerivedAlign, !NullCheckValue); } return Builder.CreateBitCast(Value, BasePtrTy); } @@ -196,14 +324,14 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull"); endBB = createBasicBlock("cast.end"); - llvm::Value *isNull = Builder.CreateIsNull(Value); + llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer()); Builder.CreateCondBr(isNull, endBB, notNullBB); EmitBlock(notNullBB); } if (sanitizePerformTypeCheck()) { - EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc, Value, - DerivedTy, DerivedAlign, true); + EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc, + Value.getPointer(), DerivedTy, DerivedAlign, true); } // Compute the virtual offset. @@ -214,9 +342,8 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( } // Apply both offsets. - Value = ApplyNonVirtualAndVirtualOffset(*this, Value, - NonVirtualOffset, - VirtualOffset); + Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset, + VirtualOffset, Derived, VBase); // Cast to the destination type. Value = Builder.CreateBitCast(Value, BasePtrTy); @@ -228,16 +355,16 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( EmitBlock(endBB); llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result"); - PHI->addIncoming(Value, notNullBB); + PHI->addIncoming(Value.getPointer(), notNullBB); PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB); - Value = PHI; + Value = Address(PHI, Value.getAlignment()); } return Value; } -llvm::Value * -CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value, +Address +CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, const CXXRecordDecl *Derived, CastExpr::path_const_iterator PathBegin, CastExpr::path_const_iterator PathEnd, @@ -253,7 +380,7 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value, if (!NonVirtualOffset) { // No offset, we can just cast back. - return Builder.CreateBitCast(Value, DerivedPtrTy); + return Builder.CreateBitCast(BaseAddr, DerivedPtrTy); } llvm::BasicBlock *CastNull = nullptr; @@ -265,19 +392,20 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value, CastNotNull = createBasicBlock("cast.notnull"); CastEnd = createBasicBlock("cast.end"); - llvm::Value *IsNull = Builder.CreateIsNull(Value); + llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer()); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } // Apply the offset. - Value = Builder.CreateBitCast(Value, Int8PtrTy); + llvm::Value *Value = Builder.CreateBitCast(BaseAddr.getPointer(), Int8PtrTy); Value = Builder.CreateGEP(Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr"); // Just cast. Value = Builder.CreateBitCast(Value, DerivedPtrTy); + // Produce a PHI if we had a null-check. if (NullCheckValue) { Builder.CreateBr(CastEnd); EmitBlock(CastNull); @@ -286,12 +414,11 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value, llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2); PHI->addIncoming(Value, CastNotNull); - PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), - CastNull); + PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull); Value = PHI; } - return Value; + return Address(Value, CGM.getClassPointerAlignment(Derived)); } llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, @@ -345,7 +472,7 @@ llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, namespace { /// Call the destructor for a direct base class. - struct CallBaseDtor : EHScopeStack::Cleanup { + struct CallBaseDtor final : EHScopeStack::Cleanup { const CXXRecordDecl *BaseClass; bool BaseIsVirtual; CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual) @@ -356,8 +483,8 @@ namespace { cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent(); const CXXDestructorDecl *D = BaseClass->getDestructor(); - llvm::Value *Addr = - CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThis(), + Address Addr = + CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(), DerivedClass, BaseClass, BaseIsVirtual); CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual, @@ -381,7 +508,7 @@ namespace { // external code might potentially access the vtable. void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; } }; -} +} // end anonymous namespace static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) { DynamicThisUseChecker Checker(C); @@ -396,7 +523,7 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, assert(BaseInit->isBaseInitializer() && "Must have base initializer!"); - llvm::Value *ThisPtr = CGF.LoadCXXThis(); + Address ThisPtr = CGF.LoadCXXThisAddress(); const Type *BaseType = BaseInit->getBaseClass(); CXXRecordDecl *BaseClassDecl = @@ -416,13 +543,12 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, // We can pretend to be a complete class because it only matters for // virtual bases, and we only do virtual bases for complete ctors. - llvm::Value *V = + Address V = CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl, BaseClassDecl, isBaseVirtual); - CharUnits Alignment = CGF.getContext().getTypeAlignInChars(BaseType); AggValueSlot AggSlot = - AggValueSlot::forAddr(V, Alignment, Qualifiers(), + AggValueSlot::forAddr(V, Qualifiers(), AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased); @@ -438,17 +564,17 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, static void EmitAggMemberInitializer(CodeGenFunction &CGF, LValue LHS, Expr *Init, - llvm::Value *ArrayIndexVar, + Address ArrayIndexVar, QualType T, ArrayRef<VarDecl *> ArrayIndexes, unsigned Index) { if (Index == ArrayIndexes.size()) { LValue LV = LHS; - if (ArrayIndexVar) { + if (ArrayIndexVar.isValid()) { // If we have an array index variable, load it and use it as an offset. // Then, increment the value. - llvm::Value *Dest = LHS.getAddress(); + llvm::Value *Dest = LHS.getPointer(); llvm::Value *ArrayIndex = CGF.Builder.CreateLoad(ArrayIndexVar); Dest = CGF.Builder.CreateInBoundsGEP(Dest, ArrayIndex, "destaddress"); llvm::Value *Next = llvm::ConstantInt::get(ArrayIndex->getType(), 1); @@ -456,9 +582,9 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF, CGF.Builder.CreateStore(Next, ArrayIndexVar); // Update the LValue. - LV.setAddress(Dest); - CharUnits Align = CGF.getContext().getTypeAlignInChars(T); - LV.setAlignment(std::min(Align, LV.getAlignment())); + CharUnits EltSize = CGF.getContext().getTypeSizeInChars(T); + CharUnits Align = LV.getAlignment().alignmentOfArrayElement(EltSize); + LV.setAddress(Address(Dest, Align)); } switch (CGF.getEvaluationKind(T)) { @@ -485,14 +611,11 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF, const ConstantArrayType *Array = CGF.getContext().getAsConstantArrayType(T); assert(Array && "Array initialization without the array type?"); - llvm::Value *IndexVar - = CGF.GetAddrOfLocalVar(ArrayIndexes[Index]); - assert(IndexVar && "Array index variable not loaded"); + Address IndexVar = CGF.GetAddrOfLocalVar(ArrayIndexes[Index]); // Initialize this index variable to zero. llvm::Value* Zero - = llvm::Constant::getNullValue( - CGF.ConvertType(CGF.getContext().getSizeType())); + = llvm::Constant::getNullValue(IndexVar.getElementType()); CGF.Builder.CreateStore(Zero, IndexVar); // Start the loop with a block that tests the condition. @@ -626,9 +749,8 @@ static void EmitMemberInitializer(CodeGenFunction &CGF, CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit(), ArrayIndexes); } -void CodeGenFunction::EmitInitializerForField( - FieldDecl *Field, LValue LHS, Expr *Init, - ArrayRef<VarDecl *> ArrayIndexes) { +void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS, + Expr *Init, ArrayRef<VarDecl *> ArrayIndexes) { QualType FieldType = Field->getType(); switch (getEvaluationKind(FieldType)) { case TEK_Scalar: @@ -643,26 +765,23 @@ void CodeGenFunction::EmitInitializerForField( EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true); break; case TEK_Aggregate: { - llvm::Value *ArrayIndexVar = nullptr; + Address ArrayIndexVar = Address::invalid(); if (ArrayIndexes.size()) { - llvm::Type *SizeTy = ConvertType(getContext().getSizeType()); - // The LHS is a pointer to the first object we'll be constructing, as // a flat array. QualType BaseElementTy = getContext().getBaseElementType(FieldType); llvm::Type *BasePtr = ConvertType(BaseElementTy); BasePtr = llvm::PointerType::getUnqual(BasePtr); - llvm::Value *BaseAddrPtr = Builder.CreateBitCast(LHS.getAddress(), - BasePtr); + Address BaseAddrPtr = Builder.CreateBitCast(LHS.getAddress(), BasePtr); LHS = MakeAddrLValue(BaseAddrPtr, BaseElementTy); // Create an array index that will be used to walk over all of the // objects we're constructing. - ArrayIndexVar = CreateTempAlloca(SizeTy, "object.index"); - llvm::Value *Zero = llvm::Constant::getNullValue(SizeTy); + ArrayIndexVar = CreateMemTemp(getContext().getSizeType(), "object.index"); + llvm::Value *Zero = + llvm::Constant::getNullValue(ArrayIndexVar.getElementType()); Builder.CreateStore(Zero, ArrayIndexVar); - // Emit the block variables for the array indices, if any. for (unsigned I = 0, N = ArrayIndexes.size(); I != N; ++I) EmitAutoVarDecl(*ArrayIndexes[I]); @@ -811,7 +930,7 @@ void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) { return; } - const FunctionDecl *Definition = 0; + const FunctionDecl *Definition = nullptr; Stmt *Body = Ctor->getBody(Definition); assert(Definition == Ctor && "emitting wrong constructor body"); @@ -868,7 +987,7 @@ namespace { SanitizerSet OldSanOpts; }; } - + namespace { class FieldMemcpyizer { public: @@ -930,19 +1049,16 @@ namespace { CharUnits MemcpySize = getMemcpySize(FirstByteOffset); QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl); - llvm::Value *ThisPtr = CGF.LoadCXXThis(); - LValue DestLV = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy); + Address ThisPtr = CGF.LoadCXXThisAddress(); + LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy); LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField); llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec)); LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy); LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField); - CharUnits Offset = CGF.getContext().toCharUnitsFromBits(FirstByteOffset); - CharUnits Alignment = DestLV.getAlignment().alignmentAtOffset(Offset); - - emitMemcpyIR(Dest.isBitField() ? Dest.getBitFieldAddr() : Dest.getAddress(), - Src.isBitField() ? Src.getBitFieldAddr() : Src.getAddress(), - MemcpySize, Alignment); + emitMemcpyIR(Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(), + Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(), + MemcpySize); reset(); } @@ -956,20 +1072,18 @@ namespace { private: - void emitMemcpyIR(llvm::Value *DestPtr, llvm::Value *SrcPtr, - CharUnits Size, CharUnits Alignment) { - llvm::PointerType *DPT = cast<llvm::PointerType>(DestPtr->getType()); + void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) { + llvm::PointerType *DPT = DestPtr.getType(); llvm::Type *DBP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), DPT->getAddressSpace()); DestPtr = CGF.Builder.CreateBitCast(DestPtr, DBP); - llvm::PointerType *SPT = cast<llvm::PointerType>(SrcPtr->getType()); + llvm::PointerType *SPT = SrcPtr.getType(); llvm::Type *SBP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), SPT->getAddressSpace()); SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, SBP); - CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity(), - Alignment.getQuantity()); + CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity()); } void addInitialField(FieldDecl *F) { @@ -1089,9 +1203,9 @@ namespace { } void pushEHDestructors() { - llvm::Value *ThisPtr = CGF.LoadCXXThis(); + Address ThisPtr = CGF.LoadCXXThisAddress(); QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl); - LValue LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy); + LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy); for (unsigned i = 0; i < AggregatedInits.size(); ++i) { CXXCtorInitializer *MemberInit = AggregatedInits[i]; @@ -1228,7 +1342,13 @@ namespace { emitAggregatedStmts(); } }; +} // end anonymous namespace +static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) { + const Type *BaseType = BaseInit->getBaseClass(); + const auto *BaseClassDecl = + cast<CXXRecordDecl>(BaseType->getAs<RecordType>()->getDecl()); + return BaseClassDecl->isDynamicClass(); } /// EmitCtorPrologue - This routine generates necessary code to initialize @@ -1254,8 +1374,13 @@ void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD, assert(BaseCtorContinueBB); } + llvm::Value *const OldThis = CXXThisValue; // Virtual base initializers first. for (; B != E && (*B)->isBaseInitializer() && (*B)->isBaseVirtual(); B++) { + if (CGM.getCodeGenOpts().StrictVTablePointers && + CGM.getCodeGenOpts().OptimizationLevel > 0 && + isInitializerOfDynamicClass(*B)) + CXXThisValue = Builder.CreateInvariantGroupBarrier(LoadCXXThis()); EmitBaseInitializer(*this, ClassDecl, *B, CtorType); } @@ -1268,13 +1393,20 @@ void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD, // Then, non-virtual base initializers. for (; B != E && (*B)->isBaseInitializer(); B++) { assert(!(*B)->isBaseVirtual()); + + if (CGM.getCodeGenOpts().StrictVTablePointers && + CGM.getCodeGenOpts().OptimizationLevel > 0 && + isInitializerOfDynamicClass(*B)) + CXXThisValue = Builder.CreateInvariantGroupBarrier(LoadCXXThis()); EmitBaseInitializer(*this, ClassDecl, *B, CtorType); } + CXXThisValue = OldThis; + InitializeVTablePointers(ClassDecl); // And finally, initialize class members. - FieldConstructionScope FCS(*this, CXXThisValue); + FieldConstructionScope FCS(*this, LoadCXXThisAddress()); ConstructorMemcpyizer CM(*this, CD, Args); for (; B != E; B++) { CXXCtorInitializer *Member = (*B); @@ -1334,7 +1466,7 @@ HasTrivialDestructorBody(ASTContext &Context, static bool FieldHasTrivialDestructorBody(ASTContext &Context, - const FieldDecl *Field) + const FieldDecl *Field) { QualType FieldBaseElementType = Context.getBaseElementType(Field->getType()); @@ -1353,39 +1485,23 @@ FieldHasTrivialDestructorBody(ASTContext &Context, /// CanSkipVTablePointerInitialization - Check whether we need to initialize /// any vtable pointers before calling this destructor. -static bool CanSkipVTablePointerInitialization(ASTContext &Context, +static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor) { + const CXXRecordDecl *ClassDecl = Dtor->getParent(); + if (!ClassDecl->isDynamicClass()) + return true; + if (!Dtor->hasTrivialBody()) return false; // Check the fields. - const CXXRecordDecl *ClassDecl = Dtor->getParent(); for (const auto *Field : ClassDecl->fields()) - if (!FieldHasTrivialDestructorBody(Context, Field)) + if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field)) return false; return true; } -// Generates function call for handling object poisoning, passing in -// references to 'this' and its size as arguments. -static void EmitDtorSanitizerCallback(CodeGenFunction &CGF, - const CXXDestructorDecl *Dtor) { - const ASTRecordLayout &Layout = - CGF.getContext().getASTRecordLayout(Dtor->getParent()); - - llvm::Value *Args[] = { - CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.VoidPtrTy), - llvm::ConstantInt::get(CGF.SizeTy, Layout.getSize().getQuantity())}; - llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy}; - - llvm::FunctionType *FnType = - llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false); - llvm::Value *Fn = - CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback"); - CGF.EmitNounwindRuntimeCall(Fn, Args); -} - /// EmitDestructorBody - Emits the body of the current destructor. void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl()); @@ -1402,7 +1518,7 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { if (DtorType == Dtor_Deleting) { EnterDtorCleanups(Dtor, Dtor_Deleting); EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false, - /*Delegating=*/false, LoadCXXThis()); + /*Delegating=*/false, LoadCXXThisAddress()); PopCleanupBlock(); return; } @@ -1437,7 +1553,7 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { if (!isTryBody) { EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false, - /*Delegating=*/false, LoadCXXThis()); + /*Delegating=*/false, LoadCXXThisAddress()); break; } // Fallthrough: act like we're in the base variant. @@ -1449,8 +1565,14 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { EnterDtorCleanups(Dtor, Dtor_Base); // Initialize the vtable pointers before entering the body. - if (!CanSkipVTablePointerInitialization(getContext(), Dtor)) - InitializeVTablePointers(Dtor->getParent()); + if (!CanSkipVTablePointerInitialization(*this, Dtor)) { + // Insert the llvm.invariant.group.barrier intrinsic before initializing + // the vptrs to cancel any previous assumptions we might have made. + if (CGM.getCodeGenOpts().StrictVTablePointers && + CGM.getCodeGenOpts().OptimizationLevel > 0) + CXXThisValue = Builder.CreateInvariantGroupBarrier(LoadCXXThis()); + InitializeVTablePointers(Dtor->getParent()); + } if (isTryBody) EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock()); @@ -1464,6 +1586,7 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { // the caller's body. if (getLangOpts().AppleKext) CurFn->addFnAttr(llvm::Attribute::AlwaysInline); + break; } @@ -1473,10 +1596,6 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { // Exit the try if applicable. if (isTryBody) ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true); - - // Insert memory-poisoning instrumentation. - if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor) - EmitDtorSanitizerCallback(*this, Dtor); } void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) { @@ -1496,7 +1615,7 @@ void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) namespace { /// Call the operator delete associated with the current destructor. - struct CallDtorDelete : EHScopeStack::Cleanup { + struct CallDtorDelete final : EHScopeStack::Cleanup { CallDtorDelete() {} void Emit(CodeGenFunction &CGF, Flags flags) override { @@ -1507,11 +1626,11 @@ namespace { } }; - struct CallDtorDeleteConditional : EHScopeStack::Cleanup { + struct CallDtorDeleteConditional final : EHScopeStack::Cleanup { llvm::Value *ShouldDeleteCondition; public: CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition) - : ShouldDeleteCondition(ShouldDeleteCondition) { + : ShouldDeleteCondition(ShouldDeleteCondition) { assert(ShouldDeleteCondition != nullptr); } @@ -1533,7 +1652,7 @@ namespace { } }; - class DestroyField : public EHScopeStack::Cleanup { + class DestroyField final : public EHScopeStack::Cleanup { const FieldDecl *field; CodeGenFunction::Destroyer *destroyer; bool useEHCleanupForArray; @@ -1541,12 +1660,12 @@ namespace { public: DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer, bool useEHCleanupForArray) - : field(field), destroyer(destroyer), - useEHCleanupForArray(useEHCleanupForArray) {} + : field(field), destroyer(destroyer), + useEHCleanupForArray(useEHCleanupForArray) {} void Emit(CodeGenFunction &CGF, Flags flags) override { // Find the address of the field. - llvm::Value *thisValue = CGF.LoadCXXThis(); + Address thisValue = CGF.LoadCXXThisAddress(); QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent()); LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy); LValue LV = CGF.EmitLValueForField(ThisLV, field); @@ -1556,7 +1675,133 @@ namespace { flags.isForNormalCleanup() && useEHCleanupForArray); } }; -} + + static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr, + CharUnits::QuantityType PoisonSize) { + // Pass in void pointer and size of region as arguments to runtime + // function + llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy), + llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)}; + + llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy}; + + llvm::FunctionType *FnType = + llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false); + llvm::Value *Fn = + CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback"); + CGF.EmitNounwindRuntimeCall(Fn, Args); + } + + class SanitizeDtorMembers final : public EHScopeStack::Cleanup { + const CXXDestructorDecl *Dtor; + + public: + SanitizeDtorMembers(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {} + + // Generate function call for handling object poisoning. + // Disables tail call elimination, to prevent the current stack frame + // from disappearing from the stack trace. + void Emit(CodeGenFunction &CGF, Flags flags) override { + const ASTRecordLayout &Layout = + CGF.getContext().getASTRecordLayout(Dtor->getParent()); + + // Nothing to poison. + if (Layout.getFieldCount() == 0) + return; + + // Prevent the current stack frame from disappearing from the stack trace. + CGF.CurFn->addFnAttr("disable-tail-calls", "true"); + + // Construct pointer to region to begin poisoning, and calculate poison + // size, so that only members declared in this class are poisoned. + ASTContext &Context = CGF.getContext(); + unsigned fieldIndex = 0; + int startIndex = -1; + // RecordDecl::field_iterator Field; + for (const FieldDecl *Field : Dtor->getParent()->fields()) { + // Poison field if it is trivial + if (FieldHasTrivialDestructorBody(Context, Field)) { + // Start sanitizing at this field + if (startIndex < 0) + startIndex = fieldIndex; + + // Currently on the last field, and it must be poisoned with the + // current block. + if (fieldIndex == Layout.getFieldCount() - 1) { + PoisonMembers(CGF, startIndex, Layout.getFieldCount()); + } + } else if (startIndex >= 0) { + // No longer within a block of memory to poison, so poison the block + PoisonMembers(CGF, startIndex, fieldIndex); + // Re-set the start index + startIndex = -1; + } + fieldIndex += 1; + } + } + + private: + /// \param layoutStartOffset index of the ASTRecordLayout field to + /// start poisoning (inclusive) + /// \param layoutEndOffset index of the ASTRecordLayout field to + /// end poisoning (exclusive) + void PoisonMembers(CodeGenFunction &CGF, unsigned layoutStartOffset, + unsigned layoutEndOffset) { + ASTContext &Context = CGF.getContext(); + const ASTRecordLayout &Layout = + Context.getASTRecordLayout(Dtor->getParent()); + + llvm::ConstantInt *OffsetSizePtr = llvm::ConstantInt::get( + CGF.SizeTy, + Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset)) + .getQuantity()); + + llvm::Value *OffsetPtr = CGF.Builder.CreateGEP( + CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.Int8PtrTy), + OffsetSizePtr); + + CharUnits::QuantityType PoisonSize; + if (layoutEndOffset >= Layout.getFieldCount()) { + PoisonSize = Layout.getNonVirtualSize().getQuantity() - + Context.toCharUnitsFromBits( + Layout.getFieldOffset(layoutStartOffset)) + .getQuantity(); + } else { + PoisonSize = Context.toCharUnitsFromBits( + Layout.getFieldOffset(layoutEndOffset) - + Layout.getFieldOffset(layoutStartOffset)) + .getQuantity(); + } + + if (PoisonSize == 0) + return; + + EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize); + } + }; + + class SanitizeDtorVTable final : public EHScopeStack::Cleanup { + const CXXDestructorDecl *Dtor; + + public: + SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {} + + // Generate function call for handling vtable pointer poisoning. + void Emit(CodeGenFunction &CGF, Flags flags) override { + assert(Dtor->getParent()->isDynamicClass()); + (void)Dtor; + ASTContext &Context = CGF.getContext(); + // Poison vtable and vtable ptr if they exist for this class. + llvm::Value *VTablePtr = CGF.LoadCXXThis(); + + CharUnits::QuantityType PoisonSize = + Context.toCharUnitsFromBits(CGF.PointerWidthInBits).getQuantity(); + // Pass in void pointer and size of region as arguments to runtime + // function + EmitSanitizerDtorCallback(CGF, VTablePtr, PoisonSize); + } + }; +} // end anonymous namespace /// \brief Emit all code that comes at the end of class's /// destructor. This is to call destructors on members and base classes @@ -1590,6 +1835,12 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD, // The complete-destructor phase just destructs all the virtual bases. if (DtorType == Dtor_Complete) { + // Poison the vtable pointer such that access after the base + // and member destructors are invoked is invalid. + if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor && + SanOpts.has(SanitizerKind::Memory) && ClassDecl->getNumVBases() && + ClassDecl->isPolymorphic()) + EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD); // We push them in the forward order so that they'll be popped in // the reverse order. @@ -1610,6 +1861,12 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD, } assert(DtorType == Dtor_Base); + // Poison the vtable pointer if it has no virtual bases, but inherits + // virtual functions. + if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor && + SanOpts.has(SanitizerKind::Memory) && !ClassDecl->getNumVBases() && + ClassDecl->isPolymorphic()) + EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD); // Destroy non-virtual bases. for (const auto &Base : ClassDecl->bases()) { @@ -1628,6 +1885,12 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD, /*BaseIsVirtual*/ false); } + // Poison fields such that access after their destructors are + // invoked, and before the base class destructor runs, is invalid. + if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor && + SanOpts.has(SanitizerKind::Memory)) + EHStack.pushCleanup<SanitizeDtorMembers>(NormalAndEHCleanup, DD); + // Destroy direct fields. for (const auto *Field : ClassDecl->fields()) { QualType type = Field->getType(); @@ -1655,7 +1918,7 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD, /// zero-initialized before it is constructed void CodeGenFunction::EmitCXXAggrConstructorCall( const CXXConstructorDecl *ctor, const ConstantArrayType *arrayType, - llvm::Value *arrayBegin, const CXXConstructExpr *E, bool zeroInitialize) { + Address arrayBegin, const CXXConstructExpr *E, bool zeroInitialize) { QualType elementType; llvm::Value *numElements = emitArrayLength(arrayType, elementType, arrayBegin); @@ -1669,15 +1932,14 @@ void CodeGenFunction::EmitCXXAggrConstructorCall( /// \param ctor the constructor to call for each element /// \param numElements the number of elements in the array; /// may be zero -/// \param arrayBegin a T*, where T is the type constructed by ctor +/// \param arrayBase a T*, where T is the type constructed by ctor /// \param zeroInitialize true if each element should be /// zero-initialized before it is constructed void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, llvm::Value *numElements, - llvm::Value *arrayBegin, + Address arrayBase, const CXXConstructExpr *E, bool zeroInitialize) { - // It's legal for numElements to be zero. This can happen both // dynamically, because x can be zero in 'new A[x]', and statically, // because of GCC extensions that permit zero-length arrays. There @@ -1701,6 +1963,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, } // Find the end of the array. + llvm::Value *arrayBegin = arrayBase.getPointer(); llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(arrayBegin, numElements, "arrayctor.end"); @@ -1714,11 +1977,21 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, // Inside the loop body, emit the constructor call on the array element. + // The alignment of the base, adjusted by the size of a single element, + // provides a conservative estimate of the alignment of every element. + // (This assumes we never start tracking offsetted alignments.) + // + // Note that these are complete objects and so we don't need to + // use the non-virtual size or alignment. QualType type = getContext().getTypeDeclType(ctor->getParent()); + CharUnits eltAlignment = + arrayBase.getAlignment() + .alignmentOfArrayElement(getContext().getTypeSizeInChars(type)); + Address curAddr = Address(cur, eltAlignment); // Zero initialize the storage, if requested. if (zeroInitialize) - EmitNullInitialization(cur, type); + EmitNullInitialization(curAddr, type); // C++ [class.temporary]p4: // There are two contexts in which temporaries are destroyed at a different @@ -1736,11 +2009,12 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, if (getLangOpts().Exceptions && !ctor->getParent()->hasTrivialDestructor()) { Destroyer *destroyer = destroyCXXObject; - pushRegularPartialArrayCleanup(arrayBegin, cur, type, *destroyer); + pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment, + *destroyer); } EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false, - /*Delegating=*/false, cur, E); + /*Delegating=*/false, curAddr, E); } // Go to the next element. @@ -1761,7 +2035,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, } void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF, - llvm::Value *addr, + Address addr, QualType type) { const RecordType *rtype = type->castAs<RecordType>(); const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl()); @@ -1774,14 +2048,16 @@ void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF, void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type, bool ForVirtualBase, - bool Delegating, llvm::Value *This, + bool Delegating, Address This, const CXXConstructExpr *E) { + const CXXRecordDecl *ClassDecl = D->getParent(); + // C++11 [class.mfct.non-static]p2: // If a non-static member function of a class X is called for an object that // is not of type X, or of a type derived from X, the behavior is undefined. // FIXME: Provide a source location here. - EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, SourceLocation(), This, - getContext().getRecordType(D->getParent())); + EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, SourceLocation(), + This.getPointer(), getContext().getRecordType(ClassDecl)); if (D->isTrivial() && D->isDefaultConstructor()) { assert(E->getNumArgs() == 0 && "trivial default ctor with args"); @@ -1796,8 +2072,8 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, const Expr *Arg = E->getArg(0); QualType SrcTy = Arg->getType(); - llvm::Value *Src = EmitLValue(Arg).getAddress(); - QualType DestTy = getContext().getTypeDeclType(D->getParent()); + Address Src = EmitLValue(Arg).getAddress(); + QualType DestTy = getContext().getTypeDeclType(ClassDecl); EmitAggregateCopyCtor(This, Src, DestTy, SrcTy); return; } @@ -1805,11 +2081,11 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, CallArgList Args; // Push the this ptr. - Args.add(RValue::get(This), D->getThisType(getContext())); + Args.add(RValue::get(This.getPointer()), D->getThisType(getContext())); // Add the rest of the user-supplied arguments. const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>(); - EmitCallArgs(Args, FPT, E->arg_begin(), E->arg_end(), E->getConstructor()); + EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor()); // Insert any ABI-specific implicit constructor arguments. unsigned ExtraArgs = CGM.getCXXABI().addImplicitConstructorArgs( @@ -1820,19 +2096,64 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(Args, D, Type, ExtraArgs); EmitCall(Info, Callee, ReturnValueSlot(), Args, D); + + // Generate vtable assumptions if we're constructing a complete object + // with a vtable. We don't do this for base subobjects for two reasons: + // first, it's incorrect for classes with virtual bases, and second, we're + // about to overwrite the vptrs anyway. + // We also have to make sure if we can refer to vtable: + // - Otherwise we can refer to vtable if it's safe to speculatively emit. + // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are + // sure that definition of vtable is not hidden, + // then we are always safe to refer to it. + // FIXME: It looks like InstCombine is very inefficient on dealing with + // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily. + if (CGM.getCodeGenOpts().OptimizationLevel > 0 && + ClassDecl->isDynamicClass() && Type != Ctor_Base && + CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl) && + CGM.getCodeGenOpts().StrictVTablePointers) + EmitVTableAssumptionLoads(ClassDecl, This); +} + +void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) { + llvm::Value *VTableGlobal = + CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass); + if (!VTableGlobal) + return; + + // We can just use the base offset in the complete class. + CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset(); + + if (!NonVirtualOffset.isZero()) + This = + ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr, + Vptr.VTableClass, Vptr.NearestVBase); + + llvm::Value *VPtrValue = + GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass); + llvm::Value *Cmp = + Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables"); + Builder.CreateAssumption(Cmp); +} + +void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, + Address This) { + if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl)) + for (const VPtr &Vptr : getVTablePointers(ClassDecl)) + EmitVTableAssumptionLoad(Vptr, This); } void CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, - llvm::Value *This, llvm::Value *Src, - const CXXConstructExpr *E) { + Address This, Address Src, + const CXXConstructExpr *E) { if (isMemcpyEquivalentSpecialMember(D)) { assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor"); assert(D->isCopyOrMoveConstructor() && "trivial 1-arg ctor not a copy/move ctor"); EmitAggregateCopyCtor(This, Src, getContext().getTypeDeclType(D->getParent()), - E->arg_begin()->getType()); + (*E->arg_begin())->getType()); return; } llvm::Value *Callee = CGM.getAddrOfCXXStructor(D, StructorType::Complete); @@ -1844,16 +2165,16 @@ CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, CallArgList Args; // Push the this ptr. - Args.add(RValue::get(This), D->getThisType(getContext())); + Args.add(RValue::get(This.getPointer()), D->getThisType(getContext())); // Push the src ptr. QualType QT = *(FPT->param_type_begin()); llvm::Type *t = CGM.getTypes().ConvertType(QT); Src = Builder.CreateBitCast(Src, t); - Args.add(RValue::get(Src), QT); + Args.add(RValue::get(Src.getPointer()), QT); // Skip over first argument (Src). - EmitCallArgs(Args, FPT, E->arg_begin() + 1, E->arg_end(), E->getConstructor(), + EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(), /*ParamsToSkip*/ 1); EmitCall(CGM.getTypes().arrangeCXXMethodCall(Args, FPT, RequiredArgs::All), @@ -1903,12 +2224,12 @@ CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor, } namespace { - struct CallDelegatingCtorDtor : EHScopeStack::Cleanup { + struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup { const CXXDestructorDecl *Dtor; - llvm::Value *Addr; + Address Addr; CXXDtorType Type; - CallDelegatingCtorDtor(const CXXDestructorDecl *D, llvm::Value *Addr, + CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr, CXXDtorType Type) : Dtor(D), Addr(Addr), Type(Type) {} @@ -1917,19 +2238,17 @@ namespace { /*Delegating=*/true, Addr); } }; -} +} // end anonymous namespace void CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor, const FunctionArgList &Args) { assert(Ctor->isDelegatingConstructor()); - llvm::Value *ThisPtr = LoadCXXThis(); + Address ThisPtr = LoadCXXThisAddress(); - QualType Ty = getContext().getTagDeclType(Ctor->getParent()); - CharUnits Alignment = getContext().getTypeAlignInChars(Ty); AggValueSlot AggSlot = - AggValueSlot::forAddr(ThisPtr, Alignment, Qualifiers(), + AggValueSlot::forAddr(ThisPtr, Qualifiers(), AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased); @@ -1951,17 +2270,17 @@ void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD, CXXDtorType Type, bool ForVirtualBase, bool Delegating, - llvm::Value *This) { + Address This) { CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase, Delegating, This); } namespace { - struct CallLocalDtor : EHScopeStack::Cleanup { + struct CallLocalDtor final : EHScopeStack::Cleanup { const CXXDestructorDecl *Dtor; - llvm::Value *Addr; + Address Addr; - CallLocalDtor(const CXXDestructorDecl *D, llvm::Value *Addr) + CallLocalDtor(const CXXDestructorDecl *D, Address Addr) : Dtor(D), Addr(Addr) {} void Emit(CodeGenFunction &CGF, Flags flags) override { @@ -1973,11 +2292,11 @@ namespace { } void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D, - llvm::Value *Addr) { + Address Addr) { EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr); } -void CodeGenFunction::PushDestructorCleanup(QualType T, llvm::Value *Addr) { +void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) { CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl(); if (!ClassDecl) return; if (ClassDecl->hasTrivialDestructor()) return; @@ -1987,24 +2306,12 @@ void CodeGenFunction::PushDestructorCleanup(QualType T, llvm::Value *Addr) { PushDestructorCleanup(D, Addr); } -void -CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, - const CXXRecordDecl *NearestVBase, - CharUnits OffsetFromNearestVBase, - const CXXRecordDecl *VTableClass) { - const CXXRecordDecl *RD = Base.getBase(); - - // Don't initialize the vtable pointer if the class is marked with the - // 'novtable' attribute. - if ((RD == VTableClass || RD == NearestVBase) && - VTableClass->hasAttr<MSNoVTableAttr>()) - return; - +void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) { // Compute the address point. - bool NeedsVirtualOffset; llvm::Value *VTableAddressPoint = CGM.getCXXABI().getVTableAddressPointInStructor( - *this, VTableClass, Base, NearestVBase, NeedsVirtualOffset); + *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase); + if (!VTableAddressPoint) return; @@ -2012,26 +2319,25 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, llvm::Value *VirtualOffset = nullptr; CharUnits NonVirtualOffset = CharUnits::Zero(); - if (NeedsVirtualOffset) { + if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) { // We need to use the virtual base offset offset because the virtual base // might have a different offset in the most derived class. - VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(*this, - LoadCXXThis(), - VTableClass, - NearestVBase); - NonVirtualOffset = OffsetFromNearestVBase; + + VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset( + *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase); + NonVirtualOffset = Vptr.OffsetFromNearestVBase; } else { // We can just use the base offset in the complete class. - NonVirtualOffset = Base.getBaseOffset(); + NonVirtualOffset = Vptr.Base.getBaseOffset(); } // Apply the offsets. - llvm::Value *VTableField = LoadCXXThis(); + Address VTableField = LoadCXXThisAddress(); if (!NonVirtualOffset.isZero() || VirtualOffset) - VTableField = ApplyNonVirtualAndVirtualOffset(*this, VTableField, - NonVirtualOffset, - VirtualOffset); + VTableField = ApplyNonVirtualAndVirtualOffset( + *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass, + Vptr.NearestVBase); // Finally, store the address point. Use the same LLVM types as the field to // support optimization. @@ -2041,23 +2347,39 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, ->getPointerTo(); VTableField = Builder.CreateBitCast(VTableField, VTablePtrTy->getPointerTo()); VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy); + llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField); - CGM.DecorateInstruction(Store, CGM.getTBAAInfoForVTablePtr()); + CGM.DecorateInstructionWithTBAA(Store, CGM.getTBAAInfoForVTablePtr()); + if (CGM.getCodeGenOpts().OptimizationLevel > 0 && + CGM.getCodeGenOpts().StrictVTablePointers) + CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass); } -void -CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, - const CXXRecordDecl *NearestVBase, - CharUnits OffsetFromNearestVBase, - bool BaseIsNonVirtualPrimaryBase, - const CXXRecordDecl *VTableClass, - VisitedVirtualBasesSetTy& VBases) { +CodeGenFunction::VPtrsVector +CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) { + CodeGenFunction::VPtrsVector VPtrsResult; + VisitedVirtualBasesSetTy VBases; + getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()), + /*NearestVBase=*/nullptr, + /*OffsetFromNearestVBase=*/CharUnits::Zero(), + /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases, + VPtrsResult); + return VPtrsResult; +} + +void CodeGenFunction::getVTablePointers(BaseSubobject Base, + const CXXRecordDecl *NearestVBase, + CharUnits OffsetFromNearestVBase, + bool BaseIsNonVirtualPrimaryBase, + const CXXRecordDecl *VTableClass, + VisitedVirtualBasesSetTy &VBases, + VPtrsVector &Vptrs) { // If this base is a non-virtual primary base the address point has already // been set. if (!BaseIsNonVirtualPrimaryBase) { // Initialize the vtable pointer for this base. - InitializeVTablePointer(Base, NearestVBase, OffsetFromNearestVBase, - VTableClass); + VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass}; + Vptrs.push_back(Vptr); } const CXXRecordDecl *RD = Base.getBase(); @@ -2095,11 +2417,10 @@ CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl; } - InitializeVTablePointers(BaseSubobject(BaseDecl, BaseOffset), - I.isVirtual() ? BaseDecl : NearestVBase, - BaseOffsetFromNearestVBase, - BaseDeclIsNonVirtualPrimaryBase, - VTableClass, VBases); + getVTablePointers( + BaseSubobject(BaseDecl, BaseOffset), + I.isVirtual() ? BaseDecl : NearestVBase, BaseOffsetFromNearestVBase, + BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs); } } @@ -2109,21 +2430,25 @@ void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) { return; // Initialize the vtable pointers for this class and all of its bases. - VisitedVirtualBasesSetTy VBases; - InitializeVTablePointers(BaseSubobject(RD, CharUnits::Zero()), - /*NearestVBase=*/nullptr, - /*OffsetFromNearestVBase=*/CharUnits::Zero(), - /*BaseIsNonVirtualPrimaryBase=*/false, RD, VBases); + if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD)) + for (const VPtr &Vptr : getVTablePointers(RD)) + InitializeVTablePointer(Vptr); if (RD->getNumVBases()) CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD); } -llvm::Value *CodeGenFunction::GetVTablePtr(llvm::Value *This, - llvm::Type *Ty) { - llvm::Value *VTablePtrSrc = Builder.CreateBitCast(This, Ty->getPointerTo()); +llvm::Value *CodeGenFunction::GetVTablePtr(Address This, + llvm::Type *VTableTy, + const CXXRecordDecl *RD) { + Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy); llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable"); - CGM.DecorateInstruction(VTable, CGM.getTBAAInfoForVTablePtr()); + CGM.DecorateInstructionWithTBAA(VTable, CGM.getTBAAInfoForVTablePtr()); + + if (CGM.getCodeGenOpts().OptimizationLevel > 0 && + CGM.getCodeGenOpts().StrictVTablePointers) + CGM.DecorateInstructionWithInvariantGroup(VTable, RD); + return VTable; } @@ -2190,19 +2515,10 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass()) return; - SmallString<64> MangledName; - llvm::raw_svector_ostream Out(MangledName); - CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T.getUnqualifiedType(), - Out); - - // Blacklist based on the mangled type. - if (CGM.getContext().getSanitizerBlacklist().isBlacklistedType(Out.str())) - return; - if (!SanOpts.has(SanitizerKind::CFICastStrict)) ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl); - llvm::BasicBlock *ContBlock = 0; + llvm::BasicBlock *ContBlock = nullptr; if (MayBeNull) { llvm::Value *DerivedNotNull = @@ -2216,7 +2532,9 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, EmitBlock(CheckBlock); } - llvm::Value *VTable = GetVTablePtr(Derived, Int8PtrTy); + llvm::Value *VTable = + GetVTablePtr(Address(Derived, getPointerAlign()), Int8PtrTy, ClassDecl); + EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc); if (MayBeNull) { @@ -2234,18 +2552,22 @@ void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD, SanitizerScope SanScope(this); - std::string OutName; - llvm::raw_string_ostream Out(OutName); - CGM.getCXXABI().getMangleContext().mangleCXXVTableBitSet(RD, Out); - - llvm::Value *BitSetName = llvm::MetadataAsValue::get( - getLLVMContext(), llvm::MDString::get(getLLVMContext(), Out.str())); + llvm::Metadata *MD = + CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); + llvm::Value *BitSetName = llvm::MetadataAsValue::get(getLLVMContext(), MD); llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy); llvm::Value *BitSetTest = Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::bitset_test), {CastedVTable, BitSetName}); + if (CGM.getCodeGenOpts().SanitizeCfiCrossDso) { + if (auto TypeId = CGM.CreateCfiIdForTypeMetadata(MD)) { + EmitCfiSlowPathCheck(BitSetTest, TypeId, CastedVTable); + return; + } + } + SanitizerMask M; switch (TCK) { case CFITCK_VCall: @@ -2263,9 +2585,9 @@ void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD, } llvm::Constant *StaticData[] = { - EmitCheckSourceLocation(Loc), - EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)), - llvm::ConstantInt::get(Int8Ty, TCK), + EmitCheckSourceLocation(Loc), + EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)), + llvm::ConstantInt::get(Int8Ty, TCK), }; EmitCheck(std::make_pair(BitSetTest, M), "cfi_bad_type", StaticData, CastedVTable); @@ -2405,8 +2727,8 @@ void CodeGenFunction::EmitLambdaBlockInvokeBody() { CallArgList CallArgs; QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda)); - llvm::Value *ThisPtr = GetAddrOfBlockDecl(variable, false); - CallArgs.add(RValue::get(ThisPtr), ThisType); + Address ThisPtr = GetAddrOfBlockDecl(variable, false); + CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); // Add the rest of the parameters. for (auto param : BD->params()) diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp index d97e405..ba7dcf7 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp @@ -19,6 +19,7 @@ #include "CGCleanup.h" #include "CodeGenFunction.h" +#include "llvm/Support/SaveAndRestore.h" using namespace clang; using namespace CodeGen; @@ -27,7 +28,7 @@ bool DominatingValue<RValue>::saved_type::needsSaving(RValue rv) { if (rv.isScalar()) return DominatingLLVMValue::needsSaving(rv.getScalarVal()); if (rv.isAggregate()) - return DominatingLLVMValue::needsSaving(rv.getAggregateAddr()); + return DominatingLLVMValue::needsSaving(rv.getAggregatePointer()); return true; } @@ -41,9 +42,10 @@ DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) { return saved_type(V, ScalarLiteral); // Everything else needs an alloca. - llvm::Value *addr = CGF.CreateTempAlloca(V->getType(), "saved-rvalue"); + Address addr = + CGF.CreateDefaultAlignTempAlloca(V->getType(), "saved-rvalue"); CGF.Builder.CreateStore(V, addr); - return saved_type(addr, ScalarAddress); + return saved_type(addr.getPointer(), ScalarAddress); } if (rv.isComplex()) { @@ -51,42 +53,56 @@ DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) { llvm::Type *ComplexTy = llvm::StructType::get(V.first->getType(), V.second->getType(), (void*) nullptr); - llvm::Value *addr = CGF.CreateTempAlloca(ComplexTy, "saved-complex"); + Address addr = CGF.CreateDefaultAlignTempAlloca(ComplexTy, "saved-complex"); CGF.Builder.CreateStore(V.first, - CGF.Builder.CreateStructGEP(ComplexTy, addr, 0)); + CGF.Builder.CreateStructGEP(addr, 0, CharUnits())); + CharUnits offset = CharUnits::fromQuantity( + CGF.CGM.getDataLayout().getTypeAllocSize(V.first->getType())); CGF.Builder.CreateStore(V.second, - CGF.Builder.CreateStructGEP(ComplexTy, addr, 1)); - return saved_type(addr, ComplexAddress); + CGF.Builder.CreateStructGEP(addr, 1, offset)); + return saved_type(addr.getPointer(), ComplexAddress); } assert(rv.isAggregate()); - llvm::Value *V = rv.getAggregateAddr(); // TODO: volatile? - if (!DominatingLLVMValue::needsSaving(V)) - return saved_type(V, AggregateLiteral); - - llvm::Value *addr = CGF.CreateTempAlloca(V->getType(), "saved-rvalue"); - CGF.Builder.CreateStore(V, addr); - return saved_type(addr, AggregateAddress); + Address V = rv.getAggregateAddress(); // TODO: volatile? + if (!DominatingLLVMValue::needsSaving(V.getPointer())) + return saved_type(V.getPointer(), AggregateLiteral, + V.getAlignment().getQuantity()); + + Address addr = + CGF.CreateTempAlloca(V.getType(), CGF.getPointerAlign(), "saved-rvalue"); + CGF.Builder.CreateStore(V.getPointer(), addr); + return saved_type(addr.getPointer(), AggregateAddress, + V.getAlignment().getQuantity()); } /// Given a saved r-value produced by SaveRValue, perform the code /// necessary to restore it to usability at the current insertion /// point. RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) { + auto getSavingAddress = [&](llvm::Value *value) { + auto alignment = cast<llvm::AllocaInst>(value)->getAlignment(); + return Address(value, CharUnits::fromQuantity(alignment)); + }; switch (K) { case ScalarLiteral: return RValue::get(Value); case ScalarAddress: - return RValue::get(CGF.Builder.CreateLoad(Value)); + return RValue::get(CGF.Builder.CreateLoad(getSavingAddress(Value))); case AggregateLiteral: - return RValue::getAggregate(Value); - case AggregateAddress: - return RValue::getAggregate(CGF.Builder.CreateLoad(Value)); + return RValue::getAggregate(Address(Value, CharUnits::fromQuantity(Align))); + case AggregateAddress: { + auto addr = CGF.Builder.CreateLoad(getSavingAddress(Value)); + return RValue::getAggregate(Address(addr, CharUnits::fromQuantity(Align))); + } case ComplexAddress: { - llvm::Value *real = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(nullptr, Value, 0)); - llvm::Value *imag = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(nullptr, Value, 1)); + Address address = getSavingAddress(Value); + llvm::Value *real = CGF.Builder.CreateLoad( + CGF.Builder.CreateStructGEP(address, 0, CharUnits())); + CharUnits offset = CharUnits::fromQuantity( + CGF.CGM.getDataLayout().getTypeAllocSize(real->getType())); + llvm::Value *imag = CGF.Builder.CreateLoad( + CGF.Builder.CreateStructGEP(address, 1, offset)); return RValue::getComplex(real, imag); } } @@ -96,6 +112,7 @@ RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) { /// Push an entry of the given size onto this protected-scope stack. char *EHScopeStack::allocate(size_t Size) { + Size = llvm::RoundUpToAlignment(Size, ScopeStackAlignment); if (!StartOfBuffer) { unsigned Capacity = 1024; while (Capacity < Size) Capacity *= 2; @@ -125,6 +142,10 @@ char *EHScopeStack::allocate(size_t Size) { return StartOfData; } +void EHScopeStack::deallocate(size_t Size) { + StartOfData += llvm::RoundUpToAlignment(Size, ScopeStackAlignment); +} + bool EHScopeStack::containsOnlyLifetimeMarkers( EHScopeStack::stable_iterator Old) const { for (EHScopeStack::iterator it = begin(); stabilize(it) != Old; it++) { @@ -147,26 +168,8 @@ EHScopeStack::getInnermostActiveNormalCleanup() const { return stable_end(); } -EHScopeStack::stable_iterator EHScopeStack::getInnermostActiveEHScope() const { - for (stable_iterator si = getInnermostEHScope(), se = stable_end(); - si != se; ) { - // Skip over inactive cleanups. - EHCleanupScope *cleanup = dyn_cast<EHCleanupScope>(&*find(si)); - if (cleanup && !cleanup->isActive()) { - si = cleanup->getEnclosingEHScope(); - continue; - } - - // All other scopes are always active. - return si; - } - - return stable_end(); -} - void *EHScopeStack::pushCleanup(CleanupKind Kind, size_t Size) { - assert(((Size % sizeof(void*)) == 0) && "cleanup type is misaligned"); char *Buffer = allocate(EHCleanupScope::getSizeForCleanupSize(Size)); bool IsNormalCleanup = Kind & NormalCleanup; bool IsEHCleanup = Kind & EHCleanup; @@ -194,7 +197,7 @@ void EHScopeStack::popCleanup() { EHCleanupScope &Cleanup = cast<EHCleanupScope>(*begin()); InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup(); InnermostEHScope = Cleanup.getEnclosingEHScope(); - StartOfData += Cleanup.getAllocatedSize(); + deallocate(Cleanup.getAllocatedSize()); // Destroy the cleanup. Cleanup.Destroy(); @@ -224,7 +227,7 @@ void EHScopeStack::popFilter() { assert(!empty() && "popping exception stack when not empty"); EHFilterScope &filter = cast<EHFilterScope>(*begin()); - StartOfData += EHFilterScope::getSizeForNumFilters(filter.getNumFilters()); + deallocate(EHFilterScope::getSizeForNumFilters(filter.getNumFilters())); InnermostEHScope = filter.getEnclosingEHScope(); } @@ -264,8 +267,8 @@ void EHScopeStack::popNullFixups() { void CodeGenFunction::initFullExprCleanup() { // Create a variable to decide whether the cleanup needs to be run. - llvm::AllocaInst *active - = CreateTempAlloca(Builder.getInt1Ty(), "cleanup.cond"); + Address active = CreateTempAlloca(Builder.getInt1Ty(), CharUnits::One(), + "cleanup.cond"); // Initialize it to false at a site that's guaranteed to be run // before each evaluation. @@ -276,7 +279,7 @@ void CodeGenFunction::initFullExprCleanup() { // Set that as the active flag in the cleanup. EHCleanupScope &cleanup = cast<EHCleanupScope>(*EHStack.begin()); - assert(!cleanup.getActiveFlag() && "cleanup already has active flag?"); + assert(!cleanup.hasActiveFlag() && "cleanup already has active flag?"); cleanup.setActiveFlag(active); if (cleanup.isNormalCleanup()) cleanup.setTestFlagInNormalCleanup(); @@ -285,6 +288,19 @@ void CodeGenFunction::initFullExprCleanup() { void EHScopeStack::Cleanup::anchor() {} +static void createStoreInstBefore(llvm::Value *value, Address addr, + llvm::Instruction *beforeInst) { + auto store = new llvm::StoreInst(value, addr.getPointer(), beforeInst); + store->setAlignment(addr.getAlignment().getQuantity()); +} + +static llvm::LoadInst *createLoadInstBefore(Address addr, const Twine &name, + llvm::Instruction *beforeInst) { + auto load = new llvm::LoadInst(addr.getPointer(), name, beforeInst); + load->setAlignment(addr.getAlignment().getQuantity()); + return load; +} + /// All the branch fixups on the EH stack have propagated out past the /// outermost normal cleanup; resolve them all by adding cases to the /// given switch instruction. @@ -307,9 +323,9 @@ static void ResolveAllBranchFixups(CodeGenFunction &CGF, // i.e. where there's an unresolved fixup inside a single cleanup // entry which we're currently popping. if (Fixup.OptimisticBranchBlock == nullptr) { - new llvm::StoreInst(CGF.Builder.getInt32(Fixup.DestinationIndex), - CGF.getNormalCleanupDestSlot(), - Fixup.InitialBranch); + createStoreInstBefore(CGF.Builder.getInt32(Fixup.DestinationIndex), + CGF.getNormalCleanupDestSlot(), + Fixup.InitialBranch); Fixup.InitialBranch->setSuccessor(0, CleanupEntry); } @@ -335,8 +351,8 @@ static llvm::SwitchInst *TransitionToCleanupSwitch(CodeGenFunction &CGF, if (llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Term)) { assert(Br->isUnconditional()); - llvm::LoadInst *Load = - new llvm::LoadInst(CGF.getNormalCleanupDestSlot(), "cleanup.dest", Term); + auto Load = createLoadInstBefore(CGF.getNormalCleanupDestSlot(), + "cleanup.dest", Term); llvm::SwitchInst *Switch = llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block); Br->eraseFromParent(); @@ -481,20 +497,11 @@ static llvm::BasicBlock *SimplifyCleanupEntry(CodeGenFunction &CGF, static void EmitCleanup(CodeGenFunction &CGF, EHScopeStack::Cleanup *Fn, EHScopeStack::Cleanup::Flags flags, - llvm::Value *ActiveFlag) { - // Itanium EH cleanups occur within a terminate scope. Microsoft SEH doesn't - // have this behavior, and the Microsoft C++ runtime will call terminate for - // us if the cleanup throws. - bool PushedTerminate = false; - if (flags.isForEHCleanup() && !CGF.getTarget().getCXXABI().isMicrosoft()) { - CGF.EHStack.pushTerminate(); - PushedTerminate = true; - } - + Address ActiveFlag) { // If there's an active flag, load it and skip the cleanup if it's // false. llvm::BasicBlock *ContBB = nullptr; - if (ActiveFlag) { + if (ActiveFlag.isValid()) { ContBB = CGF.createBasicBlock("cleanup.done"); llvm::BasicBlock *CleanupBB = CGF.createBasicBlock("cleanup.action"); llvm::Value *IsActive @@ -508,12 +515,8 @@ static void EmitCleanup(CodeGenFunction &CGF, assert(CGF.HaveInsertPoint() && "cleanup ended with no insertion point?"); // Emit the continuation block if there was an active flag. - if (ActiveFlag) + if (ActiveFlag.isValid()) CGF.EmitBlock(ContBB); - - // Leave the terminate scope. - if (PushedTerminate) - CGF.EHStack.popTerminate(); } static void ForwardPrebranchedFallthrough(llvm::BasicBlock *Exit, @@ -588,10 +591,12 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { // Remember activation information. bool IsActive = Scope.isActive(); - llvm::Value *NormalActiveFlag = - Scope.shouldTestFlagInNormalCleanup() ? Scope.getActiveFlag() : nullptr; - llvm::Value *EHActiveFlag = - Scope.shouldTestFlagInEHCleanup() ? Scope.getActiveFlag() : nullptr; + Address NormalActiveFlag = + Scope.shouldTestFlagInNormalCleanup() ? Scope.getActiveFlag() + : Address::invalid(); + Address EHActiveFlag = + Scope.shouldTestFlagInEHCleanup() ? Scope.getActiveFlag() + : Address::invalid(); // Check whether we need an EH cleanup. This is only true if we've // generated a lazy EH cleanup block. @@ -671,16 +676,25 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { return; } - // Copy the cleanup emission data out. Note that SmallVector - // guarantees maximal alignment for its buffer regardless of its - // type parameter. - SmallVector<char, 8*sizeof(void*)> CleanupBuffer; - CleanupBuffer.reserve(Scope.getCleanupSize()); - memcpy(CleanupBuffer.data(), - Scope.getCleanupBuffer(), Scope.getCleanupSize()); - CleanupBuffer.set_size(Scope.getCleanupSize()); - EHScopeStack::Cleanup *Fn = - reinterpret_cast<EHScopeStack::Cleanup*>(CleanupBuffer.data()); + // Copy the cleanup emission data out. This uses either a stack + // array or malloc'd memory, depending on the size, which is + // behavior that SmallVector would provide, if we could use it + // here. Unfortunately, if you ask for a SmallVector<char>, the + // alignment isn't sufficient. + auto *CleanupSource = reinterpret_cast<char *>(Scope.getCleanupBuffer()); + llvm::AlignedCharArray<EHScopeStack::ScopeStackAlignment, 8 * sizeof(void *)> CleanupBufferStack; + std::unique_ptr<char[]> CleanupBufferHeap; + size_t CleanupSize = Scope.getCleanupSize(); + EHScopeStack::Cleanup *Fn; + + if (CleanupSize <= sizeof(CleanupBufferStack)) { + memcpy(CleanupBufferStack.buffer, CleanupSource, CleanupSize); + Fn = reinterpret_cast<EHScopeStack::Cleanup *>(CleanupBufferStack.buffer); + } else { + CleanupBufferHeap.reset(new char[CleanupSize]); + memcpy(CleanupBufferHeap.get(), CleanupSource, CleanupSize); + Fn = reinterpret_cast<EHScopeStack::Cleanup *>(CleanupBufferHeap.get()); + } EHScopeStack::Cleanup::Flags cleanupFlags; if (Scope.isNormalCleanup()) @@ -761,7 +775,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { // Clean up the possibly dead store to the cleanup dest slot. llvm::Instruction *NormalCleanupDestSlot = - cast<llvm::Instruction>(getNormalCleanupDestSlot()); + cast<llvm::Instruction>(getNormalCleanupDestSlot().getPointer()); if (NormalCleanupDestSlot->hasOneUse()) { NormalCleanupDestSlot->user_back()->eraseFromParent(); NormalCleanupDestSlot->eraseFromParent(); @@ -787,7 +801,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { const unsigned SwitchCapacity = 10; llvm::LoadInst *Load = - new llvm::LoadInst(getNormalCleanupDestSlot(), "cleanup.dest"); + createLoadInstBefore(getNormalCleanupDestSlot(), "cleanup.dest", + nullptr); llvm::SwitchInst *Switch = llvm::SwitchInst::Create(Load, Default, SwitchCapacity); @@ -833,9 +848,9 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { BranchFixup &Fixup = EHStack.getBranchFixup(I); if (!Fixup.Destination) continue; if (!Fixup.OptimisticBranchBlock) { - new llvm::StoreInst(Builder.getInt32(Fixup.DestinationIndex), - getNormalCleanupDestSlot(), - Fixup.InitialBranch); + createStoreInstBefore(Builder.getInt32(Fixup.DestinationIndex), + getNormalCleanupDestSlot(), + Fixup.InitialBranch); Fixup.InitialBranch->setSuccessor(0, NormalEntry); } Fixup.OptimisticBranchBlock = NormalExit; @@ -893,15 +908,40 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { EmitBlock(EHEntry); + llvm::BasicBlock *NextAction = getEHDispatchBlock(EHParent); + + // Push a terminate scope or cleanupendpad scope around the potentially + // throwing cleanups. For funclet EH personalities, the cleanupendpad models + // program termination when cleanups throw. + bool PushedTerminate = false; + SaveAndRestore<llvm::Instruction *> RestoreCurrentFuncletPad( + CurrentFuncletPad); + llvm::CleanupPadInst *CPI = nullptr; + if (!EHPersonality::get(*this).usesFuncletPads()) { + EHStack.pushTerminate(); + PushedTerminate = true; + } else { + llvm::Value *ParentPad = CurrentFuncletPad; + if (!ParentPad) + ParentPad = llvm::ConstantTokenNone::get(CGM.getLLVMContext()); + CurrentFuncletPad = CPI = Builder.CreateCleanupPad(ParentPad); + } + // We only actually emit the cleanup code if the cleanup is either // active or was used before it was deactivated. - if (EHActiveFlag || IsActive) { - + if (EHActiveFlag.isValid() || IsActive) { cleanupFlags.setIsForEHCleanup(); EmitCleanup(*this, Fn, cleanupFlags, EHActiveFlag); } - Builder.CreateBr(getEHDispatchBlock(EHParent)); + if (CPI) + Builder.CreateCleanupRet(CPI, NextAction); + else + Builder.CreateBr(NextAction); + + // Leave the terminate scope. + if (PushedTerminate) + EHStack.popTerminate(); Builder.restoreIP(SavedIP); @@ -977,7 +1017,7 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) { // Store the index at the start. llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex()); - new llvm::StoreInst(Index, getNormalCleanupDestSlot(), BI); + createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI); // Adjust BI to point to the first cleanup block. { @@ -1096,23 +1136,24 @@ static void SetupCleanupBlockActivation(CodeGenFunction &CGF, // If it hasn't yet been used as either, we're done. if (!needFlag) return; - llvm::AllocaInst *var = Scope.getActiveFlag(); - if (!var) { - var = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(), "cleanup.isactive"); + Address var = Scope.getActiveFlag(); + if (!var.isValid()) { + var = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(), CharUnits::One(), + "cleanup.isactive"); Scope.setActiveFlag(var); assert(dominatingIP && "no existing variable and no dominating IP!"); // Initialize to true or false depending on whether it was // active up to this point. - llvm::Value *value = CGF.Builder.getInt1(kind == ForDeactivation); + llvm::Constant *value = CGF.Builder.getInt1(kind == ForDeactivation); // If we're in a conditional block, ignore the dominating IP and // use the outermost conditional branch. if (CGF.isInConditionalBranch()) { CGF.setBeforeOutermostConditional(value, var); } else { - new llvm::StoreInst(value, var, dominatingIP); + createStoreInstBefore(value, var, dominatingIP); } } @@ -1154,17 +1195,17 @@ void CodeGenFunction::DeactivateCleanupBlock(EHScopeStack::stable_iterator C, Scope.setActive(false); } -llvm::Value *CodeGenFunction::getNormalCleanupDestSlot() { +Address CodeGenFunction::getNormalCleanupDestSlot() { if (!NormalCleanupDest) NormalCleanupDest = CreateTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot"); - return NormalCleanupDest; + return Address(NormalCleanupDest, CharUnits::fromQuantity(4)); } /// Emits all the code to cause the given temporary to be cleaned up. void CodeGenFunction::EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType, - llvm::Value *Ptr) { + Address Ptr) { pushDestroy(NormalAndEHCleanup, Ptr, TempType, destroyCXXObject, /*useEHCleanup*/ true); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h index 81c6412..909f00b 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h @@ -15,6 +15,8 @@ #define LLVM_CLANG_LIB_CODEGEN_CGCLEANUP_H #include "EHScopeStack.h" + +#include "Address.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" @@ -26,7 +28,17 @@ class AllocaInst; } namespace clang { +class FunctionDecl; namespace CodeGen { +class CodeGenModule; +class CodeGenFunction; + +/// The MS C++ ABI needs a pointer to RTTI data plus some flags to describe the +/// type of a catch handler, so we use this wrapper. +struct CatchTypeInfo { + llvm::Constant *RTTI; + unsigned Flags; +}; /// A protected scope for zero-cost EH handling. class EHScope { @@ -37,9 +49,9 @@ class EHScope { class CommonBitFields { friend class EHScope; - unsigned Kind : 2; + unsigned Kind : 3; }; - enum { NumCommonBits = 2 }; + enum { NumCommonBits = 3 }; protected: class CatchBitFields { @@ -78,7 +90,7 @@ protected: /// The number of fixups required by enclosing scopes (not including /// this one). If this is the top cleanup scope, all the fixups /// from this index onwards belong to this scope. - unsigned FixupDepth : 32 - 18 - NumCommonBits; // currently 13 + unsigned FixupDepth : 32 - 18 - NumCommonBits; // currently 12 }; class FilterBitFields { @@ -96,7 +108,7 @@ protected: }; public: - enum Kind { Cleanup, Catch, Terminate, Filter }; + enum Kind { Cleanup, Catch, Terminate, Filter, PadEnd }; EHScope(Kind kind, EHScopeStack::stable_iterator enclosingEHScope) : CachedLandingPad(nullptr), CachedEHDispatchBlock(nullptr), @@ -148,12 +160,12 @@ public: struct Handler { /// A type info value, or null (C++ null, not an LLVM null pointer) /// for a catch-all. - llvm::Constant *Type; + CatchTypeInfo Type; /// The catch handler for this type. llvm::BasicBlock *Block; - bool isCatchAll() const { return Type == nullptr; } + bool isCatchAll() const { return Type.RTTI == nullptr; } }; private: @@ -183,11 +195,17 @@ public: } void setCatchAllHandler(unsigned I, llvm::BasicBlock *Block) { - setHandler(I, /*catchall*/ nullptr, Block); + setHandler(I, CatchTypeInfo{nullptr, 0}, Block); } void setHandler(unsigned I, llvm::Constant *Type, llvm::BasicBlock *Block) { assert(I < getNumHandlers()); + getHandlers()[I].Type = CatchTypeInfo{Type, 0}; + getHandlers()[I].Block = Block; + } + + void setHandler(unsigned I, CatchTypeInfo Type, llvm::BasicBlock *Block) { + assert(I < getNumHandlers()); getHandlers()[I].Type = Type; getHandlers()[I].Block = Block; } @@ -216,7 +234,7 @@ public: }; /// A cleanup scope which generates the cleanup blocks lazily. -class EHCleanupScope : public EHScope { +class LLVM_ALIGNAS(/*alignof(uint64_t)*/ 8) EHCleanupScope : public EHScope { /// The nearest normal cleanup scope enclosing this one. EHScopeStack::stable_iterator EnclosingNormal; @@ -302,8 +320,14 @@ public: bool isLifetimeMarker() const { return CleanupBits.IsLifetimeMarker; } void setLifetimeMarker() { CleanupBits.IsLifetimeMarker = true; } - llvm::AllocaInst *getActiveFlag() const { return ActiveFlag; } - void setActiveFlag(llvm::AllocaInst *Var) { ActiveFlag = Var; } + bool hasActiveFlag() const { return ActiveFlag != nullptr; } + Address getActiveFlag() const { + return Address(ActiveFlag, CharUnits::One()); + } + void setActiveFlag(Address Var) { + assert(Var.getAlignment().isOne()); + ActiveFlag = cast<llvm::AllocaInst>(Var.getPointer()); + } void setTestFlagInNormalCleanup() { CleanupBits.TestFlagInNormalCleanup = true; @@ -396,6 +420,15 @@ public: return (Scope->getKind() == Cleanup); } }; +// NOTE: there's a bunch of different data classes tacked on after an +// EHCleanupScope. It is asserted (in EHScopeStack::pushCleanup*) that +// they don't require greater alignment than ScopeStackAlignment. So, +// EHCleanupScope ought to have alignment equal to that -- not more +// (would be misaligned by the stack allocator), and not less (would +// break the appended classes). +static_assert(llvm::AlignOf<EHCleanupScope>::Alignment == + EHScopeStack::ScopeStackAlignment, + "EHCleanupScope expected alignment"); /// An exceptions scope which filters exceptions thrown through it. /// Only exceptions matching the filter types will be permitted to be @@ -454,6 +487,17 @@ public: } }; +class EHPadEndScope : public EHScope { +public: + EHPadEndScope(EHScopeStack::stable_iterator enclosingEHScope) + : EHScope(PadEnd, enclosingEHScope) {} + static size_t getSize() { return sizeof(EHPadEndScope); } + + static bool classof(const EHScope *scope) { + return scope->getKind() == PadEnd; + } +}; + /// A non-stable pointer into the scope stack. class EHScopeStack::iterator { char *Ptr; @@ -472,27 +516,31 @@ public: EHScope &operator*() const { return *get(); } iterator &operator++() { + size_t Size; switch (get()->getKind()) { case EHScope::Catch: - Ptr += EHCatchScope::getSizeForNumHandlers( - static_cast<const EHCatchScope*>(get())->getNumHandlers()); + Size = EHCatchScope::getSizeForNumHandlers( + static_cast<const EHCatchScope *>(get())->getNumHandlers()); break; case EHScope::Filter: - Ptr += EHFilterScope::getSizeForNumFilters( - static_cast<const EHFilterScope*>(get())->getNumFilters()); + Size = EHFilterScope::getSizeForNumFilters( + static_cast<const EHFilterScope *>(get())->getNumFilters()); break; case EHScope::Cleanup: - Ptr += static_cast<const EHCleanupScope*>(get()) - ->getAllocatedSize(); + Size = static_cast<const EHCleanupScope *>(get())->getAllocatedSize(); break; case EHScope::Terminate: - Ptr += EHTerminateScope::getSize(); + Size = EHTerminateScope::getSize(); break; - } + case EHScope::PadEnd: + Size = EHPadEndScope::getSize(); + break; + } + Ptr += llvm::RoundUpToAlignment(Size, ScopeStackAlignment); return *this; } @@ -528,7 +576,7 @@ inline void EHScopeStack::popCatch() { EHCatchScope &scope = cast<EHCatchScope>(*begin()); InnermostEHScope = scope.getEnclosingEHScope(); - StartOfData += EHCatchScope::getSizeForNumHandlers(scope.getNumHandlers()); + deallocate(EHCatchScope::getSizeForNumHandlers(scope.getNumHandlers())); } inline void EHScopeStack::popTerminate() { @@ -536,7 +584,7 @@ inline void EHScopeStack::popTerminate() { EHTerminateScope &scope = cast<EHTerminateScope>(*begin()); InnermostEHScope = scope.getEnclosingEHScope(); - StartOfData += EHTerminateScope::getSize(); + deallocate(EHTerminateScope::getSize()); } inline EHScopeStack::iterator EHScopeStack::find(stable_iterator sp) const { @@ -551,6 +599,43 @@ EHScopeStack::stabilize(iterator ir) const { return stable_iterator(EndOfBuffer - ir.Ptr); } +/// The exceptions personality for a function. +struct EHPersonality { + const char *PersonalityFn; + + // If this is non-null, this personality requires a non-standard + // function for rethrowing an exception after a catchall cleanup. + // This function must have prototype void(void*). + const char *CatchallRethrowFn; + + static const EHPersonality &get(CodeGenModule &CGM, const FunctionDecl *FD); + static const EHPersonality &get(CodeGenFunction &CGF); + + static const EHPersonality GNU_C; + static const EHPersonality GNU_C_SJLJ; + static const EHPersonality GNU_C_SEH; + static const EHPersonality GNU_ObjC; + static const EHPersonality GNUstep_ObjC; + static const EHPersonality GNU_ObjCXX; + static const EHPersonality NeXT_ObjC; + static const EHPersonality GNU_CPlusPlus; + static const EHPersonality GNU_CPlusPlus_SJLJ; + static const EHPersonality GNU_CPlusPlus_SEH; + static const EHPersonality MSVC_except_handler; + static const EHPersonality MSVC_C_specific_handler; + static const EHPersonality MSVC_CxxFrameHandler3; + + /// Does this personality use landingpads or the family of pad instructions + /// designed to form funclets? + bool usesFuncletPads() const { return isMSVCPersonality(); } + + bool isMSVCPersonality() const { + return this == &MSVC_except_handler || this == &MSVC_C_specific_handler || + this == &MSVC_CxxFrameHandler3; + } + + bool isMSVCXXPersonality() const { return this == &MSVC_CxxFrameHandler3; } +}; } } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp index 93a2287..78e3978 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp @@ -28,6 +28,7 @@ #include "clang/Basic/Version.h" #include "clang/Frontend/CodeGenOptions.h" #include "clang/Lex/HeaderSearchOptions.h" +#include "clang/Lex/ModuleMap.h" #include "clang/Lex/PreprocessorOptions.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" @@ -37,7 +38,6 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Module.h" -#include "llvm/Support/Dwarf.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" using namespace clang; @@ -45,7 +45,10 @@ using namespace clang::CodeGen; CGDebugInfo::CGDebugInfo(CodeGenModule &CGM) : CGM(CGM), DebugKind(CGM.getCodeGenOpts().getDebugInfo()), + DebugTypeExtRefs(CGM.getCodeGenOpts().DebugTypeExtRefs), DBuilder(CGM.getModule()) { + for (const auto &KV : CGM.getCodeGenOpts().DebugPrefixMap) + DebugPrefixMap[KV.first] = KV.second; CreateCompileUnit(); } @@ -56,54 +59,63 @@ CGDebugInfo::~CGDebugInfo() { ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, SourceLocation TemporaryLocation) - : CGF(CGF) { + : CGF(&CGF) { init(TemporaryLocation); } ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, bool DefaultToEmpty, SourceLocation TemporaryLocation) - : CGF(CGF) { + : CGF(&CGF) { init(TemporaryLocation, DefaultToEmpty); } void ApplyDebugLocation::init(SourceLocation TemporaryLocation, bool DefaultToEmpty) { - if (auto *DI = CGF.getDebugInfo()) { - OriginalLocation = CGF.Builder.getCurrentDebugLocation(); - if (TemporaryLocation.isInvalid()) { - if (DefaultToEmpty) - CGF.Builder.SetCurrentDebugLocation(llvm::DebugLoc()); - else { - // Construct a location that has a valid scope, but no line info. - assert(!DI->LexicalBlockStack.empty()); - CGF.Builder.SetCurrentDebugLocation( - llvm::DebugLoc::get(0, 0, DI->LexicalBlockStack.back())); - } - } else - DI->EmitLocation(CGF.Builder, TemporaryLocation); + auto *DI = CGF->getDebugInfo(); + if (!DI) { + CGF = nullptr; + return; + } + + OriginalLocation = CGF->Builder.getCurrentDebugLocation(); + if (TemporaryLocation.isValid()) { + DI->EmitLocation(CGF->Builder, TemporaryLocation); + return; } + + if (DefaultToEmpty) { + CGF->Builder.SetCurrentDebugLocation(llvm::DebugLoc()); + return; + } + + // Construct a location that has a valid scope, but no line info. + assert(!DI->LexicalBlockStack.empty()); + CGF->Builder.SetCurrentDebugLocation( + llvm::DebugLoc::get(0, 0, DI->LexicalBlockStack.back())); } ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, const Expr *E) - : CGF(CGF) { + : CGF(&CGF) { init(E->getExprLoc()); } ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, llvm::DebugLoc Loc) - : CGF(CGF) { - if (CGF.getDebugInfo()) { - OriginalLocation = CGF.Builder.getCurrentDebugLocation(); - if (Loc) - CGF.Builder.SetCurrentDebugLocation(std::move(Loc)); + : CGF(&CGF) { + if (!CGF.getDebugInfo()) { + this->CGF = nullptr; + return; } + OriginalLocation = CGF.Builder.getCurrentDebugLocation(); + if (Loc) + CGF.Builder.SetCurrentDebugLocation(std::move(Loc)); } ApplyDebugLocation::~ApplyDebugLocation() { // Query CGF so the location isn't overwritten when location updates are // temporarily disabled (for C++ default function arguments) - if (CGF.getDebugInfo()) - CGF.Builder.SetCurrentDebugLocation(std::move(OriginalLocation)); + if (CGF) + CGF->Builder.SetCurrentDebugLocation(std::move(OriginalLocation)); } void CGDebugInfo::setLocation(SourceLocation Loc) { @@ -138,9 +150,16 @@ void CGDebugInfo::setLocation(SourceLocation Loc) { } } -llvm::DIScope *CGDebugInfo::getContextDescriptor(const Decl *Context) { +llvm::DIScope *CGDebugInfo::getDeclContextDescriptor(const Decl *D) { + llvm::DIScope *Mod = getParentModuleOrNull(D); + return getContextDescriptor(cast<Decl>(D->getDeclContext()), + Mod ? Mod : TheCU); +} + +llvm::DIScope *CGDebugInfo::getContextDescriptor(const Decl *Context, + llvm::DIScope *Default) { if (!Context) - return TheCU; + return Default; auto I = RegionMap.find(Context); if (I != RegionMap.end()) { @@ -156,7 +175,7 @@ llvm::DIScope *CGDebugInfo::getContextDescriptor(const Decl *Context) { if (!RDecl->isDependentType()) return getOrCreateType(CGM.getContext().getTypeDeclType(RDecl), getOrCreateMainFile()); - return TheCU; + return Default; } StringRef CGDebugInfo::getFunctionName(const FunctionDecl *FD) { @@ -164,22 +183,31 @@ StringRef CGDebugInfo::getFunctionName(const FunctionDecl *FD) { IdentifierInfo *FII = FD->getIdentifier(); FunctionTemplateSpecializationInfo *Info = FD->getTemplateSpecializationInfo(); - if (!Info && FII) + + if (!Info && FII && !CGM.getCodeGenOpts().EmitCodeView) return FII->getName(); // Otherwise construct human readable name for debug info. SmallString<128> NS; llvm::raw_svector_ostream OS(NS); - FD->printName(OS); + PrintingPolicy Policy(CGM.getLangOpts()); - // Add any template specialization args. - if (Info) { - const TemplateArgumentList *TArgs = Info->TemplateArguments; - const TemplateArgument *Args = TArgs->data(); - unsigned NumArgs = TArgs->size(); - PrintingPolicy Policy(CGM.getLangOpts()); - TemplateSpecializationType::PrintTemplateArgumentList(OS, Args, NumArgs, - Policy); + if (CGM.getCodeGenOpts().EmitCodeView) { + // Print a fully qualified name like MSVC would. + Policy.MSVCFormatting = true; + FD->printQualifiedName(OS, Policy); + } else { + // Print the unqualified name with some template arguments. This is what + // DWARF-based debuggers expect. + FD->printName(OS); + // Add any template specialization args. + if (Info) { + const TemplateArgumentList *TArgs = Info->TemplateArguments; + const TemplateArgument *Args = TArgs->data(); + unsigned NumArgs = TArgs->size(); + TemplateSpecializationType::PrintTemplateArgumentList(OS, Args, NumArgs, + Policy); + } } // Copy this name on the side and use its reference. @@ -197,6 +225,13 @@ StringRef CGDebugInfo::getObjCMethodName(const ObjCMethodDecl *OMD) { } else if (const ObjCInterfaceDecl *OID = dyn_cast<const ObjCInterfaceDecl>(DC)) { OS << OID->getName(); + } else if (const ObjCCategoryDecl *OC = dyn_cast<ObjCCategoryDecl>(DC)) { + if (OC->IsClassExtension()) { + OS << OC->getClassInterface()->getName(); + } else { + OS << ((const NamedDecl *)OC)->getIdentifier()->getNameStart() << '(' + << OC->getIdentifier()->getNameStart() << ')'; + } } else if (const ObjCCategoryImplDecl *OCD = dyn_cast<const ObjCCategoryImplDecl>(DC)) { OS << ((const NamedDecl *)OCD)->getIdentifier()->getNameStart() << '(' @@ -238,14 +273,16 @@ StringRef CGDebugInfo::getClassName(const RecordDecl *RD) { llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) { if (!Loc.isValid()) // If Location is not valid then use main input file. - return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory()); + return DBuilder.createFile(remapDIPath(TheCU->getFilename()), + remapDIPath(TheCU->getDirectory())); SourceManager &SM = CGM.getContext().getSourceManager(); PresumedLoc PLoc = SM.getPresumedLoc(Loc); if (PLoc.isInvalid() || StringRef(PLoc.getFilename()).empty()) // If the location is not valid then use main input file. - return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory()); + return DBuilder.createFile(remapDIPath(TheCU->getFilename()), + remapDIPath(TheCU->getDirectory())); // Cache the results. const char *fname = PLoc.getFilename(); @@ -257,15 +294,23 @@ llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) { return cast<llvm::DIFile>(V); } - llvm::DIFile *F = - DBuilder.createFile(PLoc.getFilename(), getCurrentDirname()); + llvm::DIFile *F = DBuilder.createFile(remapDIPath(PLoc.getFilename()), + remapDIPath(getCurrentDirname())); DIFileCache[fname].reset(F); return F; } llvm::DIFile *CGDebugInfo::getOrCreateMainFile() { - return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory()); + return DBuilder.createFile(remapDIPath(TheCU->getFilename()), + remapDIPath(TheCU->getDirectory())); +} + +std::string CGDebugInfo::remapDIPath(StringRef Path) const { + for (const auto &Entry : DebugPrefixMap) + if (Path.startswith(Entry.first)) + return (Twine(Entry.second) + Path.substr(Entry.first.size())).str(); + return Path.str(); } unsigned CGDebugInfo::getLineNumber(SourceLocation Loc) { @@ -321,7 +366,7 @@ void CGDebugInfo::CreateCompileUnit() { // file to determine the real absolute path for the file. std::string MainFileDir; if (const FileEntry *MainFile = SM.getFileEntryForID(SM.getMainFileID())) { - MainFileDir = MainFile->getDir()->getName(); + MainFileDir = remapDIPath(MainFile->getDir()->getName()); if (MainFileDir != ".") { llvm::SmallString<1024> MainFileDirSS(MainFileDir); llvm::sys::path::append(MainFileDirSS, MainFileName); @@ -329,13 +374,6 @@ void CGDebugInfo::CreateCompileUnit() { } } - // Save filename string. - StringRef Filename = internString(MainFileName); - - // Save split dwarf file string. - std::string SplitDwarfFile = CGM.getCodeGenOpts().SplitDwarfFile; - StringRef SplitDwarfFilename = internString(SplitDwarfFile); - llvm::dwarf::SourceLanguage LangTag; const LangOptions &LO = CGM.getLangOpts(); if (LO.CPlusPlus) { @@ -361,13 +399,13 @@ void CGDebugInfo::CreateCompileUnit() { // Create new compile unit. // FIXME - Eliminate TheCU. TheCU = DBuilder.createCompileUnit( - LangTag, Filename, getCurrentDirname(), Producer, LO.Optimize, - CGM.getCodeGenOpts().DwarfDebugFlags, RuntimeVers, SplitDwarfFilename, + LangTag, remapDIPath(MainFileName), remapDIPath(getCurrentDirname()), + Producer, LO.Optimize, CGM.getCodeGenOpts().DwarfDebugFlags, RuntimeVers, + CGM.getCodeGenOpts().SplitDwarfFile, DebugKind <= CodeGenOptions::DebugLineTablesOnly ? llvm::DIBuilder::LineTablesOnly : llvm::DIBuilder::FullDebug, - 0 /* DWOid */, - DebugKind != CodeGenOptions::LocTrackingOnly); + 0 /* DWOid */, DebugKind != CodeGenOptions::LocTrackingOnly); } llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { @@ -438,6 +476,24 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { case BuiltinType::OCLImage2dArray: return getOrCreateStructPtrType("opencl_image2d_array_t", OCLImage2dArrayDITy); + case BuiltinType::OCLImage2dDepth: + return getOrCreateStructPtrType("opencl_image2d_depth_t", + OCLImage2dDepthDITy); + case BuiltinType::OCLImage2dArrayDepth: + return getOrCreateStructPtrType("opencl_image2d_array_depth_t", + OCLImage2dArrayDepthDITy); + case BuiltinType::OCLImage2dMSAA: + return getOrCreateStructPtrType("opencl_image2d_msaa_t", + OCLImage2dMSAADITy); + case BuiltinType::OCLImage2dArrayMSAA: + return getOrCreateStructPtrType("opencl_image2d_array_msaa_t", + OCLImage2dArrayMSAADITy); + case BuiltinType::OCLImage2dMSAADepth: + return getOrCreateStructPtrType("opencl_image2d_msaa_depth_t", + OCLImage2dMSAADepthDITy); + case BuiltinType::OCLImage2dArrayMSAADepth: + return getOrCreateStructPtrType("opencl_image2d_array_msaa_depth_t", + OCLImage2dArrayMSAADepthDITy); case BuiltinType::OCLImage3d: return getOrCreateStructPtrType("opencl_image3d_t", OCLImage3dDITy); case BuiltinType::OCLSampler: @@ -446,6 +502,14 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { CGM.getContext().getTypeAlign(BT), llvm::dwarf::DW_ATE_unsigned); case BuiltinType::OCLEvent: return getOrCreateStructPtrType("opencl_event_t", OCLEventDITy); + case BuiltinType::OCLClkEvent: + return getOrCreateStructPtrType("opencl_clk_event_t", OCLClkEventDITy); + case BuiltinType::OCLQueue: + return getOrCreateStructPtrType("opencl_queue_t", OCLQueueDITy); + case BuiltinType::OCLNDRange: + return getOrCreateStructPtrType("opencl_ndrange_t", OCLNDRangeDITy); + case BuiltinType::OCLReserveID: + return getOrCreateStructPtrType("opencl_reserve_id_t", OCLReserveIDDITy); case BuiltinType::UChar: case BuiltinType::Char_U: @@ -604,7 +668,6 @@ static SmallString<256> getUniqueTagTypeName(const TagType *Ty, // a unique string for a type? llvm::raw_svector_ostream Out(FullName); CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(QualType(Ty, 0), Out); - Out.flush(); return FullName; } @@ -658,10 +721,6 @@ llvm::DIType *CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag, const Type *Ty, QualType PointeeTy, llvm::DIFile *Unit) { - if (Tag == llvm::dwarf::DW_TAG_reference_type || - Tag == llvm::dwarf::DW_TAG_rvalue_reference_type) - return DBuilder.createReferenceType(Tag, getOrCreateType(PointeeTy, Unit)); - // Bit size, align and offset of the type. // Size is always the size of a pointer. We can't use getTypeSize here // because that does not return the correct value for references. @@ -669,8 +728,13 @@ llvm::DIType *CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag, uint64_t Size = CGM.getTarget().getPointerWidth(AS); uint64_t Align = CGM.getContext().getTypeAlign(Ty); - return DBuilder.createPointerType(getOrCreateType(PointeeTy, Unit), Size, - Align); + if (Tag == llvm::dwarf::DW_TAG_reference_type || + Tag == llvm::dwarf::DW_TAG_rvalue_reference_type) + return DBuilder.createReferenceType(Tag, getOrCreateType(PointeeTy, Unit), + Size, Align); + else + return DBuilder.createPointerType(getOrCreateType(PointeeTy, Unit), Size, + Align); } llvm::DIType *CGDebugInfo::getOrCreateStructPtrType(StringRef Name, @@ -760,9 +824,9 @@ llvm::DIType *CGDebugInfo::CreateType(const TemplateSpecializationType *Ty, Ty->getTemplateName().getAsTemplateDecl())->getTemplatedDecl(); SourceLocation Loc = AliasDecl->getLocation(); - return DBuilder.createTypedef( - Src, internString(OS.str()), getOrCreateFile(Loc), getLineNumber(Loc), - getContextDescriptor(cast<Decl>(AliasDecl->getDeclContext()))); + return DBuilder.createTypedef(Src, OS.str(), getOrCreateFile(Loc), + getLineNumber(Loc), + getDeclContextDescriptor(AliasDecl)); } llvm::DIType *CGDebugInfo::CreateType(const TypedefType *Ty, @@ -775,7 +839,7 @@ llvm::DIType *CGDebugInfo::CreateType(const TypedefType *Ty, return DBuilder.createTypedef( getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit), Ty->getDecl()->getName(), getOrCreateFile(Loc), getLineNumber(Loc), - getContextDescriptor(cast<Decl>(Ty->getDecl()->getDeclContext()))); + getDeclContextDescriptor(Ty->getDecl())); } llvm::DIType *CGDebugInfo::CreateType(const FunctionType *Ty, @@ -797,7 +861,7 @@ llvm::DIType *CGDebugInfo::CreateType(const FunctionType *Ty, } llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); - return DBuilder.createSubroutineType(Unit, EltTypeArray); + return DBuilder.createSubroutineType(EltTypeArray); } /// Convert an AccessSpecifier into the corresponding DINode flag. @@ -972,7 +1036,7 @@ void CGDebugInfo::CollectRecordFields( if (MI != StaticDataMemberCache.end()) { assert(MI->second && "Static data member declaration should still exist"); - elements.push_back(cast<llvm::DIDerivedTypeBase>(MI->second)); + elements.push_back(MI->second); } else { auto Field = CreateRecordStaticField(V, RecordTy, record); elements.push_back(Field); @@ -1048,7 +1112,7 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType( if (Func->getExtProtoInfo().RefQualifier == RQ_RValue) Flags |= llvm::DINode::FlagRValueReference; - return DBuilder.createSubroutineType(Unit, EltTypeArray, Flags); + return DBuilder.createSubroutineType(EltTypeArray, Flags); } /// isFunctionLocalClass - Return true if CXXRecordDecl is defined @@ -1129,7 +1193,7 @@ llvm::DISubprogram *CGDebugInfo::CreateCXXMemberFunction( RecordTy, MethodName, MethodLinkageName, MethodDefUnit, MethodLine, MethodTy, /*isLocalToUnit=*/false, /* isDefinition=*/false, Virtuality, VIndex, ContainingType, Flags, - CGM.getLangOpts().Optimize, nullptr, TParamsArray.get()); + CGM.getLangOpts().Optimize, TParamsArray.get()); SPCache[Method->getCanonicalDecl()].reset(SP); @@ -1348,7 +1412,7 @@ llvm::DIType *CGDebugInfo::getOrCreateVTablePtrType(llvm::DIFile *Unit) { /* Function type */ llvm::Metadata *STy = getOrCreateType(Context.IntTy, Unit); llvm::DITypeRefArray SElements = DBuilder.getOrCreateTypeArray(STy); - llvm::DIType *SubTy = DBuilder.createSubroutineType(Unit, SElements); + llvm::DIType *SubTy = DBuilder.createSubroutineType(SElements); unsigned Size = Context.getTypeSize(Context.VoidPtrTy); llvm::DIType *vtbl_ptr_type = DBuilder.createPointerType(SubTy, Size, 0, "__vtbl_ptr_type"); @@ -1389,8 +1453,21 @@ llvm::DIType *CGDebugInfo::getOrCreateRecordType(QualType RTy, llvm::DIType *CGDebugInfo::getOrCreateInterfaceType(QualType D, SourceLocation Loc) { + return getOrCreateStandaloneType(D, Loc); +} + +llvm::DIType *CGDebugInfo::getOrCreateStandaloneType(QualType D, + SourceLocation Loc) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); + assert(!D.isNull() && "null type"); llvm::DIType *T = getOrCreateType(D, getOrCreateFile(Loc)); + assert(T && "could not create debug info for type"); + + // Composite types with UIDs were already retained by DIBuilder + // because they are only referenced by name in the IR. + if (auto *CTy = dyn_cast<llvm::DICompositeType>(T)) + if (!CTy->getIdentifier().empty()) + return T; RetainedTypes.push_back(D.getAsOpaquePtr()); return T; } @@ -1422,6 +1499,9 @@ void CGDebugInfo::completeRequiredType(const RecordDecl *RD) { if (CXXDecl->isDynamicClass()) return; + if (DebugTypeExtRefs && RD->isFromASTFile()) + return; + QualType Ty = CGM.getContext().getRecordType(RD); llvm::DIType *T = getTypeOrNull(Ty); if (T && T->isForwardDecl()) @@ -1452,8 +1532,13 @@ static bool hasExplicitMemberDefinition(CXXRecordDecl::method_iterator I, } static bool shouldOmitDefinition(CodeGenOptions::DebugInfoKind DebugKind, + bool DebugTypeExtRefs, const RecordDecl *RD, const LangOptions &LangOpts) { + // Does the type exist in an imported clang module? + if (DebugTypeExtRefs && RD->isFromASTFile() && RD->getDefinition()) + return true; + if (DebugKind > CodeGenOptions::LimitedDebugInfo) return false; @@ -1487,10 +1572,10 @@ static bool shouldOmitDefinition(CodeGenOptions::DebugInfoKind DebugKind, llvm::DIType *CGDebugInfo::CreateType(const RecordType *Ty) { RecordDecl *RD = Ty->getDecl(); llvm::DIType *T = cast_or_null<llvm::DIType>(getTypeOrNull(QualType(Ty, 0))); - if (T || shouldOmitDefinition(DebugKind, RD, CGM.getLangOpts())) { + if (T || shouldOmitDefinition(DebugKind, DebugTypeExtRefs, RD, + CGM.getLangOpts())) { if (!T) - T = getOrCreateRecordFwdDecl( - Ty, getContextDescriptor(cast<Decl>(RD->getDeclContext()))); + T = getOrCreateRecordFwdDecl(Ty, getDeclContextDescriptor(RD)); return T; } @@ -1509,9 +1594,7 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) { // its members. Finally, we create a descriptor for the complete type (which // may refer to the forward decl if the struct is recursive) and replace all // uses of the forward declaration with the final definition. - - auto *FwdDecl = - cast<llvm::DICompositeType>(getOrCreateLimitedType(Ty, DefUnit)); + llvm::DICompositeType *FwdDecl = getOrCreateLimitedType(Ty, DefUnit); const RecordDecl *D = RD->getDefinition(); if (!D || !D->isCompleteDefinition()) @@ -1593,6 +1676,12 @@ llvm::DIType *CGDebugInfo::CreateType(const ObjCInterfaceType *Ty, if (!ID) return nullptr; + // Return a forward declaration if this type was imported from a clang module. + if (DebugTypeExtRefs && ID->isFromASTFile() && ID->getDefinition()) + return DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type, + ID->getName(), + getDeclContextDescriptor(ID), Unit, 0); + // Get overall information about the record type for the debug info. llvm::DIFile *DefUnit = getOrCreateFile(ID->getLocation()); unsigned Line = getLineNumber(ID->getLocation()); @@ -1603,9 +1692,10 @@ llvm::DIType *CGDebugInfo::CreateType(const ObjCInterfaceType *Ty, // debug type since we won't be able to lay out the entire type. ObjCInterfaceDecl *Def = ID->getDefinition(); if (!Def || !Def->getImplementation()) { + llvm::DIScope *Mod = getParentModuleOrNull(ID); llvm::DIType *FwdDecl = DBuilder.createReplaceableCompositeType( - llvm::dwarf::DW_TAG_structure_type, ID->getName(), TheCU, DefUnit, Line, - RuntimeLang); + llvm::dwarf::DW_TAG_structure_type, ID->getName(), Mod ? Mod : TheCU, + DefUnit, Line, RuntimeLang); ObjCInterfaceCache.push_back(ObjCInterfaceCacheEntry(Ty, FwdDecl, Unit)); return FwdDecl; } @@ -1614,10 +1704,15 @@ llvm::DIType *CGDebugInfo::CreateType(const ObjCInterfaceType *Ty, } llvm::DIModule * -CGDebugInfo::getOrCreateModuleRef(ExternalASTSource::ASTSourceDescriptor Mod) { - auto it = ModuleRefCache.find(Mod.Signature); - if (it != ModuleRefCache.end()) - return it->second; +CGDebugInfo::getOrCreateModuleRef(ExternalASTSource::ASTSourceDescriptor Mod, + bool CreateSkeletonCU) { + // Use the Module pointer as the key into the cache. This is a + // nullptr if the "Module" is a PCH, which is safe because we don't + // support chained PCH debug info, so there can only be a single PCH. + const Module *M = Mod.getModuleOrNull(); + auto ModRef = ModuleCache.find(M); + if (ModRef != ModuleCache.end()) + return cast<llvm::DIModule>(ModRef->second); // Macro definitions that were defined with "-D" on the command line. SmallString<128> ConfigMacros; @@ -1641,17 +1736,26 @@ CGDebugInfo::getOrCreateModuleRef(ExternalASTSource::ASTSourceDescriptor Mod) { OS << '\"'; } } - llvm::DIBuilder DIB(CGM.getModule()); - auto *CU = DIB.createCompileUnit( - TheCU->getSourceLanguage(), internString(Mod.ModuleName), - internString(Mod.Path), TheCU->getProducer(), true, StringRef(), 0, - internString(Mod.ASTFile), llvm::DIBuilder::FullDebug, Mod.Signature); - llvm::DIModule *ModuleRef = - DIB.createModule(CU, Mod.ModuleName, ConfigMacros, internString(Mod.Path), - internString(CGM.getHeaderSearchOpts().Sysroot)); - DIB.finalize(); - ModuleRefCache.insert(std::make_pair(Mod.Signature, ModuleRef)); - return ModuleRef; + + bool IsRootModule = M ? !M->Parent : true; + if (CreateSkeletonCU && IsRootModule) { + llvm::DIBuilder DIB(CGM.getModule()); + DIB.createCompileUnit(TheCU->getSourceLanguage(), Mod.getModuleName(), + Mod.getPath(), TheCU->getProducer(), true, + StringRef(), 0, Mod.getASTFile(), + llvm::DIBuilder::FullDebug, Mod.getSignature()); + DIB.finalize(); + } + llvm::DIModule *Parent = + IsRootModule ? nullptr + : getOrCreateModuleRef( + ExternalASTSource::ASTSourceDescriptor(*M->Parent), + CreateSkeletonCU); + llvm::DIModule *DIMod = + DBuilder.createModule(Parent, Mod.getModuleName(), ConfigMacros, + Mod.getPath(), CGM.getHeaderSearchOpts().Sysroot); + ModuleCache[M].reset(DIMod); + return DIMod; } llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, @@ -1669,9 +1773,10 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, if (ID->getImplementation()) Flags |= llvm::DINode::FlagObjcClassComplete; + llvm::DIScope *Mod = getParentModuleOrNull(ID); llvm::DICompositeType *RealDecl = DBuilder.createStructType( - Unit, ID->getName(), DefUnit, Line, Size, Align, Flags, nullptr, - llvm::DINodeArray(), RuntimeLang); + Mod ? Mod : Unit, ID->getName(), DefUnit, Line, Size, Align, Flags, + nullptr, llvm::DINodeArray(), RuntimeLang); QualType QTy(Ty, 0); TypeCache[QTy.getAsOpaquePtr()].reset(RealDecl); @@ -1695,7 +1800,7 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, } // Create entries for all of the properties. - for (const auto *PD : ID->properties()) { + auto AddProperty = [&](const ObjCPropertyDecl *PD) { SourceLocation Loc = PD->getLocation(); llvm::DIFile *PUnit = getOrCreateFile(Loc); unsigned PLine = getLineNumber(Loc); @@ -1709,6 +1814,21 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, : getSelectorName(PD->getSetterName()), PD->getPropertyAttributes(), getOrCreateType(PD->getType(), PUnit)); EltTys.push_back(PropertyNode); + }; + { + llvm::SmallPtrSet<const IdentifierInfo*, 16> PropertySet; + for (const ObjCCategoryDecl *ClassExt : ID->known_extensions()) + for (auto *PD : ClassExt->properties()) { + PropertySet.insert(PD->getIdentifier()); + AddProperty(PD); + } + for (const auto *PD : ID->properties()) { + // Don't emit duplicate metadata for properties that were already in a + // class extension. + if (!PropertySet.insert(PD->getIdentifier()).second) + continue; + AddProperty(PD); + } } const ASTRecordLayout &RL = CGM.getContext().getASTObjCInterfaceLayout(ID); @@ -1883,9 +2003,8 @@ llvm::DIType *CGDebugInfo::CreateType(const RValueReferenceType *Ty, llvm::DIType *CGDebugInfo::CreateType(const MemberPointerType *Ty, llvm::DIFile *U) { - uint64_t Size = CGM.getCXXABI().isTypeInfoCalculable(QualType(Ty, 0)) - ? CGM.getContext().getTypeSize(Ty) - : 0; + uint64_t Size = + !Ty->isIncompleteType() ? CGM.getContext().getTypeSize(Ty) : 0; llvm::DIType *ClassType = getOrCreateType(QualType(Ty->getClass(), 0), U); if (Ty->isMemberDataPointerType()) return DBuilder.createMemberPointerType( @@ -1908,6 +2027,7 @@ llvm::DIType *CGDebugInfo::CreateType(const AtomicType *Ty, llvm::DIFile *U) { llvm::DIType *CGDebugInfo::CreateEnumType(const EnumType *Ty) { const EnumDecl *ED = Ty->getDecl(); + uint64_t Size = 0; uint64_t Align = 0; if (!ED->getTypeForDecl()->isIncompleteType()) { @@ -1917,11 +2037,13 @@ llvm::DIType *CGDebugInfo::CreateEnumType(const EnumType *Ty) { SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU); + bool isImportedFromModule = + DebugTypeExtRefs && ED->isFromASTFile() && ED->getDefinition(); + // If this is just a forward declaration, construct an appropriately // marked node and just return it. - if (!ED->getDefinition()) { - llvm::DIScope *EDContext = - getContextDescriptor(cast<Decl>(ED->getDeclContext())); + if (isImportedFromModule || !ED->getDefinition()) { + llvm::DIScope *EDContext = getDeclContextDescriptor(ED); llvm::DIFile *DefUnit = getOrCreateFile(ED->getLocation()); unsigned Line = getLineNumber(ED->getLocation()); StringRef EDName = ED->getName(); @@ -1961,8 +2083,7 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) { llvm::DIFile *DefUnit = getOrCreateFile(ED->getLocation()); unsigned Line = getLineNumber(ED->getLocation()); - llvm::DIScope *EnumContext = - getContextDescriptor(cast<Decl>(ED->getDeclContext())); + llvm::DIScope *EnumContext = getDeclContextDescriptor(ED); llvm::DIType *ClassTy = ED->isFixed() ? getOrCreateType(ED->getIntegerType(), DefUnit) : nullptr; return DBuilder.createEnumerationType(EnumContext, ED->getName(), DefUnit, @@ -2061,9 +2182,8 @@ llvm::DIType *CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile *Unit) { if (auto *T = getTypeOrNull(Ty)) return T; - // Otherwise create the type. llvm::DIType *Res = CreateTypeNode(Ty, Unit); - void *TyPtr = Ty.getAsOpaquePtr(); + void* TyPtr = Ty.getAsOpaquePtr(); // And update the type cache. TypeCache[TyPtr].reset(Res); @@ -2071,28 +2191,36 @@ llvm::DIType *CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile *Unit) { return Res; } -unsigned CGDebugInfo::Checksum(const ObjCInterfaceDecl *ID) { - // The assumption is that the number of ivars can only increase - // monotonically, so it is safe to just use their current number as - // a checksum. - unsigned Sum = 0; - for (const ObjCIvarDecl *Ivar = ID->all_declared_ivar_begin(); - Ivar != nullptr; Ivar = Ivar->getNextIvar()) - ++Sum; - - return Sum; -} - -ObjCInterfaceDecl *CGDebugInfo::getObjCInterfaceDecl(QualType Ty) { - switch (Ty->getTypeClass()) { - case Type::ObjCObjectPointer: - return getObjCInterfaceDecl( - cast<ObjCObjectPointerType>(Ty)->getPointeeType()); - case Type::ObjCInterface: - return cast<ObjCInterfaceType>(Ty)->getDecl(); - default: +llvm::DIModule *CGDebugInfo::getParentModuleOrNull(const Decl *D) { + // A forward declaration inside a module header does not belong to the module. + if (isa<RecordDecl>(D) && !cast<RecordDecl>(D)->getDefinition()) return nullptr; + if (DebugTypeExtRefs && D->isFromASTFile()) { + // Record a reference to an imported clang module or precompiled header. + auto *Reader = CGM.getContext().getExternalSource(); + auto Idx = D->getOwningModuleID(); + auto Info = Reader->getSourceDescriptor(Idx); + if (Info) + return getOrCreateModuleRef(*Info, /*SkeletonCU=*/true); + } else if (ClangModuleMap) { + // We are building a clang module or a precompiled header. + // + // TODO: When D is a CXXRecordDecl or a C++ Enum, the ODR applies + // and it wouldn't be necessary to specify the parent scope + // because the type is already unique by definition (it would look + // like the output of -fno-standalone-debug). On the other hand, + // the parent scope helps a consumer to quickly locate the object + // file where the type's definition is located, so it might be + // best to make this behavior a command line or debugger tuning + // option. + FullSourceLoc Loc(D->getLocation(), CGM.getContext().getSourceManager()); + if (Module *M = ClangModuleMap->inferModuleFromLocation(Loc)) { + auto Info = ExternalASTSource::ASTSourceDescriptor(*M); + return getOrCreateModuleRef(Info, /*SkeletonCU=*/false); + } } + + return nullptr; } llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) { @@ -2175,11 +2303,11 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) { llvm_unreachable("type should have been unwrapped!"); } -llvm::DIType *CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty, - llvm::DIFile *Unit) { +llvm::DICompositeType *CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty, + llvm::DIFile *Unit) { QualType QTy(Ty, 0); - auto *T = cast_or_null<llvm::DICompositeTypeBase>(getTypeOrNull(QTy)); + auto *T = cast_or_null<llvm::DICompositeType>(getTypeOrNull(QTy)); // We may have cached a forward decl when we could have created // a non-forward decl. Go ahead and create a non-forward decl @@ -2209,8 +2337,7 @@ llvm::DICompositeType *CGDebugInfo::CreateLimitedType(const RecordType *Ty) { unsigned Line = getLineNumber(RD->getLocation()); StringRef RDName = getClassName(RD); - llvm::DIScope *RDContext = - getContextDescriptor(cast<Decl>(RD->getDeclContext())); + llvm::DIScope *RDContext = getDeclContextDescriptor(RD); // If we ended up creating the type during the context chain construction, // just return that. @@ -2306,8 +2433,10 @@ void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD, llvm::DIFile *Unit, dyn_cast_or_null<NamespaceDecl>(FD->getDeclContext())) FDContext = getOrCreateNameSpace(NSDecl); else if (const RecordDecl *RDecl = - dyn_cast_or_null<RecordDecl>(FD->getDeclContext())) - FDContext = getContextDescriptor(cast<Decl>(RDecl)); + dyn_cast_or_null<RecordDecl>(FD->getDeclContext())) { + llvm::DIScope *Mod = getParentModuleOrNull(RDecl); + FDContext = getContextDescriptor(RDecl, Mod ? Mod : TheCU); + } // Collect template parameters. TParamsArray = CollectFunctionTemplateParams(FD, Unit); } @@ -2355,7 +2484,9 @@ void CGDebugInfo::collectVarDeclProps(const VarDecl *VD, llvm::DIFile *&Unit, // outside the class by putting it in the global scope. if (DC->isRecord()) DC = CGM.getContext().getTranslationUnitDecl(); - VDContext = getContextDescriptor(dyn_cast<Decl>(DC)); + + llvm::DIScope *Mod = getParentModuleOrNull(VD); + VDContext = getContextDescriptor(cast<Decl>(DC), Mod ? Mod : TheCU); } llvm::DISubprogram * @@ -2380,7 +2511,7 @@ CGDebugInfo::getFunctionForwardDeclaration(const FunctionDecl *FD) { llvm::DISubprogram *SP = DBuilder.createTempFunctionFwdDecl( DContext, Name, LinkageName, Unit, Line, getOrCreateFunctionType(FD, FnType, Unit), !FD->isExternallyVisible(), - false /*declaration*/, 0, Flags, CGM.getLangOpts().Optimize, nullptr, + /* isDefinition = */ false, 0, Flags, CGM.getLangOpts().Optimize, TParamsArray.get(), getFunctionDeclaration(FD)); const FunctionDecl *CanonDecl = cast<FunctionDecl>(FD->getCanonicalDecl()); FwdDeclReplaceMap.emplace_back(std::piecewise_construct, @@ -2441,7 +2572,7 @@ llvm::DISubprogram *CGDebugInfo::getFunctionDeclaration(const Decl *D) { return nullptr; // Setup context. - auto *S = getContextDescriptor(cast<Decl>(D->getDeclContext())); + auto *S = getDeclContextDescriptor(D); auto MI = SPCache.find(FD->getCanonicalDecl()); if (MI == SPCache.end()) { @@ -2476,8 +2607,7 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D, if (!D || DebugKind <= CodeGenOptions::DebugLineTablesOnly) // Create fake but valid subroutine type. Otherwise -verify would fail, and // subprogram DIE will miss DW_AT_decl_file and DW_AT_decl_line fields. - return DBuilder.createSubroutineType(F, - DBuilder.getOrCreateTypeArray(None)); + return DBuilder.createSubroutineType(DBuilder.getOrCreateTypeArray(None)); if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D)) return getOrCreateMethodType(Method, F); @@ -2495,11 +2625,17 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D, Elts.push_back(getOrCreateType(ResultTy, F)); // "self" pointer is always first argument. - QualType SelfDeclTy = OMethod->getSelfDecl()->getType(); - Elts.push_back(CreateSelfType(SelfDeclTy, getOrCreateType(SelfDeclTy, F))); + QualType SelfDeclTy; + if (auto *SelfDecl = OMethod->getSelfDecl()) + SelfDeclTy = SelfDecl->getType(); + else if (auto *FPT = dyn_cast<FunctionProtoType>(FnType)) + if (FPT->getNumParams() > 1) + SelfDeclTy = FPT->getParamType(0); + if (!SelfDeclTy.isNull()) + Elts.push_back(CreateSelfType(SelfDeclTy, getOrCreateType(SelfDeclTy, F))); // "_cmd" pointer is always second argument. Elts.push_back(DBuilder.createArtificialType( - getOrCreateType(OMethod->getCmdDecl()->getType(), F))); + getOrCreateType(CGM.getContext().getObjCSelType(), F))); // Get rest of the arguments. for (const auto *PI : OMethod->params()) Elts.push_back(getOrCreateType(PI->getType(), F)); @@ -2508,7 +2644,7 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D, Elts.push_back(DBuilder.createUnspecifiedParameter()); llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts); - return DBuilder.createSubroutineType(F, EltTypeArray); + return DBuilder.createSubroutineType(EltTypeArray); } // Handle variadic function types; they need an additional @@ -2522,7 +2658,7 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D, EltTys.push_back(getOrCreateType(FPT->getParamType(i), F)); EltTys.push_back(DBuilder.createUnspecifiedParameter()); llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); - return DBuilder.createSubroutineType(F, EltTypeArray); + return DBuilder.createSubroutineType(EltTypeArray); } return cast<llvm::DISubroutineType>(getOrCreateType(FnType, F)); @@ -2588,8 +2724,9 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, llvm::DISubprogram *SP = DBuilder.createFunction( FDContext, Name, LinkageName, Unit, LineNo, getOrCreateFunctionType(D, FnType, Unit), Fn->hasInternalLinkage(), - true /*definition*/, ScopeLine, Flags, CGM.getLangOpts().Optimize, Fn, + true /*definition*/, ScopeLine, Flags, CGM.getLangOpts().Optimize, TParamsArray.get(), getFunctionDeclaration(D)); + Fn->setSubprogram(SP); // We might get here with a VarDecl in the case we're generating // code for the initialization of globals. Do not record these decls // as they will overwrite the actual VarDecl Decl in the cache. @@ -2603,6 +2740,48 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, RegionMap[D].reset(SP); } +void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc, + QualType FnType) { + StringRef Name; + StringRef LinkageName; + + const Decl *D = GD.getDecl(); + if (!D) + return; + + unsigned Flags = 0; + llvm::DIFile *Unit = getOrCreateFile(Loc); + llvm::DIScope *FDContext = getDeclContextDescriptor(D); + llvm::DINodeArray TParamsArray; + if (isa<FunctionDecl>(D)) { + // If there is a DISubprogram for this function available then use it. + collectFunctionDeclProps(GD, Unit, Name, LinkageName, FDContext, + TParamsArray, Flags); + } else if (const ObjCMethodDecl *OMD = dyn_cast<ObjCMethodDecl>(D)) { + Name = getObjCMethodName(OMD); + Flags |= llvm::DINode::FlagPrototyped; + } else { + llvm_unreachable("not a function or ObjC method"); + } + if (!Name.empty() && Name[0] == '\01') + Name = Name.substr(1); + + if (D->isImplicit()) { + Flags |= llvm::DINode::FlagArtificial; + // Artificial functions without a location should not silently reuse CurLoc. + if (Loc.isInvalid()) + CurLoc = SourceLocation(); + } + unsigned LineNo = getLineNumber(Loc); + unsigned ScopeLine = 0; + + DBuilder.createFunction(FDContext, Name, LinkageName, Unit, LineNo, + getOrCreateFunctionType(D, FnType, Unit), + false /*internalLinkage*/, true /*definition*/, + ScopeLine, Flags, CGM.getLangOpts().Optimize, + TParamsArray.get(), getFunctionDeclaration(D)); +} + void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc) { // Update our current location setLocation(Loc); @@ -2740,8 +2919,8 @@ llvm::DIType *CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD, nullptr, Elements); } -void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::Tag Tag, - llvm::Value *Storage, unsigned ArgNo, +void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::Value *Storage, + llvm::Optional<unsigned> ArgNo, CGBuilderTy &Builder) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!"); @@ -2780,7 +2959,7 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::Tag Tag, // FIXME: There has to be a better way to do this, but for static // functions there won't be an implicit param at arg1 and // otherwise it is 'self' or 'this'. - if (isa<ImplicitParamDecl>(VD) && ArgNo == 1) + if (isa<ImplicitParamDecl>(VD) && ArgNo && *ArgNo == 1) Flags |= llvm::DINode::FlagObjectPointer; if (llvm::Argument *Arg = dyn_cast<llvm::Argument>(Storage)) if (Arg->getType()->isPointerTy() && !Arg->hasByValAttr() && @@ -2805,8 +2984,11 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::Tag Tag, Expr.push_back(offset.getQuantity()); // Create the descriptor for the variable. - auto *D = DBuilder.createLocalVariable(Tag, Scope, VD->getName(), Unit, - Line, Ty, ArgNo); + auto *D = ArgNo + ? DBuilder.createParameterVariable(Scope, VD->getName(), + *ArgNo, Unit, Line, Ty) + : DBuilder.createAutoVariable(Scope, VD->getName(), Unit, + Line, Ty); // Insert an llvm.dbg.declare into the current block. DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), @@ -2836,10 +3018,9 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::Tag Tag, continue; // Use VarDecl's Tag, Scope and Line number. - auto *D = DBuilder.createLocalVariable( - Tag, Scope, FieldName, Unit, Line, FieldTy, - CGM.getLangOpts().Optimize, Flags | llvm::DINode::FlagArtificial, - ArgNo); + auto *D = DBuilder.createAutoVariable( + Scope, FieldName, Unit, Line, FieldTy, CGM.getLangOpts().Optimize, + Flags | llvm::DINode::FlagArtificial); // Insert an llvm.dbg.declare into the current block. DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), @@ -2851,8 +3032,12 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::Tag Tag, // Create the descriptor for the variable. auto *D = - DBuilder.createLocalVariable(Tag, Scope, Name, Unit, Line, Ty, - CGM.getLangOpts().Optimize, Flags, ArgNo); + ArgNo + ? DBuilder.createParameterVariable(Scope, Name, *ArgNo, Unit, Line, + Ty, CGM.getLangOpts().Optimize, + Flags) + : DBuilder.createAutoVariable(Scope, Name, Unit, Line, Ty, + CGM.getLangOpts().Optimize, Flags); // Insert an llvm.dbg.declare into the current block. DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), @@ -2864,7 +3049,7 @@ void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD, llvm::Value *Storage, CGBuilderTy &Builder) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); - EmitDeclare(VD, llvm::dwarf::DW_TAG_auto_variable, Storage, 0, Builder); + EmitDeclare(VD, Storage, llvm::None, Builder); } llvm::DIType *CGDebugInfo::CreateSelfType(const QualType &QualTy, @@ -2929,8 +3114,7 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable( } // Create the descriptor for the variable. - auto *D = DBuilder.createLocalVariable( - llvm::dwarf::DW_TAG_auto_variable, + auto *D = DBuilder.createAutoVariable( cast<llvm::DILocalScope>(LexicalBlockStack.back()), VD->getName(), Unit, Line, Ty); @@ -2948,7 +3132,7 @@ void CGDebugInfo::EmitDeclareOfArgVariable(const VarDecl *VD, llvm::Value *AI, unsigned ArgNo, CGBuilderTy &Builder) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); - EmitDeclare(VD, llvm::dwarf::DW_TAG_arg_variable, AI, ArgNo, Builder); + EmitDeclare(VD, AI, ArgNo, Builder); } namespace { @@ -2977,7 +3161,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, unsigned column = getColumnNumber(loc); // Build the debug-info type for the block literal. - getContextDescriptor(cast<Decl>(blockDecl->getDeclContext())); + getDeclContextDescriptor(blockDecl); const llvm::StructLayout *blockLayout = CGM.getDataLayout().getStructLayout(block.StructureType); @@ -3090,9 +3274,9 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, auto *scope = cast<llvm::DILocalScope>(LexicalBlockStack.back()); // Create the descriptor for the parameter. - auto *debugVar = DBuilder.createLocalVariable( - llvm::dwarf::DW_TAG_arg_variable, scope, Arg->getName(), tunit, line, - type, CGM.getLangOpts().Optimize, flags, ArgNo); + auto *debugVar = DBuilder.createParameterVariable( + scope, Arg->getName(), ArgNo, tunit, line, type, + CGM.getLangOpts().Optimize, flags); if (LocalAddr) { // Insert an llvm.dbg.value into the current block. @@ -3115,14 +3299,13 @@ CGDebugInfo::getOrCreateStaticDataMemberDeclarationOrNull(const VarDecl *D) { auto MI = StaticDataMemberCache.find(D->getCanonicalDecl()); if (MI != StaticDataMemberCache.end()) { assert(MI->second && "Static data member declaration should still exist"); - return cast<llvm::DIDerivedType>(MI->second); + return MI->second; } // If the member wasn't found in the cache, lazily construct and add it to the // type (used when a limited form of the type is emitted). auto DC = D->getDeclContext(); - auto *Ctxt = - cast<llvm::DICompositeType>(getContextDescriptor(cast<Decl>(DC))); + auto *Ctxt = cast<llvm::DICompositeType>(getDeclContextDescriptor(D)); return CreateRecordStaticField(D, Ctxt, cast<RecordDecl>(DC)); } @@ -3170,7 +3353,7 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var, // variable for each member of the anonymous union so that it's possible // to find the name of any field in the union. if (T->isUnionType() && DeclName.empty()) { - const RecordDecl *RD = cast<RecordType>(T)->getDecl(); + const RecordDecl *RD = T->castAs<RecordType>()->getDecl(); assert(RD->isAnonymousStructOrUnion() && "unnamed non-anonymous struct or union?"); GV = CollectAnonRecordDecls(RD, Unit, LineNo, LinkageName, Var, DContext); @@ -3207,15 +3390,14 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, auto *VarD = cast<VarDecl>(VD); if (VarD->isStaticDataMember()) { auto *RD = cast<RecordDecl>(VarD->getDeclContext()); - getContextDescriptor(RD); + getDeclContextDescriptor(VarD); // Ensure that the type is retained even though it's otherwise unreferenced. RetainedTypes.push_back( CGM.getContext().getRecordType(RD).getAsOpaquePtr()); return; } - llvm::DIScope *DContext = - getContextDescriptor(dyn_cast<Decl>(VD->getDeclContext())); + llvm::DIScope *DContext = getDeclContextDescriptor(VD); auto &GV = DeclCache[VD]; if (GV) @@ -3228,16 +3410,21 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, llvm::DIScope *CGDebugInfo::getCurrentContextDescriptor(const Decl *D) { if (!LexicalBlockStack.empty()) return LexicalBlockStack.back(); - return getContextDescriptor(D); + llvm::DIScope *Mod = getParentModuleOrNull(D); + return getContextDescriptor(D, Mod ? Mod : TheCU); } void CGDebugInfo::EmitUsingDirective(const UsingDirectiveDecl &UD) { if (CGM.getCodeGenOpts().getDebugInfo() < CodeGenOptions::LimitedDebugInfo) return; - DBuilder.createImportedModule( - getCurrentContextDescriptor(cast<Decl>(UD.getDeclContext())), - getOrCreateNameSpace(UD.getNominatedNamespace()), - getLineNumber(UD.getLocation())); + const NamespaceDecl *NSDecl = UD.getNominatedNamespace(); + if (!NSDecl->isAnonymousNamespace() || + CGM.getCodeGenOpts().DebugExplicitImport) { + DBuilder.createImportedModule( + getCurrentContextDescriptor(cast<Decl>(UD.getDeclContext())), + getOrCreateNameSpace(NSDecl), + getLineNumber(UD.getLocation())); + } } void CGDebugInfo::EmitUsingDecl(const UsingDecl &UD) { @@ -3256,12 +3443,13 @@ void CGDebugInfo::EmitUsingDecl(const UsingDecl &UD) { } void CGDebugInfo::EmitImportDecl(const ImportDecl &ID) { - auto *Reader = CGM.getContext().getExternalSource(); - auto Info = Reader->getSourceDescriptor(*ID.getImportedModule()); - DBuilder.createImportedDeclaration( - getCurrentContextDescriptor(cast<Decl>(ID.getDeclContext())), - getOrCreateModuleRef(Info), - getLineNumber(ID.getLocation())); + if (Module *M = ID.getImportedModule()) { + auto Info = ExternalASTSource::ASTSourceDescriptor(*M); + DBuilder.createImportedDeclaration( + getCurrentContextDescriptor(cast<Decl>(ID.getDeclContext())), + getOrCreateModuleRef(Info, DebugTypeExtRefs), + getLineNumber(ID.getLocation())); + } } llvm::DIImportedEntity * @@ -3297,14 +3485,19 @@ CGDebugInfo::getOrCreateNameSpace(const NamespaceDecl *NSDecl) { unsigned LineNo = getLineNumber(NSDecl->getLocation()); llvm::DIFile *FileD = getOrCreateFile(NSDecl->getLocation()); - llvm::DIScope *Context = - getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext())); + llvm::DIScope *Context = getDeclContextDescriptor(NSDecl); llvm::DINamespace *NS = DBuilder.createNameSpace(Context, NSDecl->getName(), FileD, LineNo); NameSpaceCache[NSDecl].reset(NS); return NS; } +void CGDebugInfo::setDwoId(uint64_t Signature) { + assert(TheCU && "no main compile unit"); + TheCU->setDWOId(Signature); +} + + void CGDebugInfo::finalize() { // Creating types might create further types - invalidating the current // element and the size(), so don't cache/reference them. @@ -3348,9 +3541,9 @@ void CGDebugInfo::finalize() { // We keep our own list of retained types, because we need to look // up the final type in the type cache. - for (std::vector<void *>::const_iterator RI = RetainedTypes.begin(), - RE = RetainedTypes.end(); RI != RE; ++RI) - DBuilder.retainType(cast<llvm::DIType>(TypeCache[*RI])); + for (auto &RT : RetainedTypes) + if (auto MD = TypeCache[RT]) + DBuilder.retainType(cast<llvm::DIType>(MD)); DBuilder.finalize(); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h index 82680a8..57d5c80 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h @@ -20,6 +20,7 @@ #include "clang/Basic/SourceLocation.h" #include "clang/Frontend/CodeGenOptions.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/ValueHandle.h" @@ -31,12 +32,13 @@ class MDNode; namespace clang { class CXXMethodDecl; -class VarDecl; -class ObjCInterfaceDecl; -class ObjCIvarDecl; class ClassTemplateSpecializationDecl; class GlobalDecl; +class ModuleMap; +class ObjCInterfaceDecl; +class ObjCIvarDecl; class UsingDecl; +class VarDecl; namespace CodeGen { class CodeGenModule; @@ -51,8 +53,10 @@ class CGDebugInfo { friend class SaveAndRestoreLocation; CodeGenModule &CGM; const CodeGenOptions::DebugInfoKind DebugKind; + bool DebugTypeExtRefs; llvm::DIBuilder DBuilder; llvm::DICompileUnit *TheCU = nullptr; + ModuleMap *ClangModuleMap = nullptr; SourceLocation CurLoc; llvm::DIType *VTablePtrType = nullptr; llvm::DIType *ClassTy = nullptr; @@ -63,12 +67,24 @@ class CGDebugInfo { llvm::DIType *OCLImage1dBufferDITy = nullptr; llvm::DIType *OCLImage2dDITy = nullptr; llvm::DIType *OCLImage2dArrayDITy = nullptr; + llvm::DIType *OCLImage2dDepthDITy = nullptr; + llvm::DIType *OCLImage2dArrayDepthDITy = nullptr; + llvm::DIType *OCLImage2dMSAADITy = nullptr; + llvm::DIType *OCLImage2dArrayMSAADITy = nullptr; + llvm::DIType *OCLImage2dMSAADepthDITy = nullptr; + llvm::DIType *OCLImage2dArrayMSAADepthDITy = nullptr; llvm::DIType *OCLImage3dDITy = nullptr; llvm::DIType *OCLEventDITy = nullptr; + llvm::DIType *OCLClkEventDITy = nullptr; + llvm::DIType *OCLQueueDITy = nullptr; + llvm::DIType *OCLNDRangeDITy = nullptr; + llvm::DIType *OCLReserveIDDITy = nullptr; /// Cache of previously constructed Types. llvm::DenseMap<const void *, llvm::TrackingMDRef> TypeCache; + llvm::SmallDenseMap<llvm::StringRef, llvm::StringRef> DebugPrefixMap; + struct ObjCInterfaceCacheEntry { const ObjCInterfaceType *Type; llvm::DIType *Decl; @@ -81,8 +97,8 @@ class CGDebugInfo { /// Cache of previously constructed interfaces which may change. llvm::SmallVector<ObjCInterfaceCacheEntry, 32> ObjCInterfaceCache; - /// Cache of references to AST files such as PCHs or modules. - llvm::DenseMap<uint64_t, llvm::DIModule *> ModuleRefCache; + /// Cache of references to clang modules and precompiled headers. + llvm::DenseMap<const Module *, llvm::TrackingMDRef> ModuleCache; /// List of interfaces we want to keep even if orphaned. std::vector<void *> RetainedTypes; @@ -117,13 +133,13 @@ class CGDebugInfo { llvm::DenseMap<const NamespaceDecl *, llvm::TrackingMDRef> NameSpaceCache; llvm::DenseMap<const NamespaceAliasDecl *, llvm::TrackingMDRef> NamespaceAliasCache; - llvm::DenseMap<const Decl *, llvm::TrackingMDRef> StaticDataMemberCache; + llvm::DenseMap<const Decl *, llvm::TypedTrackingMDRef<llvm::DIDerivedType>> + StaticDataMemberCache; /// Helper functions for getOrCreateType. /// @{ /// Currently the checksum of an interface includes the number of /// ivars and property accessors. - unsigned Checksum(const ObjCInterfaceDecl *InterfaceDecl); llvm::DIType *CreateType(const BuiltinType *Ty); llvm::DIType *CreateType(const ComplexType *Ty); llvm::DIType *CreateQualifiedType(QualType Ty, llvm::DIFile *Fg); @@ -182,11 +198,8 @@ class CGDebugInfo { llvm::DIType *getOrCreateVTablePtrType(llvm::DIFile *F); /// \return namespace descriptor for the given namespace decl. llvm::DINamespace *getOrCreateNameSpace(const NamespaceDecl *N); - llvm::DIType *getOrCreateTypeDeclaration(QualType PointeeTy, llvm::DIFile *F); llvm::DIType *CreatePointerLikeType(llvm::dwarf::Tag Tag, const Type *Ty, QualType PointeeTy, llvm::DIFile *F); - - llvm::Value *getCachedInterfaceTypeOrNull(const QualType Ty); llvm::DIType *getOrCreateStructPtrType(StringRef Name, llvm::DIType *&Cache); /// A helper function to create a subprogram for a single member @@ -261,6 +274,14 @@ public: void finalize(); + /// Set the main CU's DwoId field to \p Signature. + void setDwoId(uint64_t Signature); + + /// When generating debug information for a clang module or + /// precompiled header, this module map will be used to determine + /// the module of origin of each Decl. + void setModuleMap(ModuleMap &MMap) { ClangModuleMap = &MMap; } + /// Update the current source location. If \arg loc is invalid it is /// ignored. void setLocation(SourceLocation Loc); @@ -278,6 +299,9 @@ public: SourceLocation ScopeLoc, QualType FnType, llvm::Function *Fn, CGBuilderTy &Builder); + /// Emit debug info for a function declaration. + void EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc, QualType FnType); + /// Constructs the debug code for exiting a function. void EmitFunctionEnd(CGBuilderTy &Builder); @@ -300,7 +324,7 @@ public: llvm::Value *storage, CGBuilderTy &Builder, const CGBlockInfo &blockInfo, - llvm::Instruction *InsertPoint = 0); + llvm::Instruction *InsertPoint = nullptr); /// Emit call to \c llvm.dbg.declare for an argument variable /// declaration. @@ -341,6 +365,9 @@ public: /// Emit an Objective-C interface type standalone debug info. llvm::DIType *getOrCreateInterfaceType(QualType Ty, SourceLocation Loc); + /// Emit standalone debug info for a type. + llvm::DIType *getOrCreateStandaloneType(QualType Ty, SourceLocation Loc); + void completeType(const EnumDecl *ED); void completeType(const RecordDecl *RD); void completeRequiredType(const RecordDecl *RD); @@ -350,17 +377,18 @@ public: private: /// Emit call to llvm.dbg.declare for a variable declaration. - /// Tag accepts custom types DW_TAG_arg_variable and DW_TAG_auto_variable, - /// otherwise would be of type llvm::dwarf::Tag. - void EmitDeclare(const VarDecl *decl, llvm::dwarf::Tag Tag, llvm::Value *AI, - unsigned ArgNo, CGBuilderTy &Builder); + void EmitDeclare(const VarDecl *decl, llvm::Value *AI, + llvm::Optional<unsigned> ArgNo, CGBuilderTy &Builder); /// Build up structure info for the byref. See \a BuildByRefType. llvm::DIType *EmitTypeForVarWithBlocksAttr(const VarDecl *VD, uint64_t *OffSet); - /// Get context info for the decl. - llvm::DIScope *getContextDescriptor(const Decl *Decl); + /// Get context info for the DeclContext of \p Decl. + llvm::DIScope *getDeclContextDescriptor(const Decl *D); + /// Get context info for a given DeclContext \p Decl. + llvm::DIScope *getContextDescriptor(const Decl *Context, + llvm::DIScope *Default); llvm::DIScope *getCurrentContextDescriptor(const Decl *Decl); @@ -374,6 +402,9 @@ private: /// Create new compile unit. void CreateCompileUnit(); + /// Remap a given path with the current debug prefix map + std::string remapDIPath(StringRef) const; + /// Get the file debug info descriptor for the input location. llvm::DIFile *getOrCreateFile(SourceLocation Loc); @@ -383,21 +414,23 @@ private: /// Get the type from the cache or create a new type if necessary. llvm::DIType *getOrCreateType(QualType Ty, llvm::DIFile *Fg); - /// Get a reference to a clang module. + /// Get a reference to a clang module. If \p CreateSkeletonCU is true, + /// this also creates a split dwarf skeleton compile unit. llvm::DIModule * - getOrCreateModuleRef(ExternalASTSource::ASTSourceDescriptor Mod); + getOrCreateModuleRef(ExternalASTSource::ASTSourceDescriptor Mod, + bool CreateSkeletonCU); + + /// DebugTypeExtRefs: If \p D originated in a clang module, return it. + llvm::DIModule *getParentModuleOrNull(const Decl *D); /// Get the type from the cache or create a new partial type if /// necessary. - llvm::DIType *getOrCreateLimitedType(const RecordType *Ty, llvm::DIFile *F); + llvm::DICompositeType *getOrCreateLimitedType(const RecordType *Ty, + llvm::DIFile *F); /// Create type metadata for a source language type. llvm::DIType *CreateTypeNode(QualType Ty, llvm::DIFile *Fg); - /// Return the underlying ObjCInterfaceDecl if \arg Ty is an - /// ObjCInterface or a pointer to one. - ObjCInterfaceDecl *getObjCInterfaceDecl(QualType Ty); - /// Create new member and increase Offset by FType's size. llvm::DIType *CreateMemberType(llvm::DIFile *Unit, QualType FType, StringRef Name, uint64_t *Offset); @@ -501,13 +534,16 @@ private: SourceLocation TemporaryLocation); llvm::DebugLoc OriginalLocation; - CodeGenFunction &CGF; + CodeGenFunction *CGF; public: /// Set the location to the (valid) TemporaryLocation. ApplyDebugLocation(CodeGenFunction &CGF, SourceLocation TemporaryLocation); ApplyDebugLocation(CodeGenFunction &CGF, const Expr *E); ApplyDebugLocation(CodeGenFunction &CGF, llvm::DebugLoc Loc); + ApplyDebugLocation(ApplyDebugLocation &&Other) : CGF(Other.CGF) { + Other.CGF = nullptr; + } ~ApplyDebugLocation(); @@ -538,20 +574,14 @@ public: /// passing an empty SourceLocation to \a CGDebugInfo::setLocation() /// will result in the last valid location being reused. Note that /// all instructions that do not have a location at the beginning of - /// a function are counted towards to funciton prologue. + /// a function are counted towards to function prologue. static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF) { return ApplyDebugLocation(CGF, true, SourceLocation()); } - /// \brief Apply TemporaryLocation if it is valid. Otherwise set the IRBuilder - /// to not attach debug locations. - static ApplyDebugLocation - CreateDefaultEmpty(CodeGenFunction &CGF, SourceLocation TemporaryLocation) { - return ApplyDebugLocation(CGF, true, TemporaryLocation); - } }; } // namespace CodeGen } // namespace clang -#endif +#endif // LLVM_CLANG_LIB_CODEGEN_CGDEBUGINFO_H diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp index 96aa8c68..b78e80d 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "CodeGenFunction.h" +#include "CGBlocks.h" #include "CGCleanup.h" #include "CGDebugInfo.h" #include "CGOpenCLRuntime.h" @@ -34,6 +35,7 @@ using namespace CodeGen; void CodeGenFunction::EmitDecl(const Decl &D) { switch (D.getKind()) { + case Decl::BuiltinTemplate: case Decl::TranslationUnit: case Decl::ExternCContext: case Decl::Namespace: @@ -142,7 +144,7 @@ void CodeGenFunction::EmitVarDecl(const VarDecl &D) { // Don't emit it now, allow it to be emitted lazily on its first use. return; - if (D.getStorageClass() == SC_OpenCLWorkGroupLocal) + if (D.getType().getAddressSpace() == LangAS::opencl_local) return CGM.getOpenCLRuntime().EmitWorkGroupLocalVarDecl(*this, D); assert(D.hasLocalStorage()); @@ -311,6 +313,7 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D, OldGV->getThreadLocalMode(), CGM.getContext().getTargetAddressSpace(D.getType())); GV->setVisibility(OldGV->getVisibility()); + GV->setComdat(OldGV->getComdat()); // Steal the name of the old global GV->takeName(OldGV); @@ -339,17 +342,15 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D, void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D, llvm::GlobalValue::LinkageTypes Linkage) { - llvm::Value *&DMEntry = LocalDeclMap[&D]; - assert(!DMEntry && "Decl already exists in localdeclmap!"); - // Check to see if we already have a global variable for this // declaration. This can happen when double-emitting function // bodies, e.g. with complete and base constructors. llvm::Constant *addr = CGM.getOrCreateStaticVarDecl(D, Linkage); + CharUnits alignment = getContext().getDeclAlign(&D); // Store into LocalDeclMap before generating initializer to handle // circular references. - DMEntry = addr; + setAddrOfLocalVar(&D, Address(addr, alignment)); // We can't have a VLA here, but we can have a pointer to a VLA, // even though that doesn't really make any sense. @@ -366,7 +367,7 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D, if (D.getInit()) var = AddInitializerToStaticVarDecl(D, var); - var->setAlignment(getContext().getDeclAlign(&D).getQuantity()); + var->setAlignment(alignment.getQuantity()); if (D.hasAttr<AnnotateAttr>()) CGM.AddGlobalAnnotations(&D, var); @@ -384,7 +385,8 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D, // RAUW's the GV uses of this constant will be invalid. llvm::Constant *castedAddr = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(var, expectedType); - DMEntry = castedAddr; + if (var != castedAddr) + LocalDeclMap.find(&D)->second = Address(castedAddr, alignment); CGM.setStaticLocalDeclAddress(&D, castedAddr); CGM.getSanitizerMetadata()->reportGlobalToASan(var, D); @@ -399,14 +401,14 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D, } namespace { - struct DestroyObject : EHScopeStack::Cleanup { - DestroyObject(llvm::Value *addr, QualType type, + struct DestroyObject final : EHScopeStack::Cleanup { + DestroyObject(Address addr, QualType type, CodeGenFunction::Destroyer *destroyer, bool useEHCleanupForArray) : addr(addr), type(type), destroyer(destroyer), useEHCleanupForArray(useEHCleanupForArray) {} - llvm::Value *addr; + Address addr; QualType type; CodeGenFunction::Destroyer *destroyer; bool useEHCleanupForArray; @@ -420,15 +422,15 @@ namespace { } }; - struct DestroyNRVOVariable : EHScopeStack::Cleanup { - DestroyNRVOVariable(llvm::Value *addr, + struct DestroyNRVOVariable final : EHScopeStack::Cleanup { + DestroyNRVOVariable(Address addr, const CXXDestructorDecl *Dtor, llvm::Value *NRVOFlag) : Dtor(Dtor), NRVOFlag(NRVOFlag), Loc(addr) {} const CXXDestructorDecl *Dtor; llvm::Value *NRVOFlag; - llvm::Value *Loc; + Address Loc; void Emit(CodeGenFunction &CGF, Flags flags) override { // Along the exceptions path we always execute the dtor. @@ -439,7 +441,8 @@ namespace { // If we exited via NRVO, we skip the destructor call. llvm::BasicBlock *RunDtorBB = CGF.createBasicBlock("nrvo.unused"); SkipDtorBB = CGF.createBasicBlock("nrvo.skipdtor"); - llvm::Value *DidNRVO = CGF.Builder.CreateLoad(NRVOFlag, "nrvo.val"); + llvm::Value *DidNRVO = + CGF.Builder.CreateFlagLoad(NRVOFlag, "nrvo.val"); CGF.Builder.CreateCondBr(DidNRVO, SkipDtorBB, RunDtorBB); CGF.EmitBlock(RunDtorBB); } @@ -453,9 +456,9 @@ namespace { } }; - struct CallStackRestore : EHScopeStack::Cleanup { - llvm::Value *Stack; - CallStackRestore(llvm::Value *Stack) : Stack(Stack) {} + struct CallStackRestore final : EHScopeStack::Cleanup { + Address Stack; + CallStackRestore(Address Stack) : Stack(Stack) {} void Emit(CodeGenFunction &CGF, Flags flags) override { llvm::Value *V = CGF.Builder.CreateLoad(Stack); llvm::Value *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore); @@ -463,7 +466,7 @@ namespace { } }; - struct ExtendGCLifetime : EHScopeStack::Cleanup { + struct ExtendGCLifetime final : EHScopeStack::Cleanup { const VarDecl &Var; ExtendGCLifetime(const VarDecl *var) : Var(*var) {} @@ -478,7 +481,7 @@ namespace { } }; - struct CallCleanupFunction : EHScopeStack::Cleanup { + struct CallCleanupFunction final : EHScopeStack::Cleanup { llvm::Constant *CleanupFn; const CGFunctionInfo &FnInfo; const VarDecl &Var; @@ -492,7 +495,7 @@ namespace { Var.getType(), VK_LValue, SourceLocation()); // Compute the address of the local variable, in case it's a byref // or something. - llvm::Value *Addr = CGF.EmitDeclRefLValue(&DRE).getAddress(); + llvm::Value *Addr = CGF.EmitDeclRefLValue(&DRE).getPointer(); // In some cases, the type of the function argument will be different from // the type of the pointer. An example of this is @@ -512,12 +515,12 @@ namespace { }; /// A cleanup to call @llvm.lifetime.end. - class CallLifetimeEnd : public EHScopeStack::Cleanup { + class CallLifetimeEnd final : public EHScopeStack::Cleanup { llvm::Value *Addr; llvm::Value *Size; public: - CallLifetimeEnd(llvm::Value *addr, llvm::Value *size) - : Addr(addr), Size(size) {} + CallLifetimeEnd(Address addr, llvm::Value *size) + : Addr(addr.getPointer()), Size(size) {} void Emit(CodeGenFunction &CGF, Flags flags) override { CGF.EmitLifetimeEnd(Size, Addr); @@ -528,7 +531,7 @@ namespace { /// EmitAutoVarWithLifetime - Does the setup required for an automatic /// variable with lifetime. static void EmitAutoVarWithLifetime(CodeGenFunction &CGF, const VarDecl &var, - llvm::Value *addr, + Address addr, Qualifiers::ObjCLifetime lifetime) { switch (lifetime) { case Qualifiers::OCL_None: @@ -595,10 +598,61 @@ static bool isAccessedBy(const ValueDecl *decl, const Expr *e) { return isAccessedBy(*var, e); } +static bool tryEmitARCCopyWeakInit(CodeGenFunction &CGF, + const LValue &destLV, const Expr *init) { + bool needsCast = false; + + while (auto castExpr = dyn_cast<CastExpr>(init->IgnoreParens())) { + switch (castExpr->getCastKind()) { + // Look through casts that don't require representation changes. + case CK_NoOp: + case CK_BitCast: + case CK_BlockPointerToObjCPointerCast: + needsCast = true; + break; + + // If we find an l-value to r-value cast from a __weak variable, + // emit this operation as a copy or move. + case CK_LValueToRValue: { + const Expr *srcExpr = castExpr->getSubExpr(); + if (srcExpr->getType().getObjCLifetime() != Qualifiers::OCL_Weak) + return false; + + // Emit the source l-value. + LValue srcLV = CGF.EmitLValue(srcExpr); + + // Handle a formal type change to avoid asserting. + auto srcAddr = srcLV.getAddress(); + if (needsCast) { + srcAddr = CGF.Builder.CreateElementBitCast(srcAddr, + destLV.getAddress().getElementType()); + } + + // If it was an l-value, use objc_copyWeak. + if (srcExpr->getValueKind() == VK_LValue) { + CGF.EmitARCCopyWeak(destLV.getAddress(), srcAddr); + } else { + assert(srcExpr->getValueKind() == VK_XValue); + CGF.EmitARCMoveWeak(destLV.getAddress(), srcAddr); + } + return true; + } + + // Stop at anything else. + default: + return false; + } + + init = castExpr->getSubExpr(); + continue; + } + return false; +} + static void drillIntoBlockVariable(CodeGenFunction &CGF, LValue &lvalue, const VarDecl *var) { - lvalue.setAddress(CGF.BuildBlockByrefAddress(lvalue.getAddress(), var)); + lvalue.setAddress(CGF.emitBlockByrefAddress(lvalue.getAddress(), var)); } void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D, @@ -636,15 +690,12 @@ void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D, if (capturedByInit) { // We can use a simple GEP for this because it can't have been // moved yet. - tempLV.setAddress(Builder.CreateStructGEP( - nullptr, tempLV.getAddress(), - getByRefValueLLVMField(cast<VarDecl>(D)).second)); + tempLV.setAddress(emitBlockByrefAddress(tempLV.getAddress(), + cast<VarDecl>(D), + /*follow*/ false)); } - llvm::PointerType *ty - = cast<llvm::PointerType>(tempLV.getAddress()->getType()); - ty = cast<llvm::PointerType>(ty->getElementType()); - + auto ty = cast<llvm::PointerType>(tempLV.getAddress().getElementType()); llvm::Value *zero = llvm::ConstantPointerNull::get(ty); // If __weak, we want to use a barrier under certain conditions. @@ -674,6 +725,12 @@ void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D, } case Qualifiers::OCL_Weak: { + // If it's not accessed by the initializer, try to emit the + // initialization with a copy or move. + if (!accessedByInit && tryEmitARCCopyWeakInit(*this, lvalue, init)) { + return; + } + // No way to optimize a producing initializer into this. It's not // worth optimizing for, because the value will immediately // disappear in the common case. @@ -788,7 +845,7 @@ static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc, if (isa<llvm::ConstantInt>(Init) || isa<llvm::ConstantFP>(Init) || isa<llvm::ConstantVector>(Init) || isa<llvm::BlockAddress>(Init) || isa<llvm::ConstantExpr>(Init)) { - Builder.CreateStore(Init, Loc, isVolatile); + Builder.CreateDefaultAlignedStore(Init, Loc, isVolatile); return; } @@ -891,13 +948,12 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { emission.IsByRef = isByRef; CharUnits alignment = getContext().getDeclAlign(&D); - emission.Alignment = alignment; // If the type is variably-modified, emit all the VLA sizes for it. if (Ty->isVariablyModifiedType()) EmitVariablyModifiedType(Ty); - llvm::Value *DeclPtr; + Address address = Address::invalid(); if (Ty->isConstantSizeType()) { bool NRVO = getLangOpts().ElideConstructors && D.isNRVOVariable(); @@ -923,7 +979,8 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { CGM.isTypeConstant(Ty, true)) { EmitStaticVarDecl(D, llvm::GlobalValue::InternalLinkage); - emission.Address = nullptr; // signal this condition to later callbacks + // Signal this condition to later callbacks. + emission.Addr = Address::invalid(); assert(emission.wasEmittedAsGlobal()); return emission; } @@ -934,13 +991,12 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // A normal fixed sized variable becomes an alloca in the entry block, // unless it's an NRVO variable. - llvm::Type *LTy = ConvertTypeForMem(Ty); if (NRVO) { // The named return value optimization: allocate this variable in the // return slot, so that we can elide the copy when returning this // variable (C++0x [class.copy]p34). - DeclPtr = ReturnValue; + address = ReturnValue; if (const RecordType *RecordTy = Ty->getAs<RecordType>()) { if (!cast<CXXRecordDecl>(RecordTy->getDecl())->hasTrivialDestructor()) { @@ -948,34 +1004,46 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // to this variable. Set it to zero to indicate that NRVO was not // applied. llvm::Value *Zero = Builder.getFalse(); - llvm::Value *NRVOFlag = CreateTempAlloca(Zero->getType(), "nrvo"); + Address NRVOFlag = + CreateTempAlloca(Zero->getType(), CharUnits::One(), "nrvo"); EnsureInsertPoint(); Builder.CreateStore(Zero, NRVOFlag); // Record the NRVO flag for this variable. - NRVOFlags[&D] = NRVOFlag; - emission.NRVOFlag = NRVOFlag; + NRVOFlags[&D] = NRVOFlag.getPointer(); + emission.NRVOFlag = NRVOFlag.getPointer(); } } } else { - if (isByRef) - LTy = BuildByRefType(&D); + CharUnits allocaAlignment; + llvm::Type *allocaTy; + if (isByRef) { + auto &byrefInfo = getBlockByrefInfo(&D); + allocaTy = byrefInfo.Type; + allocaAlignment = byrefInfo.ByrefAlignment; + } else { + allocaTy = ConvertTypeForMem(Ty); + allocaAlignment = alignment; + } - llvm::AllocaInst *Alloc = CreateTempAlloca(LTy); - Alloc->setName(D.getName()); + // Create the alloca. Note that we set the name separately from + // building the instruction so that it's there even in no-asserts + // builds. + address = CreateTempAlloca(allocaTy, allocaAlignment); + address.getPointer()->setName(D.getName()); - CharUnits allocaAlignment = alignment; - if (isByRef) - allocaAlignment = std::max(allocaAlignment, - getContext().toCharUnitsFromBits(getTarget().getPointerAlign(0))); - Alloc->setAlignment(allocaAlignment.getQuantity()); - DeclPtr = Alloc; + // Don't emit lifetime markers for MSVC catch parameters. The lifetime of + // the catch parameter starts in the catchpad instruction, and we can't + // insert code in those basic blocks. + bool IsMSCatchParam = + D.isExceptionVariable() && getTarget().getCXXABI().isMicrosoft(); // Emit a lifetime intrinsic if meaningful. There's no point // in doing this if we don't have a valid insertion point (?). - uint64_t size = CGM.getDataLayout().getTypeAllocSize(LTy); - if (HaveInsertPoint()) { - emission.SizeForLifetimeMarkers = EmitLifetimeStart(size, Alloc); + if (HaveInsertPoint() && !IsMSCatchParam) { + uint64_t size = CGM.getDataLayout().getTypeAllocSize(allocaTy); + emission.SizeForLifetimeMarkers = + EmitLifetimeStart(size, address.getPointer()); } else { assert(!emission.useLifetimeMarkers()); } @@ -985,11 +1053,11 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { if (!DidCallStackSave) { // Save the stack. - llvm::Value *Stack = CreateTempAlloca(Int8PtrTy, "saved_stack"); + Address Stack = + CreateTempAlloca(Int8PtrTy, getPointerAlign(), "saved_stack"); llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::stacksave); llvm::Value *V = Builder.CreateCall(F); - Builder.CreateStore(V, Stack); DidCallStackSave = true; @@ -1009,13 +1077,11 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { llvm::AllocaInst *vla = Builder.CreateAlloca(llvmTy, elementCount, "vla"); vla->setAlignment(alignment.getQuantity()); - DeclPtr = vla; + address = Address(vla, alignment); } - llvm::Value *&DMEntry = LocalDeclMap[&D]; - assert(!DMEntry && "Decl already exists in localdeclmap!"); - DMEntry = DeclPtr; - emission.Address = DeclPtr; + setAddrOfLocalVar(&D, address); + emission.Addr = address; // Emit debug info for local var declaration. if (HaveInsertPoint()) @@ -1023,12 +1089,12 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { if (CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo) { DI->setLocation(D.getLocation()); - DI->EmitDeclareOfAutoVariable(&D, DeclPtr, Builder); + DI->EmitDeclareOfAutoVariable(&D, address.getPointer(), Builder); } } if (D.hasAttr<AnnotateAttr>()) - EmitVarAnnotations(&D, emission.Address); + EmitVarAnnotations(&D, address.getPointer()); return emission; } @@ -1124,15 +1190,13 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) { if (isTrivialInitializer(Init)) return; - CharUnits alignment = emission.Alignment; - // Check whether this is a byref variable that's potentially // captured and moved by its own initializer. If so, we'll need to // emit the initializer first, then copy into the variable. bool capturedByInit = emission.IsByRef && isCapturedBy(D, Init); - llvm::Value *Loc = - capturedByInit ? emission.Address : emission.getObjectAddress(*this); + Address Loc = + capturedByInit ? emission.Addr : emission.getObjectAddress(*this); llvm::Constant *constant = nullptr; if (emission.IsConstantAggregate || D.isConstexpr()) { @@ -1141,14 +1205,14 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) { } if (!constant) { - LValue lv = MakeAddrLValue(Loc, type, alignment); + LValue lv = MakeAddrLValue(Loc, type); lv.setNonGC(true); return EmitExprAsInit(Init, &D, lv, capturedByInit); } if (!emission.IsConstantAggregate) { // For simple scalar/complex initialization, store the value directly. - LValue lv = MakeAddrLValue(Loc, type, alignment); + LValue lv = MakeAddrLValue(Loc, type); lv.setNonGC(true); return EmitStoreThroughLValue(RValue::get(constant), lv, true); } @@ -1162,7 +1226,7 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) { getContext().getTypeSizeInChars(type).getQuantity()); llvm::Type *BP = Int8PtrTy; - if (Loc->getType() != BP) + if (Loc.getType() != BP) Loc = Builder.CreateBitCast(Loc, BP); // If the initializer is all or mostly zeros, codegen with memset then do @@ -1170,11 +1234,12 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) { if (shouldUseMemSetPlusStoresToInitialize(constant, CGM.getDataLayout().getTypeAllocSize(constant->getType()))) { Builder.CreateMemSet(Loc, llvm::ConstantInt::get(Int8Ty, 0), SizeVal, - alignment.getQuantity(), isVolatile); + isVolatile); // Zero and undef don't require a stores. if (!constant->isNullValue() && !isa<llvm::UndefValue>(constant)) { Loc = Builder.CreateBitCast(Loc, constant->getType()->getPointerTo()); - emitStoresForInitAfterMemset(constant, Loc, isVolatile, Builder); + emitStoresForInitAfterMemset(constant, Loc.getPointer(), + isVolatile, Builder); } } else { // Otherwise, create a temporary global with the initializer then @@ -1184,15 +1249,14 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) { new llvm::GlobalVariable(CGM.getModule(), constant->getType(), true, llvm::GlobalValue::PrivateLinkage, constant, Name); - GV->setAlignment(alignment.getQuantity()); + GV->setAlignment(Loc.getAlignment().getQuantity()); GV->setUnnamedAddr(true); - llvm::Value *SrcPtr = GV; - if (SrcPtr->getType() != BP) + Address SrcPtr = Address(GV, Loc.getAlignment()); + if (SrcPtr.getType() != BP) SrcPtr = Builder.CreateBitCast(SrcPtr, BP); - Builder.CreateMemCpy(Loc, SrcPtr, SizeVal, alignment.getQuantity(), - isVolatile); + Builder.CreateMemCpy(Loc, SrcPtr, SizeVal, isVolatile); } } @@ -1253,7 +1317,7 @@ void CodeGenFunction::emitAutoVarTypeCleanup( // Note that for __block variables, we want to destroy the // original stack object, not the possibly forwarded object. - llvm::Value *addr = emission.getObjectAddress(*this); + Address addr = emission.getObjectAddress(*this); const VarDecl *var = emission.Variable; QualType type = var->getType(); @@ -1271,8 +1335,8 @@ void CodeGenFunction::emitAutoVarTypeCleanup( if (emission.NRVOFlag) { assert(!type->isArrayType()); CXXDestructorDecl *dtor = type->getAsCXXRecordDecl()->getDestructor(); - EHStack.pushCleanup<DestroyNRVOVariable>(cleanupKind, addr, dtor, - emission.NRVOFlag); + EHStack.pushCleanup<DestroyNRVOVariable>(cleanupKind, addr, + dtor, emission.NRVOFlag); return; } break; @@ -1369,7 +1433,7 @@ CodeGenFunction::getDestroyer(QualType::DestructionKind kind) { /// pushEHDestroy - Push the standard destructor for the given type as /// an EH-only cleanup. void CodeGenFunction::pushEHDestroy(QualType::DestructionKind dtorKind, - llvm::Value *addr, QualType type) { + Address addr, QualType type) { assert(dtorKind && "cannot push destructor for trivial type"); assert(needsEHCleanup(dtorKind)); @@ -1379,7 +1443,7 @@ void CodeGenFunction::pushEHDestroy(QualType::DestructionKind dtorKind, /// pushDestroy - Push the standard destructor for the given type as /// at least a normal cleanup. void CodeGenFunction::pushDestroy(QualType::DestructionKind dtorKind, - llvm::Value *addr, QualType type) { + Address addr, QualType type) { assert(dtorKind && "cannot push destructor for trivial type"); CleanupKind cleanupKind = getCleanupKind(dtorKind); @@ -1387,19 +1451,19 @@ void CodeGenFunction::pushDestroy(QualType::DestructionKind dtorKind, cleanupKind & EHCleanup); } -void CodeGenFunction::pushDestroy(CleanupKind cleanupKind, llvm::Value *addr, +void CodeGenFunction::pushDestroy(CleanupKind cleanupKind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray) { pushFullExprCleanup<DestroyObject>(cleanupKind, addr, type, destroyer, useEHCleanupForArray); } -void CodeGenFunction::pushStackRestore(CleanupKind Kind, llvm::Value *SPMem) { +void CodeGenFunction::pushStackRestore(CleanupKind Kind, Address SPMem) { EHStack.pushCleanup<CallStackRestore>(Kind, SPMem); } void CodeGenFunction::pushLifetimeExtendedDestroy( - CleanupKind cleanupKind, llvm::Value *addr, QualType type, + CleanupKind cleanupKind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray) { assert(!isInConditionalBranch() && "performing lifetime extension from within conditional"); @@ -1429,15 +1493,18 @@ void CodeGenFunction::pushLifetimeExtendedDestroy( /// \param useEHCleanupForArray - whether an EH cleanup should be /// used when destroying array elements, in case one of the /// destructions throws an exception -void CodeGenFunction::emitDestroy(llvm::Value *addr, QualType type, +void CodeGenFunction::emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray) { const ArrayType *arrayType = getContext().getAsArrayType(type); if (!arrayType) return destroyer(*this, addr, type); - llvm::Value *begin = addr; - llvm::Value *length = emitArrayLength(arrayType, type, begin); + llvm::Value *length = emitArrayLength(arrayType, type, addr); + + CharUnits elementAlign = + addr.getAlignment() + .alignmentOfArrayElement(getContext().getTypeSizeInChars(type)); // Normally we have to check whether the array is zero-length. bool checkZeroLength = true; @@ -1449,8 +1516,9 @@ void CodeGenFunction::emitDestroy(llvm::Value *addr, QualType type, checkZeroLength = false; } + llvm::Value *begin = addr.getPointer(); llvm::Value *end = Builder.CreateInBoundsGEP(begin, length); - emitArrayDestroy(begin, end, type, destroyer, + emitArrayDestroy(begin, end, type, elementAlign, destroyer, checkZeroLength, useEHCleanupForArray); } @@ -1459,18 +1527,19 @@ void CodeGenFunction::emitDestroy(llvm::Value *addr, QualType type, /// /// \param begin - a type* denoting the first element of the array /// \param end - a type* denoting one past the end of the array -/// \param type - the element type of the array +/// \param elementType - the element type of the array /// \param destroyer - the function to call to destroy elements /// \param useEHCleanup - whether to push an EH cleanup to destroy /// the remaining elements in case the destruction of a single /// element throws void CodeGenFunction::emitArrayDestroy(llvm::Value *begin, llvm::Value *end, - QualType type, + QualType elementType, + CharUnits elementAlign, Destroyer *destroyer, bool checkZeroLength, bool useEHCleanup) { - assert(!type->isArrayType()); + assert(!elementType->isArrayType()); // The basic structure here is a do-while loop, because we don't // need to check for the zero-element case. @@ -1496,10 +1565,11 @@ void CodeGenFunction::emitArrayDestroy(llvm::Value *begin, "arraydestroy.element"); if (useEHCleanup) - pushRegularPartialArrayCleanup(begin, element, type, destroyer); + pushRegularPartialArrayCleanup(begin, element, elementType, elementAlign, + destroyer); // Perform the actual destruction there. - destroyer(*this, element, type); + destroyer(*this, Address(element, elementAlign), elementType); if (useEHCleanup) PopCleanupBlock(); @@ -1517,7 +1587,7 @@ void CodeGenFunction::emitArrayDestroy(llvm::Value *begin, /// emitArrayDestroy, the element type here may still be an array type. static void emitPartialArrayDestroy(CodeGenFunction &CGF, llvm::Value *begin, llvm::Value *end, - QualType type, + QualType type, CharUnits elementAlign, CodeGenFunction::Destroyer *destroyer) { // If the element type is itself an array, drill down. unsigned arrayDepth = 0; @@ -1529,9 +1599,9 @@ static void emitPartialArrayDestroy(CodeGenFunction &CGF, } if (arrayDepth) { - llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, arrayDepth+1); + llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); - SmallVector<llvm::Value*,4> gepIndices(arrayDepth, zero); + SmallVector<llvm::Value*,4> gepIndices(arrayDepth+1, zero); begin = CGF.Builder.CreateInBoundsGEP(begin, gepIndices, "pad.arraybegin"); end = CGF.Builder.CreateInBoundsGEP(end, gepIndices, "pad.arrayend"); } @@ -1539,7 +1609,7 @@ static void emitPartialArrayDestroy(CodeGenFunction &CGF, // Destroy the array. We don't ever need an EH cleanup because we // assume that we're in an EH cleanup ourselves, so a throwing // destructor causes an immediate terminate. - CGF.emitArrayDestroy(begin, end, type, destroyer, + CGF.emitArrayDestroy(begin, end, type, elementAlign, destroyer, /*checkZeroLength*/ true, /*useEHCleanup*/ false); } @@ -1547,44 +1617,49 @@ namespace { /// RegularPartialArrayDestroy - a cleanup which performs a partial /// array destroy where the end pointer is regularly determined and /// does not need to be loaded from a local. - class RegularPartialArrayDestroy : public EHScopeStack::Cleanup { + class RegularPartialArrayDestroy final : public EHScopeStack::Cleanup { llvm::Value *ArrayBegin; llvm::Value *ArrayEnd; QualType ElementType; CodeGenFunction::Destroyer *Destroyer; + CharUnits ElementAlign; public: RegularPartialArrayDestroy(llvm::Value *arrayBegin, llvm::Value *arrayEnd, - QualType elementType, + QualType elementType, CharUnits elementAlign, CodeGenFunction::Destroyer *destroyer) : ArrayBegin(arrayBegin), ArrayEnd(arrayEnd), - ElementType(elementType), Destroyer(destroyer) {} + ElementType(elementType), Destroyer(destroyer), + ElementAlign(elementAlign) {} void Emit(CodeGenFunction &CGF, Flags flags) override { emitPartialArrayDestroy(CGF, ArrayBegin, ArrayEnd, - ElementType, Destroyer); + ElementType, ElementAlign, Destroyer); } }; /// IrregularPartialArrayDestroy - a cleanup which performs a /// partial array destroy where the end pointer is irregularly /// determined and must be loaded from a local. - class IrregularPartialArrayDestroy : public EHScopeStack::Cleanup { + class IrregularPartialArrayDestroy final : public EHScopeStack::Cleanup { llvm::Value *ArrayBegin; - llvm::Value *ArrayEndPointer; + Address ArrayEndPointer; QualType ElementType; CodeGenFunction::Destroyer *Destroyer; + CharUnits ElementAlign; public: IrregularPartialArrayDestroy(llvm::Value *arrayBegin, - llvm::Value *arrayEndPointer, + Address arrayEndPointer, QualType elementType, + CharUnits elementAlign, CodeGenFunction::Destroyer *destroyer) : ArrayBegin(arrayBegin), ArrayEndPointer(arrayEndPointer), - ElementType(elementType), Destroyer(destroyer) {} + ElementType(elementType), Destroyer(destroyer), + ElementAlign(elementAlign) {} void Emit(CodeGenFunction &CGF, Flags flags) override { llvm::Value *arrayEnd = CGF.Builder.CreateLoad(ArrayEndPointer); emitPartialArrayDestroy(CGF, ArrayBegin, arrayEnd, - ElementType, Destroyer); + ElementType, ElementAlign, Destroyer); } }; } @@ -1596,12 +1671,14 @@ namespace { /// \param elementType - the immediate element type of the array; /// possibly still an array type void CodeGenFunction::pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin, - llvm::Value *arrayEndPointer, + Address arrayEndPointer, QualType elementType, + CharUnits elementAlign, Destroyer *destroyer) { pushFullExprCleanup<IrregularPartialArrayDestroy>(EHCleanup, arrayBegin, arrayEndPointer, - elementType, destroyer); + elementType, elementAlign, + destroyer); } /// pushRegularPartialArrayCleanup - Push an EH cleanup to destroy @@ -1613,10 +1690,12 @@ void CodeGenFunction::pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin, void CodeGenFunction::pushRegularPartialArrayCleanup(llvm::Value *arrayBegin, llvm::Value *arrayEnd, QualType elementType, + CharUnits elementAlign, Destroyer *destroyer) { pushFullExprCleanup<RegularPartialArrayDestroy>(EHCleanup, arrayBegin, arrayEnd, - elementType, destroyer); + elementType, elementAlign, + destroyer); } /// Lazily declare the @llvm.lifetime.start intrinsic. @@ -1640,7 +1719,7 @@ namespace { /// function. This is used to balance out the incoming +1 of a /// ns_consumed argument when we can't reasonably do that just by /// not doing the initial retain for a __block argument. - struct ConsumeARCParameter : EHScopeStack::Cleanup { + struct ConsumeARCParameter final : EHScopeStack::Cleanup { ConsumeARCParameter(llvm::Value *param, ARCPreciseLifetime_t precise) : Param(param), Precise(precise) {} @@ -1656,56 +1735,38 @@ namespace { /// Emit an alloca (or GlobalValue depending on target) /// for the specified parameter and set up LocalDeclMap. -void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg, - bool ArgIsPointer, unsigned ArgNo) { +void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, + unsigned ArgNo) { // FIXME: Why isn't ImplicitParamDecl a ParmVarDecl? assert((isa<ParmVarDecl>(D) || isa<ImplicitParamDecl>(D)) && "Invalid argument to EmitParmDecl"); - Arg->setName(D.getName()); + Arg.getAnyValue()->setName(D.getName()); QualType Ty = D.getType(); // Use better IR generation for certain implicit parameters. - if (isa<ImplicitParamDecl>(D)) { + if (auto IPD = dyn_cast<ImplicitParamDecl>(&D)) { // The only implicit argument a block has is its literal. + // We assume this is always passed directly. if (BlockInfo) { - LocalDeclMap[&D] = Arg; - llvm::Value *LocalAddr = nullptr; - if (CGM.getCodeGenOpts().OptimizationLevel == 0) { - // Allocate a stack slot to let the debug info survive the RA. - llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty), - D.getName() + ".addr"); - Alloc->setAlignment(getContext().getDeclAlign(&D).getQuantity()); - LValue lv = MakeAddrLValue(Alloc, Ty, getContext().getDeclAlign(&D)); - EmitStoreOfScalar(Arg, lv, /* isInitialization */ true); - LocalAddr = Builder.CreateLoad(Alloc); - } - - if (CGDebugInfo *DI = getDebugInfo()) { - if (CGM.getCodeGenOpts().getDebugInfo() - >= CodeGenOptions::LimitedDebugInfo) { - DI->setLocation(D.getLocation()); - DI->EmitDeclareOfBlockLiteralArgVariable(*BlockInfo, Arg, ArgNo, - LocalAddr, Builder); - } - } - + setBlockContextParameter(IPD, ArgNo, Arg.getDirectValue()); return; } } - llvm::Value *DeclPtr; + Address DeclPtr = Address::invalid(); bool DoStore = false; bool IsScalar = hasScalarEvaluationKind(Ty); - CharUnits Align = getContext().getDeclAlign(&D); // If we already have a pointer to the argument, reuse the input pointer. - if (ArgIsPointer) { + if (Arg.isIndirect()) { + DeclPtr = Arg.getIndirectAddress(); // If we have a prettier pointer type at this point, bitcast to that. - unsigned AS = cast<llvm::PointerType>(Arg->getType())->getAddressSpace(); + unsigned AS = DeclPtr.getType()->getAddressSpace(); llvm::Type *IRTy = ConvertTypeForMem(Ty)->getPointerTo(AS); - DeclPtr = Arg->getType() == IRTy ? Arg : Builder.CreateBitCast(Arg, IRTy, - D.getName()); + if (DeclPtr.getType() != IRTy) + DeclPtr = Builder.CreateBitCast(DeclPtr, IRTy, D.getName()); + // Push a destructor cleanup for this parameter if the ABI requires it. // Don't push a cleanup in a thunk for a method that will also emit a // cleanup. @@ -1717,14 +1778,14 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg, } } else { // Otherwise, create a temporary to hold the value. - llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty), - D.getName() + ".addr"); - Alloc->setAlignment(Align.getQuantity()); - DeclPtr = Alloc; + DeclPtr = CreateMemTemp(Ty, getContext().getDeclAlign(&D), + D.getName() + ".addr"); DoStore = true; } - LValue lv = MakeAddrLValue(DeclPtr, Ty, Align); + llvm::Value *ArgVal = (DoStore ? Arg.getDirectValue() : nullptr); + + LValue lv = MakeAddrLValue(DeclPtr, Ty); if (IsScalar) { Qualifiers qs = Ty.getQualifiers(); if (Qualifiers::ObjCLifetime lt = qs.getObjCLifetime()) { @@ -1754,26 +1815,26 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg, // objc_storeStrong attempts to release its old value. llvm::Value *Null = CGM.EmitNullConstant(D.getType()); EmitStoreOfScalar(Null, lv, /* isInitialization */ true); - EmitARCStoreStrongCall(lv.getAddress(), Arg, true); + EmitARCStoreStrongCall(lv.getAddress(), ArgVal, true); DoStore = false; } else // Don't use objc_retainBlock for block pointers, because we // don't want to Block_copy something just because we got it // as a parameter. - Arg = EmitARCRetainNonBlock(Arg); + ArgVal = EmitARCRetainNonBlock(ArgVal); } } else { // Push the cleanup for a consumed parameter. if (isConsumed) { ARCPreciseLifetime_t precise = (D.hasAttr<ObjCPreciseLifetimeAttr>() ? ARCPreciseLifetime : ARCImpreciseLifetime); - EHStack.pushCleanup<ConsumeARCParameter>(getARCCleanupKind(), Arg, + EHStack.pushCleanup<ConsumeARCParameter>(getARCCleanupKind(), ArgVal, precise); } if (lt == Qualifiers::OCL_Weak) { - EmitARCInitWeak(DeclPtr, Arg); + EmitARCInitWeak(DeclPtr, ArgVal); DoStore = false; // The weak init is a store, no need to do two. } } @@ -1785,20 +1846,18 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg, // Store the initial value into the alloca. if (DoStore) - EmitStoreOfScalar(Arg, lv, /* isInitialization */ true); + EmitStoreOfScalar(ArgVal, lv, /* isInitialization */ true); - llvm::Value *&DMEntry = LocalDeclMap[&D]; - assert(!DMEntry && "Decl already exists in localdeclmap!"); - DMEntry = DeclPtr; + setAddrOfLocalVar(&D, DeclPtr); // Emit debug info for param declaration. if (CGDebugInfo *DI = getDebugInfo()) { if (CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo) { - DI->EmitDeclareOfArgVariable(&D, DeclPtr, ArgNo, Builder); + DI->EmitDeclareOfArgVariable(&D, DeclPtr.getPointer(), ArgNo, Builder); } } if (D.hasAttr<AnnotateAttr>()) - EmitVarAnnotations(&D, DeclPtr); + EmitVarAnnotations(&D, DeclPtr.getPointer()); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp index 00d6d5c..adba731 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp @@ -24,16 +24,13 @@ using namespace clang; using namespace CodeGen; static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D, - llvm::Constant *DeclPtr) { + ConstantAddress DeclPtr) { assert(D.hasGlobalStorage() && "VarDecl must have global storage!"); assert(!D.getType()->isReferenceType() && "Should not call EmitDeclInit on a reference!"); - ASTContext &Context = CGF.getContext(); - - CharUnits alignment = Context.getDeclAlign(&D); QualType type = D.getType(); - LValue lv = CGF.MakeAddrLValue(DeclPtr, type, alignment); + LValue lv = CGF.MakeAddrLValue(DeclPtr, type); const Expr *Init = D.getInit(); switch (CGF.getEvaluationKind(type)) { @@ -64,7 +61,7 @@ static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D, /// Emit code to cause the destruction of the given variable with /// static storage duration. static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D, - llvm::Constant *addr) { + ConstantAddress addr) { CodeGenModule &CGM = CGF.CGM; // FIXME: __attribute__((cleanup)) ? @@ -99,7 +96,7 @@ static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D, function = CGM.getAddrOfCXXStructor(dtor, StructorType::Complete); argument = llvm::ConstantExpr::getBitCast( - addr, CGF.getTypes().ConvertType(type)->getPointerTo()); + addr.getPointer(), CGF.getTypes().ConvertType(type)->getPointerTo()); // Otherwise, the standard logic requires a helper function. } else { @@ -162,25 +159,26 @@ void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D, DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(DeclPtr, PTy); } + ConstantAddress DeclAddr(DeclPtr, getContext().getDeclAlign(&D)); + if (!T->isReferenceType()) { if (getLangOpts().OpenMP && D.hasAttr<OMPThreadPrivateDeclAttr>()) (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition( - &D, DeclPtr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(), + &D, DeclAddr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(), PerformInit, this); if (PerformInit) - EmitDeclInit(*this, D, DeclPtr); + EmitDeclInit(*this, D, DeclAddr); if (CGM.isTypeConstant(D.getType(), true)) EmitDeclInvariant(*this, D, DeclPtr); else - EmitDeclDestroy(*this, D, DeclPtr); + EmitDeclDestroy(*this, D, DeclAddr); return; } assert(PerformInit && "cannot have constant initializer which needs " "destruction for reference"); - unsigned Alignment = getContext().getDeclAlign(&D).getQuantity(); RValue RV = EmitReferenceBindingToExpr(Init); - EmitStoreOfScalar(RV.getScalarVal(), DeclPtr, false, Alignment, T); + EmitStoreOfScalar(RV.getScalarVal(), DeclAddr, false, T); } /// Create a stub function, suitable for being passed to atexit, @@ -195,13 +193,15 @@ llvm::Constant *CodeGenFunction::createAtExitStub(const VarDecl &VD, llvm::raw_svector_ostream Out(FnName); CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(&VD, Out); } + + const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction(); llvm::Function *fn = CGM.CreateGlobalInitOrDestructFunction(ty, FnName.str(), + FI, VD.getLocation()); CodeGenFunction CGF(CGM); - CGF.StartFunction(&VD, CGM.getContext().VoidTy, fn, - CGM.getTypes().arrangeNullaryFunction(), FunctionArgList()); + CGF.StartFunction(&VD, CGM.getContext().VoidTy, fn, FI, FunctionArgList()); llvm::CallInst *call = CGF.Builder.CreateCall(dtor, addr); @@ -249,7 +249,8 @@ void CodeGenFunction::EmitCXXGuardedInit(const VarDecl &D, } llvm::Function *CodeGenModule::CreateGlobalInitOrDestructFunction( - llvm::FunctionType *FTy, const Twine &Name, SourceLocation Loc, bool TLS) { + llvm::FunctionType *FTy, const Twine &Name, const CGFunctionInfo &FI, + SourceLocation Loc, bool TLS) { llvm::Function *Fn = llvm::Function::Create(FTy, llvm::GlobalValue::InternalLinkage, Name, &getModule()); @@ -259,7 +260,7 @@ llvm::Function *CodeGenModule::CreateGlobalInitOrDestructFunction( Fn->setSection(Section); } - SetLLVMFunctionAttributes(nullptr, getTypes().arrangeNullaryFunction(), Fn); + SetInternalFunctionAttributes(nullptr, Fn, FI); Fn->setCallingConv(getRuntimeCC()); @@ -317,7 +318,9 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D, // Create a variable initialization function. llvm::Function *Fn = - CreateGlobalInitOrDestructFunction(FTy, FnName.str(), D->getLocation()); + CreateGlobalInitOrDestructFunction(FTy, FnName.str(), + getTypes().arrangeNullaryFunction(), + D->getLocation()); auto *ISA = D->getAttr<InitSegAttr>(); CodeGenFunction(*this).GenerateCXXGlobalVarDeclInitFunc(Fn, D, Addr, @@ -334,7 +337,7 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D, // FIXME: We only need to register one __cxa_thread_atexit function for the // entire TU. CXXThreadLocalInits.push_back(Fn); - CXXThreadLocalInitVars.push_back(Addr); + CXXThreadLocalInitVars.push_back(D); } else if (PerformInit && ISA) { EmitPointerToInitFunc(D, Addr, Fn, ISA); } else if (auto *IPA = D->getAttr<InitPriorityAttr>()) { @@ -392,7 +395,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() { return; llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false); - + const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction(); // Create our global initialization function. if (!PrioritizedCXXGlobalInits.empty()) { @@ -416,7 +419,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() { // Priority is always <= 65535 (enforced by sema). PrioritySuffix = std::string(6-PrioritySuffix.size(), '0')+PrioritySuffix; llvm::Function *Fn = CreateGlobalInitOrDestructFunction( - FTy, "_GLOBAL__I_" + PrioritySuffix); + FTy, "_GLOBAL__I_" + PrioritySuffix, FI); for (; I < PrioE; ++I) LocalCXXGlobalInits.push_back(I->second); @@ -446,7 +449,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() { } llvm::Function *Fn = CreateGlobalInitOrDestructFunction( - FTy, llvm::Twine("_GLOBAL__sub_I_", FileName)); + FTy, llvm::Twine("_GLOBAL__sub_I_", FileName), FI); CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXGlobalInits); AddGlobalCtor(Fn); @@ -461,7 +464,9 @@ void CodeGenModule::EmitCXXGlobalDtorFunc() { llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false); // Create our global destructor function. - llvm::Function *Fn = CreateGlobalInitOrDestructFunction(FTy, "_GLOBAL__D_a"); + const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction(); + llvm::Function *Fn = + CreateGlobalInitOrDestructFunction(FTy, "_GLOBAL__D_a", FI); CodeGenFunction(*this).GenerateCXXGlobalDtorsFunc(Fn, CXXGlobalDtors); AddGlobalDtor(Fn); @@ -498,7 +503,7 @@ void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, void CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn, ArrayRef<llvm::Function *> Decls, - llvm::GlobalVariable *Guard) { + Address Guard) { { auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(GlobalDecl(), getContext().VoidTy, Fn, @@ -507,20 +512,20 @@ CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn, auto AL = ApplyDebugLocation::CreateArtificial(*this); llvm::BasicBlock *ExitBlock = nullptr; - if (Guard) { + if (Guard.isValid()) { // If we have a guard variable, check whether we've already performed // these initializations. This happens for TLS initialization functions. llvm::Value *GuardVal = Builder.CreateLoad(Guard); llvm::Value *Uninit = Builder.CreateIsNull(GuardVal, "guard.uninitialized"); - // Mark as initialized before initializing anything else. If the - // initializers use previously-initialized thread_local vars, that's - // probably supposed to be OK, but the standard doesn't say. - Builder.CreateStore(llvm::ConstantInt::get(GuardVal->getType(),1), Guard); llvm::BasicBlock *InitBlock = createBasicBlock("init"); ExitBlock = createBasicBlock("exit"); Builder.CreateCondBr(Uninit, InitBlock, ExitBlock); EmitBlock(InitBlock); + // Mark as initialized before initializing anything else. If the + // initializers use previously-initialized thread_local vars, that's + // probably supposed to be OK, but the standard doesn't say. + Builder.CreateStore(llvm::ConstantInt::get(GuardVal->getType(),1), Guard); } RunCleanupsScope Scope(*this); @@ -572,9 +577,10 @@ void CodeGenFunction::GenerateCXXGlobalDtorsFunc(llvm::Function *Fn, } /// generateDestroyHelper - Generates a helper function which, when -/// invoked, destroys the given object. +/// invoked, destroys the given object. The address of the object +/// should be in global memory. llvm::Function *CodeGenFunction::generateDestroyHelper( - llvm::Constant *addr, QualType type, Destroyer *destroyer, + Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray, const VarDecl *VD) { FunctionArgList args; ImplicitParamDecl dst(getContext(), nullptr, SourceLocation(), nullptr, @@ -585,7 +591,7 @@ llvm::Function *CodeGenFunction::generateDestroyHelper( getContext().VoidTy, args, FunctionType::ExtInfo(), /*variadic=*/false); llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI); llvm::Function *fn = CGM.CreateGlobalInitOrDestructFunction( - FTy, "__cxx_global_array_dtor", VD->getLocation()); + FTy, "__cxx_global_array_dtor", FI, VD->getLocation()); CurEHLocation = VD->getLocStart(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp index 7b8368e..fce2e75 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp @@ -1,4 +1,4 @@ -//===--- CGException.cpp - Emit LLVM Code for C++ exceptions --------------===// +//===--- CGException.cpp - Emit LLVM Code for C++ exceptions ----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -81,38 +81,6 @@ static llvm::Constant *getCatchallRethrowFn(CodeGenModule &CGM, return CGM.CreateRuntimeFunction(FTy, Name); } -namespace { - /// The exceptions personality for a function. - struct EHPersonality { - const char *PersonalityFn; - - // If this is non-null, this personality requires a non-standard - // function for rethrowing an exception after a catchall cleanup. - // This function must have prototype void(void*). - const char *CatchallRethrowFn; - - static const EHPersonality &get(CodeGenModule &CGM, - const FunctionDecl *FD); - static const EHPersonality &get(CodeGenFunction &CGF) { - return get(CGF.CGM, dyn_cast_or_null<FunctionDecl>(CGF.CurCodeDecl)); - } - - static const EHPersonality GNU_C; - static const EHPersonality GNU_C_SJLJ; - static const EHPersonality GNU_C_SEH; - static const EHPersonality GNU_ObjC; - static const EHPersonality GNUstep_ObjC; - static const EHPersonality GNU_ObjCXX; - static const EHPersonality NeXT_ObjC; - static const EHPersonality GNU_CPlusPlus; - static const EHPersonality GNU_CPlusPlus_SJLJ; - static const EHPersonality GNU_CPlusPlus_SEH; - static const EHPersonality MSVC_except_handler; - static const EHPersonality MSVC_C_specific_handler; - static const EHPersonality MSVC_CxxFrameHandler3; - }; -} - const EHPersonality EHPersonality::GNU_C = { "__gcc_personality_v0", nullptr }; const EHPersonality EHPersonality::GNU_C_SJLJ = { "__gcc_personality_sj0", nullptr }; @@ -161,6 +129,7 @@ static const EHPersonality &getObjCPersonality(const llvm::Triple &T, return getCPersonality(T, L); case ObjCRuntime::MacOSX: case ObjCRuntime::iOS: + case ObjCRuntime::WatchOS: return EHPersonality::NeXT_ObjC; case ObjCRuntime::GNUstep: if (L.ObjCRuntime.getVersion() >= VersionTuple(1, 7)) @@ -192,6 +161,7 @@ static const EHPersonality &getObjCXXPersonality(const llvm::Triple &T, // function on targets using (backend-driven) SJLJ EH. case ObjCRuntime::MacOSX: case ObjCRuntime::iOS: + case ObjCRuntime::WatchOS: return EHPersonality::NeXT_ObjC; // In the fragile ABI, just use C++ exception handling and hope @@ -221,14 +191,16 @@ const EHPersonality &EHPersonality::get(CodeGenModule &CGM, const llvm::Triple &T = CGM.getTarget().getTriple(); const LangOptions &L = CGM.getLangOpts(); + // Functions using SEH get an SEH personality. + if (FD && FD->usesSEHTry()) + return getSEHPersonalityMSVC(T); + // Try to pick a personality function that is compatible with MSVC if we're // not compiling Obj-C. Obj-C users better have an Obj-C runtime that supports // the GCC-style personality function. if (T.isWindowsMSVCEnvironment() && !L.ObjC1) { if (L.SjLjExceptions) return EHPersonality::GNU_CPlusPlus_SJLJ; - else if (FD && FD->usesSEHTry()) - return getSEHPersonalityMSVC(T); else return EHPersonality::MSVC_CxxFrameHandler3; } @@ -243,6 +215,10 @@ const EHPersonality &EHPersonality::get(CodeGenModule &CGM, return getCPersonality(T, L); } +const EHPersonality &EHPersonality::get(CodeGenFunction &CGF) { + return get(CGF.CGM, dyn_cast_or_null<FunctionDecl>(CGF.CurCodeDecl)); +} + static llvm::Constant *getPersonalityFn(CodeGenModule &CGM, const EHPersonality &Personality) { llvm::Constant *Fn = @@ -257,6 +233,36 @@ static llvm::Constant *getOpaquePersonalityFn(CodeGenModule &CGM, return llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy); } +/// Check whether a landingpad instruction only uses C++ features. +static bool LandingPadHasOnlyCXXUses(llvm::LandingPadInst *LPI) { + for (unsigned I = 0, E = LPI->getNumClauses(); I != E; ++I) { + // Look for something that would've been returned by the ObjC + // runtime's GetEHType() method. + llvm::Value *Val = LPI->getClause(I)->stripPointerCasts(); + if (LPI->isCatch(I)) { + // Check if the catch value has the ObjC prefix. + if (llvm::GlobalVariable *GV = dyn_cast<llvm::GlobalVariable>(Val)) + // ObjC EH selector entries are always global variables with + // names starting like this. + if (GV->getName().startswith("OBJC_EHTYPE")) + return false; + } else { + // Check if any of the filter values have the ObjC prefix. + llvm::Constant *CVal = cast<llvm::Constant>(Val); + for (llvm::User::op_iterator + II = CVal->op_begin(), IE = CVal->op_end(); II != IE; ++II) { + if (llvm::GlobalVariable *GV = + cast<llvm::GlobalVariable>((*II)->stripPointerCasts())) + // ObjC EH selector entries are always global variables with + // names starting like this. + if (GV->getName().startswith("OBJC_EHTYPE")) + return false; + } + } + } + return true; +} + /// Check whether a personality function could reasonably be swapped /// for a C++ personality function. static bool PersonalityHasOnlyCXXUses(llvm::Constant *Fn) { @@ -269,34 +275,14 @@ static bool PersonalityHasOnlyCXXUses(llvm::Constant *Fn) { continue; } - // Otherwise, it has to be a landingpad instruction. - llvm::LandingPadInst *LPI = dyn_cast<llvm::LandingPadInst>(U); - if (!LPI) return false; - - for (unsigned I = 0, E = LPI->getNumClauses(); I != E; ++I) { - // Look for something that would've been returned by the ObjC - // runtime's GetEHType() method. - llvm::Value *Val = LPI->getClause(I)->stripPointerCasts(); - if (LPI->isCatch(I)) { - // Check if the catch value has the ObjC prefix. - if (llvm::GlobalVariable *GV = dyn_cast<llvm::GlobalVariable>(Val)) - // ObjC EH selector entries are always global variables with - // names starting like this. - if (GV->getName().startswith("OBJC_EHTYPE")) - return false; - } else { - // Check if any of the filter values have the ObjC prefix. - llvm::Constant *CVal = cast<llvm::Constant>(Val); - for (llvm::User::op_iterator - II = CVal->op_begin(), IE = CVal->op_end(); II != IE; ++II) { - if (llvm::GlobalVariable *GV = - cast<llvm::GlobalVariable>((*II)->stripPointerCasts())) - // ObjC EH selector entries are always global variables with - // names starting like this. - if (GV->getName().startswith("OBJC_EHTYPE")) - return false; - } - } + // Otherwise it must be a function. + llvm::Function *F = dyn_cast<llvm::Function>(U); + if (!F) return false; + + for (auto BB = F->begin(), E = F->end(); BB != E; ++BB) { + if (BB->isLandingPad()) + if (!LandingPadHasOnlyCXXUses(BB->getLandingPadInst())) + return false; } } @@ -355,29 +341,29 @@ static llvm::Constant *getCatchAllValue(CodeGenFunction &CGF) { namespace { /// A cleanup to free the exception object if its initialization /// throws. - struct FreeException : EHScopeStack::Cleanup { + struct FreeException final : EHScopeStack::Cleanup { llvm::Value *exn; FreeException(llvm::Value *exn) : exn(exn) {} void Emit(CodeGenFunction &CGF, Flags flags) override { CGF.EmitNounwindRuntimeCall(getFreeExceptionFn(CGF.CGM), exn); } }; -} +} // end anonymous namespace // Emits an exception expression into the given location. This // differs from EmitAnyExprToMem only in that, if a final copy-ctor // call is required, an exception within that copy ctor causes // std::terminate to be invoked. -void CodeGenFunction::EmitAnyExprToExn(const Expr *e, llvm::Value *addr) { +void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) { // Make sure the exception object is cleaned up if there's an // exception during initialization. - pushFullExprCleanup<FreeException>(EHCleanup, addr); + pushFullExprCleanup<FreeException>(EHCleanup, addr.getPointer()); EHScopeStack::stable_iterator cleanup = EHStack.stable_begin(); // __cxa_allocate_exception returns a void*; we need to cast this // to the appropriate type for the object. llvm::Type *ty = ConvertTypeForMem(e->getType())->getPointerTo(); - llvm::Value *typedAddr = Builder.CreateBitCast(addr, ty); + Address typedAddr = Builder.CreateBitCast(addr, ty); // FIXME: this isn't quite right! If there's a final unelided call // to a copy constructor, then according to [except.terminate]p1 we @@ -390,19 +376,20 @@ void CodeGenFunction::EmitAnyExprToExn(const Expr *e, llvm::Value *addr) { /*IsInit*/ true); // Deactivate the cleanup block. - DeactivateCleanupBlock(cleanup, cast<llvm::Instruction>(typedAddr)); + DeactivateCleanupBlock(cleanup, + cast<llvm::Instruction>(typedAddr.getPointer())); } -llvm::Value *CodeGenFunction::getExceptionSlot() { +Address CodeGenFunction::getExceptionSlot() { if (!ExceptionSlot) ExceptionSlot = CreateTempAlloca(Int8PtrTy, "exn.slot"); - return ExceptionSlot; + return Address(ExceptionSlot, getPointerAlign()); } -llvm::Value *CodeGenFunction::getEHSelectorSlot() { +Address CodeGenFunction::getEHSelectorSlot() { if (!EHSelectorSlot) EHSelectorSlot = CreateTempAlloca(Int32Ty, "ehselector.slot"); - return EHSelectorSlot; + return Address(EHSelectorSlot, CharUnits::fromQuantity(4)); } llvm::Value *CodeGenFunction::getExceptionFromSlot() { @@ -571,22 +558,25 @@ void CodeGenFunction::EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) { QualType CaughtType = CGM.getContext().getUnqualifiedArrayType( C->getCaughtType().getNonReferenceType(), CaughtTypeQuals); - llvm::Constant *TypeInfo = nullptr; + CatchTypeInfo TypeInfo{nullptr, 0}; if (CaughtType->isObjCObjectPointerType()) - TypeInfo = CGM.getObjCRuntime().GetEHType(CaughtType); + TypeInfo.RTTI = CGM.getObjCRuntime().GetEHType(CaughtType); else - TypeInfo = - CGM.getAddrOfCXXCatchHandlerType(CaughtType, C->getCaughtType()); + TypeInfo = CGM.getCXXABI().getAddrOfCXXCatchHandlerType( + CaughtType, C->getCaughtType()); CatchScope->setHandler(I, TypeInfo, Handler); } else { // No exception decl indicates '...', a catch-all. - CatchScope->setCatchAllHandler(I, Handler); + CatchScope->setHandler(I, CGM.getCXXABI().getCatchAllTypeInfo(), Handler); } } } llvm::BasicBlock * CodeGenFunction::getEHDispatchBlock(EHScopeStack::stable_iterator si) { + if (EHPersonality::get(*this).usesFuncletPads()) + return getMSVCDispatchBlock(si); + // The dispatch block for the end of the scope chain is a block that // just resumes unwinding. if (si == EHStack.stable_end()) @@ -623,12 +613,58 @@ CodeGenFunction::getEHDispatchBlock(EHScopeStack::stable_iterator si) { case EHScope::Terminate: dispatchBlock = getTerminateHandler(); break; + + case EHScope::PadEnd: + llvm_unreachable("PadEnd unnecessary for Itanium!"); } scope.setCachedEHDispatchBlock(dispatchBlock); } return dispatchBlock; } +llvm::BasicBlock * +CodeGenFunction::getMSVCDispatchBlock(EHScopeStack::stable_iterator SI) { + // Returning nullptr indicates that the previous dispatch block should unwind + // to caller. + if (SI == EHStack.stable_end()) + return nullptr; + + // Otherwise, we should look at the actual scope. + EHScope &EHS = *EHStack.find(SI); + + llvm::BasicBlock *DispatchBlock = EHS.getCachedEHDispatchBlock(); + if (DispatchBlock) + return DispatchBlock; + + if (EHS.getKind() == EHScope::Terminate) + DispatchBlock = getTerminateHandler(); + else + DispatchBlock = createBasicBlock(); + CGBuilderTy Builder(*this, DispatchBlock); + + switch (EHS.getKind()) { + case EHScope::Catch: + DispatchBlock->setName("catch.dispatch"); + break; + + case EHScope::Cleanup: + DispatchBlock->setName("ehcleanup"); + break; + + case EHScope::Filter: + llvm_unreachable("exception specifications not handled yet!"); + + case EHScope::Terminate: + DispatchBlock->setName("terminate"); + break; + + case EHScope::PadEnd: + llvm_unreachable("PadEnd dispatch block missing!"); + } + EHS.setCachedEHDispatchBlock(DispatchBlock); + return DispatchBlock; +} + /// Check whether this is a non-EH scope, i.e. a scope which doesn't /// affect exception handling. Currently, the only non-EH scopes are /// normal-only cleanup scopes. @@ -639,6 +675,7 @@ static bool isNonEHScope(const EHScope &S) { case EHScope::Filter: case EHScope::Catch: case EHScope::Terminate: + case EHScope::PadEnd: return false; } @@ -664,8 +701,19 @@ llvm::BasicBlock *CodeGenFunction::getInvokeDestImpl() { llvm::BasicBlock *LP = EHStack.begin()->getCachedLandingPad(); if (LP) return LP; - // Build the landing pad for this scope. - LP = EmitLandingPad(); + const EHPersonality &Personality = EHPersonality::get(*this); + + if (!CurFn->hasPersonalityFn()) + CurFn->setPersonalityFn(getOpaquePersonalityFn(CGM, Personality)); + + if (Personality.usesFuncletPads()) { + // We don't need separate landing pads in the funclet model. + LP = getEHDispatchBlock(EHStack.getInnermostEHScope()); + } else { + // Build the landing pad for this scope. + LP = EmitLandingPad(); + } + assert(LP); // Cache the landing pad on the innermost scope. If this is a @@ -686,6 +734,9 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() { case EHScope::Terminate: return getTerminateLandingPad(); + case EHScope::PadEnd: + llvm_unreachable("PadEnd unnecessary for Itanium!"); + case EHScope::Catch: case EHScope::Cleanup: case EHScope::Filter: @@ -697,11 +748,6 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() { CGBuilderTy::InsertPoint savedIP = Builder.saveAndClearIP(); auto DL = ApplyDebugLocation::CreateDefaultArtificial(*this, CurEHLocation); - const EHPersonality &personality = EHPersonality::get(*this); - - if (!CurFn->hasPersonalityFn()) - CurFn->setPersonalityFn(getOpaquePersonalityFn(CGM, personality)); - // Create and configure the landing pad. llvm::BasicBlock *lpad = createBasicBlock("lpad"); EmitBlock(lpad); @@ -756,23 +802,28 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() { case EHScope::Catch: break; + + case EHScope::PadEnd: + llvm_unreachable("PadEnd unnecessary for Itanium!"); } EHCatchScope &catchScope = cast<EHCatchScope>(*I); for (unsigned hi = 0, he = catchScope.getNumHandlers(); hi != he; ++hi) { EHCatchScope::Handler handler = catchScope.getHandler(hi); + assert(handler.Type.Flags == 0 && + "landingpads do not support catch handler flags"); // If this is a catch-all, register that and abort. - if (!handler.Type) { + if (!handler.Type.RTTI) { assert(!hasCatchAll); hasCatchAll = true; goto done; } // Check whether we already have a handler for this type. - if (catchTypes.insert(handler.Type).second) + if (catchTypes.insert(handler.Type.RTTI).second) // If not, add it directly to the landingpad. - LPadInst->addClause(handler.Type); + LPadInst->addClause(handler.Type.RTTI); } } @@ -820,10 +871,53 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() { return lpad; } +static void emitCatchPadBlock(CodeGenFunction &CGF, EHCatchScope &CatchScope) { + llvm::BasicBlock *DispatchBlock = CatchScope.getCachedEHDispatchBlock(); + assert(DispatchBlock); + + CGBuilderTy::InsertPoint SavedIP = CGF.Builder.saveIP(); + CGF.EmitBlockAfterUses(DispatchBlock); + + llvm::Value *ParentPad = CGF.CurrentFuncletPad; + if (!ParentPad) + ParentPad = llvm::ConstantTokenNone::get(CGF.getLLVMContext()); + llvm::BasicBlock *UnwindBB = + CGF.getEHDispatchBlock(CatchScope.getEnclosingEHScope()); + + unsigned NumHandlers = CatchScope.getNumHandlers(); + llvm::CatchSwitchInst *CatchSwitch = + CGF.Builder.CreateCatchSwitch(ParentPad, UnwindBB, NumHandlers); + + // Test against each of the exception types we claim to catch. + for (unsigned I = 0; I < NumHandlers; ++I) { + const EHCatchScope::Handler &Handler = CatchScope.getHandler(I); + + CatchTypeInfo TypeInfo = Handler.Type; + if (!TypeInfo.RTTI) + TypeInfo.RTTI = llvm::Constant::getNullValue(CGF.VoidPtrTy); + + CGF.Builder.SetInsertPoint(Handler.Block); + + if (EHPersonality::get(CGF).isMSVCXXPersonality()) { + CGF.Builder.CreateCatchPad( + CatchSwitch, {TypeInfo.RTTI, CGF.Builder.getInt32(TypeInfo.Flags), + llvm::Constant::getNullValue(CGF.VoidPtrTy)}); + } else { + CGF.Builder.CreateCatchPad(CatchSwitch, {TypeInfo.RTTI}); + } + + CatchSwitch->addHandler(Handler.Block); + } + CGF.Builder.restoreIP(SavedIP); +} + /// Emit the structure of the dispatch block for the given catch scope. /// It is an invariant that the dispatch block already exists. static void emitCatchDispatchBlock(CodeGenFunction &CGF, EHCatchScope &catchScope) { + if (EHPersonality::get(CGF).usesFuncletPads()) + return emitCatchPadBlock(CGF, catchScope); + llvm::BasicBlock *dispatchBlock = catchScope.getCachedEHDispatchBlock(); assert(dispatchBlock); @@ -850,7 +944,9 @@ static void emitCatchDispatchBlock(CodeGenFunction &CGF, assert(i < e && "ran off end of handlers!"); const EHCatchScope::Handler &handler = catchScope.getHandler(i); - llvm::Value *typeValue = handler.Type; + llvm::Value *typeValue = handler.Type.RTTI; + assert(handler.Type.Flags == 0 && + "landingpads do not support catch handler flags"); assert(typeValue && "fell into catch-all case!"); typeValue = CGF.Builder.CreateBitCast(typeValue, CGF.Int8PtrTy); @@ -919,9 +1015,8 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) { // Copy the handler blocks off before we pop the EH stack. Emitting // the handlers might scribble on this memory. - SmallVector<EHCatchScope::Handler, 8> Handlers(NumHandlers); - memcpy(Handlers.data(), CatchScope.begin(), - NumHandlers * sizeof(EHCatchScope::Handler)); + SmallVector<EHCatchScope::Handler, 8> Handlers( + CatchScope.begin(), CatchScope.begin() + NumHandlers); EHStack.popCatch(); @@ -958,6 +1053,8 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) { RunCleanupsScope CatchScope(*this); // Initialize the catch variable and set up the cleanups. + SaveAndRestore<llvm::Instruction *> RestoreCurrentFuncletPad( + CurrentFuncletPad); CGM.getCXXABI().emitBeginCatch(*this, C); // Emit the PGO counter increment. @@ -994,7 +1091,7 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) { } namespace { - struct CallEndCatchForFinally : EHScopeStack::Cleanup { + struct CallEndCatchForFinally final : EHScopeStack::Cleanup { llvm::Value *ForEHVar; llvm::Value *EndCatchFn; CallEndCatchForFinally(llvm::Value *ForEHVar, llvm::Value *EndCatchFn) @@ -1006,7 +1103,7 @@ namespace { CGF.createBasicBlock("finally.cleanup.cont"); llvm::Value *ShouldEndCatch = - CGF.Builder.CreateLoad(ForEHVar, "finally.endcatch"); + CGF.Builder.CreateFlagLoad(ForEHVar, "finally.endcatch"); CGF.Builder.CreateCondBr(ShouldEndCatch, EndCatchBB, CleanupContBB); CGF.EmitBlock(EndCatchBB); CGF.EmitRuntimeCallOrInvoke(EndCatchFn); // catch-all, so might throw @@ -1014,7 +1111,7 @@ namespace { } }; - struct PerformFinally : EHScopeStack::Cleanup { + struct PerformFinally final : EHScopeStack::Cleanup { const Stmt *Body; llvm::Value *ForEHVar; llvm::Value *EndCatchFn; @@ -1049,13 +1146,13 @@ namespace { llvm::BasicBlock *ContBB = CGF.createBasicBlock("finally.cont"); llvm::Value *ShouldRethrow = - CGF.Builder.CreateLoad(ForEHVar, "finally.shouldthrow"); + CGF.Builder.CreateFlagLoad(ForEHVar, "finally.shouldthrow"); CGF.Builder.CreateCondBr(ShouldRethrow, RethrowBB, ContBB); CGF.EmitBlock(RethrowBB); if (SavedExnVar) { CGF.EmitRuntimeCallOrInvoke(RethrowFn, - CGF.Builder.CreateLoad(SavedExnVar)); + CGF.Builder.CreateAlignedLoad(SavedExnVar, CGF.getPointerAlign())); } else { CGF.EmitRuntimeCallOrInvoke(RethrowFn); } @@ -1082,7 +1179,7 @@ namespace { CGF.EnsureInsertPoint(); } }; -} +} // end anonymous namespace /// Enters a finally block for an implementation using zero-cost /// exceptions. This is mostly general, but hard-codes some @@ -1130,7 +1227,7 @@ void CodeGenFunction::FinallyInfo::enter(CodeGenFunction &CGF, // Whether the finally block is being executed for EH purposes. ForEHVar = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(), "finally.for-eh"); - CGF.Builder.CreateStore(CGF.Builder.getFalse(), ForEHVar); + CGF.Builder.CreateFlagStore(false, ForEHVar); // Enter a normal cleanup which will perform the @finally block. CGF.EHStack.pushCleanup<PerformFinally>(NormalCleanup, body, @@ -1168,11 +1265,11 @@ void CodeGenFunction::FinallyInfo::exit(CodeGenFunction &CGF) { // If we need to remember the exception pointer to rethrow later, do so. if (SavedExnVar) { if (!exn) exn = CGF.getExceptionFromSlot(); - CGF.Builder.CreateStore(exn, SavedExnVar); + CGF.Builder.CreateAlignedStore(exn, SavedExnVar, CGF.getPointerAlign()); } // Tell the cleanups in the finally block that we're do this for EH. - CGF.Builder.CreateStore(CGF.Builder.getTrue(), ForEHVar); + CGF.Builder.CreateFlagStore(true, ForEHVar); // Thread a jump through the finally cleanup. CGF.EmitBranchThroughCleanup(RethrowDest); @@ -1204,7 +1301,7 @@ llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() { llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), 0); LPadInst->addClause(getCatchAllValue(*this)); - llvm::Value *Exn = 0; + llvm::Value *Exn = nullptr; if (getLangOpts().CPlusPlus) Exn = Builder.CreateExtractValue(LPadInst, 0); llvm::CallInst *terminateCall = @@ -1228,9 +1325,16 @@ llvm::BasicBlock *CodeGenFunction::getTerminateHandler() { // end of the function by FinishFunction. TerminateHandler = createBasicBlock("terminate.handler"); Builder.SetInsertPoint(TerminateHandler); - llvm::Value *Exn = 0; - if (getLangOpts().CPlusPlus) - Exn = getExceptionFromSlot(); + llvm::Value *Exn = nullptr; + if (EHPersonality::get(*this).usesFuncletPads()) { + llvm::Value *ParentPad = CurrentFuncletPad; + if (!ParentPad) + ParentPad = llvm::ConstantTokenNone::get(CGM.getLLVMContext()); + Builder.CreateCleanupPad(ParentPad); + } else { + if (getLangOpts().CPlusPlus) + Exn = getExceptionFromSlot(); + } llvm::CallInst *terminateCall = CGM.getCXXABI().emitTerminateForUnexpectedException(*this, Exn); terminateCall->setDoesNotReturn(); @@ -1297,7 +1401,7 @@ void CodeGenFunction::EmitSEHTryStmt(const SEHTryStmt &S) { } namespace { -struct PerformSEHFinally : EHScopeStack::Cleanup { +struct PerformSEHFinally final : EHScopeStack::Cleanup { llvm::Function *OutlinedFinally; PerformSEHFinally(llvm::Function *OutlinedFinally) : OutlinedFinally(OutlinedFinally) {} @@ -1328,21 +1432,21 @@ struct PerformSEHFinally : EHScopeStack::Cleanup { CGF.EmitCall(FnInfo, OutlinedFinally, ReturnValueSlot(), Args); } }; -} +} // end anonymous namespace namespace { /// Find all local variable captures in the statement. struct CaptureFinder : ConstStmtVisitor<CaptureFinder> { CodeGenFunction &ParentCGF; const VarDecl *ParentThis; - SmallVector<const VarDecl *, 4> Captures; - llvm::Value *SEHCodeSlot = nullptr; + llvm::SmallSetVector<const VarDecl *, 4> Captures; + Address SEHCodeSlot = Address::invalid(); CaptureFinder(CodeGenFunction &ParentCGF, const VarDecl *ParentThis) : ParentCGF(ParentCGF), ParentThis(ParentThis) {} // Return true if we need to do any capturing work. bool foundCaptures() { - return !Captures.empty() || SEHCodeSlot; + return !Captures.empty() || SEHCodeSlot.isValid(); } void Visit(const Stmt *S) { @@ -1356,17 +1460,17 @@ struct CaptureFinder : ConstStmtVisitor<CaptureFinder> { void VisitDeclRefExpr(const DeclRefExpr *E) { // If this is already a capture, just make sure we capture 'this'. if (E->refersToEnclosingVariableOrCapture()) { - Captures.push_back(ParentThis); + Captures.insert(ParentThis); return; } const auto *D = dyn_cast<VarDecl>(E->getDecl()); if (D && D->isLocalVarDeclOrParm() && D->hasLocalStorage()) - Captures.push_back(D); + Captures.insert(D); } void VisitCXXThisExpr(const CXXThisExpr *E) { - Captures.push_back(ParentThis); + Captures.insert(ParentThis); } void VisitCallExpr(const CallExpr *E) { @@ -1381,19 +1485,20 @@ struct CaptureFinder : ConstStmtVisitor<CaptureFinder> { // This is the simple case where we are the outermost finally. All we // have to do here is make sure we escape this and recover it in the // outlined handler. - if (!SEHCodeSlot) + if (!SEHCodeSlot.isValid()) SEHCodeSlot = ParentCGF.SEHCodeSlotStack.back(); break; } } }; -} +} // end anonymous namespace -llvm::Value *CodeGenFunction::recoverAddrOfEscapedLocal( - CodeGenFunction &ParentCGF, llvm::Value *ParentVar, llvm::Value *ParentFP) { +Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, + Address ParentVar, + llvm::Value *ParentFP) { llvm::CallInst *RecoverCall = nullptr; - CGBuilderTy Builder(AllocaInsertPt); - if (auto *ParentAlloca = dyn_cast<llvm::AllocaInst>(ParentVar)) { + CGBuilderTy Builder(*this, AllocaInsertPt); + if (auto *ParentAlloca = dyn_cast<llvm::AllocaInst>(ParentVar.getPointer())) { // Mark the variable escaped if nobody else referenced it and compute the // localescape index. auto InsertPair = ParentCGF.EscapedLocals.insert( @@ -1413,7 +1518,7 @@ llvm::Value *CodeGenFunction::recoverAddrOfEscapedLocal( // Just clone the existing localrecover call, but tweak the FP argument to // use our FP value. All other arguments are constants. auto *ParentRecover = - cast<llvm::IntrinsicInst>(ParentVar->stripPointerCasts()); + cast<llvm::IntrinsicInst>(ParentVar.getPointer()->stripPointerCasts()); assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::localrecover && "expected alloca or localrecover in parent LocalDeclMap"); RecoverCall = cast<llvm::CallInst>(ParentRecover->clone()); @@ -1423,9 +1528,9 @@ llvm::Value *CodeGenFunction::recoverAddrOfEscapedLocal( // Bitcast the variable, rename it, and insert it in the local decl map. llvm::Value *ChildVar = - Builder.CreateBitCast(RecoverCall, ParentVar->getType()); - ChildVar->setName(ParentVar->getName()); - return ChildVar; + Builder.CreateBitCast(RecoverCall, ParentVar.getType()); + ChildVar->setName(ParentVar.getName()); + return Address(ChildVar, ParentVar.getAlignment()); } void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, @@ -1444,27 +1549,32 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, return; } - llvm::Value *EntryEBP = nullptr; - llvm::Value *ParentFP; + llvm::Value *EntryFP = nullptr; + CGBuilderTy Builder(CGM, AllocaInsertPt); if (IsFilter && CGM.getTarget().getTriple().getArch() == llvm::Triple::x86) { // 32-bit SEH filters need to be careful about FP recovery. The end of the // EH registration is passed in as the EBP physical register. We can - // recover that with llvm.frameaddress(1), and adjust that to recover the - // parent's true frame pointer. - CGBuilderTy Builder(AllocaInsertPt); - EntryEBP = Builder.CreateCall( + // recover that with llvm.frameaddress(1). + EntryFP = Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::frameaddress), {Builder.getInt32(1)}); - llvm::Function *RecoverFPIntrin = - CGM.getIntrinsic(llvm::Intrinsic::x86_seh_recoverfp); - llvm::Constant *ParentI8Fn = - llvm::ConstantExpr::getBitCast(ParentCGF.CurFn, Int8PtrTy); - ParentFP = Builder.CreateCall(RecoverFPIntrin, {ParentI8Fn, EntryEBP}); } else { // Otherwise, for x64 and 32-bit finally functions, the parent FP is the // second parameter. auto AI = CurFn->arg_begin(); ++AI; - ParentFP = AI; + EntryFP = &*AI; + } + + llvm::Value *ParentFP = EntryFP; + if (IsFilter) { + // Given whatever FP the runtime provided us in EntryFP, recover the true + // frame pointer of the parent function. We only need to do this in filters, + // since finally funclets recover the parent FP for us. + llvm::Function *RecoverFPIntrin = + CGM.getIntrinsic(llvm::Intrinsic::x86_seh_recoverfp); + llvm::Constant *ParentI8Fn = + llvm::ConstantExpr::getBitCast(ParentCGF.CurFn, Int8PtrTy); + ParentFP = Builder.CreateCall(RecoverFPIntrin, {ParentI8Fn, EntryFP}); } // Create llvm.localrecover calls for all captures. @@ -1486,19 +1596,19 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, auto I = ParentCGF.LocalDeclMap.find(VD); if (I == ParentCGF.LocalDeclMap.end()) continue; - llvm::Value *ParentVar = I->second; - LocalDeclMap[VD] = - recoverAddrOfEscapedLocal(ParentCGF, ParentVar, ParentFP); + Address ParentVar = I->second; + setAddrOfLocalVar( + VD, recoverAddrOfEscapedLocal(ParentCGF, ParentVar, ParentFP)); } - if (Finder.SEHCodeSlot) { + if (Finder.SEHCodeSlot.isValid()) { SEHCodeSlotStack.push_back( recoverAddrOfEscapedLocal(ParentCGF, Finder.SEHCodeSlot, ParentFP)); } if (IsFilter) - EmitSEHExceptionCodeSave(ParentCGF, ParentFP, EntryEBP); + EmitSEHExceptionCodeSave(ParentCGF, ParentFP, EntryFP); } /// Arrange a function prototype that can be called by Windows exception @@ -1614,13 +1724,12 @@ CodeGenFunction::GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF, void CodeGenFunction::EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF, llvm::Value *ParentFP, - llvm::Value *EntryEBP) { + llvm::Value *EntryFP) { // Get the pointer to the EXCEPTION_POINTERS struct. This is returned by the // __exception_info intrinsic. if (CGM.getTarget().getTriple().getArch() != llvm::Triple::x86) { // On Win64, the info is passed as the first parameter to the filter. - auto AI = CurFn->arg_begin(); - SEHInfo = AI; + SEHInfo = &*CurFn->arg_begin(); SEHCodeSlotStack.push_back( CreateMemTemp(getContext().IntTy, "__exception_code")); } else { @@ -1628,9 +1737,9 @@ void CodeGenFunction::EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF, // exception registration object. It contains 6 32-bit fields, and the info // pointer is stored in the second field. So, GEP 20 bytes backwards and // load the pointer. - SEHInfo = Builder.CreateConstInBoundsGEP1_32(Int8Ty, EntryEBP, -20); + SEHInfo = Builder.CreateConstInBoundsGEP1_32(Int8Ty, EntryFP, -20); SEHInfo = Builder.CreateBitCast(SEHInfo, Int8PtrTy->getPointerTo()); - SEHInfo = Builder.CreateLoad(Int8PtrTy, SEHInfo); + SEHInfo = Builder.CreateAlignedLoad(Int8PtrTy, SEHInfo, getPointerAlign()); SEHCodeSlotStack.push_back(recoverAddrOfEscapedLocal( ParentCGF, ParentCGF.SEHCodeSlotStack.back(), ParentFP)); } @@ -1646,8 +1755,8 @@ void CodeGenFunction::EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF, llvm::Type *PtrsTy = llvm::StructType::get(RecordTy, CGM.VoidPtrTy, nullptr); llvm::Value *Ptrs = Builder.CreateBitCast(SEHInfo, PtrsTy->getPointerTo()); llvm::Value *Rec = Builder.CreateStructGEP(PtrsTy, Ptrs, 0); - Rec = Builder.CreateLoad(Rec); - llvm::Value *Code = Builder.CreateLoad(Rec); + Rec = Builder.CreateAlignedLoad(Rec, getPointerAlign()); + llvm::Value *Code = Builder.CreateAlignedLoad(Rec, getIntAlign()); assert(!SEHCodeSlotStack.empty() && "emitting EH code outside of __except"); Builder.CreateStore(Code, SEHCodeSlotStack.back()); } @@ -1663,7 +1772,7 @@ llvm::Value *CodeGenFunction::EmitSEHExceptionInfo() { llvm::Value *CodeGenFunction::EmitSEHExceptionCode() { assert(!SEHCodeSlotStack.empty() && "emitting EH code outside of __except"); - return Builder.CreateLoad(Int32Ty, SEHCodeSlotStack.back()); + return Builder.CreateLoad(SEHCodeSlotStack.back()); } llvm::Value *CodeGenFunction::EmitSEHAbnormalTermination() { @@ -1709,7 +1818,7 @@ void CodeGenFunction::EnterSEHTryStmt(const SEHTryStmt &S) { HelperCGF.GenerateSEHFilterFunction(*this, *Except); llvm::Constant *OpaqueFunc = llvm::ConstantExpr::getBitCast(FilterFunc, Int8PtrTy); - CatchScope->setHandler(0, OpaqueFunc, createBasicBlock("__except")); + CatchScope->setHandler(0, OpaqueFunc, createBasicBlock("__except.ret")); } void CodeGenFunction::ExitSEHTryStmt(const SEHTryStmt &S) { @@ -1745,16 +1854,24 @@ void CodeGenFunction::ExitSEHTryStmt(const SEHTryStmt &S) { emitCatchDispatchBlock(*this, CatchScope); // Grab the block before we pop the handler. - llvm::BasicBlock *ExceptBB = CatchScope.getHandler(0).Block; + llvm::BasicBlock *CatchPadBB = CatchScope.getHandler(0).Block; EHStack.popCatch(); - EmitBlockAfterUses(ExceptBB); + EmitBlockAfterUses(CatchPadBB); + + // __except blocks don't get outlined into funclets, so immediately do a + // catchret. + llvm::CatchPadInst *CPI = + cast<llvm::CatchPadInst>(CatchPadBB->getFirstNonPHI()); + llvm::BasicBlock *ExceptBB = createBasicBlock("__except"); + Builder.CreateCatchRet(CPI, ExceptBB); + EmitBlock(ExceptBB); - // On Win64, the exception pointer is the exception code. Copy it to the slot. + // On Win64, the exception code is returned in EAX. Copy it into the slot. if (CGM.getTarget().getTriple().getArch() != llvm::Triple::x86) { - llvm::Value *Code = - Builder.CreatePtrToInt(getExceptionFromSlot(), IntPtrTy); - Code = Builder.CreateTrunc(Code, Int32Ty); + llvm::Function *SEHCodeIntrin = + CGM.getIntrinsic(llvm::Intrinsic::eh_exceptioncode); + llvm::Value *Code = Builder.CreateCall(SEHCodeIntrin, {CPI}); Builder.CreateStore(Code, SEHCodeSlotStack.back()); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp index 9c3dfe5..dabd2b1 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp @@ -54,6 +54,15 @@ llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) { /// CreateTempAlloca - This creates a alloca and inserts it into the entry /// block. +Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, + const Twine &Name) { + auto Alloca = CreateTempAlloca(Ty, Name); + Alloca->setAlignment(Align.getQuantity()); + return Address(Alloca, Align); +} + +/// CreateTempAlloca - This creates a alloca and inserts it into the entry +/// block. llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, const Twine &Name) { if (!Builder.isNamePreserving()) @@ -61,29 +70,38 @@ llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, return new llvm::AllocaInst(Ty, nullptr, Name, AllocaInsertPt); } -void CodeGenFunction::InitTempAlloca(llvm::AllocaInst *Var, - llvm::Value *Init) { - auto *Store = new llvm::StoreInst(Init, Var); +/// CreateDefaultAlignTempAlloca - This creates an alloca with the +/// default alignment of the corresponding LLVM type, which is *not* +/// guaranteed to be related in any way to the expected alignment of +/// an AST type that might have been lowered to Ty. +Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, + const Twine &Name) { + CharUnits Align = + CharUnits::fromQuantity(CGM.getDataLayout().getABITypeAlignment(Ty)); + return CreateTempAlloca(Ty, Align, Name); +} + +void CodeGenFunction::InitTempAlloca(Address Var, llvm::Value *Init) { + assert(isa<llvm::AllocaInst>(Var.getPointer())); + auto *Store = new llvm::StoreInst(Init, Var.getPointer()); + Store->setAlignment(Var.getAlignment().getQuantity()); llvm::BasicBlock *Block = AllocaInsertPt->getParent(); - Block->getInstList().insertAfter(&*AllocaInsertPt, Store); + Block->getInstList().insertAfter(AllocaInsertPt->getIterator(), Store); } -llvm::AllocaInst *CodeGenFunction::CreateIRTemp(QualType Ty, - const Twine &Name) { - llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertType(Ty), Name); - // FIXME: Should we prefer the preferred type alignment here? +Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { CharUnits Align = getContext().getTypeAlignInChars(Ty); - Alloc->setAlignment(Align.getQuantity()); - return Alloc; + return CreateTempAlloca(ConvertType(Ty), Align, Name); } -llvm::AllocaInst *CodeGenFunction::CreateMemTemp(QualType Ty, - const Twine &Name) { - llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty), Name); +Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name) { // FIXME: Should we prefer the preferred type alignment here? - CharUnits Align = getContext().getTypeAlignInChars(Ty); - Alloc->setAlignment(Align.getQuantity()); - return Alloc; + return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name); +} + +Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, + const Twine &Name) { + return CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name); } /// EvaluateExprAsBool - Perform the usual unary conversions on the specified @@ -96,10 +114,12 @@ llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) { } QualType BoolTy = getContext().BoolTy; + SourceLocation Loc = E->getExprLoc(); if (!E->getType()->isAnyComplexType()) - return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy); + return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy, Loc); - return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(),BoolTy); + return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(), BoolTy, + Loc); } /// EmitIgnoredExpr - Emit code to compute the specified expression, @@ -146,20 +166,18 @@ RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) { /// EmitAnyExprToMem - Evaluate an expression into a given memory /// location. void CodeGenFunction::EmitAnyExprToMem(const Expr *E, - llvm::Value *Location, + Address Location, Qualifiers Quals, bool IsInit) { // FIXME: This function should take an LValue as an argument. switch (getEvaluationKind(E->getType())) { case TEK_Complex: - EmitComplexExprIntoLValue(E, - MakeNaturalAlignAddrLValue(Location, E->getType()), + EmitComplexExprIntoLValue(E, MakeAddrLValue(Location, E->getType()), /*isInit*/ false); return; case TEK_Aggregate: { - CharUnits Alignment = getContext().getTypeAlignInChars(E->getType()); - EmitAggExpr(E, AggValueSlot::forAddr(Location, Alignment, Quals, + EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals, AggValueSlot::IsDestructed_t(IsInit), AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsAliased_t(!IsInit))); @@ -178,17 +196,14 @@ void CodeGenFunction::EmitAnyExprToMem(const Expr *E, static void pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, - const Expr *E, llvm::Value *ReferenceTemporary) { + const Expr *E, Address ReferenceTemporary) { // Objective-C++ ARC: // If we are binding a reference to a temporary that has ownership, we // need to perform retain/release operations on the temporary. // // FIXME: This should be looking at E, not M. - if (CGF.getLangOpts().ObjCAutoRefCount && - M->getType()->isObjCLifetimeType()) { - QualType ObjCARCReferenceLifetimeType = M->getType(); - switch (Qualifiers::ObjCLifetime Lifetime = - ObjCARCReferenceLifetimeType.getObjCLifetime()) { + if (auto Lifetime = M->getType().getObjCLifetime()) { + switch (Lifetime) { case Qualifiers::OCL_None: case Qualifiers::OCL_ExplicitNone: // Carry on to normal cleanup handling. @@ -229,11 +244,11 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, } if (Duration == SD_FullExpression) CGF.pushDestroy(CleanupKind, ReferenceTemporary, - ObjCARCReferenceLifetimeType, *Destroy, + M->getType(), *Destroy, CleanupKind & EHCleanup); else CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary, - ObjCARCReferenceLifetimeType, + M->getType(), *Destroy, CleanupKind & EHCleanup); return; @@ -264,14 +279,14 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, llvm::Constant *CleanupArg; if (E->getType()->isArrayType()) { CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper( - cast<llvm::Constant>(ReferenceTemporary), E->getType(), + ReferenceTemporary, E->getType(), CodeGenFunction::destroyCXXObject, CGF.getLangOpts().Exceptions, dyn_cast_or_null<VarDecl>(M->getExtendingDecl())); CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy); } else { CleanupFn = CGF.CGM.getAddrOfCXXStructor(ReferenceTemporaryDtor, StructorType::Complete); - CleanupArg = cast<llvm::Constant>(ReferenceTemporary); + CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer()); } CGF.CGM.getCXXABI().registerGlobalDtor( CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg); @@ -296,7 +311,7 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, } } -static llvm::Value * +static Address createReferenceTemporary(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, const Expr *Inner) { switch (M->getStorageDuration()) { @@ -314,10 +329,10 @@ createReferenceTemporary(CodeGenFunction &CGF, auto *GV = new llvm::GlobalVariable( CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp"); - GV->setAlignment( - CGF.getContext().getTypeAlignInChars(Ty).getQuantity()); + CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty); + GV->setAlignment(alignment.getQuantity()); // FIXME: Should we put the new global into a COMDAT? - return GV; + return Address(GV, alignment); } return CGF.CreateMemTemp(Ty, "ref.tmp"); } @@ -337,20 +352,22 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so // as that will cause the lifetime adjustment to be lost for ARC - if (getLangOpts().ObjCAutoRefCount && - M->getType()->isObjCLifetimeType() && - M->getType().getObjCLifetime() != Qualifiers::OCL_None && - M->getType().getObjCLifetime() != Qualifiers::OCL_ExplicitNone) { - llvm::Value *Object = createReferenceTemporary(*this, M, E); - if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) { - Object = llvm::ConstantExpr::getBitCast( - Var, ConvertTypeForMem(E->getType())->getPointerTo()); + auto ownership = M->getType().getObjCLifetime(); + if (ownership != Qualifiers::OCL_None && + ownership != Qualifiers::OCL_ExplicitNone) { + Address Object = createReferenceTemporary(*this, M, E); + if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) { + Object = Address(llvm::ConstantExpr::getBitCast(Var, + ConvertTypeForMem(E->getType()) + ->getPointerTo(Object.getAddressSpace())), + Object.getAlignment()); // We should not have emitted the initializer for this temporary as a // constant. assert(!Var->hasInitializer()); Var->setInitializer(CGM.EmitNullConstant(E->getType())); } - LValue RefTempDst = MakeAddrLValue(Object, M->getType()); + LValue RefTempDst = MakeAddrLValue(Object, M->getType(), + AlignmentSource::Decl); switch (getEvaluationKind(E->getType())) { default: llvm_unreachable("expected scalar or aggregate expression"); @@ -358,8 +375,7 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false); break; case TEK_Aggregate: { - CharUnits Alignment = getContext().getTypeAlignInChars(E->getType()); - EmitAggExpr(E, AggValueSlot::forAddr(Object, Alignment, + EmitAggExpr(E, AggValueSlot::forAddr(Object, E->getType().getQualifiers(), AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, @@ -387,10 +403,11 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { } // Create and initialize the reference temporary. - llvm::Value *Object = createReferenceTemporary(*this, M, E); - if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) { - Object = llvm::ConstantExpr::getBitCast( - Var, ConvertTypeForMem(E->getType())->getPointerTo()); + Address Object = createReferenceTemporary(*this, M, E); + if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) { + Object = Address(llvm::ConstantExpr::getBitCast( + Var, ConvertTypeForMem(E->getType())->getPointerTo()), + Object.getAlignment()); // If the temporary is a global and has a constant initializer or is a // constant temporary that we promoted to a global, we may have already // initialized it. @@ -418,7 +435,8 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { break; case SubobjectAdjustment::FieldAdjustment: { - LValue LV = MakeAddrLValue(Object, E->getType()); + LValue LV = MakeAddrLValue(Object, E->getType(), + AlignmentSource::Decl); LV = EmitLValueForField(LV, Adjustment.Field); assert(LV.isSimple() && "materialized temporary field is not a simple lvalue"); @@ -428,14 +446,14 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { case SubobjectAdjustment::MemberPointerAdjustment: { llvm::Value *Ptr = EmitScalarExpr(Adjustment.Ptr.RHS); - Object = CGM.getCXXABI().EmitMemberDataPointerAddress( - *this, E, Object, Ptr, Adjustment.Ptr.MPT); + Object = EmitCXXMemberDataPointerAddress(E, Object, Ptr, + Adjustment.Ptr.MPT); break; } } } - return MakeAddrLValue(Object, M->getType()); + return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl); } RValue @@ -443,7 +461,7 @@ CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) { // Emit the expression as an lvalue. LValue LV = EmitLValue(E); assert(LV.isSimple()); - llvm::Value *Value = LV.getAddress(); + llvm::Value *Value = LV.getPointer(); if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) { // C++11 [dcl.ref]p5 (as amended by core issue 453): @@ -487,7 +505,7 @@ bool CodeGenFunction::sanitizePerformTypeCheck() const { } void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, - llvm::Value *Address, QualType Ty, + llvm::Value *Ptr, QualType Ty, CharUnits Alignment, bool SkipNullCheck) { if (!sanitizePerformTypeCheck()) return; @@ -495,7 +513,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, // Don't check pointers outside the default address space. The null check // isn't correct, the object-size check isn't supported by LLVM, and we can't // communicate the addresses to the runtime handler for the vptr check. - if (Address->getType()->getPointerAddressSpace()) + if (Ptr->getType()->getPointerAddressSpace()) return; SanitizerScope SanScope(this); @@ -508,8 +526,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) && !SkipNullCheck) { // The glvalue must not be an empty glvalue. - llvm::Value *IsNonNull = Builder.CreateICmpNE( - Address, llvm::Constant::getNullValue(Address->getType())); + llvm::Value *IsNonNull = Builder.CreateIsNotNull(Ptr); if (AllowNullPointers) { // When performing pointer casts, it's OK if the value is null. @@ -533,7 +550,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy }; llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys); llvm::Value *Min = Builder.getFalse(); - llvm::Value *CastAddr = Builder.CreateBitCast(Address, Int8PtrTy); + llvm::Value *CastAddr = Builder.CreateBitCast(Ptr, Int8PtrTy); llvm::Value *LargeEnough = Builder.CreateICmpUGE(Builder.CreateCall(F, {CastAddr, Min}), llvm::ConstantInt::get(IntPtrTy, Size)); @@ -550,7 +567,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, // The glvalue must be suitably aligned. if (AlignVal) { llvm::Value *Align = - Builder.CreateAnd(Builder.CreatePtrToInt(Address, IntPtrTy), + Builder.CreateAnd(Builder.CreatePtrToInt(Ptr, IntPtrTy), llvm::ConstantInt::get(IntPtrTy, AlignVal - 1)); llvm::Value *Aligned = Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0)); @@ -565,7 +582,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::ConstantInt::get(SizeTy, AlignVal), llvm::ConstantInt::get(Int8Ty, TCK) }; - EmitCheck(Checks, "type_mismatch", StaticData, Address); + EmitCheck(Checks, "type_mismatch", StaticData, Ptr); } // If possible, check that the vptr indicates that there is a subobject of @@ -600,7 +617,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, // Load the vptr, and compute hash_16_bytes(TypeHash, vptr). llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash); llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0); - llvm::Value *VPtrAddr = Builder.CreateBitCast(Address, VPtrTy); + Address VPtrAddr(Builder.CreateBitCast(Ptr, VPtrTy), getPointerAlign()); llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr); llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty); @@ -617,7 +634,8 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, CacheSize-1)); llvm::Value *Indices[] = { Builder.getInt32(0), Slot }; llvm::Value *CacheVal = - Builder.CreateLoad(Builder.CreateInBoundsGEP(Cache, Indices)); + Builder.CreateAlignedLoad(Builder.CreateInBoundsGEP(Cache, Indices), + getPointerAlign()); // If the hash isn't in the cache, call a runtime handler to perform the // hard work of checking whether the vptr is for an object of the right @@ -630,7 +648,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, CGM.GetAddrOfRTTIDescriptor(Ty.getUnqualifiedType()), llvm::ConstantInt::get(Int8Ty, TCK) }; - llvm::Value *DynamicData[] = { Address, Hash }; + llvm::Value *DynamicData[] = { Ptr, Hash }; EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr), "dynamic_type_cache_miss", StaticData, DynamicData); } @@ -758,10 +776,104 @@ EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV, return isPre ? IncVal : InVal; } +void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E, + CodeGenFunction *CGF) { + // Bind VLAs in the cast type. + if (CGF && E->getType()->isVariablyModifiedType()) + CGF->EmitVariablyModifiedType(E->getType()); + + if (CGDebugInfo *DI = getModuleDebugInfo()) + DI->EmitExplicitCastType(E->getType()); +} + //===----------------------------------------------------------------------===// // LValue Expression Emission //===----------------------------------------------------------------------===// +/// EmitPointerWithAlignment - Given an expression of pointer type, try to +/// derive a more accurate bound on the alignment of the pointer. +Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, + AlignmentSource *Source) { + // We allow this with ObjC object pointers because of fragile ABIs. + assert(E->getType()->isPointerType() || + E->getType()->isObjCObjectPointerType()); + E = E->IgnoreParens(); + + // Casts: + if (const CastExpr *CE = dyn_cast<CastExpr>(E)) { + if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE)) + CGM.EmitExplicitCastExprType(ECE, this); + + switch (CE->getCastKind()) { + // Non-converting casts (but not C's implicit conversion from void*). + case CK_BitCast: + case CK_NoOp: + if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) { + if (PtrTy->getPointeeType()->isVoidType()) + break; + + AlignmentSource InnerSource; + Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), &InnerSource); + if (Source) *Source = InnerSource; + + // If this is an explicit bitcast, and the source l-value is + // opaque, honor the alignment of the casted-to type. + if (isa<ExplicitCastExpr>(CE) && + InnerSource != AlignmentSource::Decl) { + Addr = Address(Addr.getPointer(), + getNaturalPointeeTypeAlignment(E->getType(), Source)); + } + + if (SanOpts.has(SanitizerKind::CFIUnrelatedCast)) { + if (auto PT = E->getType()->getAs<PointerType>()) + EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr.getPointer(), + /*MayBeNull=*/true, + CodeGenFunction::CFITCK_UnrelatedCast, + CE->getLocStart()); + } + + return Builder.CreateBitCast(Addr, ConvertType(E->getType())); + } + break; + + // Array-to-pointer decay. + case CK_ArrayToPointerDecay: + return EmitArrayToPointerDecay(CE->getSubExpr(), Source); + + // Derived-to-base conversions. + case CK_UncheckedDerivedToBase: + case CK_DerivedToBase: { + Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), Source); + auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl(); + return GetAddressOfBaseClass(Addr, Derived, + CE->path_begin(), CE->path_end(), + ShouldNullCheckClassCastValue(CE), + CE->getExprLoc()); + } + + // TODO: Is there any reason to treat base-to-derived conversions + // specially? + default: + break; + } + } + + // Unary &. + if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) { + if (UO->getOpcode() == UO_AddrOf) { + LValue LV = EmitLValue(UO->getSubExpr()); + if (Source) *Source = LV.getAlignmentSource(); + return LV.getAddress(); + } + } + + // TODO: conditional operators, comma. + + // Otherwise, use the alignment of the type. + CharUnits Align = getNaturalPointeeTypeAlignment(E->getType(), Source); + return Address(EmitScalarExpr(E), Align); +} + RValue CodeGenFunction::GetUndefRValue(QualType Ty) { if (Ty->isVoidType()) return RValue::get(nullptr); @@ -778,7 +890,7 @@ RValue CodeGenFunction::GetUndefRValue(QualType Ty) { // identifiable address. Just because the contents of the value are undefined // doesn't mean that the address can't be taken and compared. case TEK_Aggregate: { - llvm::Value *DestPtr = CreateMemTemp(Ty, "undef.agg.tmp"); + Address DestPtr = CreateMemTemp(Ty, "undef.agg.tmp"); return RValue::getAggregate(DestPtr); } @@ -798,7 +910,8 @@ LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E, const char *Name) { ErrorUnsupported(E, Name); llvm::Type *Ty = llvm::PointerType::getUnqual(ConvertType(E->getType())); - return MakeAddrLValue(llvm::UndefValue::get(Ty), E->getType()); + return MakeAddrLValue(Address(llvm::UndefValue::get(Ty), CharUnits::One()), + E->getType()); } LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { @@ -808,7 +921,7 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { else LV = EmitLValue(E); if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple()) - EmitTypeCheck(TCK, E->getExprLoc(), LV.getAddress(), + EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(), E->getType(), LV.getAlignment()); return LV; } @@ -909,6 +1022,8 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { return EmitUnaryOpLValue(cast<UnaryOperator>(E)); case Expr::ArraySubscriptExprClass: return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E)); + case Expr::OMPArraySectionExprClass: + return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E)); case Expr::ExtVectorElementExprClass: return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E)); case Expr::MemberExprClass: @@ -1055,9 +1170,10 @@ CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) { llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue, SourceLocation Loc) { return EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(), - lvalue.getAlignment().getQuantity(), - lvalue.getType(), Loc, lvalue.getTBAAInfo(), - lvalue.getTBAABaseType(), lvalue.getTBAAOffset()); + lvalue.getType(), Loc, lvalue.getAlignmentSource(), + lvalue.getTBAAInfo(), + lvalue.getTBAABaseType(), lvalue.getTBAAOffset(), + lvalue.isNontemporal()); } static bool hasBooleanRepresentation(QualType Ty) { @@ -1117,68 +1233,56 @@ llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) { return MDHelper.createRange(Min, End); } -llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile, - unsigned Alignment, QualType Ty, +llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, + QualType Ty, SourceLocation Loc, + AlignmentSource AlignSource, llvm::MDNode *TBAAInfo, QualType TBAABaseType, - uint64_t TBAAOffset) { + uint64_t TBAAOffset, + bool isNontemporal) { // For better performance, handle vector loads differently. if (Ty->isVectorType()) { - llvm::Value *V; - const llvm::Type *EltTy = - cast<llvm::PointerType>(Addr->getType())->getElementType(); + const llvm::Type *EltTy = Addr.getElementType(); const auto *VTy = cast<llvm::VectorType>(EltTy); - // Handle vectors of size 3, like size 4 for better performance. + // Handle vectors of size 3 like size 4 for better performance. if (VTy->getNumElements() == 3) { // Bitcast to vec4 type. llvm::VectorType *vec4Ty = llvm::VectorType::get(VTy->getElementType(), 4); - llvm::PointerType *ptVec4Ty = - llvm::PointerType::get(vec4Ty, - (cast<llvm::PointerType>( - Addr->getType()))->getAddressSpace()); - llvm::Value *Cast = Builder.CreateBitCast(Addr, ptVec4Ty, - "castToVec4"); + Address Cast = Builder.CreateElementBitCast(Addr, vec4Ty, "castToVec4"); // Now load value. - llvm::Value *LoadVal = Builder.CreateLoad(Cast, Volatile, "loadVec4"); + llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4"); // Shuffle vector to get vec3. - llvm::Constant *Mask[] = { - llvm::ConstantInt::get(llvm::Type::getInt32Ty(getLLVMContext()), 0), - llvm::ConstantInt::get(llvm::Type::getInt32Ty(getLLVMContext()), 1), - llvm::ConstantInt::get(llvm::Type::getInt32Ty(getLLVMContext()), 2) - }; - - llvm::Value *MaskV = llvm::ConstantVector::get(Mask); - V = Builder.CreateShuffleVector(LoadVal, - llvm::UndefValue::get(vec4Ty), - MaskV, "extractVec"); + V = Builder.CreateShuffleVector(V, llvm::UndefValue::get(vec4Ty), + {0, 1, 2}, "extractVec"); return EmitFromMemory(V, Ty); } } // Atomic operations have to be done on integral types. if (Ty->isAtomicType() || typeIsSuitableForInlineAtomic(Ty, Volatile)) { - LValue lvalue = LValue::MakeAddr(Addr, Ty, - CharUnits::fromQuantity(Alignment), - getContext(), TBAAInfo); + LValue lvalue = + LValue::MakeAddr(Addr, Ty, getContext(), AlignSource, TBAAInfo); return EmitAtomicLoad(lvalue, Loc).getScalarVal(); } - llvm::LoadInst *Load = Builder.CreateLoad(Addr); - if (Volatile) - Load->setVolatile(true); - if (Alignment) - Load->setAlignment(Alignment); + llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile); + if (isNontemporal) { + llvm::MDNode *Node = llvm::MDNode::get( + Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); + Load->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); + } if (TBAAInfo) { llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo, TBAAOffset); if (TBAAPath) - CGM.DecorateInstruction(Load, TBAAPath, false/*ConvertTypeToTag*/); + CGM.DecorateInstructionWithTBAA(Load, TBAAPath, + false /*ConvertTypeToTag*/); } bool NeedsBoolCheck = @@ -1241,11 +1345,13 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) { return Value; } -void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr, - bool Volatile, unsigned Alignment, - QualType Ty, llvm::MDNode *TBAAInfo, +void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, + bool Volatile, QualType Ty, + AlignmentSource AlignSource, + llvm::MDNode *TBAAInfo, bool isInit, QualType TBAABaseType, - uint64_t TBAAOffset) { + uint64_t TBAAOffset, + bool isNontemporal) { // Handle vectors differently to get better performance. if (Ty->isVectorType()) { @@ -1253,29 +1359,18 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr, auto *VecTy = cast<llvm::VectorType>(SrcTy); // Handle vec3 special. if (VecTy->getNumElements() == 3) { - llvm::LLVMContext &VMContext = getLLVMContext(); - // Our source is a vec3, do a shuffle vector to make it a vec4. - SmallVector<llvm::Constant*, 4> Mask; - Mask.push_back(llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), - 0)); - Mask.push_back(llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), - 1)); - Mask.push_back(llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), - 2)); - Mask.push_back(llvm::UndefValue::get(llvm::Type::getInt32Ty(VMContext))); - + llvm::Constant *Mask[] = {Builder.getInt32(0), Builder.getInt32(1), + Builder.getInt32(2), + llvm::UndefValue::get(Builder.getInt32Ty())}; llvm::Value *MaskV = llvm::ConstantVector::get(Mask); Value = Builder.CreateShuffleVector(Value, llvm::UndefValue::get(VecTy), MaskV, "extractVec"); SrcTy = llvm::VectorType::get(VecTy->getElementType(), 4); } - auto *DstPtr = cast<llvm::PointerType>(Addr->getType()); - if (DstPtr->getElementType() != SrcTy) { - llvm::Type *MemTy = - llvm::PointerType::get(SrcTy, DstPtr->getAddressSpace()); - Addr = Builder.CreateBitCast(Addr, MemTy, "storetmp"); + if (Addr.getElementType() != SrcTy) { + Addr = Builder.CreateElementBitCast(Addr, SrcTy, "storetmp"); } } @@ -1284,30 +1379,34 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr, if (Ty->isAtomicType() || (!isInit && typeIsSuitableForInlineAtomic(Ty, Volatile))) { EmitAtomicStore(RValue::get(Value), - LValue::MakeAddr(Addr, Ty, - CharUnits::fromQuantity(Alignment), - getContext(), TBAAInfo), + LValue::MakeAddr(Addr, Ty, getContext(), + AlignSource, TBAAInfo), isInit); return; } llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile); - if (Alignment) - Store->setAlignment(Alignment); + if (isNontemporal) { + llvm::MDNode *Node = + llvm::MDNode::get(Store->getContext(), + llvm::ConstantAsMetadata::get(Builder.getInt32(1))); + Store->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); + } if (TBAAInfo) { llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo, TBAAOffset); if (TBAAPath) - CGM.DecorateInstruction(Store, TBAAPath, false/*ConvertTypeToTag*/); + CGM.DecorateInstructionWithTBAA(Store, TBAAPath, + false /*ConvertTypeToTag*/); } } void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue, bool isInit) { EmitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(), - lvalue.getAlignment().getQuantity(), lvalue.getType(), + lvalue.getType(), lvalue.getAlignmentSource(), lvalue.getTBAAInfo(), isInit, lvalue.getTBAABaseType(), - lvalue.getTBAAOffset()); + lvalue.getTBAAOffset(), lvalue.isNontemporal()); } /// EmitLoadOfLValue - Given an expression that represents a value lvalue, this @@ -1316,11 +1415,17 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue, RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) { if (LV.isObjCWeak()) { // load of a __weak object. - llvm::Value *AddrWeakObj = LV.getAddress(); + Address AddrWeakObj = LV.getAddress(); return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this, AddrWeakObj)); } if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) { + // In MRC mode, we do a load+autorelease. + if (!getLangOpts().ObjCAutoRefCount) { + return RValue::get(EmitARCLoadWeak(LV.getAddress())); + } + + // In ARC mode, we load retained and then consume the value. llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress()); Object = EmitObjCConsumeObject(LV.getType(), Object); return RValue::get(Object); @@ -1334,9 +1439,8 @@ RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) { } if (LV.isVectorElt()) { - llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddr(), + llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddress(), LV.isVolatileQualified()); - Load->setAlignment(LV.getAlignment().getQuantity()); return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(), "vecext")); } @@ -1356,15 +1460,12 @@ RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) { RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV) { const CGBitFieldInfo &Info = LV.getBitFieldInfo(); - CharUnits Align = LV.getAlignment().alignmentAtOffset(Info.StorageOffset); // Get the output type. llvm::Type *ResLTy = ConvertType(LV.getType()); - llvm::Value *Ptr = LV.getBitFieldAddr(); - llvm::Value *Val = Builder.CreateAlignedLoad(Ptr, Align.getQuantity(), - LV.isVolatileQualified(), - "bf.load"); + Address Ptr = LV.getBitFieldAddress(); + llvm::Value *Val = Builder.CreateLoad(Ptr, LV.isVolatileQualified(), "bf.load"); if (Info.IsSigned) { assert(static_cast<unsigned>(Info.Offset + Info.Size) <= Info.StorageSize); @@ -1389,10 +1490,8 @@ RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV) { // If this is a reference to a subset of the elements of a vector, create an // appropriate shufflevector. RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) { - llvm::LoadInst *Load = Builder.CreateLoad(LV.getExtVectorAddr(), - LV.isVolatileQualified()); - Load->setAlignment(LV.getAlignment().getQuantity()); - llvm::Value *Vec = Load; + llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(), + LV.isVolatileQualified()); const llvm::Constant *Elts = LV.getExtVectorElts(); @@ -1419,24 +1518,24 @@ RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) { } /// @brief Generates lvalue for partial ext_vector access. -llvm::Value *CodeGenFunction::EmitExtVectorElementLValue(LValue LV) { - llvm::Value *VectorAddress = LV.getExtVectorAddr(); +Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) { + Address VectorAddress = LV.getExtVectorAddress(); const VectorType *ExprVT = LV.getType()->getAs<VectorType>(); QualType EQT = ExprVT->getElementType(); llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT); - llvm::Type *VectorElementPtrToTy = VectorElementTy->getPointerTo(); - llvm::Value *CastToPointerElement = - Builder.CreateBitCast(VectorAddress, - VectorElementPtrToTy, "conv.ptr.element"); + Address CastToPointerElement = + Builder.CreateElementBitCast(VectorAddress, VectorElementTy, + "conv.ptr.element"); const llvm::Constant *Elts = LV.getExtVectorElts(); unsigned ix = getAccessedFieldNo(0, Elts); - llvm::Value *VectorBasePtrPlusIx = - Builder.CreateInBoundsGEP(CastToPointerElement, - llvm::ConstantInt::get(SizeTy, ix), "add.ptr"); - + Address VectorBasePtrPlusIx = + Builder.CreateConstInBoundsGEP(CastToPointerElement, ix, + getContext().getTypeSizeInChars(EQT), + "vector.elt"); + return VectorBasePtrPlusIx; } @@ -1471,15 +1570,12 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, if (!Dst.isSimple()) { if (Dst.isVectorElt()) { // Read/modify/write the vector, inserting the new element. - llvm::LoadInst *Load = Builder.CreateLoad(Dst.getVectorAddr(), - Dst.isVolatileQualified()); - Load->setAlignment(Dst.getAlignment().getQuantity()); - llvm::Value *Vec = Load; + llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(), + Dst.isVolatileQualified()); Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(), Dst.getVectorIdx(), "vecins"); - llvm::StoreInst *Store = Builder.CreateStore(Vec, Dst.getVectorAddr(), - Dst.isVolatileQualified()); - Store->setAlignment(Dst.getAlignment().getQuantity()); + Builder.CreateStore(Vec, Dst.getVectorAddress(), + Dst.isVolatileQualified()); return; } @@ -1523,7 +1619,7 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, if (Dst.isObjCWeak() && !Dst.isNonGC()) { // load of a __weak object. - llvm::Value *LvalueDst = Dst.getAddress(); + Address LvalueDst = Dst.getAddress(); llvm::Value *src = Src.getScalarVal(); CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst); return; @@ -1531,16 +1627,17 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, if (Dst.isObjCStrong() && !Dst.isNonGC()) { // load of a __strong object. - llvm::Value *LvalueDst = Dst.getAddress(); + Address LvalueDst = Dst.getAddress(); llvm::Value *src = Src.getScalarVal(); if (Dst.isObjCIvar()) { assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL"); - llvm::Type *ResultType = ConvertType(getContext().LongTy); - llvm::Value *RHS = EmitScalarExpr(Dst.getBaseIvarExp()); - llvm::Value *dst = RHS; + llvm::Type *ResultType = IntPtrTy; + Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp()); + llvm::Value *RHS = dst.getPointer(); RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast"); llvm::Value *LHS = - Builder.CreatePtrToInt(LvalueDst, ResultType, "sub.ptr.lhs.cast"); + Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType, + "sub.ptr.lhs.cast"); llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset"); CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, BytesBetween); @@ -1560,16 +1657,14 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, llvm::Value **Result) { const CGBitFieldInfo &Info = Dst.getBitFieldInfo(); - CharUnits Align = Dst.getAlignment().alignmentAtOffset(Info.StorageOffset); llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType()); - llvm::Value *Ptr = Dst.getBitFieldAddr(); + Address Ptr = Dst.getBitFieldAddress(); // Get the source value, truncated to the width of the bit-field. llvm::Value *SrcVal = Src.getScalarVal(); // Cast the source to the storage type and shift it into place. - SrcVal = Builder.CreateIntCast(SrcVal, - Ptr->getType()->getPointerElementType(), + SrcVal = Builder.CreateIntCast(SrcVal, Ptr.getElementType(), /*IsSigned=*/false); llvm::Value *MaskedVal = SrcVal; @@ -1577,9 +1672,8 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, // and mask together with source before storing. if (Info.StorageSize != Info.Size) { assert(Info.StorageSize > Info.Size && "Invalid bitfield size."); - llvm::Value *Val = Builder.CreateAlignedLoad(Ptr, Align.getQuantity(), - Dst.isVolatileQualified(), - "bf.load"); + llvm::Value *Val = + Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), "bf.load"); // Mask the source value as needed. if (!hasBooleanRepresentation(Dst.getType())) @@ -1605,8 +1699,7 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, } // Write the new value back out. - Builder.CreateAlignedStore(SrcVal, Ptr, Align.getQuantity(), - Dst.isVolatileQualified()); + Builder.CreateStore(SrcVal, Ptr, Dst.isVolatileQualified()); // Return the new value of the bit-field, if requested. if (Result) { @@ -1632,10 +1725,8 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst) { // This access turns into a read/modify/write of the vector. Load the input // value now. - llvm::LoadInst *Load = Builder.CreateLoad(Dst.getExtVectorAddr(), - Dst.isVolatileQualified()); - Load->setAlignment(Dst.getAlignment().getQuantity()); - llvm::Value *Vec = Load; + llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddress(), + Dst.isVolatileQualified()); const llvm::Constant *Elts = Dst.getExtVectorElts(); llvm::Value *SrcVal = Src.getScalarVal(); @@ -1697,9 +1788,8 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src, Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt); } - llvm::StoreInst *Store = Builder.CreateStore(Vec, Dst.getExtVectorAddr(), - Dst.isVolatileQualified()); - Store->setAlignment(Dst.getAlignment().getQuantity()); + Builder.CreateStore(Vec, Dst.getExtVectorAddress(), + Dst.isVolatileQualified()); } /// @brief Store of global named registers are always calls to intrinsics. @@ -1834,11 +1924,27 @@ EmitBitCastOfLValueToProperType(CodeGenFunction &CGF, } static LValue EmitThreadPrivateVarDeclLValue( - CodeGenFunction &CGF, const VarDecl *VD, QualType T, llvm::Value *V, - llvm::Type *RealVarTy, CharUnits Alignment, SourceLocation Loc) { - V = CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, V, Loc); - V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy); - return CGF.MakeAddrLValue(V, T, Alignment); + CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr, + llvm::Type *RealVarTy, SourceLocation Loc) { + Addr = CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc); + Addr = CGF.Builder.CreateElementBitCast(Addr, RealVarTy); + return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl); +} + +Address CodeGenFunction::EmitLoadOfReference(Address Addr, + const ReferenceType *RefTy, + AlignmentSource *Source) { + llvm::Value *Ptr = Builder.CreateLoad(Addr); + return Address(Ptr, getNaturalTypeAlignment(RefTy->getPointeeType(), + Source, /*forPointee*/ true)); + +} + +LValue CodeGenFunction::EmitLoadOfReferenceLValue(Address RefAddr, + const ReferenceType *RefTy) { + AlignmentSource Source; + Address Addr = EmitLoadOfReference(RefAddr, RefTy, &Source); + return MakeAddrLValue(Addr, RefTy->getPointeeType(), Source); } static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF, @@ -1854,19 +1960,17 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF, llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType()); V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy); CharUnits Alignment = CGF.getContext().getDeclAlign(VD); + Address Addr(V, Alignment); LValue LV; // Emit reference to the private copy of the variable if it is an OpenMP // threadprivate variable. if (CGF.getLangOpts().OpenMP && VD->hasAttr<OMPThreadPrivateDeclAttr>()) - return EmitThreadPrivateVarDeclLValue(CGF, VD, T, V, RealVarTy, Alignment, + return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy, E->getExprLoc()); - if (VD->getType()->isReferenceType()) { - llvm::LoadInst *LI = CGF.Builder.CreateLoad(V); - LI->setAlignment(Alignment.getQuantity()); - V = LI; - LV = CGF.MakeNaturalAlignAddrLValue(V, T); + if (auto RefTy = VD->getType()->getAs<ReferenceType>()) { + LV = CGF.EmitLoadOfReferenceLValue(Addr, RefTy); } else { - LV = CGF.MakeAddrLValue(V, T, Alignment); + LV = CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl); } setObjCGCLValueClass(CGF.getContext(), E, LV); return LV; @@ -1888,7 +1992,7 @@ static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, } } CharUnits Alignment = CGF.getContext().getDeclAlign(FD); - return CGF.MakeAddrLValue(V, E->getType(), Alignment); + return CGF.MakeAddrLValue(V, E->getType(), Alignment, AlignmentSource::Decl); } static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD, @@ -1904,9 +2008,7 @@ static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD, /// So far, only the name is being passed down, but other options such as /// register type, allocation type or even optimization options could be /// passed down via the metadata node. -static LValue EmitGlobalNamedRegister(const VarDecl *VD, - CodeGenModule &CGM, - CharUnits Alignment) { +static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) { SmallString<64> Name("llvm.named.register."); AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>(); assert(Asm->getLabel().size() < 64-Name.size() && @@ -1920,33 +2022,43 @@ static LValue EmitGlobalNamedRegister(const VarDecl *VD, llvm::Metadata *Ops[] = {Str}; M->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops)); } - return LValue::MakeGlobalReg( - llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0)), - VD->getType(), Alignment); + + CharUnits Alignment = CGM.getContext().getDeclAlign(VD); + + llvm::Value *Ptr = + llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0)); + return LValue::MakeGlobalReg(Address(Ptr, Alignment), VD->getType()); } LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { const NamedDecl *ND = E->getDecl(); - CharUnits Alignment = getContext().getDeclAlign(ND); QualType T = E->getType(); if (const auto *VD = dyn_cast<VarDecl>(ND)) { // Global Named registers access via intrinsics only if (VD->getStorageClass() == SC_Register && VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl()) - return EmitGlobalNamedRegister(VD, CGM, Alignment); + return EmitGlobalNamedRegister(VD, CGM); // A DeclRefExpr for a reference initialized by a constant expression can // appear without being odr-used. Directly emit the constant initializer. const Expr *Init = VD->getAnyInitializer(VD); if (Init && !isa<ParmVarDecl>(VD) && VD->getType()->isReferenceType() && VD->isUsableInConstantExpressions(getContext()) && - VD->checkInitIsICE()) { + VD->checkInitIsICE() && + // Do not emit if it is private OpenMP variable. + !(E->refersToEnclosingVariableOrCapture() && CapturedStmtInfo && + LocalDeclMap.count(VD))) { llvm::Constant *Val = CGM.EmitConstantValue(*VD->evaluateValue(), VD->getType(), this); assert(Val && "failed to emit reference constant expression"); // FIXME: Eventually we will want to emit vector element references. - return MakeAddrLValue(Val, T, Alignment); + + // Should we be using the alignment of the constant pointer we emitted? + CharUnits Alignment = getNaturalTypeAlignment(E->getType(), nullptr, + /*pointee*/ true); + + return MakeAddrLValue(Address(Val, Alignment), T, AlignmentSource::Decl); } // Check for captured variables. @@ -1954,15 +2066,24 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { if (auto *FD = LambdaCaptureFields.lookup(VD)) return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue); else if (CapturedStmtInfo) { - if (auto *V = LocalDeclMap.lookup(VD)) - return MakeAddrLValue(V, T, Alignment); - else - return EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD), - CapturedStmtInfo->getContextValue()); + auto it = LocalDeclMap.find(VD); + if (it != LocalDeclMap.end()) { + if (auto RefTy = VD->getType()->getAs<ReferenceType>()) { + return EmitLoadOfReferenceLValue(it->second, RefTy); + } + return MakeAddrLValue(it->second, T); + } + LValue CapLVal = + EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD), + CapturedStmtInfo->getContextValue()); + return MakeAddrLValue( + Address(CapLVal.getPointer(), getContext().getDeclAlign(VD)), + CapLVal.getType(), AlignmentSource::Decl); } + assert(isa<BlockDecl>(CurCodeDecl)); - return MakeAddrLValue(GetAddrOfBlockDecl(VD, VD->hasAttr<BlocksAttr>()), - T, Alignment); + Address addr = GetAddrOfBlockDecl(VD, VD->hasAttr<BlocksAttr>()); + return MakeAddrLValue(addr, T, AlignmentSource::Decl); } } @@ -1975,8 +2096,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { if (ND->hasAttr<WeakRefAttr>()) { const auto *VD = cast<ValueDecl>(ND); - llvm::Constant *Aliasee = CGM.GetWeakRefReference(VD); - return MakeAddrLValue(Aliasee, T, Alignment); + ConstantAddress Aliasee = CGM.GetWeakRefReference(VD); + return MakeAddrLValue(Aliasee, T, AlignmentSource::Decl); } if (const auto *VD = dyn_cast<VarDecl>(ND)) { @@ -1984,39 +2105,52 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { if (VD->hasLinkage() || VD->isStaticDataMember()) return EmitGlobalVarDeclLValue(*this, E, VD); - bool isBlockVariable = VD->hasAttr<BlocksAttr>(); + Address addr = Address::invalid(); - llvm::Value *V = LocalDeclMap.lookup(VD); - if (!V && VD->isStaticLocal()) - V = CGM.getOrCreateStaticVarDecl( - *VD, CGM.getLLVMLinkageVarDefinition(VD, /*isConstant=*/false)); + // The variable should generally be present in the local decl map. + auto iter = LocalDeclMap.find(VD); + if (iter != LocalDeclMap.end()) { + addr = iter->second; - // Check if variable is threadprivate. - if (V && getLangOpts().OpenMP && VD->hasAttr<OMPThreadPrivateDeclAttr>()) - return EmitThreadPrivateVarDeclLValue( - *this, VD, T, V, getTypes().ConvertTypeForMem(VD->getType()), - Alignment, E->getExprLoc()); + // Otherwise, it might be static local we haven't emitted yet for + // some reason; most likely, because it's in an outer function. + } else if (VD->isStaticLocal()) { + addr = Address(CGM.getOrCreateStaticVarDecl( + *VD, CGM.getLLVMLinkageVarDefinition(VD, /*isConstant=*/false)), + getContext().getDeclAlign(VD)); + + // No other cases for now. + } else { + llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?"); + } - assert(V && "DeclRefExpr not entered in LocalDeclMap?"); - if (isBlockVariable) - V = BuildBlockByrefAddress(V, VD); + // Check for OpenMP threadprivate variables. + if (getLangOpts().OpenMP && VD->hasAttr<OMPThreadPrivateDeclAttr>()) { + return EmitThreadPrivateVarDeclLValue( + *this, VD, T, addr, getTypes().ConvertTypeForMem(VD->getType()), + E->getExprLoc()); + } + // Drill into block byref variables. + bool isBlockByref = VD->hasAttr<BlocksAttr>(); + if (isBlockByref) { + addr = emitBlockByrefAddress(addr, VD); + } + + // Drill into reference types. LValue LV; - if (VD->getType()->isReferenceType()) { - llvm::LoadInst *LI = Builder.CreateLoad(V); - LI->setAlignment(Alignment.getQuantity()); - V = LI; - LV = MakeNaturalAlignAddrLValue(V, T); + if (auto RefTy = VD->getType()->getAs<ReferenceType>()) { + LV = EmitLoadOfReferenceLValue(addr, RefTy); } else { - LV = MakeAddrLValue(V, T, Alignment); + LV = MakeAddrLValue(addr, T, AlignmentSource::Decl); } bool isLocalStorage = VD->hasLocalStorage(); bool NonGCable = isLocalStorage && !VD->getType()->isReferenceType() && - !isBlockVariable; + !isBlockByref; if (NonGCable) { LV.getQuals().removeObjCGCAttr(); LV.setNonGC(true); @@ -2048,7 +2182,9 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) { QualType T = E->getSubExpr()->getType()->getPointeeType(); assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type"); - LValue LV = MakeNaturalAlignAddrLValue(EmitScalarExpr(E->getSubExpr()), T); + AlignmentSource AlignSource; + Address Addr = EmitPointerWithAlignment(E->getSubExpr(), &AlignSource); + LValue LV = MakeAddrLValue(Addr, T, AlignSource); LV.getQuals().setAddressSpace(ExprTy.getAddressSpace()); // We should not generate __weak write barrier on indirect reference @@ -2065,22 +2201,22 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) { case UO_Imag: { LValue LV = EmitLValue(E->getSubExpr()); assert(LV.isSimple() && "real/imag on non-ordinary l-value"); - llvm::Value *Addr = LV.getAddress(); // __real is valid on scalars. This is a faster way of testing that. // __imag can only produce an rvalue on scalars. if (E->getOpcode() == UO_Real && - !cast<llvm::PointerType>(Addr->getType()) - ->getElementType()->isStructTy()) { + !LV.getAddress().getElementType()->isStructTy()) { assert(E->getSubExpr()->getType()->isArithmeticType()); return LV; } assert(E->getSubExpr()->getType()->isAnyComplexType()); - unsigned Idx = E->getOpcode() == UO_Imag; - return MakeAddrLValue( - Builder.CreateStructGEP(nullptr, LV.getAddress(), Idx, "idx"), ExprTy); + Address Component = + (E->getOpcode() == UO_Real + ? emitAddrOfRealComponent(LV.getAddress(), LV.getType()) + : emitAddrOfImagComponent(LV.getAddress(), LV.getType())); + return MakeAddrLValue(Component, ExprTy, LV.getAlignmentSource()); } case UO_PreInc: case UO_PreDec: { @@ -2098,12 +2234,12 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) { LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) { return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E), - E->getType()); + E->getType(), AlignmentSource::Decl); } LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) { return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E), - E->getType()); + E->getType(), AlignmentSource::Decl); } LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) { @@ -2116,11 +2252,11 @@ LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) { PredefinedExpr::getIdentTypeName(E->getIdentType()), FnName}; std::string GVName = llvm::join(NameItems, NameItems + 2, "."); if (CurCodeDecl && isa<BlockDecl>(CurCodeDecl)) { - auto C = CGM.GetAddrOfConstantCString(FnName, GVName.c_str(), 1); - return MakeAddrLValue(C, E->getType()); + auto C = CGM.GetAddrOfConstantCString(FnName, GVName.c_str()); + return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl); } auto C = CGM.GetAddrOfConstantStringFromLiteral(SL, GVName); - return MakeAddrLValue(C, E->getType()); + return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl); } /// Emit a type description suitable for use by a runtime sanitizer library. The @@ -2194,9 +2330,9 @@ llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) { // Pointers are passed directly, everything else is passed by address. if (!V->getType()->isPointerTy()) { - llvm::Value *Ptr = CreateTempAlloca(V->getType()); + Address Ptr = CreateDefaultAlignTempAlloca(V->getType()); Builder.CreateStore(V, Ptr); - V = Ptr; + V = Ptr.getPointer(); } return Builder.CreatePtrToInt(V, TargetTy); } @@ -2217,8 +2353,9 @@ llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) { PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc); if (PLoc.isValid()) { auto FilenameGV = CGM.GetAddrOfConstantCString(PLoc.getFilename(), ".src"); - CGM.getSanitizerMetadata()->disableSanitizerForGlobal(FilenameGV); - Filename = FilenameGV; + CGM.getSanitizerMetadata()->disableSanitizerForGlobal( + cast<llvm::GlobalVariable>(FilenameGV.getPointer())); + Filename = FilenameGV.getPointer(); Line = PLoc.getLine(); Column = PLoc.getColumn(); } else { @@ -2395,6 +2532,34 @@ void CodeGenFunction::EmitCheck( EmitBlock(Cont); } +void CodeGenFunction::EmitCfiSlowPathCheck(llvm::Value *Cond, + llvm::ConstantInt *TypeId, + llvm::Value *Ptr) { + auto &Ctx = getLLVMContext(); + llvm::BasicBlock *Cont = createBasicBlock("cfi.cont"); + + llvm::BasicBlock *CheckBB = createBasicBlock("cfi.slowpath"); + llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB); + + llvm::MDBuilder MDHelper(getLLVMContext()); + llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1); + BI->setMetadata(llvm::LLVMContext::MD_prof, Node); + + EmitBlock(CheckBB); + + llvm::Constant *SlowPathFn = CGM.getModule().getOrInsertFunction( + "__cfi_slowpath", + llvm::FunctionType::get( + llvm::Type::getVoidTy(Ctx), + {llvm::Type::getInt64Ty(Ctx), + llvm::PointerType::getUnqual(llvm::Type::getInt8Ty(Ctx))}, + false)); + llvm::CallInst *CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr}); + CheckCall->setDoesNotThrow(); + + EmitBlock(Cont); +} + void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked) { llvm::BasicBlock *Cont = createBasicBlock("cont"); @@ -2426,6 +2591,33 @@ llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) { return TrapCall; } +Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E, + AlignmentSource *AlignSource) { + assert(E->getType()->isArrayType() && + "Array to pointer decay must have array source type!"); + + // Expressions of array type can't be bitfields or vector elements. + LValue LV = EmitLValue(E); + Address Addr = LV.getAddress(); + if (AlignSource) *AlignSource = LV.getAlignmentSource(); + + // If the array type was an incomplete type, we need to make sure + // the decay ends up being the right type. + llvm::Type *NewTy = ConvertType(E->getType()); + Addr = Builder.CreateElementBitCast(Addr, NewTy); + + // Note that VLA pointers are always decayed, so we don't need to do + // anything here. + if (!E->getType()->isVariableArrayType()) { + assert(isa<llvm::ArrayType>(Addr.getElementType()) && + "Expected pointer to array"); + Addr = Builder.CreateStructGEP(Addr, 0, CharUnits::Zero(), "arraydecay"); + } + + QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType(); + return Builder.CreateElementBitCast(Addr, ConvertTypeForMem(EltType)); +} + /// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an /// array to pointer, return the array subexpression. static const Expr *isSimpleArrayDecayOperand(const Expr *E) { @@ -2442,6 +2634,69 @@ static const Expr *isSimpleArrayDecayOperand(const Expr *E) { return SubExpr; } +static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF, + llvm::Value *ptr, + ArrayRef<llvm::Value*> indices, + bool inbounds, + const llvm::Twine &name = "arrayidx") { + if (inbounds) { + return CGF.Builder.CreateInBoundsGEP(ptr, indices, name); + } else { + return CGF.Builder.CreateGEP(ptr, indices, name); + } +} + +static CharUnits getArrayElementAlign(CharUnits arrayAlign, + llvm::Value *idx, + CharUnits eltSize) { + // If we have a constant index, we can use the exact offset of the + // element we're accessing. + if (auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) { + CharUnits offset = constantIdx->getZExtValue() * eltSize; + return arrayAlign.alignmentAtOffset(offset); + + // Otherwise, use the worst-case alignment for any element. + } else { + return arrayAlign.alignmentOfArrayElement(eltSize); + } +} + +static QualType getFixedSizeElementType(const ASTContext &ctx, + const VariableArrayType *vla) { + QualType eltType; + do { + eltType = vla->getElementType(); + } while ((vla = ctx.getAsVariableArrayType(eltType))); + return eltType; +} + +static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, + ArrayRef<llvm::Value*> indices, + QualType eltType, bool inbounds, + const llvm::Twine &name = "arrayidx") { + // All the indices except that last must be zero. +#ifndef NDEBUG + for (auto idx : indices.drop_back()) + assert(isa<llvm::ConstantInt>(idx) && + cast<llvm::ConstantInt>(idx)->isZero()); +#endif + + // Determine the element size of the statically-sized base. This is + // the thing that the indices are expressed in terms of. + if (auto vla = CGF.getContext().getAsVariableArrayType(eltType)) { + eltType = getFixedSizeElementType(CGF.getContext(), vla); + } + + // We can use that to compute the best alignment of the element. + CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType); + CharUnits eltAlign = + getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); + + llvm::Value *eltPtr = + emitArraySubscriptGEP(CGF, addr.getPointer(), indices, inbounds, name); + return Address(eltPtr, eltAlign); +} + LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, bool Accessed) { // The index must always be an integer, which is not an aggregate. Emit it. @@ -2460,32 +2715,34 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, LValue LHS = EmitLValue(E->getBase()); assert(LHS.isSimple() && "Can only subscript lvalue vectors here!"); return LValue::MakeVectorElt(LHS.getAddress(), Idx, - E->getBase()->getType(), LHS.getAlignment()); + E->getBase()->getType(), + LHS.getAlignmentSource()); } + // All the other cases basically behave like simple offsetting. + // Extend or truncate the index type to 32 or 64-bits. if (Idx->getType() != IntPtrTy) Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom"); - // We know that the pointer points to a type of the correct size, unless the - // size is a VLA or Objective-C interface. - llvm::Value *Address = nullptr; - CharUnits ArrayAlignment; + // Handle the extvector case we ignored above. if (isa<ExtVectorElementExpr>(E->getBase())) { LValue LV = EmitLValue(E->getBase()); - Address = EmitExtVectorElementLValue(LV); - Address = Builder.CreateInBoundsGEP(Address, Idx, "arrayidx"); - const VectorType *ExprVT = LV.getType()->getAs<VectorType>(); - QualType EQT = ExprVT->getElementType(); - return MakeAddrLValue(Address, EQT, - getContext().getTypeAlignInChars(EQT)); - } - else if (const VariableArrayType *vla = + Address Addr = EmitExtVectorElementLValue(LV); + + QualType EltType = LV.getType()->castAs<VectorType>()->getElementType(); + Addr = emitArraySubscriptGEP(*this, Addr, Idx, EltType, /*inbounds*/ true); + return MakeAddrLValue(Addr, EltType, LV.getAlignmentSource()); + } + + AlignmentSource AlignSource; + Address Addr = Address::invalid(); + if (const VariableArrayType *vla = getContext().getAsVariableArrayType(E->getType())) { // The base must be a pointer, which is not an aggregate. Emit // it. It needs to be emitted first in case it's what captures // the VLA bounds. - Address = EmitScalarExpr(E->getBase()); + Addr = EmitPointerWithAlignment(E->getBase(), &AlignSource); // The element count here is the total number of non-VLA elements. llvm::Value *numElements = getVLASize(vla).first; @@ -2496,24 +2753,40 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, // multiply. We suppress this if overflow is not undefined behavior. if (getLangOpts().isSignedOverflowDefined()) { Idx = Builder.CreateMul(Idx, numElements); - Address = Builder.CreateGEP(Address, Idx, "arrayidx"); } else { Idx = Builder.CreateNSWMul(Idx, numElements); - Address = Builder.CreateInBoundsGEP(Address, Idx, "arrayidx"); } - } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){ - // Indexing over an interface, as in "NSString *P; P[4];" - llvm::Value *InterfaceSize = - llvm::ConstantInt::get(Idx->getType(), - getContext().getTypeSizeInChars(OIT).getQuantity()); - Idx = Builder.CreateMul(Idx, InterfaceSize); + Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(), + !getLangOpts().isSignedOverflowDefined()); - // The base must be a pointer, which is not an aggregate. Emit it. - llvm::Value *Base = EmitScalarExpr(E->getBase()); - Address = EmitCastToVoidPtr(Base); - Address = Builder.CreateGEP(Address, Idx, "arrayidx"); - Address = Builder.CreateBitCast(Address, Base->getType()); + } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){ + // Indexing over an interface, as in "NSString *P; P[4];" + CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT); + llvm::Value *InterfaceSizeVal = + llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity());; + + llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal); + + // Emit the base pointer. + Addr = EmitPointerWithAlignment(E->getBase(), &AlignSource); + + // We don't necessarily build correct LLVM struct types for ObjC + // interfaces, so we can't rely on GEP to do this scaling + // correctly, so we need to cast to i8*. FIXME: is this actually + // true? A lot of other things in the fragile ABI would break... + llvm::Type *OrigBaseTy = Addr.getType(); + Addr = Builder.CreateElementBitCast(Addr, Int8Ty); + + // Do the GEP. + CharUnits EltAlign = + getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize); + llvm::Value *EltPtr = + emitArraySubscriptGEP(*this, Addr.getPointer(), ScaledIdx, false); + Addr = Address(EltPtr, EltAlign); + + // Cast back. + Addr = Builder.CreateBitCast(Addr, OrigBaseTy); } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) { // If this is A[i] where A is an array, the frontend will have decayed the // base to be a ArrayToPointerDecay implicit cast. While correct, it is @@ -2528,42 +2801,23 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true); else ArrayLV = EmitLValue(Array); - llvm::Value *ArrayPtr = ArrayLV.getAddress(); - llvm::Value *Zero = llvm::ConstantInt::get(Int32Ty, 0); - llvm::Value *Args[] = { Zero, Idx }; // Propagate the alignment from the array itself to the result. - ArrayAlignment = ArrayLV.getAlignment(); - - if (getLangOpts().isSignedOverflowDefined()) - Address = Builder.CreateGEP(ArrayPtr, Args, "arrayidx"); - else - Address = Builder.CreateInBoundsGEP(ArrayPtr, Args, "arrayidx"); + Addr = emitArraySubscriptGEP(*this, ArrayLV.getAddress(), + {CGM.getSize(CharUnits::Zero()), Idx}, + E->getType(), + !getLangOpts().isSignedOverflowDefined()); + AlignSource = ArrayLV.getAlignmentSource(); } else { - // The base must be a pointer, which is not an aggregate. Emit it. - llvm::Value *Base = EmitScalarExpr(E->getBase()); - if (getLangOpts().isSignedOverflowDefined()) - Address = Builder.CreateGEP(Base, Idx, "arrayidx"); - else - Address = Builder.CreateInBoundsGEP(Base, Idx, "arrayidx"); + // The base must be a pointer; emit it with an estimate of its alignment. + Addr = EmitPointerWithAlignment(E->getBase(), &AlignSource); + Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(), + !getLangOpts().isSignedOverflowDefined()); } - QualType T = E->getBase()->getType()->getPointeeType(); - assert(!T.isNull() && - "CodeGenFunction::EmitArraySubscriptExpr(): Illegal base type"); - + LValue LV = MakeAddrLValue(Addr, E->getType(), AlignSource); - // Limit the alignment to that of the result type. - LValue LV; - if (!ArrayAlignment.isZero()) { - CharUnits Align = getContext().getTypeAlignInChars(T); - ArrayAlignment = std::min(Align, ArrayAlignment); - LV = MakeAddrLValue(Address, T, ArrayAlignment); - } else { - LV = MakeNaturalAlignAddrLValue(Address, T); - } - - LV.getQuals().setAddressSpace(E->getBase()->getType().getAddressSpace()); + // TODO: Preserve/extend path TBAA metadata? if (getLangOpts().ObjC1 && getLangOpts().getGC() != LangOptions::NonGC) { @@ -2573,14 +2827,150 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, return LV; } -static -llvm::Constant *GenerateConstantVector(CGBuilderTy &Builder, - SmallVectorImpl<unsigned> &Elts) { - SmallVector<llvm::Constant*, 4> CElts; - for (unsigned i = 0, e = Elts.size(); i != e; ++i) - CElts.push_back(Builder.getInt32(Elts[i])); +LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, + bool IsLowerBound) { + LValue Base; + if (auto *ASE = + dyn_cast<OMPArraySectionExpr>(E->getBase()->IgnoreParenImpCasts())) + Base = EmitOMPArraySectionExpr(ASE, IsLowerBound); + else + Base = EmitLValue(E->getBase()); + QualType BaseTy = Base.getType(); + llvm::Value *Idx = nullptr; + QualType ResultExprTy; + if (auto *AT = getContext().getAsArrayType(BaseTy)) + ResultExprTy = AT->getElementType(); + else + ResultExprTy = BaseTy->getPointeeType(); + if (IsLowerBound || (!IsLowerBound && E->getColonLoc().isInvalid())) { + // Requesting lower bound or upper bound, but without provided length and + // without ':' symbol for the default length -> length = 1. + // Idx = LowerBound ?: 0; + if (auto *LowerBound = E->getLowerBound()) { + Idx = Builder.CreateIntCast( + EmitScalarExpr(LowerBound), IntPtrTy, + LowerBound->getType()->hasSignedIntegerRepresentation()); + } else + Idx = llvm::ConstantInt::getNullValue(IntPtrTy); + } else { + // Try to emit length or lower bound as constant. If this is possible, 1 is + // subtracted from constant length or lower bound. Otherwise, emit LLVM IR + // (LB + Len) - 1. + auto &C = CGM.getContext(); + auto *Length = E->getLength(); + llvm::APSInt ConstLength; + if (Length) { + // Idx = LowerBound + Length - 1; + if (Length->isIntegerConstantExpr(ConstLength, C)) { + ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits); + Length = nullptr; + } + auto *LowerBound = E->getLowerBound(); + llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false); + if (LowerBound && LowerBound->isIntegerConstantExpr(ConstLowerBound, C)) { + ConstLowerBound = ConstLowerBound.zextOrTrunc(PointerWidthInBits); + LowerBound = nullptr; + } + if (!Length) + --ConstLength; + else if (!LowerBound) + --ConstLowerBound; + + if (Length || LowerBound) { + auto *LowerBoundVal = + LowerBound + ? Builder.CreateIntCast( + EmitScalarExpr(LowerBound), IntPtrTy, + LowerBound->getType()->hasSignedIntegerRepresentation()) + : llvm::ConstantInt::get(IntPtrTy, ConstLowerBound); + auto *LengthVal = + Length + ? Builder.CreateIntCast( + EmitScalarExpr(Length), IntPtrTy, + Length->getType()->hasSignedIntegerRepresentation()) + : llvm::ConstantInt::get(IntPtrTy, ConstLength); + Idx = Builder.CreateAdd(LowerBoundVal, LengthVal, "lb_add_len", + /*HasNUW=*/false, + !getLangOpts().isSignedOverflowDefined()); + if (Length && LowerBound) { + Idx = Builder.CreateSub( + Idx, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "idx_sub_1", + /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined()); + } + } else + Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength + ConstLowerBound); + } else { + // Idx = ArraySize - 1; + if (auto *VAT = C.getAsVariableArrayType(BaseTy)) { + Length = VAT->getSizeExpr(); + if (Length->isIntegerConstantExpr(ConstLength, C)) + Length = nullptr; + } else { + auto *CAT = C.getAsConstantArrayType(BaseTy); + ConstLength = CAT->getSize(); + } + if (Length) { + auto *LengthVal = Builder.CreateIntCast( + EmitScalarExpr(Length), IntPtrTy, + Length->getType()->hasSignedIntegerRepresentation()); + Idx = Builder.CreateSub( + LengthVal, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "len_sub_1", + /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined()); + } else { + ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits); + --ConstLength; + Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength); + } + } + } + assert(Idx); - return llvm::ConstantVector::get(CElts); + llvm::Value *EltPtr; + QualType FixedSizeEltType = ResultExprTy; + if (auto *VLA = getContext().getAsVariableArrayType(ResultExprTy)) { + // The element count here is the total number of non-VLA elements. + llvm::Value *numElements = getVLASize(VLA).first; + FixedSizeEltType = getFixedSizeElementType(getContext(), VLA); + + // Effectively, the multiply by the VLA size is part of the GEP. + // GEP indexes are signed, and scaling an index isn't permitted to + // signed-overflow, so we use the same semantics for our explicit + // multiply. We suppress this if overflow is not undefined behavior. + if (getLangOpts().isSignedOverflowDefined()) { + Idx = Builder.CreateMul(Idx, numElements); + EltPtr = Builder.CreateGEP(Base.getPointer(), Idx, "arrayidx"); + } else { + Idx = Builder.CreateNSWMul(Idx, numElements); + EltPtr = Builder.CreateInBoundsGEP(Base.getPointer(), Idx, "arrayidx"); + } + } else if (BaseTy->isConstantArrayType()) { + llvm::Value *ArrayPtr = Base.getPointer(); + llvm::Value *Zero = llvm::ConstantInt::getNullValue(IntPtrTy); + llvm::Value *Args[] = {Zero, Idx}; + + if (getLangOpts().isSignedOverflowDefined()) + EltPtr = Builder.CreateGEP(ArrayPtr, Args, "arrayidx"); + else + EltPtr = Builder.CreateInBoundsGEP(ArrayPtr, Args, "arrayidx"); + } else { + // The base must be a pointer, which is not an aggregate. Emit it. + if (getLangOpts().isSignedOverflowDefined()) + EltPtr = Builder.CreateGEP(Base.getPointer(), Idx, "arrayidx"); + else + EltPtr = Builder.CreateInBoundsGEP(Base.getPointer(), Idx, "arrayidx"); + } + + CharUnits EltAlign = + Base.getAlignment().alignmentOfArrayElement( + getContext().getTypeSizeInChars(FixedSizeEltType)); + + // Limit the alignment to that of the result type. + LValue LV = MakeAddrLValue(Address(EltPtr, EltAlign), ResultExprTy, + Base.getAlignmentSource()); + + LV.getQuals().setAddressSpace(BaseTy.getAddressSpace()); + + return LV; } LValue CodeGenFunction:: @@ -2592,9 +2982,10 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) { if (E->isArrow()) { // If it is a pointer to a vector, emit the address and form an lvalue with // it. - llvm::Value *Ptr = EmitScalarExpr(E->getBase()); + AlignmentSource AlignSource; + Address Ptr = EmitPointerWithAlignment(E->getBase(), &AlignSource); const PointerType *PT = E->getBase()->getType()->getAs<PointerType>(); - Base = MakeAddrLValue(Ptr, PT->getPointeeType()); + Base = MakeAddrLValue(Ptr, PT->getPointeeType(), AlignSource); Base.getQuals().removeObjCGCAttr(); } else if (E->getBase()->isGLValue()) { // Otherwise, if the base is an lvalue ( as in the case of foo.x.x), @@ -2608,22 +2999,24 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) { llvm::Value *Vec = EmitScalarExpr(E->getBase()); // Store the vector to memory (because LValue wants an address). - llvm::Value *VecMem = CreateMemTemp(E->getBase()->getType()); + Address VecMem = CreateMemTemp(E->getBase()->getType()); Builder.CreateStore(Vec, VecMem); - Base = MakeAddrLValue(VecMem, E->getBase()->getType()); + Base = MakeAddrLValue(VecMem, E->getBase()->getType(), + AlignmentSource::Decl); } QualType type = E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers()); // Encode the element access list into a vector of unsigned indices. - SmallVector<unsigned, 4> Indices; + SmallVector<uint32_t, 4> Indices; E->getEncodedElementAccess(Indices); if (Base.isSimple()) { - llvm::Constant *CV = GenerateConstantVector(Builder, Indices); + llvm::Constant *CV = + llvm::ConstantDataVector::get(getLLVMContext(), Indices); return LValue::MakeExtVectorElt(Base.getAddress(), CV, type, - Base.getAlignment()); + Base.getAlignmentSource()); } assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!"); @@ -2633,8 +3026,8 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) { for (unsigned i = 0, e = Indices.size(); i != e; ++i) CElts.push_back(BaseElts->getAggregateElement(Indices[i])); llvm::Constant *CV = llvm::ConstantVector::get(CElts); - return LValue::MakeExtVectorElt(Base.getExtVectorAddr(), CV, type, - Base.getAlignment()); + return LValue::MakeExtVectorElt(Base.getExtVectorAddress(), CV, type, + Base.getAlignmentSource()); } LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) { @@ -2643,10 +3036,11 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) { // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar. LValue BaseLV; if (E->isArrow()) { - llvm::Value *Ptr = EmitScalarExpr(BaseExpr); + AlignmentSource AlignSource; + Address Addr = EmitPointerWithAlignment(BaseExpr, &AlignSource); QualType PtrTy = BaseExpr->getType()->getPointeeType(); - EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Ptr, PtrTy); - BaseLV = MakeNaturalAlignAddrLValue(Ptr, PtrTy); + EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy); + BaseLV = MakeAddrLValue(Addr, PtrTy, AlignSource); } else BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess); @@ -2677,41 +3071,65 @@ LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) { return EmitLValueForField(LambdaLV, Field); } +/// Drill down to the storage of a field without walking into +/// reference types. +/// +/// The resulting address doesn't necessarily have the right type. +static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base, + const FieldDecl *field) { + const RecordDecl *rec = field->getParent(); + + unsigned idx = + CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field); + + CharUnits offset; + // Adjust the alignment down to the given offset. + // As a special case, if the LLVM field index is 0, we know that this + // is zero. + assert((idx != 0 || CGF.getContext().getASTRecordLayout(rec) + .getFieldOffset(field->getFieldIndex()) == 0) && + "LLVM field at index zero had non-zero offset?"); + if (idx != 0) { + auto &recLayout = CGF.getContext().getASTRecordLayout(rec); + auto offsetInBits = recLayout.getFieldOffset(field->getFieldIndex()); + offset = CGF.getContext().toCharUnitsFromBits(offsetInBits); + } + + return CGF.Builder.CreateStructGEP(base, idx, offset, field->getName()); +} + LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field) { + AlignmentSource fieldAlignSource = + getFieldAlignmentSource(base.getAlignmentSource()); + if (field->isBitField()) { const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(field->getParent()); const CGBitFieldInfo &Info = RL.getBitFieldInfo(field); - llvm::Value *Addr = base.getAddress(); + Address Addr = base.getAddress(); unsigned Idx = RL.getLLVMFieldNo(field); if (Idx != 0) // For structs, we GEP to the field that the record layout suggests. - Addr = Builder.CreateStructGEP(nullptr, Addr, Idx, field->getName()); + Addr = Builder.CreateStructGEP(Addr, Idx, Info.StorageOffset, + field->getName()); // Get the access type. - llvm::Type *PtrTy = llvm::Type::getIntNPtrTy( - getLLVMContext(), Info.StorageSize, - CGM.getContext().getTargetAddressSpace(base.getType())); - if (Addr->getType() != PtrTy) - Addr = Builder.CreateBitCast(Addr, PtrTy); + llvm::Type *FieldIntTy = + llvm::Type::getIntNTy(getLLVMContext(), Info.StorageSize); + if (Addr.getElementType() != FieldIntTy) + Addr = Builder.CreateElementBitCast(Addr, FieldIntTy); QualType fieldType = field->getType().withCVRQualifiers(base.getVRQualifiers()); - return LValue::MakeBitfield(Addr, Info, fieldType, base.getAlignment()); + return LValue::MakeBitfield(Addr, Info, fieldType, fieldAlignSource); } const RecordDecl *rec = field->getParent(); QualType type = field->getType(); - CharUnits alignment = getContext().getDeclAlign(field); - - // FIXME: It should be impossible to have an LValue without alignment for a - // complete type. - if (!base.getAlignment().isZero()) - alignment = std::min(alignment, base.getAlignment()); bool mayAlias = rec->hasAttr<MayAliasAttr>(); - llvm::Value *addr = base.getAddress(); + Address addr = base.getAddress(); unsigned cvr = base.getVRQualifiers(); bool TBAAPath = CGM.getCodeGenOpts().StructPathTBAA; if (rec->isUnion()) { @@ -2721,14 +3139,12 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, TBAAPath = false; } else { // For structs, we GEP to the field that the record layout suggests. - unsigned idx = CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field); - addr = Builder.CreateStructGEP(nullptr, addr, idx, field->getName()); + addr = emitAddrOfFieldStorage(*this, addr, field); // If this is a reference field, load the reference right now. if (const ReferenceType *refType = type->getAs<ReferenceType>()) { llvm::LoadInst *load = Builder.CreateLoad(addr, "ref"); if (cvr & Qualifiers::Volatile) load->setVolatile(true); - load->setAlignment(alignment.getQuantity()); // Loading the reference will disable path-aware TBAA. TBAAPath = false; @@ -2739,17 +3155,20 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, else tbaa = CGM.getTBAAInfo(type); if (tbaa) - CGM.DecorateInstruction(load, tbaa); + CGM.DecorateInstructionWithTBAA(load, tbaa); } - addr = load; mayAlias = false; type = refType->getPointeeType(); - if (type->isIncompleteType()) - alignment = CharUnits(); - else - alignment = getContext().getTypeAlignInChars(type); - cvr = 0; // qualifiers don't recursively apply to referencee + + CharUnits alignment = + getNaturalTypeAlignment(type, &fieldAlignSource, /*pointee*/ true); + addr = Address(load, alignment); + + // Qualifiers on the struct don't apply to the referencee, and + // we'll pick up CVR from the actual type later, so reset these + // additional qualifiers now. + cvr = 0; } } @@ -2757,14 +3176,14 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, // for both unions and structs. A union needs a bitcast, a struct element // will need a bitcast if the LLVM type laid out doesn't match the desired // type. - addr = EmitBitCastOfLValueToProperType(*this, addr, - CGM.getTypes().ConvertTypeForMem(type), - field->getName()); + addr = Builder.CreateElementBitCast(addr, + CGM.getTypes().ConvertTypeForMem(type), + field->getName()); if (field->hasAttr<AnnotateAttr>()) addr = EmitFieldAnnotations(field, addr); - LValue LV = MakeAddrLValue(addr, type, alignment); + LValue LV = MakeAddrLValue(addr, type, fieldAlignSource); LV.getQuals().addCVRQualifiers(cvr); if (TBAAPath) { const ASTRecordLayout &Layout = @@ -2798,41 +3217,29 @@ CodeGenFunction::EmitLValueForFieldInitialization(LValue Base, if (!FieldType->isReferenceType()) return EmitLValueForField(Base, Field); - const CGRecordLayout &RL = - CGM.getTypes().getCGRecordLayout(Field->getParent()); - unsigned idx = RL.getLLVMFieldNo(Field); - llvm::Value *V = Builder.CreateStructGEP(nullptr, Base.getAddress(), idx); - assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs"); + Address V = emitAddrOfFieldStorage(*this, Base.getAddress(), Field); - // Make sure that the address is pointing to the right type. This is critical - // for both unions and structs. A union needs a bitcast, a struct element - // will need a bitcast if the LLVM type laid out doesn't match the desired - // type. + // Make sure that the address is pointing to the right type. llvm::Type *llvmType = ConvertTypeForMem(FieldType); - V = EmitBitCastOfLValueToProperType(*this, V, llvmType, Field->getName()); - - CharUnits Alignment = getContext().getDeclAlign(Field); - - // FIXME: It should be impossible to have an LValue without alignment for a - // complete type. - if (!Base.getAlignment().isZero()) - Alignment = std::min(Alignment, Base.getAlignment()); + V = Builder.CreateElementBitCast(V, llvmType, Field->getName()); - return MakeAddrLValue(V, FieldType, Alignment); + // TODO: access-path TBAA? + auto FieldAlignSource = getFieldAlignmentSource(Base.getAlignmentSource()); + return MakeAddrLValue(V, FieldType, FieldAlignSource); } LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){ if (E->isFileScope()) { - llvm::Value *GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E); - return MakeAddrLValue(GlobalPtr, E->getType()); + ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E); + return MakeAddrLValue(GlobalPtr, E->getType(), AlignmentSource::Decl); } if (E->getType()->isVariablyModifiedType()) // make sure to emit the VLA size. EmitVariablyModifiedType(E->getType()); - llvm::Value *DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral"); + Address DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral"); const Expr *InitExpr = E->getInitializer(); - LValue Result = MakeAddrLValue(DeclPtr, E->getType()); + LValue Result = MakeAddrLValue(DeclPtr, E->getType(), AlignmentSource::Decl); EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(), /*Init*/ true); @@ -2923,11 +3330,14 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) { EmitBlock(contBlock); if (lhs && rhs) { - llvm::PHINode *phi = Builder.CreatePHI(lhs->getAddress()->getType(), + llvm::PHINode *phi = Builder.CreatePHI(lhs->getPointer()->getType(), 2, "cond-lvalue"); - phi->addIncoming(lhs->getAddress(), lhsBlock); - phi->addIncoming(rhs->getAddress(), rhsBlock); - return MakeAddrLValue(phi, expr->getType()); + phi->addIncoming(lhs->getPointer(), lhsBlock); + phi->addIncoming(rhs->getPointer(), rhsBlock); + Address result(phi, std::min(lhs->getAlignment(), rhs->getAlignment())); + AlignmentSource alignSource = + std::max(lhs->getAlignmentSource(), rhs->getAlignmentSource()); + return MakeAddrLValue(result, expr->getType(), alignSource); } else { assert((lhs || rhs) && "both operands of glvalue conditional are throw-expressions?"); @@ -2996,9 +3406,9 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { case CK_Dynamic: { LValue LV = EmitLValue(E->getSubExpr()); - llvm::Value *V = LV.getAddress(); + Address V = LV.getAddress(); const auto *DCE = cast<CXXDynamicCastExpr>(E); - return MakeAddrLValue(EmitDynamicCast(V, DCE), E->getType()); + return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType()); } case CK_ConstructorConversion: @@ -3016,14 +3426,14 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl()); LValue LV = EmitLValue(E->getSubExpr()); - llvm::Value *This = LV.getAddress(); + Address This = LV.getAddress(); // Perform the derived-to-base conversion - llvm::Value *Base = GetAddressOfBaseClass( + Address Base = GetAddressOfBaseClass( This, DerivedClassDecl, E->path_begin(), E->path_end(), /*NullCheckValue=*/false, E->getExprLoc()); - return MakeAddrLValue(Base, E->getType()); + return MakeAddrLValue(Base, E->getType(), LV.getAlignmentSource()); } case CK_ToUnion: return EmitAggExprToLValue(E); @@ -3034,7 +3444,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { LValue LV = EmitLValue(E->getSubExpr()); // Perform the base-to-derived conversion - llvm::Value *Derived = + Address Derived = GetAddressOfDerivedClass(LV.getAddress(), DerivedClassDecl, E->path_begin(), E->path_end(), /*NullCheckValue=*/false); @@ -3043,34 +3453,36 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { // performed and the object is not of the derived type. if (sanitizePerformTypeCheck()) EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), - Derived, E->getType()); + Derived.getPointer(), E->getType()); if (SanOpts.has(SanitizerKind::CFIDerivedCast)) - EmitVTablePtrCheckForCast(E->getType(), Derived, /*MayBeNull=*/false, + EmitVTablePtrCheckForCast(E->getType(), Derived.getPointer(), + /*MayBeNull=*/false, CFITCK_DerivedCast, E->getLocStart()); - return MakeAddrLValue(Derived, E->getType()); + return MakeAddrLValue(Derived, E->getType(), LV.getAlignmentSource()); } case CK_LValueBitCast: { // This must be a reinterpret_cast (or c-style equivalent). const auto *CE = cast<ExplicitCastExpr>(E); + CGM.EmitExplicitCastExprType(CE, this); LValue LV = EmitLValue(E->getSubExpr()); - llvm::Value *V = Builder.CreateBitCast(LV.getAddress(), - ConvertType(CE->getTypeAsWritten())); + Address V = Builder.CreateBitCast(LV.getAddress(), + ConvertType(CE->getTypeAsWritten())); if (SanOpts.has(SanitizerKind::CFIUnrelatedCast)) - EmitVTablePtrCheckForCast(E->getType(), V, /*MayBeNull=*/false, + EmitVTablePtrCheckForCast(E->getType(), V.getPointer(), + /*MayBeNull=*/false, CFITCK_UnrelatedCast, E->getLocStart()); - return MakeAddrLValue(V, E->getType()); + return MakeAddrLValue(V, E->getType(), LV.getAlignmentSource()); } case CK_ObjCObjectLValueCast: { LValue LV = EmitLValue(E->getSubExpr()); - QualType ToType = getContext().getLValueReferenceType(E->getType()); - llvm::Value *V = Builder.CreateBitCast(LV.getAddress(), - ConvertType(ToType)); - return MakeAddrLValue(V, E->getType()); + Address V = Builder.CreateElementBitCast(LV.getAddress(), + ConvertType(E->getType())); + return MakeAddrLValue(V, E->getType(), LV.getAlignmentSource()); } case CK_ZeroToOCLEvent: llvm_unreachable("NULL to OpenCL event lvalue cast is not valid"); @@ -3129,20 +3541,17 @@ RValue CodeGenFunction::EmitCallExpr(const CallExpr *E, if (const auto *PseudoDtor = dyn_cast<CXXPseudoDestructorExpr>(E->getCallee()->IgnoreParens())) { QualType DestroyedType = PseudoDtor->getDestroyedType(); - if (getLangOpts().ObjCAutoRefCount && - DestroyedType->isObjCLifetimeType() && - (DestroyedType.getObjCLifetime() == Qualifiers::OCL_Strong || - DestroyedType.getObjCLifetime() == Qualifiers::OCL_Weak)) { + if (DestroyedType.hasStrongOrWeakObjCLifetime()) { // Automatic Reference Counting: // If the pseudo-expression names a retainable object with weak or // strong lifetime, the object shall be released. Expr *BaseExpr = PseudoDtor->getBase(); - llvm::Value *BaseValue = nullptr; + Address BaseValue = Address::invalid(); Qualifiers BaseQuals; // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar. if (PseudoDtor->isArrow()) { - BaseValue = EmitScalarExpr(BaseExpr); + BaseValue = EmitPointerWithAlignment(BaseExpr); const PointerType *PTy = BaseExpr->getType()->getAs<PointerType>(); BaseQuals = PTy->getPointeeType().getQualifiers(); } else { @@ -3152,7 +3561,7 @@ RValue CodeGenFunction::EmitCallExpr(const CallExpr *E, BaseQuals = BaseTy.getQualifiers(); } - switch (PseudoDtor->getDestroyedType().getObjCLifetime()) { + switch (DestroyedType.getObjCLifetime()) { case Qualifiers::OCL_None: case Qualifiers::OCL_ExplicitNone: case Qualifiers::OCL_Autoreleasing: @@ -3237,13 +3646,14 @@ LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) { RValue RV = EmitCallExpr(E); if (!RV.isScalar()) - return MakeAddrLValue(RV.getAggregateAddr(), E->getType()); + return MakeAddrLValue(RV.getAggregateAddress(), E->getType(), + AlignmentSource::Decl); assert(E->getCallReturnType(getContext())->isReferenceType() && "Can't have a scalar return unless the return type is a " "reference type!"); - return MakeAddrLValue(RV.getScalarVal(), E->getType()); + return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType()); } LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) { @@ -3256,21 +3666,23 @@ LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) { && "binding l-value to type which needs a temporary"); AggValueSlot Slot = CreateAggTemp(E->getType()); EmitCXXConstructExpr(E, Slot); - return MakeAddrLValue(Slot.getAddr(), E->getType()); + return MakeAddrLValue(Slot.getAddress(), E->getType(), + AlignmentSource::Decl); } LValue CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) { - return MakeAddrLValue(EmitCXXTypeidExpr(E), E->getType()); + return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType()); } -llvm::Value *CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) { - return Builder.CreateBitCast(CGM.GetAddrOfUuidDescriptor(E), - ConvertType(E->getType())->getPointerTo()); +Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) { + return Builder.CreateElementBitCast(CGM.GetAddrOfUuidDescriptor(E), + ConvertType(E->getType())); } LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) { - return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType()); + return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType(), + AlignmentSource::Decl); } LValue @@ -3278,34 +3690,37 @@ CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) { AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue"); Slot.setExternallyDestructed(); EmitAggExpr(E->getSubExpr(), Slot); - EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddr()); - return MakeAddrLValue(Slot.getAddr(), E->getType()); + EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddress()); + return MakeAddrLValue(Slot.getAddress(), E->getType(), + AlignmentSource::Decl); } LValue CodeGenFunction::EmitLambdaLValue(const LambdaExpr *E) { AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue"); EmitLambdaExpr(E, Slot); - return MakeAddrLValue(Slot.getAddr(), E->getType()); + return MakeAddrLValue(Slot.getAddress(), E->getType(), + AlignmentSource::Decl); } LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) { RValue RV = EmitObjCMessageExpr(E); if (!RV.isScalar()) - return MakeAddrLValue(RV.getAggregateAddr(), E->getType()); + return MakeAddrLValue(RV.getAggregateAddress(), E->getType(), + AlignmentSource::Decl); assert(E->getMethodDecl()->getReturnType()->isReferenceType() && "Can't have a scalar return unless the return type is a " "reference type!"); - return MakeAddrLValue(RV.getScalarVal(), E->getType()); + return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType()); } LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) { - llvm::Value *V = - CGM.getObjCRuntime().GetSelector(*this, E->getSelector(), true); - return MakeAddrLValue(V, E->getType()); + Address V = + CGM.getObjCRuntime().GetAddrOfSelector(*this, E->getSelector()); + return MakeAddrLValue(V, E->getType(), AlignmentSource::Decl); } llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface, @@ -3333,8 +3748,7 @@ LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) { BaseQuals = ObjectTy.getQualifiers(); } else { LValue BaseLV = EmitLValue(BaseExpr); - // FIXME: this isn't right for bitfields. - BaseValue = BaseLV.getAddress(); + BaseValue = BaseLV.getPointer(); ObjectTy = BaseExpr->getType(); BaseQuals = ObjectTy.getQualifiers(); } @@ -3349,17 +3763,38 @@ LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) { LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) { // Can only get l-value for message expression returning aggregate type RValue RV = EmitAnyExprToTemp(E); - return MakeAddrLValue(RV.getAggregateAddr(), E->getType()); + return MakeAddrLValue(RV.getAggregateAddress(), E->getType(), + AlignmentSource::Decl); } RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, const CallExpr *E, ReturnValueSlot ReturnValue, - const Decl *TargetDecl, llvm::Value *Chain) { + CGCalleeInfo CalleeInfo, llvm::Value *Chain) { // Get the actual function type. The callee type will always be a pointer to // function type or a block pointer type. assert(CalleeType->isFunctionPointerType() && "Call must have function pointer type!"); + // Preserve the non-canonical function type because things like exception + // specifications disappear in the canonical type. That information is useful + // to drive the generation of more accurate code for this call later on. + const FunctionProtoType *NonCanonicalFTP = CalleeType->getAs<PointerType>() + ->getPointeeType() + ->getAs<FunctionProtoType>(); + + const Decl *TargetDecl = CalleeInfo.getCalleeDecl(); + + if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) + // We can only guarantee that a function is called from the correct + // context/function based on the appropriate target attributes, + // so only check in the case where we have both always_inline and target + // since otherwise we could be making a conditional call after a check for + // the proper cpu features (and it won't cause code generation issues due to + // function based code generation). + if (TargetDecl->hasAttr<AlwaysInlineAttr>() && + TargetDecl->hasAttr<TargetAttr>()) + checkTargetFeatures(E, FD); + CalleeType = getContext().getCanonicalType(CalleeType); const auto *FnType = @@ -3383,7 +3818,8 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, Callee, llvm::PointerType::getUnqual(PrefixStructTy)); llvm::Value *CalleeSigPtr = Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 0); - llvm::Value *CalleeSig = Builder.CreateLoad(CalleeSigPtr); + llvm::Value *CalleeSig = + Builder.CreateAlignedLoad(CalleeSigPtr, getIntAlign()); llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig); llvm::BasicBlock *Cont = createBasicBlock("cont"); @@ -3393,7 +3829,8 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, EmitBlock(TypeCheck); llvm::Value *CalleeRTTIPtr = Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 1); - llvm::Value *CalleeRTTI = Builder.CreateLoad(CalleeRTTIPtr); + llvm::Value *CalleeRTTI = + Builder.CreateAlignedLoad(CalleeRTTIPtr, getPointerAlign()); llvm::Value *CalleeRTTIMatch = Builder.CreateICmpEQ(CalleeRTTI, FTRTTIConst); llvm::Constant *StaticData[] = { @@ -3408,12 +3845,39 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, } } + // If we are checking indirect calls and this call is indirect, check that the + // function pointer is a member of the bit set for the function type. + if (SanOpts.has(SanitizerKind::CFIICall) && + (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) { + SanitizerScope SanScope(this); + + llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(QualType(FnType, 0)); + llvm::Value *BitSetName = llvm::MetadataAsValue::get(getLLVMContext(), MD); + + llvm::Value *CastedCallee = Builder.CreateBitCast(Callee, Int8PtrTy); + llvm::Value *BitSetTest = + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::bitset_test), + {CastedCallee, BitSetName}); + + auto TypeId = CGM.CreateCfiIdForTypeMetadata(MD); + if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && TypeId) { + EmitCfiSlowPathCheck(BitSetTest, TypeId, CastedCallee); + } else { + llvm::Constant *StaticData[] = { + EmitCheckSourceLocation(E->getLocStart()), + EmitCheckTypeDescriptor(QualType(FnType, 0)), + }; + EmitCheck(std::make_pair(BitSetTest, SanitizerKind::CFIICall), + "cfi_bad_icall", StaticData, CastedCallee); + } + } + CallArgList Args; if (Chain) Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)), CGM.getContext().VoidPtrTy); - EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arg_begin(), - E->arg_end(), E->getDirectCallee(), /*ParamsToSkip*/ 0); + EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arguments(), + E->getDirectCallee(), /*ParamsToSkip*/ 0); const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall( Args, FnType, /*isChainCall=*/Chain); @@ -3444,34 +3908,38 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, Callee = Builder.CreateBitCast(Callee, CalleeTy, "callee.knr.cast"); } - return EmitCall(FnInfo, Callee, ReturnValue, Args, TargetDecl); + return EmitCall(FnInfo, Callee, ReturnValue, Args, + CGCalleeInfo(NonCanonicalFTP, TargetDecl)); } LValue CodeGenFunction:: EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) { - llvm::Value *BaseV; - if (E->getOpcode() == BO_PtrMemI) - BaseV = EmitScalarExpr(E->getLHS()); - else - BaseV = EmitLValue(E->getLHS()).getAddress(); + Address BaseAddr = Address::invalid(); + if (E->getOpcode() == BO_PtrMemI) { + BaseAddr = EmitPointerWithAlignment(E->getLHS()); + } else { + BaseAddr = EmitLValue(E->getLHS()).getAddress(); + } llvm::Value *OffsetV = EmitScalarExpr(E->getRHS()); const MemberPointerType *MPT = E->getRHS()->getType()->getAs<MemberPointerType>(); - llvm::Value *AddV = CGM.getCXXABI().EmitMemberDataPointerAddress( - *this, E, BaseV, OffsetV, MPT); + AlignmentSource AlignSource; + Address MemberAddr = + EmitCXXMemberDataPointerAddress(E, BaseAddr, OffsetV, MPT, + &AlignSource); - return MakeAddrLValue(AddV, MPT->getPointeeType()); + return MakeAddrLValue(MemberAddr, MPT->getPointeeType(), AlignSource); } /// Given the address of a temporary variable, produce an r-value of /// its type. -RValue CodeGenFunction::convertTempToRValue(llvm::Value *addr, +RValue CodeGenFunction::convertTempToRValue(Address addr, QualType type, SourceLocation loc) { - LValue lvalue = MakeNaturalAlignAddrLValue(addr, type); + LValue lvalue = MakeAddrLValue(addr, type, AlignmentSource::Decl); switch (getEvaluationKind(type)) { case TEK_Complex: return RValue::getComplex(EmitLoadOfComplex(lvalue, loc)); @@ -3527,7 +3995,8 @@ static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF, CodeGenFunction::hasAggregateEvaluationKind(ov->getType())) { CGF.EmitAggExpr(ov->getSourceExpr(), slot); - LValue LV = CGF.MakeAddrLValue(slot.getAddr(), ov->getType()); + LValue LV = CGF.MakeAddrLValue(slot.getAddress(), ov->getType(), + AlignmentSource::Decl); opaqueData = OVMA::bind(CGF, ov, LV); result.RV = slot.asRValue(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp index 883b76b..20838db 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp @@ -49,7 +49,8 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> { if (!shouldUseDestForReturnSlot()) return ReturnValueSlot(); - return ReturnValueSlot(Dest.getAddr(), Dest.isVolatile(), IsResultUnused); + return ReturnValueSlot(Dest.getAddress(), Dest.isVolatile(), + IsResultUnused); } AggValueSlot EnsureSlot(QualType T) { @@ -77,14 +78,13 @@ public: /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired. void EmitFinalDestCopy(QualType type, const LValue &src); - void EmitFinalDestCopy(QualType type, RValue src, - CharUnits srcAlignment = CharUnits::Zero()); + void EmitFinalDestCopy(QualType type, RValue src); void EmitCopy(QualType type, const AggValueSlot &dest, const AggValueSlot &src); void EmitMoveFromReturnSlot(const Expr *E, RValue Src); - void EmitArrayInit(llvm::Value *DestPtr, llvm::ArrayType *AType, + void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, QualType elementType, InitListExpr *E); AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) { @@ -199,7 +199,8 @@ public: // case Expr::ChooseExprClass: void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); } void VisitAtomicExpr(AtomicExpr *E) { - CGF.EmitAtomicExpr(E, EnsureSlot(E->getType()).getAddr()); + RValue Res = CGF.EmitAtomicExpr(E); + EmitFinalDestCopy(E->getType(), Res); } }; } // end anonymous namespace. @@ -259,17 +260,14 @@ void AggExprEmitter::EmitMoveFromReturnSlot(const Expr *E, RValue src) { } // Otherwise, copy from there to the destination. - assert(Dest.getAddr() != src.getAggregateAddr()); - std::pair<CharUnits, CharUnits> typeInfo = - CGF.getContext().getTypeInfoInChars(E->getType()); - EmitFinalDestCopy(E->getType(), src, typeInfo.second); + assert(Dest.getPointer() != src.getAggregatePointer()); + EmitFinalDestCopy(E->getType(), src); } /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired. -void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src, - CharUnits srcAlign) { +void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) { assert(src.isAggregate() && "value must be aggregate value!"); - LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddr(), type, srcAlign); + LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddress(), type); EmitFinalDestCopy(type, srcLV); } @@ -298,8 +296,8 @@ void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest, CharUnits sz = CGF.getContext().getTypeSizeInChars(type); llvm::Value *size = llvm::ConstantInt::get(CGF.SizeTy, sz.getQuantity()); CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF, - dest.getAddr(), - src.getAddr(), + dest.getAddress(), + src.getAddress(), size); return; } @@ -307,9 +305,8 @@ void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest, // If the result of the assignment is used, copy the LHS there also. // It's volatile if either side is. Use the minimum alignment of // the two sides. - CGF.EmitAggregateCopy(dest.getAddr(), src.getAddr(), type, - dest.isVolatile() || src.isVolatile(), - std::min(dest.getAlignment(), src.getAlignment())); + CGF.EmitAggregateCopy(dest.getAddress(), src.getAddress(), type, + dest.isVolatile() || src.isVolatile()); } /// \brief Emit the initializer for a std::initializer_list initialized with a @@ -321,7 +318,7 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { ASTContext &Ctx = CGF.getContext(); LValue Array = CGF.EmitLValue(E->getSubExpr()); assert(Array.isSimple() && "initializer_list array not a simple lvalue"); - llvm::Value *ArrayPtr = Array.getAddress(); + Address ArrayPtr = Array.getAddress(); const ConstantArrayType *ArrayType = Ctx.getAsConstantArrayType(E->getSubExpr()->getType()); @@ -344,13 +341,12 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { } AggValueSlot Dest = EnsureSlot(E->getType()); - LValue DestLV = CGF.MakeAddrLValue(Dest.getAddr(), E->getType(), - Dest.getAlignment()); + LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); LValue Start = CGF.EmitLValueForFieldInitialization(DestLV, *Field); llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0); llvm::Value *IdxStart[] = { Zero, Zero }; llvm::Value *ArrayStart = - Builder.CreateInBoundsGEP(ArrayPtr, IdxStart, "arraystart"); + Builder.CreateInBoundsGEP(ArrayPtr.getPointer(), IdxStart, "arraystart"); CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start); ++Field; @@ -367,7 +363,7 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { // End pointer. llvm::Value *IdxEnd[] = { Zero, Size }; llvm::Value *ArrayEnd = - Builder.CreateInBoundsGEP(ArrayPtr, IdxEnd, "arrayend"); + Builder.CreateInBoundsGEP(ArrayPtr.getPointer(), IdxEnd, "arrayend"); CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength); } else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) { // Length. @@ -402,7 +398,7 @@ static bool isTrivialFiller(Expr *E) { } /// \brief Emit initialization of an array from an initializer list. -void AggExprEmitter::EmitArrayInit(llvm::Value *DestPtr, llvm::ArrayType *AType, +void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, QualType elementType, InitListExpr *E) { uint64_t NumInitElements = E->getNumInits(); @@ -414,13 +410,17 @@ void AggExprEmitter::EmitArrayInit(llvm::Value *DestPtr, llvm::ArrayType *AType, llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); llvm::Value *indices[] = { zero, zero }; llvm::Value *begin = - Builder.CreateInBoundsGEP(DestPtr, indices, "arrayinit.begin"); + Builder.CreateInBoundsGEP(DestPtr.getPointer(), indices, "arrayinit.begin"); + + CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); + CharUnits elementAlign = + DestPtr.getAlignment().alignmentOfArrayElement(elementSize); // Exception safety requires us to destroy all the // already-constructed members if an initializer throws. // For that, we'll need an EH cleanup. QualType::DestructionKind dtorKind = elementType.isDestructedType(); - llvm::AllocaInst *endOfInit = nullptr; + Address endOfInit = Address::invalid(); EHScopeStack::stable_iterator cleanup; llvm::Instruction *cleanupDominator = nullptr; if (CGF.needsEHCleanup(dtorKind)) { @@ -428,10 +428,11 @@ void AggExprEmitter::EmitArrayInit(llvm::Value *DestPtr, llvm::ArrayType *AType, // directly, but the control flow can get so varied here that it // would actually be quite complex. Therefore we go through an // alloca. - endOfInit = CGF.CreateTempAlloca(begin->getType(), + endOfInit = CGF.CreateTempAlloca(begin->getType(), CGF.getPointerAlign(), "arrayinit.endOfInit"); cleanupDominator = Builder.CreateStore(begin, endOfInit); CGF.pushIrregularPartialArrayCleanup(begin, endOfInit, elementType, + elementAlign, CGF.getDestroyer(dtorKind)); cleanup = CGF.EHStack.stable_begin(); @@ -458,10 +459,11 @@ void AggExprEmitter::EmitArrayInit(llvm::Value *DestPtr, llvm::ArrayType *AType, // Tell the cleanup that it needs to destroy up to this // element. TODO: some of these stores can be trivially // observed to be unnecessary. - if (endOfInit) Builder.CreateStore(element, endOfInit); + if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit); } - LValue elementLV = CGF.MakeAddrLValue(element, elementType); + LValue elementLV = + CGF.MakeAddrLValue(Address(element, elementAlign), elementType); EmitInitializationToLValue(E->getInit(i), elementLV); } @@ -482,7 +484,7 @@ void AggExprEmitter::EmitArrayInit(llvm::Value *DestPtr, llvm::ArrayType *AType, // Advance to the start of the rest of the array. if (NumInitElements) { element = Builder.CreateInBoundsGEP(element, one, "arrayinit.start"); - if (endOfInit) Builder.CreateStore(element, endOfInit); + if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit); } // Compute the end of the array. @@ -500,7 +502,8 @@ void AggExprEmitter::EmitArrayInit(llvm::Value *DestPtr, llvm::ArrayType *AType, currentElement->addIncoming(element, entryBB); // Emit the actual filler expression. - LValue elementLV = CGF.MakeAddrLValue(currentElement, elementType); + LValue elementLV = + CGF.MakeAddrLValue(Address(currentElement, elementAlign), elementType); if (filler) EmitInitializationToLValue(filler, elementLV); else @@ -511,7 +514,7 @@ void AggExprEmitter::EmitArrayInit(llvm::Value *DestPtr, llvm::ArrayType *AType, Builder.CreateInBoundsGEP(currentElement, one, "arrayinit.next"); // Tell the EH cleanup that we finished with the last element. - if (endOfInit) Builder.CreateStore(nextElement, endOfInit); + if (endOfInit.isValid()) Builder.CreateStore(nextElement, endOfInit); // Leave the loop if we're done. llvm::Value *done = Builder.CreateICmpEQ(nextElement, end, @@ -569,6 +572,8 @@ static Expr *findPeephole(Expr *op, CastKind kind) { } void AggExprEmitter::VisitCastExpr(CastExpr *E) { + if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E)) + CGF.CGM.EmitExplicitCastExprType(ECE, &CGF); switch (E->getCastKind()) { case CK_Dynamic: { // FIXME: Can this actually happen? We have no test coverage for it. @@ -596,9 +601,8 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { // GCC union extension QualType Ty = E->getSubExpr()->getType(); - QualType PtrTy = CGF.getContext().getPointerType(Ty); - llvm::Value *CastPtr = Builder.CreateBitCast(Dest.getAddr(), - CGF.ConvertType(PtrTy)); + Address CastPtr = + Builder.CreateElementBitCast(Dest.getAddress(), CGF.ConvertType(Ty)); EmitInitializationToLValue(E->getSubExpr(), CGF.MakeAddrLValue(CastPtr, Ty)); break; @@ -649,13 +653,13 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { // Zero-initialize. (Strictly speaking, we only need to intialize // the padding at the end, but this is simpler.) if (!Dest.isZeroed()) - CGF.EmitNullInitialization(Dest.getAddr(), atomicType); + CGF.EmitNullInitialization(Dest.getAddress(), atomicType); // Build a GEP to refer to the subobject. - llvm::Value *valueAddr = - CGF.Builder.CreateStructGEP(nullptr, valueDest.getAddr(), 0); + Address valueAddr = + CGF.Builder.CreateStructGEP(valueDest.getAddress(), 0, + CharUnits()); valueDest = AggValueSlot::forAddr(valueAddr, - valueDest.getAlignment(), valueDest.getQualifiers(), valueDest.isExternallyDestructed(), valueDest.requiresGCollection(), @@ -673,8 +677,8 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { CGF.CreateAggTemp(atomicType, "atomic-to-nonatomic.temp"); CGF.EmitAggExpr(E->getSubExpr(), atomicSlot); - llvm::Value *valueAddr = - Builder.CreateStructGEP(nullptr, atomicSlot.getAddr(), 0); + Address valueAddr = + Builder.CreateStructGEP(atomicSlot.getAddress(), 0, CharUnits()); RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile()); return EmitFinalDestCopy(valueType, rvalue); } @@ -959,15 +963,15 @@ void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) { } void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) { - llvm::Value *ArgValue = CGF.EmitVAListRef(VE->getSubExpr()); - llvm::Value *ArgPtr = CGF.EmitVAArg(ArgValue, VE->getType()); + Address ArgValue = Address::invalid(); + Address ArgPtr = CGF.EmitVAArg(VE, ArgValue); - if (!ArgPtr) { + if (!ArgPtr.isValid()) { // If EmitVAArg fails, we fall back to the LLVM instruction. - llvm::Value *Val = - Builder.CreateVAArg(ArgValue, CGF.ConvertType(VE->getType())); + llvm::Value *Val = Builder.CreateVAArg(ArgValue.getPointer(), + CGF.ConvertType(VE->getType())); if (!Dest.isIgnored()) - Builder.CreateStore(Val, Dest.getAddr()); + Builder.CreateStore(Val, Dest.getAddress()); return; } @@ -987,7 +991,7 @@ void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) { // Push that destructor we promised. if (!wasExternallyDestructed) - CGF.EmitCXXTemporary(E->getTemporary(), E->getType(), Dest.getAddr()); + CGF.EmitCXXTemporary(E->getTemporary(), E->getType(), Dest.getAddress()); } void @@ -1011,13 +1015,13 @@ void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) { void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) { QualType T = E->getType(); AggValueSlot Slot = EnsureSlot(T); - EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddr(), T)); + EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T)); } void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) { QualType T = E->getType(); AggValueSlot Slot = EnsureSlot(T); - EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddr(), T)); + EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T)); } /// isSimpleZero - If emitting this value will obviously just cause a store of @@ -1135,8 +1139,7 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { AggValueSlot Dest = EnsureSlot(E->getType()); - LValue DestLV = CGF.MakeAddrLValue(Dest.getAddr(), E->getType(), - Dest.getAlignment()); + LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); // Handle initialization of an array. if (E->getType()->isArrayType()) { @@ -1146,12 +1149,8 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { QualType elementType = CGF.getContext().getAsArrayType(E->getType())->getElementType(); - llvm::PointerType *APType = - cast<llvm::PointerType>(Dest.getAddr()->getType()); - llvm::ArrayType *AType = - cast<llvm::ArrayType>(APType->getElementType()); - - EmitArrayInit(Dest.getAddr(), AType, elementType, E); + auto AType = cast<llvm::ArrayType>(Dest.getAddress().getElementType()); + EmitArrayInit(Dest.getAddress(), AType, elementType, E); return; } @@ -1175,7 +1174,7 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { RecordDecl *record = E->getType()->castAs<RecordType>()->getDecl(); // Prepare a 'this' for CXXDefaultInitExprs. - CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddr()); + CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress()); if (record->isUnion()) { // Only initialize one field of a union. The field itself is @@ -1253,7 +1252,10 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { assert(LV.isSimple()); if (CGF.needsEHCleanup(dtorKind)) { if (!cleanupDominator) - cleanupDominator = CGF.Builder.CreateUnreachable(); // placeholder + cleanupDominator = CGF.Builder.CreateAlignedLoad( + CGF.Int8Ty, + llvm::Constant::getNullValue(CGF.Int8PtrTy), + CharUnits::One()); // placeholder CGF.pushDestroy(EHCleanup, LV.getAddress(), field->getType(), CGF.getDestroyer(dtorKind), false); @@ -1266,7 +1268,7 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { // else, clean it up for -O0 builds and general tidiness. if (!pushedCleanup && LV.isSimple()) if (llvm::GetElementPtrInst *GEP = - dyn_cast<llvm::GetElementPtrInst>(LV.getAddress())) + dyn_cast<llvm::GetElementPtrInst>(LV.getPointer())) if (GEP->use_empty()) GEP->eraseFromParent(); } @@ -1284,8 +1286,7 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { void AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) { AggValueSlot Dest = EnsureSlot(E->getType()); - LValue DestLV = CGF.MakeAddrLValue(Dest.getAddr(), E->getType(), - Dest.getAlignment()); + LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); EmitInitializationToLValue(E->getBase(), DestLV); VisitInitListExpr(E->getUpdater()); } @@ -1355,7 +1356,7 @@ static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E, CodeGenFunction &CGF) { // If the slot is already known to be zeroed, nothing to do. Don't mess with // volatile stores. - if (Slot.isZeroed() || Slot.isVolatile() || Slot.getAddr() == nullptr) + if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid()) return; // C++ objects with a user-declared constructor don't need zero'ing. @@ -1368,26 +1369,22 @@ static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E, } // If the type is 16-bytes or smaller, prefer individual stores over memset. - std::pair<CharUnits, CharUnits> TypeInfo = - CGF.getContext().getTypeInfoInChars(E->getType()); - if (TypeInfo.first <= CharUnits::fromQuantity(16)) + CharUnits Size = CGF.getContext().getTypeSizeInChars(E->getType()); + if (Size <= CharUnits::fromQuantity(16)) return; // Check to see if over 3/4 of the initializer are known to be zero. If so, // we prefer to emit memset + individual stores for the rest. CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF); - if (NumNonZeroBytes*4 > TypeInfo.first) + if (NumNonZeroBytes*4 > Size) return; // Okay, it seems like a good idea to use an initial memset, emit the call. - llvm::Constant *SizeVal = CGF.Builder.getInt64(TypeInfo.first.getQuantity()); - CharUnits Align = TypeInfo.second; + llvm::Constant *SizeVal = CGF.Builder.getInt64(Size.getQuantity()); - llvm::Value *Loc = Slot.getAddr(); - - Loc = CGF.Builder.CreateBitCast(Loc, CGF.Int8PtrTy); - CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal, - Align.getQuantity(), false); + Address Loc = Slot.getAddress(); + Loc = CGF.Builder.CreateElementBitCast(Loc, CGF.Int8Ty); + CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal, false); // Tell the AggExprEmitter that the slot is known zero. Slot.setZeroed(); @@ -1403,7 +1400,7 @@ static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E, void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) { assert(E && hasAggregateEvaluationKind(E->getType()) && "Invalid aggregate expression to emit"); - assert((Slot.getAddr() != nullptr || Slot.isIgnored()) && + assert((Slot.getAddress().isValid() || Slot.isIgnored()) && "slot has bits but no address"); // Optimize the slot if possible. @@ -1414,7 +1411,7 @@ void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) { LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) { assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!"); - llvm::Value *Temp = CreateMemTemp(E->getType()); + Address Temp = CreateMemTemp(E->getType()); LValue LV = MakeAddrLValue(Temp, E->getType()); EmitAggExpr(E, AggValueSlot::forLValue(LV, AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers, @@ -1422,10 +1419,9 @@ LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) { return LV; } -void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, - llvm::Value *SrcPtr, QualType Ty, +void CodeGenFunction::EmitAggregateCopy(Address DestPtr, + Address SrcPtr, QualType Ty, bool isVolatile, - CharUnits alignment, bool isAssignment) { assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex"); @@ -1456,17 +1452,16 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, // implementation handles this case safely. If there is a libc that does not // safely handle this, we can add a target hook. - // Get data size and alignment info for this aggregate. If this is an - // assignment don't copy the tail padding. Otherwise copying it is fine. + // Get data size info for this aggregate. If this is an assignment, + // don't copy the tail padding, because we might be assigning into a + // base subobject where the tail padding is claimed. Otherwise, + // copying it is fine. std::pair<CharUnits, CharUnits> TypeInfo; if (isAssignment) TypeInfo = getContext().getTypeInfoDataSizeInChars(Ty); else TypeInfo = getContext().getTypeInfoInChars(Ty); - if (alignment.isZero()) - alignment = TypeInfo.second; - llvm::Value *SizeVal = nullptr; if (TypeInfo.first.isZero()) { // But note that getTypeInfo returns 0 for a VLA. @@ -1509,15 +1504,8 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, // we need to use a different call here. We use isVolatile to indicate when // either the source or the destination is volatile. - llvm::PointerType *DPT = cast<llvm::PointerType>(DestPtr->getType()); - llvm::Type *DBP = - llvm::Type::getInt8PtrTy(getLLVMContext(), DPT->getAddressSpace()); - DestPtr = Builder.CreateBitCast(DestPtr, DBP); - - llvm::PointerType *SPT = cast<llvm::PointerType>(SrcPtr->getType()); - llvm::Type *SBP = - llvm::Type::getInt8PtrTy(getLLVMContext(), SPT->getAddressSpace()); - SrcPtr = Builder.CreateBitCast(SrcPtr, SBP); + DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty); + SrcPtr = Builder.CreateElementBitCast(SrcPtr, Int8Ty); // Don't do any of the memmove_collectable tests if GC isn't set. if (CGM.getLangOpts().getGC() == LangOptions::NonGC) { @@ -1540,11 +1528,11 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, } } + auto Inst = Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, isVolatile); + // Determine the metadata to describe the position of any padding in this // memcpy, as well as the TBAA tags for the members of the struct, in case // the optimizer wishes to expand it in to scalar memory operations. - llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty); - - Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, alignment.getQuantity(), - isVolatile, /*TBAATag=*/nullptr, TBAAStructTag); + if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty)) + Inst->setMetadata(llvm::LLVMContext::MD_tbaa_struct, TBAAStructTag); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp index c7adcca..604cde7 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp @@ -59,7 +59,7 @@ static RequiredArgs commonEmitCXXMemberOrOperatorCall( if (CE) { // Special case: skip first argument of CXXOperatorCall (it is "this"). unsigned ArgsToSkip = isa<CXXOperatorCallExpr>(CE) ? 1 : 0; - CGF.EmitCallArgs(Args, FPT, CE->arg_begin() + ArgsToSkip, CE->arg_end(), + CGF.EmitCallArgs(Args, FPT, drop_begin(CE->arguments(), ArgsToSkip), CE->getDirectCallee()); } else { assert( @@ -166,9 +166,9 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( } } - llvm::Value *This; + Address This = Address::invalid(); if (IsArrow) - This = EmitScalarExpr(Base); + This = EmitPointerWithAlignment(Base); else This = EmitLValue(Base).getAddress(); @@ -185,19 +185,18 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( // when it isn't necessary; just produce the proper effect here. // Special case: skip first argument of CXXOperatorCall (it is "this"). unsigned ArgsToSkip = isa<CXXOperatorCallExpr>(CE) ? 1 : 0; - llvm::Value *RHS = - EmitLValue(*(CE->arg_begin() + ArgsToSkip)).getAddress(); + Address RHS = EmitLValue(*(CE->arg_begin() + ArgsToSkip)).getAddress(); EmitAggregateAssign(This, RHS, CE->getType()); - return RValue::get(This); + return RValue::get(This.getPointer()); } if (isa<CXXConstructorDecl>(MD) && cast<CXXConstructorDecl>(MD)->isCopyOrMoveConstructor()) { // Trivial move and copy ctor are the same. assert(CE->getNumArgs() == 1 && "unexpected argcount for trivial ctor"); - llvm::Value *RHS = EmitLValue(*CE->arg_begin()).getAddress(); - EmitAggregateCopy(This, RHS, CE->arg_begin()->getType()); - return RValue::get(This); + Address RHS = EmitLValue(*CE->arg_begin()).getAddress(); + EmitAggregateCopy(This, RHS, (*CE->arg_begin())->getType()); + return RValue::get(This.getPointer()); } llvm_unreachable("unknown trivial member function"); } @@ -245,7 +244,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( cast<CXXDestructorDecl>(DevirtualizedMethod); Callee = CGM.GetAddrOfFunction(GlobalDecl(DDtor, Dtor_Complete), Ty); } - EmitCXXMemberOrOperatorCall(MD, Callee, ReturnValue, This, + EmitCXXMemberOrOperatorCall(MD, Callee, ReturnValue, This.getPointer(), /*ImplicitParam=*/nullptr, QualType(), CE); } return RValue::get(nullptr); @@ -259,7 +258,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( } else { if (SanOpts.has(SanitizerKind::CFINVCall) && MD->getParent()->isDynamicClass()) { - llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy); + llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy, MD->getParent()); EmitVTablePtrCheckForCall(MD, VTable, CFITCK_NVCall, CE->getLocStart()); } @@ -277,7 +276,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( *this, MD, This, UseVirtualCall); } - return EmitCXXMemberOrOperatorCall(MD, Callee, ReturnValue, This, + return EmitCXXMemberOrOperatorCall(MD, Callee, ReturnValue, This.getPointer(), /*ImplicitParam=*/nullptr, QualType(), CE); } @@ -301,19 +300,20 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E, llvm::Value *MemFnPtr = EmitScalarExpr(MemFnExpr); // Emit the 'this' pointer. - llvm::Value *This; - + Address This = Address::invalid(); if (BO->getOpcode() == BO_PtrMemI) - This = EmitScalarExpr(BaseExpr); + This = EmitPointerWithAlignment(BaseExpr); else This = EmitLValue(BaseExpr).getAddress(); - EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This, + EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.getPointer(), QualType(MPT->getClass(), 0)); // Ask the ABI to load the callee. Note that This is modified. + llvm::Value *ThisPtrForCall = nullptr; llvm::Value *Callee = - CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(*this, BO, This, MemFnPtr, MPT); + CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(*this, BO, This, + ThisPtrForCall, MemFnPtr, MPT); CallArgList Args; @@ -321,12 +321,12 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E, getContext().getPointerType(getContext().getTagDeclType(RD)); // Push the this ptr. - Args.add(RValue::get(This), ThisType); + Args.add(RValue::get(ThisPtrForCall), ThisType); RequiredArgs required = RequiredArgs::forPrototypePlus(FPT, 1); // And the rest of the call args - EmitCallArgs(Args, FPT, E->arg_begin(), E->arg_end(), E->getDirectCallee()); + EmitCallArgs(Args, FPT, E->arguments(), E->getDirectCallee()); return EmitCall(CGM.getTypes().arrangeCXXMethodCall(Args, FPT, required), Callee, ReturnValue, Args); } @@ -348,18 +348,43 @@ RValue CodeGenFunction::EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E, } static void EmitNullBaseClassInitialization(CodeGenFunction &CGF, - llvm::Value *DestPtr, + Address DestPtr, const CXXRecordDecl *Base) { if (Base->isEmpty()) return; - DestPtr = CGF.EmitCastToVoidPtr(DestPtr); + DestPtr = CGF.Builder.CreateElementBitCast(DestPtr, CGF.Int8Ty); const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(Base); - CharUnits Size = Layout.getNonVirtualSize(); - CharUnits Align = Layout.getNonVirtualAlignment(); - - llvm::Value *SizeVal = CGF.CGM.getSize(Size); + CharUnits NVSize = Layout.getNonVirtualSize(); + + // We cannot simply zero-initialize the entire base sub-object if vbptrs are + // present, they are initialized by the most derived class before calling the + // constructor. + SmallVector<std::pair<CharUnits, CharUnits>, 1> Stores; + Stores.emplace_back(CharUnits::Zero(), NVSize); + + // Each store is split by the existence of a vbptr. + CharUnits VBPtrWidth = CGF.getPointerSize(); + std::vector<CharUnits> VBPtrOffsets = + CGF.CGM.getCXXABI().getVBPtrOffsets(Base); + for (CharUnits VBPtrOffset : VBPtrOffsets) { + std::pair<CharUnits, CharUnits> LastStore = Stores.pop_back_val(); + CharUnits LastStoreOffset = LastStore.first; + CharUnits LastStoreSize = LastStore.second; + + CharUnits SplitBeforeOffset = LastStoreOffset; + CharUnits SplitBeforeSize = VBPtrOffset - SplitBeforeOffset; + assert(!SplitBeforeSize.isNegative() && "negative store size!"); + if (!SplitBeforeSize.isZero()) + Stores.emplace_back(SplitBeforeOffset, SplitBeforeSize); + + CharUnits SplitAfterOffset = VBPtrOffset + VBPtrWidth; + CharUnits SplitAfterSize = LastStoreSize - SplitAfterOffset; + assert(!SplitAfterSize.isNegative() && "negative store size!"); + if (!SplitAfterSize.isZero()) + Stores.emplace_back(SplitAfterOffset, SplitAfterSize); + } // If the type contains a pointer to data member we can't memset it to zero. // Instead, create a null constant and copy it to the destination. @@ -367,27 +392,43 @@ static void EmitNullBaseClassInitialization(CodeGenFunction &CGF, // like -1, which happens to be the pattern used by member-pointers. // TODO: isZeroInitializable can be over-conservative in the case where a // virtual base contains a member pointer. - if (!CGF.CGM.getTypes().isZeroInitializable(Base)) { - llvm::Constant *NullConstant = CGF.CGM.EmitNullConstantForBase(Base); - - llvm::GlobalVariable *NullVariable = - new llvm::GlobalVariable(CGF.CGM.getModule(), NullConstant->getType(), - /*isConstant=*/true, - llvm::GlobalVariable::PrivateLinkage, - NullConstant, Twine()); + llvm::Constant *NullConstantForBase = CGF.CGM.EmitNullConstantForBase(Base); + if (!NullConstantForBase->isNullValue()) { + llvm::GlobalVariable *NullVariable = new llvm::GlobalVariable( + CGF.CGM.getModule(), NullConstantForBase->getType(), + /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, + NullConstantForBase, Twine()); + + CharUnits Align = std::max(Layout.getNonVirtualAlignment(), + DestPtr.getAlignment()); NullVariable->setAlignment(Align.getQuantity()); - llvm::Value *SrcPtr = CGF.EmitCastToVoidPtr(NullVariable); + + Address SrcPtr = Address(CGF.EmitCastToVoidPtr(NullVariable), Align); // Get and call the appropriate llvm.memcpy overload. - CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, Align.getQuantity()); - return; - } - + for (std::pair<CharUnits, CharUnits> Store : Stores) { + CharUnits StoreOffset = Store.first; + CharUnits StoreSize = Store.second; + llvm::Value *StoreSizeVal = CGF.CGM.getSize(StoreSize); + CGF.Builder.CreateMemCpy( + CGF.Builder.CreateConstInBoundsByteGEP(DestPtr, StoreOffset), + CGF.Builder.CreateConstInBoundsByteGEP(SrcPtr, StoreOffset), + StoreSizeVal); + } + // Otherwise, just memset the whole thing to zero. This is legal // because in LLVM, all default initializers (other than the ones we just // handled above) are guaranteed to have a bit pattern of all zeros. - CGF.Builder.CreateMemSet(DestPtr, CGF.Builder.getInt8(0), SizeVal, - Align.getQuantity()); + } else { + for (std::pair<CharUnits, CharUnits> Store : Stores) { + CharUnits StoreOffset = Store.first; + CharUnits StoreSize = Store.second; + llvm::Value *StoreSizeVal = CGF.CGM.getSize(StoreSize); + CGF.Builder.CreateMemSet( + CGF.Builder.CreateConstInBoundsByteGEP(DestPtr, StoreOffset), + CGF.Builder.getInt8(0), StoreSizeVal); + } + } } void @@ -404,11 +445,12 @@ CodeGenFunction::EmitCXXConstructExpr(const CXXConstructExpr *E, switch (E->getConstructionKind()) { case CXXConstructExpr::CK_Delegating: case CXXConstructExpr::CK_Complete: - EmitNullInitialization(Dest.getAddr(), E->getType()); + EmitNullInitialization(Dest.getAddress(), E->getType()); break; case CXXConstructExpr::CK_VirtualBase: case CXXConstructExpr::CK_NonVirtualBase: - EmitNullBaseClassInitialization(*this, Dest.getAddr(), CD->getParent()); + EmitNullBaseClassInitialization(*this, Dest.getAddress(), + CD->getParent()); break; } } @@ -431,7 +473,7 @@ CodeGenFunction::EmitCXXConstructExpr(const CXXConstructExpr *E, if (const ConstantArrayType *arrayType = getContext().getAsConstantArrayType(E->getType())) { - EmitCXXAggrConstructorCall(CD, arrayType, Dest.getAddr(), E); + EmitCXXAggrConstructorCall(CD, arrayType, Dest.getAddress(), E); } else { CXXCtorType Type = Ctor_Complete; bool ForVirtualBase = false; @@ -457,15 +499,13 @@ CodeGenFunction::EmitCXXConstructExpr(const CXXConstructExpr *E, } // Call the constructor. - EmitCXXConstructorCall(CD, Type, ForVirtualBase, Delegating, Dest.getAddr(), - E); + EmitCXXConstructorCall(CD, Type, ForVirtualBase, Delegating, + Dest.getAddress(), E); } } -void -CodeGenFunction::EmitSynthesizedCXXCopyCtor(llvm::Value *Dest, - llvm::Value *Src, - const Expr *Exp) { +void CodeGenFunction::EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, + const Expr *Exp) { if (const ExprWithCleanups *E = dyn_cast<ExprWithCleanups>(Exp)) Exp = E->getSubExpr(); assert(isa<CXXConstructExpr>(Exp) && @@ -759,22 +799,20 @@ static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF, } static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const Expr *Init, - QualType AllocType, llvm::Value *NewPtr) { + QualType AllocType, Address NewPtr) { // FIXME: Refactor with EmitExprAsInit. - CharUnits Alignment = CGF.getContext().getTypeAlignInChars(AllocType); switch (CGF.getEvaluationKind(AllocType)) { case TEK_Scalar: CGF.EmitScalarInit(Init, nullptr, - CGF.MakeAddrLValue(NewPtr, AllocType, Alignment), false); + CGF.MakeAddrLValue(NewPtr, AllocType), false); return; case TEK_Complex: - CGF.EmitComplexExprIntoLValue(Init, CGF.MakeAddrLValue(NewPtr, AllocType, - Alignment), + CGF.EmitComplexExprIntoLValue(Init, CGF.MakeAddrLValue(NewPtr, AllocType), /*isInit*/ true); return; case TEK_Aggregate: { AggValueSlot Slot - = AggValueSlot::forAddr(NewPtr, Alignment, AllocType.getQualifiers(), + = AggValueSlot::forAddr(NewPtr, AllocType.getQualifiers(), AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased); @@ -787,23 +825,27 @@ static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const Expr *Init, void CodeGenFunction::EmitNewArrayInitializer( const CXXNewExpr *E, QualType ElementType, llvm::Type *ElementTy, - llvm::Value *BeginPtr, llvm::Value *NumElements, + Address BeginPtr, llvm::Value *NumElements, llvm::Value *AllocSizeWithoutCookie) { // If we have a type with trivial initialization and no initializer, // there's nothing to do. if (!E->hasInitializer()) return; - llvm::Value *CurPtr = BeginPtr; + Address CurPtr = BeginPtr; unsigned InitListElements = 0; const Expr *Init = E->getInitializer(); - llvm::AllocaInst *EndOfInit = nullptr; + Address EndOfInit = Address::invalid(); QualType::DestructionKind DtorKind = ElementType.isDestructedType(); EHScopeStack::stable_iterator Cleanup; llvm::Instruction *CleanupDominator = nullptr; + CharUnits ElementSize = getContext().getTypeSizeInChars(ElementType); + CharUnits ElementAlign = + BeginPtr.getAlignment().alignmentOfArrayElement(ElementSize); + // If the initializer is an initializer list, first do the explicit elements. if (const InitListExpr *ILE = dyn_cast<InitListExpr>(Init)) { InitListElements = ILE->getNumInits(); @@ -813,10 +855,8 @@ void CodeGenFunction::EmitNewArrayInitializer( QualType AllocType = E->getAllocatedType(); if (const ConstantArrayType *CAT = dyn_cast_or_null<ConstantArrayType>( AllocType->getAsArrayTypeUnsafe())) { - unsigned AS = CurPtr->getType()->getPointerAddressSpace(); ElementTy = ConvertTypeForMem(AllocType); - llvm::Type *AllocPtrTy = ElementTy->getPointerTo(AS); - CurPtr = Builder.CreateBitCast(CurPtr, AllocPtrTy); + CurPtr = Builder.CreateElementBitCast(CurPtr, ElementTy); InitListElements *= getContext().getConstantArrayElementCount(CAT); } @@ -826,27 +866,34 @@ void CodeGenFunction::EmitNewArrayInitializer( // directly, but the control flow can get so varied here that it // would actually be quite complex. Therefore we go through an // alloca. - EndOfInit = CreateTempAlloca(BeginPtr->getType(), "array.init.end"); - CleanupDominator = Builder.CreateStore(BeginPtr, EndOfInit); - pushIrregularPartialArrayCleanup(BeginPtr, EndOfInit, ElementType, + EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(), + "array.init.end"); + CleanupDominator = Builder.CreateStore(BeginPtr.getPointer(), EndOfInit); + pushIrregularPartialArrayCleanup(BeginPtr.getPointer(), EndOfInit, + ElementType, ElementAlign, getDestroyer(DtorKind)); Cleanup = EHStack.stable_begin(); } + CharUnits StartAlign = CurPtr.getAlignment(); for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i) { // Tell the cleanup that it needs to destroy up to this // element. TODO: some of these stores can be trivially // observed to be unnecessary. - if (EndOfInit) - Builder.CreateStore(Builder.CreateBitCast(CurPtr, BeginPtr->getType()), - EndOfInit); + if (EndOfInit.isValid()) { + auto FinishedPtr = + Builder.CreateBitCast(CurPtr.getPointer(), BeginPtr.getType()); + Builder.CreateStore(FinishedPtr, EndOfInit); + } // FIXME: If the last initializer is an incomplete initializer list for // an array, and we have an array filler, we can fold together the two // initialization loops. StoreAnyExprIntoOneUnit(*this, ILE->getInit(i), ILE->getInit(i)->getType(), CurPtr); - CurPtr = Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr, 1, - "array.exp.next"); + CurPtr = Address(Builder.CreateInBoundsGEP(CurPtr.getPointer(), + Builder.getSize(1), + "array.exp.next"), + StartAlign.alignmentAtOffset((i + 1) * ElementSize)); } // The remaining elements are filled with the array filler expression. @@ -864,7 +911,7 @@ void CodeGenFunction::EmitNewArrayInitializer( } // Switch back to initializing one base element at a time. - CurPtr = Builder.CreateBitCast(CurPtr, BeginPtr->getType()); + CurPtr = Builder.CreateBitCast(CurPtr, BeginPtr.getType()); } // Attempt to perform zero-initialization using memset. @@ -889,9 +936,7 @@ void CodeGenFunction::EmitNewArrayInitializer( } // Create the memset. - CharUnits Alignment = getContext().getTypeAlignInChars(ElementType); - Builder.CreateMemSet(CurPtr, Builder.getInt8(0), RemainingSize, - Alignment.getQuantity(), false); + Builder.CreateMemSet(CurPtr, Builder.getInt8(0), RemainingSize, false); return true; }; @@ -925,7 +970,8 @@ void CodeGenFunction::EmitNewArrayInitializer( // // FIXME: Share this cleanup with the constructor call emission rather than // having it create a cleanup of its own. - if (EndOfInit) Builder.CreateStore(CurPtr, EndOfInit); + if (EndOfInit.isValid()) + Builder.CreateStore(CurPtr.getPointer(), EndOfInit); // Emit a constructor call loop to initialize the remaining elements. if (InitListElements) @@ -985,13 +1031,13 @@ void CodeGenFunction::EmitNewArrayInitializer( // Find the end of the array, hoisted out of the loop. llvm::Value *EndPtr = - Builder.CreateInBoundsGEP(BeginPtr, NumElements, "array.end"); + Builder.CreateInBoundsGEP(BeginPtr.getPointer(), NumElements, "array.end"); // If the number of elements isn't constant, we have to now check if there is // anything left to initialize. if (!ConstNum) { - llvm::Value *IsEmpty = Builder.CreateICmpEQ(CurPtr, EndPtr, - "array.isempty"); + llvm::Value *IsEmpty = + Builder.CreateICmpEQ(CurPtr.getPointer(), EndPtr, "array.isempty"); Builder.CreateCondBr(IsEmpty, ContBB, LoopBB); } @@ -1000,16 +1046,19 @@ void CodeGenFunction::EmitNewArrayInitializer( // Set up the current-element phi. llvm::PHINode *CurPtrPhi = - Builder.CreatePHI(CurPtr->getType(), 2, "array.cur"); - CurPtrPhi->addIncoming(CurPtr, EntryBB); - CurPtr = CurPtrPhi; + Builder.CreatePHI(CurPtr.getType(), 2, "array.cur"); + CurPtrPhi->addIncoming(CurPtr.getPointer(), EntryBB); + + CurPtr = Address(CurPtrPhi, ElementAlign); // Store the new Cleanup position for irregular Cleanups. - if (EndOfInit) Builder.CreateStore(CurPtr, EndOfInit); + if (EndOfInit.isValid()) + Builder.CreateStore(CurPtr.getPointer(), EndOfInit); // Enter a partial-destruction Cleanup if necessary. if (!CleanupDominator && needsEHCleanup(DtorKind)) { - pushRegularPartialArrayCleanup(BeginPtr, CurPtr, ElementType, + pushRegularPartialArrayCleanup(BeginPtr.getPointer(), CurPtr.getPointer(), + ElementType, ElementAlign, getDestroyer(DtorKind)); Cleanup = EHStack.stable_begin(); CleanupDominator = Builder.CreateUnreachable(); @@ -1026,7 +1075,8 @@ void CodeGenFunction::EmitNewArrayInitializer( // Advance to the next element by adjusting the pointer type as necessary. llvm::Value *NextPtr = - Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr, 1, "array.next"); + Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr.getPointer(), 1, + "array.next"); // Check whether we've gotten to the end of the array and, if so, // exit the loop. @@ -1039,7 +1089,7 @@ void CodeGenFunction::EmitNewArrayInitializer( static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E, QualType ElementType, llvm::Type *ElementTy, - llvm::Value *NewPtr, llvm::Value *NumElements, + Address NewPtr, llvm::Value *NumElements, llvm::Value *AllocSizeWithoutCookie) { ApplyDebugLocation DL(CGF, E); if (E->isArray()) @@ -1089,8 +1139,7 @@ RValue CodeGenFunction::EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, bool IsDelete) { CallArgList Args; const Stmt *ArgS = Arg; - EmitCallArgs(Args, *Type->param_type_begin(), - ConstExprIterator(&ArgS), ConstExprIterator(&ArgS + 1)); + EmitCallArgs(Args, *Type->param_type_begin(), llvm::makeArrayRef(ArgS)); // Find the allocation or deallocation function that we're calling. ASTContext &Ctx = getContext(); DeclarationName Name = Ctx.DeclarationNames @@ -1105,7 +1154,7 @@ RValue CodeGenFunction::EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, namespace { /// A cleanup to call the given 'operator delete' function upon /// abnormal exit from a new expression. - class CallDeleteDuringNew : public EHScopeStack::Cleanup { + class CallDeleteDuringNew final : public EHScopeStack::Cleanup { size_t NumPlacementArgs; const FunctionDecl *OperatorDelete; llvm::Value *Ptr; @@ -1158,7 +1207,7 @@ namespace { /// A cleanup to call the given 'operator delete' function upon /// abnormal exit from a new expression when the new expression is /// conditional. - class CallDeleteDuringConditionalNew : public EHScopeStack::Cleanup { + class CallDeleteDuringConditionalNew final : public EHScopeStack::Cleanup { size_t NumPlacementArgs; const FunctionDecl *OperatorDelete; DominatingValue<RValue>::saved_type Ptr; @@ -1219,7 +1268,7 @@ namespace { /// new-expression throws. static void EnterNewDeleteCleanup(CodeGenFunction &CGF, const CXXNewExpr *E, - llvm::Value *NewPtr, + Address NewPtr, llvm::Value *AllocSize, const CallArgList &NewArgs) { // If we're not inside a conditional branch, then the cleanup will @@ -1229,7 +1278,8 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, .pushCleanupWithExtra<CallDeleteDuringNew>(EHCleanup, E->getNumPlacementArgs(), E->getOperatorDelete(), - NewPtr, AllocSize); + NewPtr.getPointer(), + AllocSize); for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) Cleanup->setPlacementArg(I, NewArgs[I+1].RV); @@ -1238,7 +1288,7 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, // Otherwise, we need to save all this stuff. DominatingValue<RValue>::saved_type SavedNewPtr = - DominatingValue<RValue>::save(CGF, RValue::get(NewPtr)); + DominatingValue<RValue>::save(CGF, RValue::get(NewPtr.getPointer())); DominatingValue<RValue>::saved_type SavedAllocSize = DominatingValue<RValue>::save(CGF, RValue::get(AllocSize)); @@ -1261,13 +1311,6 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // 1. Build a call to the allocation function. FunctionDecl *allocator = E->getOperatorNew(); - const FunctionProtoType *allocatorType = - allocator->getType()->castAs<FunctionProtoType>(); - - CallArgList allocatorArgs; - - // The allocation size is the first argument. - QualType sizeType = getContext().getSizeType(); // If there is a brace-initializer, cannot allocate fewer elements than inits. unsigned minElements = 0; @@ -1282,24 +1325,61 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { EmitCXXNewAllocSize(*this, E, minElements, numElements, allocSizeWithoutCookie); - allocatorArgs.add(RValue::get(allocSize), sizeType); - - // We start at 1 here because the first argument (the allocation size) - // has already been emitted. - EmitCallArgs(allocatorArgs, allocatorType, E->placement_arg_begin(), - E->placement_arg_end(), /* CalleeDecl */ nullptr, - /*ParamsToSkip*/ 1); - // Emit the allocation call. If the allocator is a global placement // operator, just "inline" it directly. - RValue RV; + Address allocation = Address::invalid(); + CallArgList allocatorArgs; if (allocator->isReservedGlobalPlacementOperator()) { - assert(allocatorArgs.size() == 2); - RV = allocatorArgs[1].RV; - // TODO: kill any unnecessary computations done for the size - // argument. + assert(E->getNumPlacementArgs() == 1); + const Expr *arg = *E->placement_arguments().begin(); + + AlignmentSource alignSource; + allocation = EmitPointerWithAlignment(arg, &alignSource); + + // The pointer expression will, in many cases, be an opaque void*. + // In these cases, discard the computed alignment and use the + // formal alignment of the allocated type. + if (alignSource != AlignmentSource::Decl) { + allocation = Address(allocation.getPointer(), + getContext().getTypeAlignInChars(allocType)); + } + + // Set up allocatorArgs for the call to operator delete if it's not + // the reserved global operator. + if (E->getOperatorDelete() && + !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) { + allocatorArgs.add(RValue::get(allocSize), getContext().getSizeType()); + allocatorArgs.add(RValue::get(allocation.getPointer()), arg->getType()); + } + } else { - RV = EmitNewDeleteCall(*this, allocator, allocatorType, allocatorArgs); + const FunctionProtoType *allocatorType = + allocator->getType()->castAs<FunctionProtoType>(); + + // The allocation size is the first argument. + QualType sizeType = getContext().getSizeType(); + allocatorArgs.add(RValue::get(allocSize), sizeType); + + // We start at 1 here because the first argument (the allocation size) + // has already been emitted. + EmitCallArgs(allocatorArgs, allocatorType, E->placement_arguments(), + /* CalleeDecl */ nullptr, + /*ParamsToSkip*/ 1); + + RValue RV = + EmitNewDeleteCall(*this, allocator, allocatorType, allocatorArgs); + + // For now, only assume that the allocation function returns + // something satisfactorily aligned for the element type, plus + // the cookie if we have one. + CharUnits allocationAlign = + getContext().getTypeAlignInChars(allocType); + if (allocSize != allocSizeWithoutCookie) { + CharUnits cookieAlign = getSizeAlign(); // FIXME? + allocationAlign = std::max(allocationAlign, cookieAlign); + } + + allocation = Address(RV.getScalarVal(), allocationAlign); } // Emit a null check on the allocation result if the allocation @@ -1312,9 +1392,6 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { llvm::BasicBlock *nullCheckBB = nullptr; llvm::BasicBlock *contBB = nullptr; - llvm::Value *allocation = RV.getScalarVal(); - unsigned AS = allocation->getType()->getPointerAddressSpace(); - // The null-check means that the initializer is conditionally // evaluated. ConditionalEvaluation conditional(*this); @@ -1326,7 +1403,8 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { llvm::BasicBlock *notNullBB = createBasicBlock("new.notnull"); contBB = createBasicBlock("new.cont"); - llvm::Value *isNull = Builder.CreateIsNull(allocation, "new.isnull"); + llvm::Value *isNull = + Builder.CreateIsNull(allocation.getPointer(), "new.isnull"); Builder.CreateCondBr(isNull, contBB, notNullBB); EmitBlock(notNullBB); } @@ -1352,8 +1430,15 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { } llvm::Type *elementTy = ConvertTypeForMem(allocType); - llvm::Type *elementPtrTy = elementTy->getPointerTo(AS); - llvm::Value *result = Builder.CreateBitCast(allocation, elementPtrTy); + Address result = Builder.CreateElementBitCast(allocation, elementTy); + + // Passing pointer through invariant.group.barrier to avoid propagation of + // vptrs information which may be included in previous type. + if (CGM.getCodeGenOpts().StrictVTablePointers && + CGM.getCodeGenOpts().OptimizationLevel > 0 && + allocator->isReservedGlobalPlacementOperator()) + result = Address(Builder.CreateInvariantGroupBarrier(result.getPointer()), + result.getAlignment()); EmitNewInitializer(*this, E, allocType, elementTy, result, numElements, allocSizeWithoutCookie); @@ -1362,7 +1447,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // allocating an array of arrays, we'll need to cast back to the // array pointer type. llvm::Type *resultType = ConvertTypeForMem(E->getType()); - if (result->getType() != resultType) + if (result.getType() != resultType) result = Builder.CreateBitCast(result, resultType); } @@ -1373,21 +1458,22 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { cleanupDominator->eraseFromParent(); } + llvm::Value *resultPtr = result.getPointer(); if (nullCheck) { conditional.end(*this); llvm::BasicBlock *notNullBB = Builder.GetInsertBlock(); EmitBlock(contBB); - llvm::PHINode *PHI = Builder.CreatePHI(result->getType(), 2); - PHI->addIncoming(result, notNullBB); - PHI->addIncoming(llvm::Constant::getNullValue(result->getType()), + llvm::PHINode *PHI = Builder.CreatePHI(resultPtr->getType(), 2); + PHI->addIncoming(resultPtr, notNullBB); + PHI->addIncoming(llvm::Constant::getNullValue(resultPtr->getType()), nullCheckBB); - result = PHI; + resultPtr = PHI; } - return result; + return resultPtr; } void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD, @@ -1423,7 +1509,7 @@ void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD, namespace { /// Calls the given 'operator delete' on a single object. - struct CallObjectDelete : EHScopeStack::Cleanup { + struct CallObjectDelete final : EHScopeStack::Cleanup { llvm::Value *Ptr; const FunctionDecl *OperatorDelete; QualType ElementType; @@ -1450,7 +1536,7 @@ CodeGenFunction::pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete, /// Emit the code for deleting a single object. static void EmitObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE, - llvm::Value *Ptr, + Address Ptr, QualType ElementType) { // Find the destructor for the type, if applicable. If the // destructor is virtual, we'll just emit the vcall and return. @@ -1473,29 +1559,24 @@ static void EmitObjectDelete(CodeGenFunction &CGF, // to pop it off in a second. const FunctionDecl *OperatorDelete = DE->getOperatorDelete(); CGF.EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup, - Ptr, OperatorDelete, ElementType); + Ptr.getPointer(), + OperatorDelete, ElementType); if (Dtor) CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false, /*Delegating=*/false, Ptr); - else if (CGF.getLangOpts().ObjCAutoRefCount && - ElementType->isObjCLifetimeType()) { - switch (ElementType.getObjCLifetime()) { + else if (auto Lifetime = ElementType.getObjCLifetime()) { + switch (Lifetime) { case Qualifiers::OCL_None: case Qualifiers::OCL_ExplicitNone: case Qualifiers::OCL_Autoreleasing: break; - case Qualifiers::OCL_Strong: { - // Load the pointer value. - llvm::Value *PtrValue = CGF.Builder.CreateLoad(Ptr, - ElementType.isVolatileQualified()); - - CGF.EmitARCRelease(PtrValue, ARCPreciseLifetime); + case Qualifiers::OCL_Strong: + CGF.EmitARCDestroyStrong(Ptr, ARCPreciseLifetime); break; - } case Qualifiers::OCL_Weak: CGF.EmitARCDestroyWeak(Ptr); @@ -1508,7 +1589,7 @@ static void EmitObjectDelete(CodeGenFunction &CGF, namespace { /// Calls the given 'operator delete' on an array of objects. - struct CallArrayDelete : EHScopeStack::Cleanup { + struct CallArrayDelete final : EHScopeStack::Cleanup { llvm::Value *Ptr; const FunctionDecl *OperatorDelete; llvm::Value *NumElements; @@ -1570,7 +1651,7 @@ namespace { /// Emit the code for deleting an array of objects. static void EmitArrayDelete(CodeGenFunction &CGF, const CXXDeleteExpr *E, - llvm::Value *deletedPtr, + Address deletedPtr, QualType elementType) { llvm::Value *numElements = nullptr; llvm::Value *allocatedPtr = nullptr; @@ -1591,13 +1672,18 @@ static void EmitArrayDelete(CodeGenFunction &CGF, if (QualType::DestructionKind dtorKind = elementType.isDestructedType()) { assert(numElements && "no element count for a type with a destructor!"); + CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); + CharUnits elementAlign = + deletedPtr.getAlignment().alignmentOfArrayElement(elementSize); + + llvm::Value *arrayBegin = deletedPtr.getPointer(); llvm::Value *arrayEnd = - CGF.Builder.CreateInBoundsGEP(deletedPtr, numElements, "delete.end"); + CGF.Builder.CreateInBoundsGEP(arrayBegin, numElements, "delete.end"); // Note that it is legal to allocate a zero-length array, and we // can never fold the check away because the length should always // come from a cookie. - CGF.emitArrayDestroy(deletedPtr, arrayEnd, elementType, + CGF.emitArrayDestroy(arrayBegin, arrayEnd, elementType, elementAlign, CGF.getDestroyer(dtorKind), /*checkZeroLength*/ true, CGF.needsEHCleanup(dtorKind)); @@ -1609,13 +1695,13 @@ static void EmitArrayDelete(CodeGenFunction &CGF, void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) { const Expr *Arg = E->getArgument(); - llvm::Value *Ptr = EmitScalarExpr(Arg); + Address Ptr = EmitPointerWithAlignment(Arg); // Null check the pointer. llvm::BasicBlock *DeleteNotNull = createBasicBlock("delete.notnull"); llvm::BasicBlock *DeleteEnd = createBasicBlock("delete.end"); - llvm::Value *IsNull = Builder.CreateIsNull(Ptr, "isnull"); + llvm::Value *IsNull = Builder.CreateIsNull(Ptr.getPointer(), "isnull"); Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull); EmitBlock(DeleteNotNull); @@ -1640,11 +1726,11 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) { GEP.push_back(Zero); } - Ptr = Builder.CreateInBoundsGEP(Ptr, GEP, "del.first"); + Ptr = Address(Builder.CreateInBoundsGEP(Ptr.getPointer(), GEP, "del.first"), + Ptr.getAlignment()); } - assert(ConvertTypeForMem(DeleteTy) == - cast<llvm::PointerType>(Ptr->getType())->getElementType()); + assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType()); if (E->isArrayForm()) { EmitArrayDelete(*this, E, Ptr, DeleteTy); @@ -1690,7 +1776,7 @@ static bool isGLValueFromPointerDeref(const Expr *E) { static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E, llvm::Type *StdTypeInfoPtrTy) { // Get the vtable pointer. - llvm::Value *ThisPtr = CGF.EmitLValue(E).getAddress(); + Address ThisPtr = CGF.EmitLValue(E).getAddress(); // C++ [expr.typeid]p2: // If the glvalue expression is obtained by applying the unary * operator to @@ -1707,7 +1793,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E, CGF.createBasicBlock("typeid.bad_typeid"); llvm::BasicBlock *EndBlock = CGF.createBasicBlock("typeid.end"); - llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr); + llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr.getPointer()); CGF.Builder.CreateCondBr(IsNull, BadTypeidBlock, EndBlock); CGF.EmitBlock(BadTypeidBlock); @@ -1758,8 +1844,9 @@ static llvm::Value *EmitDynamicCastToNull(CodeGenFunction &CGF, return llvm::UndefValue::get(DestLTy); } -llvm::Value *CodeGenFunction::EmitDynamicCast(llvm::Value *Value, +llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr, const CXXDynamicCastExpr *DCE) { + CGM.EmitExplicitCastExprType(DCE, this); QualType DestTy = DCE->getTypeAsWritten(); if (DCE->isAlwaysNull()) @@ -1803,19 +1890,21 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(llvm::Value *Value, CastNull = createBasicBlock("dynamic_cast.null"); CastNotNull = createBasicBlock("dynamic_cast.notnull"); - llvm::Value *IsNull = Builder.CreateIsNull(Value); + llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr.getPointer()); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } + llvm::Value *Value; if (isDynamicCastToVoid) { - Value = CGM.getCXXABI().EmitDynamicCastToVoid(*this, Value, SrcRecordTy, + Value = CGM.getCXXABI().EmitDynamicCastToVoid(*this, ThisAddr, SrcRecordTy, DestTy); } else { assert(DestRecordTy->isRecordType() && "destination type must be a record type!"); - Value = CGM.getCXXABI().EmitDynamicCastCall(*this, Value, SrcRecordTy, + Value = CGM.getCXXABI().EmitDynamicCastCall(*this, ThisAddr, SrcRecordTy, DestTy, DestRecordTy, CastEnd); + CastNotNull = Builder.GetInsertBlock(); } if (ShouldNullCheckSrcValue) { @@ -1840,12 +1929,11 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(llvm::Value *Value, void CodeGenFunction::EmitLambdaExpr(const LambdaExpr *E, AggValueSlot Slot) { RunCleanupsScope Scope(*this); - LValue SlotLV = - MakeAddrLValue(Slot.getAddr(), E->getType(), Slot.getAlignment()); + LValue SlotLV = MakeAddrLValue(Slot.getAddress(), E->getType()); CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin(); - for (LambdaExpr::capture_init_iterator i = E->capture_init_begin(), - e = E->capture_init_end(); + for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(), + e = E->capture_init_end(); i != e; ++i, ++CurField) { // Emit initialization LValue LV = EmitLValueForFieldInitialization(SlotLV, *CurField); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp index 27d1c68..ccdb532 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp @@ -83,12 +83,12 @@ public: /// specified value pointer. void EmitStoreOfComplex(ComplexPairTy Val, LValue LV, bool isInit); - /// EmitComplexToComplexCast - Emit a cast from complex value Val to DestType. + /// Emit a cast from complex value Val to DestType. ComplexPairTy EmitComplexToComplexCast(ComplexPairTy Val, QualType SrcType, - QualType DestType); - /// EmitComplexToComplexCast - Emit a cast from scalar value Val to DestType. + QualType DestType, SourceLocation Loc); + /// Emit a cast from scalar value Val to DestType. ComplexPairTy EmitScalarToComplexCast(llvm::Value *Val, QualType SrcType, - QualType DestType); + QualType DestType, SourceLocation Loc); //===--------------------------------------------------------------------===// // Visitor Methods @@ -154,6 +154,8 @@ public: return EmitCast(E->getCastKind(), E->getSubExpr(), E->getType()); } ComplexPairTy VisitCastExpr(CastExpr *E) { + if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E)) + CGF.CGM.EmitExplicitCastExprType(ECE, &CGF); return EmitCast(E->getCastKind(), E->getSubExpr(), E->getType()); } ComplexPairTy VisitCallExpr(const CallExpr *E); @@ -298,6 +300,19 @@ public: // Utilities //===----------------------------------------------------------------------===// +Address CodeGenFunction::emitAddrOfRealComponent(Address addr, + QualType complexType) { + CharUnits offset = CharUnits::Zero(); + return Builder.CreateStructGEP(addr, 0, offset, addr.getName() + ".realp"); +} + +Address CodeGenFunction::emitAddrOfImagComponent(Address addr, + QualType complexType) { + QualType eltType = complexType->castAs<ComplexType>()->getElementType(); + CharUnits offset = getContext().getTypeSizeInChars(eltType); + return Builder.CreateStructGEP(addr, 1, offset, addr.getName() + ".imagp"); +} + /// EmitLoadOfLValue - Given an RValue reference for a complex, emit code to /// load the real and imaginary pieces, returning them as Real/Imag. ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue, @@ -306,29 +321,21 @@ ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue, if (lvalue.getType()->isAtomicType()) return CGF.EmitAtomicLoad(lvalue, loc).getComplexVal(); - llvm::Value *SrcPtr = lvalue.getAddress(); + Address SrcPtr = lvalue.getAddress(); bool isVolatile = lvalue.isVolatileQualified(); - unsigned AlignR = lvalue.getAlignment().getQuantity(); - ASTContext &C = CGF.getContext(); - QualType ComplexTy = lvalue.getType(); - unsigned ComplexAlign = C.getTypeAlignInChars(ComplexTy).getQuantity(); - unsigned AlignI = std::min(AlignR, ComplexAlign); - llvm::Value *Real=nullptr, *Imag=nullptr; + llvm::Value *Real = nullptr, *Imag = nullptr; if (!IgnoreReal || isVolatile) { - llvm::Value *RealP = Builder.CreateStructGEP(nullptr, SrcPtr, 0, - SrcPtr->getName() + ".realp"); - Real = Builder.CreateAlignedLoad(RealP, AlignR, isVolatile, - SrcPtr->getName() + ".real"); + Address RealP = CGF.emitAddrOfRealComponent(SrcPtr, lvalue.getType()); + Real = Builder.CreateLoad(RealP, isVolatile, SrcPtr.getName() + ".real"); } if (!IgnoreImag || isVolatile) { - llvm::Value *ImagP = Builder.CreateStructGEP(nullptr, SrcPtr, 1, - SrcPtr->getName() + ".imagp"); - Imag = Builder.CreateAlignedLoad(ImagP, AlignI, isVolatile, - SrcPtr->getName() + ".imag"); + Address ImagP = CGF.emitAddrOfImagComponent(SrcPtr, lvalue.getType()); + Imag = Builder.CreateLoad(ImagP, isVolatile, SrcPtr.getName() + ".imag"); } + return ComplexPairTy(Real, Imag); } @@ -340,19 +347,12 @@ void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, LValue lvalue, (!isInit && CGF.LValueIsSuitableForInlineAtomic(lvalue))) return CGF.EmitAtomicStore(RValue::getComplex(Val), lvalue, isInit); - llvm::Value *Ptr = lvalue.getAddress(); - llvm::Value *RealPtr = Builder.CreateStructGEP(nullptr, Ptr, 0, "real"); - llvm::Value *ImagPtr = Builder.CreateStructGEP(nullptr, Ptr, 1, "imag"); - unsigned AlignR = lvalue.getAlignment().getQuantity(); - ASTContext &C = CGF.getContext(); - QualType ComplexTy = lvalue.getType(); - unsigned ComplexAlign = C.getTypeAlignInChars(ComplexTy).getQuantity(); - unsigned AlignI = std::min(AlignR, ComplexAlign); - - Builder.CreateAlignedStore(Val.first, RealPtr, AlignR, - lvalue.isVolatileQualified()); - Builder.CreateAlignedStore(Val.second, ImagPtr, AlignI, - lvalue.isVolatileQualified()); + Address Ptr = lvalue.getAddress(); + Address RealPtr = CGF.emitAddrOfRealComponent(Ptr, lvalue.getType()); + Address ImagPtr = CGF.emitAddrOfImagComponent(Ptr, lvalue.getType()); + + Builder.CreateStore(Val.first, RealPtr, lvalue.isVolatileQualified()); + Builder.CreateStore(Val.second, ImagPtr, lvalue.isVolatileQualified()); } @@ -385,16 +385,17 @@ ComplexPairTy ComplexExprEmitter::VisitCallExpr(const CallExpr *E) { ComplexPairTy ComplexExprEmitter::VisitStmtExpr(const StmtExpr *E) { CodeGenFunction::StmtExprEvaluation eval(CGF); - llvm::Value *RetAlloca = CGF.EmitCompoundStmt(*E->getSubStmt(), true); - assert(RetAlloca && "Expected complex return value"); + Address RetAlloca = CGF.EmitCompoundStmt(*E->getSubStmt(), true); + assert(RetAlloca.isValid() && "Expected complex return value"); return EmitLoadOfLValue(CGF.MakeAddrLValue(RetAlloca, E->getType()), E->getExprLoc()); } -/// EmitComplexToComplexCast - Emit a cast from complex value Val to DestType. +/// Emit a cast from complex value Val to DestType. ComplexPairTy ComplexExprEmitter::EmitComplexToComplexCast(ComplexPairTy Val, QualType SrcType, - QualType DestType) { + QualType DestType, + SourceLocation Loc) { // Get the src/dest element type. SrcType = SrcType->castAs<ComplexType>()->getElementType(); DestType = DestType->castAs<ComplexType>()->getElementType(); @@ -402,17 +403,18 @@ ComplexPairTy ComplexExprEmitter::EmitComplexToComplexCast(ComplexPairTy Val, // C99 6.3.1.6: When a value of complex type is converted to another // complex type, both the real and imaginary parts follow the conversion // rules for the corresponding real types. - Val.first = CGF.EmitScalarConversion(Val.first, SrcType, DestType); - Val.second = CGF.EmitScalarConversion(Val.second, SrcType, DestType); + Val.first = CGF.EmitScalarConversion(Val.first, SrcType, DestType, Loc); + Val.second = CGF.EmitScalarConversion(Val.second, SrcType, DestType, Loc); return Val; } ComplexPairTy ComplexExprEmitter::EmitScalarToComplexCast(llvm::Value *Val, QualType SrcType, - QualType DestType) { + QualType DestType, + SourceLocation Loc) { // Convert the input element to the element type of the complex. DestType = DestType->castAs<ComplexType>()->getElementType(); - Val = CGF.EmitScalarConversion(Val, SrcType, DestType); + Val = CGF.EmitScalarConversion(Val, SrcType, DestType, Loc); // Return (realval, 0). return ComplexPairTy(Val, llvm::Constant::getNullValue(Val->getType())); @@ -434,12 +436,9 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op, case CK_LValueBitCast: { LValue origLV = CGF.EmitLValue(Op); - llvm::Value *V = origLV.getAddress(); - V = Builder.CreateBitCast(V, - CGF.ConvertType(CGF.getContext().getPointerType(DestTy))); - return EmitLoadOfLValue(CGF.MakeAddrLValue(V, DestTy, - origLV.getAlignment()), - Op->getExprLoc()); + Address V = origLV.getAddress(); + V = Builder.CreateElementBitCast(V, CGF.ConvertType(DestTy)); + return EmitLoadOfLValue(CGF.MakeAddrLValue(V, DestTy), Op->getExprLoc()); } case CK_BitCast: @@ -488,14 +487,15 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op, case CK_FloatingRealToComplex: case CK_IntegralRealToComplex: - return EmitScalarToComplexCast(CGF.EmitScalarExpr(Op), - Op->getType(), DestTy); + return EmitScalarToComplexCast(CGF.EmitScalarExpr(Op), Op->getType(), + DestTy, Op->getExprLoc()); case CK_FloatingComplexCast: case CK_FloatingComplexToIntegralComplex: case CK_IntegralComplexCast: case CK_IntegralComplexToFloatingComplex: - return EmitComplexToComplexCast(Visit(Op), Op->getType(), DestTy); + return EmitComplexToComplexCast(Visit(Op), Op->getType(), DestTy, + Op->getExprLoc()); } llvm_unreachable("unknown cast resulting in complex value"); @@ -585,19 +585,25 @@ ComplexPairTy ComplexExprEmitter::EmitComplexBinOpLibCall(StringRef LibCallName, // We *must* use the full CG function call building logic here because the // complex type has special ABI handling. We also should not forget about // special calling convention which may be used for compiler builtins. - const CGFunctionInfo &FuncInfo = - CGF.CGM.getTypes().arrangeFreeFunctionCall( - Op.Ty, Args, FunctionType::ExtInfo(/* No CC here - will be added later */), - RequiredArgs::All); + + // We create a function qualified type to state that this call does not have + // any exceptions. + FunctionProtoType::ExtProtoInfo EPI; + EPI = EPI.withExceptionSpec( + FunctionProtoType::ExceptionSpecInfo(EST_BasicNoexcept)); + SmallVector<QualType, 4> ArgsQTys( + 4, Op.Ty->castAs<ComplexType>()->getElementType()); + QualType FQTy = CGF.getContext().getFunctionType(Op.Ty, ArgsQTys, EPI); + const CGFunctionInfo &FuncInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall( + Args, cast<FunctionType>(FQTy.getTypePtr()), false); + llvm::FunctionType *FTy = CGF.CGM.getTypes().GetFunctionType(FuncInfo); llvm::Constant *Func = CGF.CGM.CreateBuiltinFunction(FTy, LibCallName); llvm::Instruction *Call; RValue Res = CGF.EmitCall(FuncInfo, Func, ReturnValueSlot(), Args, - nullptr, &Call); + FQTy->getAs<FunctionProtoType>(), &Call); cast<llvm::CallInst>(Call)->setCallingConv(CGF.CGM.getBuiltinCC()); - cast<llvm::CallInst>(Call)->setDoesNotThrow(); - return Res.getComplexVal(); } @@ -846,19 +852,20 @@ EmitCompoundAssignLValue(const CompoundAssignOperator *E, LValue LHS = CGF.EmitLValue(E->getLHS()); // Load from the l-value and convert it. + SourceLocation Loc = E->getExprLoc(); if (LHSTy->isAnyComplexType()) { - ComplexPairTy LHSVal = EmitLoadOfLValue(LHS, E->getExprLoc()); - OpInfo.LHS = EmitComplexToComplexCast(LHSVal, LHSTy, OpInfo.Ty); + ComplexPairTy LHSVal = EmitLoadOfLValue(LHS, Loc); + OpInfo.LHS = EmitComplexToComplexCast(LHSVal, LHSTy, OpInfo.Ty, Loc); } else { - llvm::Value *LHSVal = CGF.EmitLoadOfScalar(LHS, E->getExprLoc()); + llvm::Value *LHSVal = CGF.EmitLoadOfScalar(LHS, Loc); // For floating point real operands we can directly pass the scalar form // to the binary operator emission and potentially get more efficient code. if (LHSTy->isRealFloatingType()) { if (!CGF.getContext().hasSameUnqualifiedType(ComplexElementTy, LHSTy)) - LHSVal = CGF.EmitScalarConversion(LHSVal, LHSTy, ComplexElementTy); + LHSVal = CGF.EmitScalarConversion(LHSVal, LHSTy, ComplexElementTy, Loc); OpInfo.LHS = ComplexPairTy(LHSVal, nullptr); } else { - OpInfo.LHS = EmitScalarToComplexCast(LHSVal, LHSTy, OpInfo.Ty); + OpInfo.LHS = EmitScalarToComplexCast(LHSVal, LHSTy, OpInfo.Ty, Loc); } } @@ -867,12 +874,13 @@ EmitCompoundAssignLValue(const CompoundAssignOperator *E, // Truncate the result and store it into the LHS lvalue. if (LHSTy->isAnyComplexType()) { - ComplexPairTy ResVal = EmitComplexToComplexCast(Result, OpInfo.Ty, LHSTy); + ComplexPairTy ResVal = + EmitComplexToComplexCast(Result, OpInfo.Ty, LHSTy, Loc); EmitStoreOfComplex(ResVal, LHS, /*isInit*/ false); Val = RValue::getComplex(ResVal); } else { llvm::Value *ResVal = - CGF.EmitComplexToScalarConversion(Result, OpInfo.Ty, LHSTy); + CGF.EmitComplexToScalarConversion(Result, OpInfo.Ty, LHSTy, Loc); CGF.EmitStoreOfScalar(ResVal, LHS, /*isInit*/ false); Val = RValue::get(ResVal); } @@ -1011,10 +1019,10 @@ ComplexPairTy ComplexExprEmitter::VisitInitListExpr(InitListExpr *E) { } ComplexPairTy ComplexExprEmitter::VisitVAArgExpr(VAArgExpr *E) { - llvm::Value *ArgValue = CGF.EmitVAListRef(E->getSubExpr()); - llvm::Value *ArgPtr = CGF.EmitVAArg(ArgValue, E->getType()); + Address ArgValue = Address::invalid(); + Address ArgPtr = CGF.EmitVAArg(E, ArgValue); - if (!ArgPtr) { + if (!ArgPtr.isValid()) { CGF.ErrorUnsupported(E, "complex va_arg expression"); llvm::Type *EltTy = CGF.ConvertType(E->getType()->castAs<ComplexType>()->getElementType()); @@ -1022,7 +1030,7 @@ ComplexPairTy ComplexExprEmitter::VisitVAArgExpr(VAArgExpr *E) { return ComplexPairTy(U, U); } - return EmitLoadOfLValue(CGF.MakeNaturalAlignAddrLValue(ArgPtr, E->getType()), + return EmitLoadOfLValue(CGF.MakeAddrLValue(ArgPtr, E->getType()), E->getExprLoc()); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp index a15c151..3839ab7 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp @@ -636,6 +636,8 @@ public: } llvm::Constant *VisitCastExpr(CastExpr* E) { + if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E)) + CGM.EmitExplicitCastExprType(ECE, CGF); Expr *subExpr = E->getSubExpr(); llvm::Constant *C = CGM.EmitConstantExpr(subExpr, subExpr->getType(), CGF); if (!C) return nullptr; @@ -977,23 +979,26 @@ public: } public: - llvm::Constant *EmitLValue(APValue::LValueBase LVBase) { + ConstantAddress EmitLValue(APValue::LValueBase LVBase) { if (const ValueDecl *Decl = LVBase.dyn_cast<const ValueDecl*>()) { if (Decl->hasAttr<WeakRefAttr>()) return CGM.GetWeakRefReference(Decl); if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Decl)) - return CGM.GetAddrOfFunction(FD); + return ConstantAddress(CGM.GetAddrOfFunction(FD), CharUnits::One()); if (const VarDecl* VD = dyn_cast<VarDecl>(Decl)) { // We can never refer to a variable with local storage. if (!VD->hasLocalStorage()) { + CharUnits Align = CGM.getContext().getDeclAlign(VD); if (VD->isFileVarDecl() || VD->hasExternalStorage()) - return CGM.GetAddrOfGlobalVar(VD); - else if (VD->isLocalVarDecl()) - return CGM.getOrCreateStaticVarDecl( + return ConstantAddress(CGM.GetAddrOfGlobalVar(VD), Align); + else if (VD->isLocalVarDecl()) { + auto Ptr = CGM.getOrCreateStaticVarDecl( *VD, CGM.getLLVMLinkageVarDefinition(VD, /*isConstant=*/false)); + return ConstantAddress(Ptr, Align); + } } } - return nullptr; + return ConstantAddress::invalid(); } Expr *E = const_cast<Expr*>(LVBase.get<const Expr*>()); @@ -1006,14 +1011,18 @@ public: llvm::Constant* C = CGM.EmitConstantExpr(CLE->getInitializer(), CLE->getType(), CGF); // FIXME: "Leaked" on failure. - if (C) - C = new llvm::GlobalVariable(CGM.getModule(), C->getType(), + if (!C) return ConstantAddress::invalid(); + + CharUnits Align = CGM.getContext().getTypeAlignInChars(E->getType()); + + auto GV = new llvm::GlobalVariable(CGM.getModule(), C->getType(), E->getType().isConstant(CGM.getContext()), llvm::GlobalValue::InternalLinkage, C, ".compoundliteral", nullptr, llvm::GlobalVariable::NotThreadLocal, CGM.getContext().getTargetAddressSpace(E->getType())); - return C; + GV->setAlignment(Align.getQuantity()); + return ConstantAddress(GV, Align); } case Expr::StringLiteralClass: return CGM.GetAddrOfConstantStringFromLiteral(cast<StringLiteral>(E)); @@ -1021,15 +1030,15 @@ public: return CGM.GetAddrOfConstantStringFromObjCEncode(cast<ObjCEncodeExpr>(E)); case Expr::ObjCStringLiteralClass: { ObjCStringLiteral* SL = cast<ObjCStringLiteral>(E); - llvm::Constant *C = + ConstantAddress C = CGM.getObjCRuntime().GenerateConstantString(SL->getString()); - return llvm::ConstantExpr::getBitCast(C, ConvertType(E->getType())); + return C.getElementBitCast(ConvertType(E->getType())); } case Expr::PredefinedExprClass: { unsigned Type = cast<PredefinedExpr>(E)->getIdentType(); if (CGF) { LValue Res = CGF->EmitPredefinedLValue(cast<PredefinedExpr>(E)); - return cast<llvm::Constant>(Res.getAddress()); + return cast<ConstantAddress>(Res.getAddress()); } else if (Type == PredefinedExpr::PrettyFunction) { return CGM.GetAddrOfConstantCString("top level", ".tmp"); } @@ -1040,7 +1049,8 @@ public: assert(CGF && "Invalid address of label expression outside function."); llvm::Constant *Ptr = CGF->GetAddrOfLabel(cast<AddrLabelExpr>(E)->getLabel()); - return llvm::ConstantExpr::getBitCast(Ptr, ConvertType(E->getType())); + Ptr = llvm::ConstantExpr::getBitCast(Ptr, ConvertType(E->getType())); + return ConstantAddress(Ptr, CharUnits::One()); } case Expr::CallExprClass: { CallExpr* CE = cast<CallExpr>(E); @@ -1066,7 +1076,10 @@ public: else FunctionName = "global"; - return CGM.GetAddrOfGlobalBlock(cast<BlockExpr>(E), FunctionName.c_str()); + // This is not really an l-value. + llvm::Constant *Ptr = + CGM.GetAddrOfGlobalBlock(cast<BlockExpr>(E), FunctionName.c_str()); + return ConstantAddress(Ptr, CGM.getPointerAlign()); } case Expr::CXXTypeidExprClass: { CXXTypeidExpr *Typeid = cast<CXXTypeidExpr>(E); @@ -1075,7 +1088,8 @@ public: T = Typeid->getTypeOperand(CGM.getContext()); else T = Typeid->getExprOperand()->getType(); - return CGM.GetAddrOfRTTIDescriptor(T); + return ConstantAddress(CGM.GetAddrOfRTTIDescriptor(T), + CGM.getPointerAlign()); } case Expr::CXXUuidofExprClass: { return CGM.GetAddrOfUuidDescriptor(cast<CXXUuidofExpr>(E)); @@ -1091,7 +1105,7 @@ public: } } - return nullptr; + return ConstantAddress::invalid(); } }; @@ -1255,7 +1269,7 @@ llvm::Constant *CodeGenModule::EmitConstantValue(const APValue &Value, llvm::Constant *Offset = llvm::ConstantInt::get(Int64Ty, Value.getLValueOffset().getQuantity()); - llvm::Constant *C; + llvm::Constant *C = nullptr; if (APValue::LValueBase LVBase = Value.getLValueBase()) { // An array can be represented as an lvalue referring to the base. if (isa<llvm::ArrayType>(DestTy)) { @@ -1264,7 +1278,7 @@ llvm::Constant *CodeGenModule::EmitConstantValue(const APValue &Value, const_cast<Expr*>(LVBase.get<const Expr*>())); } - C = ConstExprEmitter(*this, CGF).EmitLValue(LVBase); + C = ConstExprEmitter(*this, CGF).EmitLValue(LVBase).getPointer(); // Apply offset if necessary. if (!Offset->isNullValue()) { @@ -1336,15 +1350,17 @@ llvm::Constant *CodeGenModule::EmitConstantValue(const APValue &Value, return llvm::ConstantStruct::get(STy, Complex); } case APValue::Vector: { - SmallVector<llvm::Constant *, 4> Inits; unsigned NumElts = Value.getVectorLength(); + SmallVector<llvm::Constant *, 4> Inits(NumElts); - for (unsigned i = 0; i != NumElts; ++i) { - const APValue &Elt = Value.getVectorElt(i); + for (unsigned I = 0; I != NumElts; ++I) { + const APValue &Elt = Value.getVectorElt(I); if (Elt.isInt()) - Inits.push_back(llvm::ConstantInt::get(VMContext, Elt.getInt())); + Inits[I] = llvm::ConstantInt::get(VMContext, Elt.getInt()); + else if (Elt.isFloat()) + Inits[I] = llvm::ConstantFP::get(VMContext, Elt.getFloat()); else - Inits.push_back(llvm::ConstantFP::get(VMContext, Elt.getFloat())); + llvm_unreachable("unsupported vector element type"); } return llvm::ConstantVector::get(Inits); } @@ -1438,7 +1454,7 @@ CodeGenModule::EmitConstantValueForMemory(const APValue &Value, return C; } -llvm::Constant * +ConstantAddress CodeGenModule::GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr *E) { assert(E->isFileScope() && "not a file-scope compound literal expr"); return ConstExprEmitter(*this, nullptr).EmitLValue(E); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp index 74f6019..725d96f 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp @@ -140,21 +140,25 @@ public: /// boolean (i1) truth value. This is equivalent to "Val != 0". Value *EmitConversionToBool(Value *Src, QualType DstTy); - /// \brief Emit a check that a conversion to or from a floating-point type - /// does not overflow. + /// Emit a check that a conversion to or from a floating-point type does not + /// overflow. void EmitFloatConversionCheck(Value *OrigSrc, QualType OrigSrcType, - Value *Src, QualType SrcType, - QualType DstType, llvm::Type *DstTy); + Value *Src, QualType SrcType, QualType DstType, + llvm::Type *DstTy, SourceLocation Loc); - /// EmitScalarConversion - Emit a conversion from the specified type to the - /// specified destination type, both of which are LLVM scalar types. - Value *EmitScalarConversion(Value *Src, QualType SrcTy, QualType DstTy); + /// Emit a conversion from the specified type to the specified destination + /// type, both of which are LLVM scalar types. + Value *EmitScalarConversion(Value *Src, QualType SrcTy, QualType DstTy, + SourceLocation Loc); - /// EmitComplexToScalarConversion - Emit a conversion from the specified - /// complex type to the specified destination type, where the destination type - /// is an LLVM scalar type. + Value *EmitScalarConversion(Value *Src, QualType SrcTy, QualType DstTy, + SourceLocation Loc, bool TreatBooleanAsSigned); + + /// Emit a conversion from the specified complex type to the specified + /// destination type, where the destination type is an LLVM scalar type. Value *EmitComplexToScalarConversion(CodeGenFunction::ComplexPairTy Src, - QualType SrcTy, QualType DstTy); + QualType SrcTy, QualType DstTy, + SourceLocation Loc); /// EmitNullValue - Emit a value that corresponds to null for the given type. Value *EmitNullValue(QualType Ty); @@ -310,12 +314,7 @@ public: return EmitNullValue(E->getType()); } Value *VisitExplicitCastExpr(ExplicitCastExpr *E) { - if (E->getType()->isVariablyModifiedType()) - CGF.EmitVariablyModifiedType(E->getType()); - - if (CGDebugInfo *DI = CGF.getDebugInfo()) - DI->EmitExplicitCastType(E->getType()); - + CGF.CGM.EmitExplicitCastExprType(E, &CGF); return VisitCastExpr(E); } Value *VisitCastExpr(CastExpr *E); @@ -362,7 +361,7 @@ public: if (isa<MemberPointerType>(E->getType())) // never sugared return CGF.CGM.getMemberPointerConstant(E); - return EmitLValue(E->getSubExpr()).getAddress(); + return EmitLValue(E->getSubExpr()).getPointer(); } Value *VisitUnaryDeref(const UnaryOperator *E) { if (E->getType()->isVoidType()) @@ -524,8 +523,9 @@ public: #undef HANDLEBINOP // Comparisons. - Value *EmitCompare(const BinaryOperator *E, unsigned UICmpOpc, - unsigned SICmpOpc, unsigned FCmpOpc); + Value *EmitCompare(const BinaryOperator *E, llvm::CmpInst::Predicate UICmpOpc, + llvm::CmpInst::Predicate SICmpOpc, + llvm::CmpInst::Predicate FCmpOpc); #define VISITCOMP(CODE, UI, SI, FP) \ Value *VisitBin##CODE(const BinaryOperator *E) { \ return EmitCompare(E, llvm::ICmpInst::UI, llvm::ICmpInst::SI, \ @@ -594,11 +594,9 @@ Value *ScalarExprEmitter::EmitConversionToBool(Value *Src, QualType SrcType) { return EmitPointerToBoolConversion(Src); } -void ScalarExprEmitter::EmitFloatConversionCheck(Value *OrigSrc, - QualType OrigSrcType, - Value *Src, QualType SrcType, - QualType DstType, - llvm::Type *DstTy) { +void ScalarExprEmitter::EmitFloatConversionCheck( + Value *OrigSrc, QualType OrigSrcType, Value *Src, QualType SrcType, + QualType DstType, llvm::Type *DstTy, SourceLocation Loc) { CodeGenFunction::SanitizerScope SanScope(&CGF); using llvm::APFloat; using llvm::APSInt; @@ -722,19 +720,25 @@ void ScalarExprEmitter::EmitFloatConversionCheck(Value *OrigSrc, } } - // FIXME: Provide a SourceLocation. - llvm::Constant *StaticArgs[] = { - CGF.EmitCheckTypeDescriptor(OrigSrcType), - CGF.EmitCheckTypeDescriptor(DstType) - }; + llvm::Constant *StaticArgs[] = {CGF.EmitCheckSourceLocation(Loc), + CGF.EmitCheckTypeDescriptor(OrigSrcType), + CGF.EmitCheckTypeDescriptor(DstType)}; CGF.EmitCheck(std::make_pair(Check, SanitizerKind::FloatCastOverflow), "float_cast_overflow", StaticArgs, OrigSrc); } -/// EmitScalarConversion - Emit a conversion from the specified type to the -/// specified destination type, both of which are LLVM scalar types. +/// Emit a conversion from the specified type to the specified destination type, +/// both of which are LLVM scalar types. +Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, + QualType DstType, + SourceLocation Loc) { + return EmitScalarConversion(Src, SrcType, DstType, Loc, false); +} + Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, - QualType DstType) { + QualType DstType, + SourceLocation Loc, + bool TreatBooleanAsSigned) { SrcType = CGF.getContext().getCanonicalType(SrcType); DstType = CGF.getContext().getCanonicalType(DstType); if (SrcType == DstType) return Src; @@ -809,7 +813,8 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, if (DstType->isExtVectorType() && !SrcType->isVectorType()) { // Cast the scalar to element type QualType EltTy = DstType->getAs<ExtVectorType>()->getElementType(); - llvm::Value *Elt = EmitScalarConversion(Src, SrcType, EltTy); + llvm::Value *Elt = EmitScalarConversion( + Src, SrcType, EltTy, Loc, CGF.getContext().getLangOpts().OpenCL); // Splat the element across to all elements unsigned NumElements = cast<llvm::VectorType>(DstTy)->getNumElements(); @@ -829,8 +834,8 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, // or the destination type is a floating-point type. if (CGF.SanOpts.has(SanitizerKind::FloatCastOverflow) && (OrigSrcType->isFloatingType() || DstType->isFloatingType())) - EmitFloatConversionCheck(OrigSrc, OrigSrcType, Src, SrcType, DstType, - DstTy); + EmitFloatConversionCheck(OrigSrc, OrigSrcType, Src, SrcType, DstType, DstTy, + Loc); // Cast to half through float if half isn't a native type. if (DstType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) { @@ -849,6 +854,9 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, if (isa<llvm::IntegerType>(SrcTy)) { bool InputSigned = SrcType->isSignedIntegerOrEnumerationType(); + if (SrcType->isBooleanType() && TreatBooleanAsSigned) { + InputSigned = true; + } if (isa<llvm::IntegerType>(DstTy)) Res = Builder.CreateIntCast(Src, DstTy, InputSigned, "conv"); else if (InputSigned) @@ -884,20 +892,19 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, return Res; } -/// EmitComplexToScalarConversion - Emit a conversion from the specified complex -/// type to the specified destination type, where the destination type is an -/// LLVM scalar type. -Value *ScalarExprEmitter:: -EmitComplexToScalarConversion(CodeGenFunction::ComplexPairTy Src, - QualType SrcTy, QualType DstTy) { +/// Emit a conversion from the specified complex type to the specified +/// destination type, where the destination type is an LLVM scalar type. +Value *ScalarExprEmitter::EmitComplexToScalarConversion( + CodeGenFunction::ComplexPairTy Src, QualType SrcTy, QualType DstTy, + SourceLocation Loc) { // Get the source element type. SrcTy = SrcTy->castAs<ComplexType>()->getElementType(); // Handle conversions to bool first, they are special: comparisons against 0. if (DstTy->isBooleanType()) { // Complex != 0 -> (Real != 0) | (Imag != 0) - Src.first = EmitScalarConversion(Src.first, SrcTy, DstTy); - Src.second = EmitScalarConversion(Src.second, SrcTy, DstTy); + Src.first = EmitScalarConversion(Src.first, SrcTy, DstTy, Loc); + Src.second = EmitScalarConversion(Src.second, SrcTy, DstTy, Loc); return Builder.CreateOr(Src.first, Src.second, "tobool"); } @@ -905,7 +912,7 @@ EmitComplexToScalarConversion(CodeGenFunction::ComplexPairTy Src, // the imaginary part of the complex value is discarded and the value of the // real part is converted according to the conversion rules for the // corresponding real type. - return EmitScalarConversion(Src.first, SrcTy, DstTy); + return EmitScalarConversion(Src.first, SrcTy, DstTy, Loc); } Value *ScalarExprEmitter::EmitNullValue(QualType Ty) { @@ -1003,14 +1010,10 @@ Value *ScalarExprEmitter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) { } llvm::VectorType *MTy = cast<llvm::VectorType>(Mask->getType()); - llvm::Constant* EltMask; - - EltMask = llvm::ConstantInt::get(MTy->getElementType(), - llvm::NextPowerOf2(LHSElts-1)-1); // Mask off the high bits of each shuffle index. - Value *MaskBits = llvm::ConstantVector::getSplat(MTy->getNumElements(), - EltMask); + Value *MaskBits = + llvm::ConstantInt::get(MTy, llvm::NextPowerOf2(LHSElts - 1) - 1); Mask = Builder.CreateAnd(Mask, MaskBits, "mask"); // newv = undef @@ -1334,13 +1337,13 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) { return V; } -static bool ShouldNullCheckClassCastValue(const CastExpr *CE) { +bool CodeGenFunction::ShouldNullCheckClassCastValue(const CastExpr *CE) { const Expr *E = CE->getSubExpr(); if (CE->getCastKind() == CK_UncheckedDerivedToBase) return false; - if (isa<CXXThisExpr>(E)) { + if (isa<CXXThisExpr>(E->IgnoreParens())) { // We always assume that 'this' is never null. return false; } @@ -1375,11 +1378,10 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { case CK_LValueBitCast: case CK_ObjCObjectLValueCast: { - Value *V = EmitLValue(E).getAddress(); - V = Builder.CreateBitCast(V, - ConvertType(CGF.getContext().getPointerType(DestTy))); - return EmitLoadOfLValue(CGF.MakeNaturalAlignAddrLValue(V, DestTy), - CE->getExprLoc()); + Address Addr = EmitLValue(E).getAddress(); + Addr = Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(DestTy)); + LValue LV = CGF.MakeAddrLValue(Addr, DestTy); + return EmitLoadOfLValue(LV, CE->getExprLoc()); } case CK_CPointerToObjCPointerCast: @@ -1419,68 +1421,44 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { const CXXRecordDecl *DerivedClassDecl = DestTy->getPointeeCXXRecordDecl(); assert(DerivedClassDecl && "BaseToDerived arg isn't a C++ object pointer!"); - llvm::Value *V = Visit(E); - - llvm::Value *Derived = - CGF.GetAddressOfDerivedClass(V, DerivedClassDecl, + Address Base = CGF.EmitPointerWithAlignment(E); + Address Derived = + CGF.GetAddressOfDerivedClass(Base, DerivedClassDecl, CE->path_begin(), CE->path_end(), - ShouldNullCheckClassCastValue(CE)); + CGF.ShouldNullCheckClassCastValue(CE)); // C++11 [expr.static.cast]p11: Behavior is undefined if a downcast is // performed and the object is not of the derived type. if (CGF.sanitizePerformTypeCheck()) CGF.EmitTypeCheck(CodeGenFunction::TCK_DowncastPointer, CE->getExprLoc(), - Derived, DestTy->getPointeeType()); + Derived.getPointer(), DestTy->getPointeeType()); if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast)) - CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived, + CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), + Derived.getPointer(), /*MayBeNull=*/true, CodeGenFunction::CFITCK_DerivedCast, CE->getLocStart()); - return Derived; + return Derived.getPointer(); } case CK_UncheckedDerivedToBase: case CK_DerivedToBase: { - const CXXRecordDecl *DerivedClassDecl = - E->getType()->getPointeeCXXRecordDecl(); - assert(DerivedClassDecl && "DerivedToBase arg isn't a C++ object pointer!"); - - return CGF.GetAddressOfBaseClass( - Visit(E), DerivedClassDecl, CE->path_begin(), CE->path_end(), - ShouldNullCheckClassCastValue(CE), CE->getExprLoc()); + // The EmitPointerWithAlignment path does this fine; just discard + // the alignment. + return CGF.EmitPointerWithAlignment(CE).getPointer(); } + case CK_Dynamic: { - Value *V = Visit(const_cast<Expr*>(E)); + Address V = CGF.EmitPointerWithAlignment(E); const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(CE); return CGF.EmitDynamicCast(V, DCE); } - case CK_ArrayToPointerDecay: { - assert(E->getType()->isArrayType() && - "Array to pointer decay must have array source type!"); - - Value *V = EmitLValue(E).getAddress(); // Bitfields can't be arrays. - - // Note that VLA pointers are always decayed, so we don't need to do - // anything here. - if (!E->getType()->isVariableArrayType()) { - assert(isa<llvm::PointerType>(V->getType()) && "Expected pointer"); - llvm::Type *NewTy = ConvertType(E->getType()); - V = CGF.Builder.CreatePointerCast( - V, NewTy->getPointerTo(V->getType()->getPointerAddressSpace())); - - assert(isa<llvm::ArrayType>(V->getType()->getPointerElementType()) && - "Expected pointer to array"); - V = Builder.CreateStructGEP(NewTy, V, 0, "arraydecay"); - } - - // Make sure the array decay ends up being the right type. This matters if - // the array type was of an incomplete type. - return CGF.Builder.CreatePointerCast(V, ConvertType(CE->getType())); - } + case CK_ArrayToPointerDecay: + return CGF.EmitArrayToPointerDecay(E).getPointer(); case CK_FunctionToPointerDecay: - return EmitLValue(E).getAddress(); + return EmitLValue(E).getPointer(); case CK_NullToPointer: if (MustVisitNullValue(E)) @@ -1563,9 +1541,14 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { } case CK_VectorSplat: { llvm::Type *DstTy = ConvertType(DestTy); - Value *Elt = Visit(const_cast<Expr*>(E)); - Elt = EmitScalarConversion(Elt, E->getType(), - DestTy->getAs<VectorType>()->getElementType()); + // Need an IgnoreImpCasts here as by default a boolean will be promoted to + // an int, which will not perform the sign extension, so if we know we are + // going to cast to a vector we have to strip the implicit cast off. + Value *Elt = Visit(const_cast<Expr*>(E->IgnoreImpCasts())); + Elt = EmitScalarConversion(Elt, E->IgnoreImpCasts()->getType(), + DestTy->getAs<VectorType>()->getElementType(), + CE->getExprLoc(), + CGF.getContext().getLangOpts().OpenCL); // Splat the element across to all elements unsigned NumElements = cast<llvm::VectorType>(DstTy)->getNumElements(); @@ -1576,7 +1559,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { case CK_IntegralToFloating: case CK_FloatingToIntegral: case CK_FloatingCast: - return EmitScalarConversion(Visit(E), E->getType(), DestTy); + return EmitScalarConversion(Visit(E), E->getType(), DestTy, + CE->getExprLoc()); case CK_IntegralToBoolean: return EmitIntToBoolConversion(Visit(E)); case CK_PointerToBoolean: @@ -1598,7 +1582,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { CodeGenFunction::ComplexPairTy V = CGF.EmitComplexExpr(E); // TODO: kill this function off, inline appropriate case here - return EmitComplexToScalarConversion(V, E->getType(), DestTy); + return EmitComplexToScalarConversion(V, E->getType(), DestTy, + CE->getExprLoc()); } case CK_ZeroToOCLEvent: { @@ -1613,9 +1598,9 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { Value *ScalarExprEmitter::VisitStmtExpr(const StmtExpr *E) { CodeGenFunction::StmtExprEvaluation eval(CGF); - llvm::Value *RetAlloca = CGF.EmitCompoundStmt(*E->getSubStmt(), - !E->getType()->isVoidType()); - if (!RetAlloca) + Address RetAlloca = CGF.EmitCompoundStmt(*E->getSubStmt(), + !E->getType()->isVoidType()); + if (!RetAlloca.isValid()) return nullptr; return CGF.EmitLoadOfScalar(CGF.MakeAddrLValue(RetAlloca, E->getType()), E->getExprLoc()); @@ -1671,16 +1656,14 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, if (isInc && type->isBooleanType()) { llvm::Value *True = CGF.EmitToMemory(Builder.getTrue(), type); if (isPre) { - Builder.Insert(new llvm::StoreInst(True, - LV.getAddress(), LV.isVolatileQualified(), - LV.getAlignment().getQuantity(), - llvm::SequentiallyConsistent)); + Builder.CreateStore(True, LV.getAddress(), LV.isVolatileQualified()) + ->setAtomic(llvm::SequentiallyConsistent); return Builder.getTrue(); } // For atomic bool increment, we just store true and return it for // preincrement, do an atomic swap with true for postincrement return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, - LV.getAddress(), True, llvm::SequentiallyConsistent); + LV.getPointer(), True, llvm::SequentiallyConsistent); } // Special case for atomic increment / decrement on integers, emit // atomicrmw instructions. We skip this if we want to be doing overflow @@ -1697,7 +1680,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, llvm::Value *amt = CGF.EmitToMemory( llvm::ConstantInt::get(ConvertType(type), 1, true), type); llvm::Value *old = Builder.CreateAtomicRMW(aop, - LV.getAddress(), amt, llvm::SequentiallyConsistent); + LV.getPointer(), amt, llvm::SequentiallyConsistent); return isPre ? Builder.CreateBinOp(op, old, amt) : old; } value = EmitLoadOfLValue(LV, E->getExprLoc()); @@ -1941,10 +1924,10 @@ Value *ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *E) { llvm::Value* Result = llvm::Constant::getNullValue(ResultType); QualType CurrentType = E->getTypeSourceInfo()->getType(); for (unsigned i = 0; i != n; ++i) { - OffsetOfExpr::OffsetOfNode ON = E->getComponent(i); + OffsetOfNode ON = E->getComponent(i); llvm::Value *Offset = nullptr; switch (ON.getKind()) { - case OffsetOfExpr::OffsetOfNode::Array: { + case OffsetOfNode::Array: { // Compute the index Expr *IdxExpr = E->getIndexExpr(ON.getArrayExprIndex()); llvm::Value* Idx = CGF.EmitScalarExpr(IdxExpr); @@ -1964,7 +1947,7 @@ Value *ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *E) { break; } - case OffsetOfExpr::OffsetOfNode::Field: { + case OffsetOfNode::Field: { FieldDecl *MemberDecl = ON.getField(); RecordDecl *RD = CurrentType->getAs<RecordType>()->getDecl(); const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(RD); @@ -1990,10 +1973,10 @@ Value *ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *E) { break; } - case OffsetOfExpr::OffsetOfNode::Identifier: + case OffsetOfNode::Identifier: llvm_unreachable("dependent __builtin_offsetof"); - case OffsetOfExpr::OffsetOfNode::Base: { + case OffsetOfNode::Base: { if (ON.getBase()->isVirtual()) { CGF.ErrorUnsupported(E, "virtual base in offsetof"); continue; @@ -2134,7 +2117,7 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( OpInfo.RHS = Visit(E->getRHS()); OpInfo.Ty = E->getComputationResultType(); OpInfo.Opcode = E->getOpcode(); - OpInfo.FPContractable = false; + OpInfo.FPContractable = E->isFPContractable(); OpInfo.E = E; // Load/convert the LHS. LValue LHSLV = EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store); @@ -2174,9 +2157,11 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( llvm_unreachable("Invalid compound assignment type"); } if (aop != llvm::AtomicRMWInst::BAD_BINOP) { - llvm::Value *amt = CGF.EmitToMemory(EmitScalarConversion(OpInfo.RHS, - E->getRHS()->getType(), LHSTy), LHSTy); - Builder.CreateAtomicRMW(aop, LHSLV.getAddress(), amt, + llvm::Value *amt = CGF.EmitToMemory( + EmitScalarConversion(OpInfo.RHS, E->getRHS()->getType(), LHSTy, + E->getExprLoc()), + LHSTy); + Builder.CreateAtomicRMW(aop, LHSLV.getPointer(), amt, llvm::SequentiallyConsistent); return LHSLV; } @@ -2196,14 +2181,16 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( else OpInfo.LHS = EmitLoadOfLValue(LHSLV, E->getExprLoc()); - OpInfo.LHS = EmitScalarConversion(OpInfo.LHS, LHSTy, - E->getComputationLHSType()); + SourceLocation Loc = E->getExprLoc(); + OpInfo.LHS = + EmitScalarConversion(OpInfo.LHS, LHSTy, E->getComputationLHSType(), Loc); // Expand the binary operator. Result = (this->*Func)(OpInfo); // Convert the result back to the LHS type. - Result = EmitScalarConversion(Result, E->getComputationResultType(), LHSTy); + Result = + EmitScalarConversion(Result, E->getComputationResultType(), LHSTy, Loc); if (atomicPHI) { llvm::BasicBlock *opBB = Builder.GetInsertBlock(); @@ -2389,9 +2376,9 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) { // Branch in case of overflow. llvm::BasicBlock *initialBB = Builder.GetInsertBlock(); - llvm::Function::iterator insertPt = initialBB; + llvm::Function::iterator insertPt = initialBB->getIterator(); llvm::BasicBlock *continueBB = CGF.createBasicBlock("nooverflow", CGF.CurFn, - std::next(insertPt)); + &*std::next(insertPt)); llvm::BasicBlock *overflowBB = CGF.createBasicBlock("overflow", CGF.CurFn); Builder.CreateCondBr(overflow, overflowBB, continueBB); @@ -2578,19 +2565,17 @@ static Value* tryEmitFMulAdd(const BinOpInfo &op, return nullptr; // We have a potentially fusable op. Look for a mul on one of the operands. - if (llvm::BinaryOperator* LHSBinOp = dyn_cast<llvm::BinaryOperator>(op.LHS)) { - if (LHSBinOp->getOpcode() == llvm::Instruction::FMul) { - assert(LHSBinOp->getNumUses() == 0 && - "Operations with multiple uses shouldn't be contracted."); + // Also, make sure that the mul result isn't used directly. In that case, + // there's no point creating a muladd operation. + if (auto *LHSBinOp = dyn_cast<llvm::BinaryOperator>(op.LHS)) { + if (LHSBinOp->getOpcode() == llvm::Instruction::FMul && + LHSBinOp->use_empty()) return buildFMulAdd(LHSBinOp, op.RHS, CGF, Builder, false, isSub); - } - } else if (llvm::BinaryOperator* RHSBinOp = - dyn_cast<llvm::BinaryOperator>(op.RHS)) { - if (RHSBinOp->getOpcode() == llvm::Instruction::FMul) { - assert(RHSBinOp->getNumUses() == 0 && - "Operations with multiple uses shouldn't be contracted."); + } + if (auto *RHSBinOp = dyn_cast<llvm::BinaryOperator>(op.RHS)) { + if (RHSBinOp->getOpcode() == llvm::Instruction::FMul && + RHSBinOp->use_empty()) return buildFMulAdd(RHSBinOp, op.LHS, CGF, Builder, isSub, false); - } } return nullptr; @@ -2848,8 +2833,10 @@ static llvm::Intrinsic::ID GetIntrinsic(IntrinsicType IT, } } -Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc, - unsigned SICmpOpc, unsigned FCmpOpc) { +Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E, + llvm::CmpInst::Predicate UICmpOpc, + llvm::CmpInst::Predicate SICmpOpc, + llvm::CmpInst::Predicate FCmpOpc) { TestAndClearIgnoreResultAssign(); Value *Result; QualType LHSTy = E->getLHS()->getType(); @@ -2927,19 +2914,17 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc, Value *CR6Param = Builder.getInt32(CR6); llvm::Function *F = CGF.CGM.getIntrinsic(ID); Result = Builder.CreateCall(F, {CR6Param, FirstVecArg, SecondVecArg}); - return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType()); + return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType(), + E->getExprLoc()); } if (LHS->getType()->isFPOrFPVectorTy()) { - Result = Builder.CreateFCmp((llvm::CmpInst::Predicate)FCmpOpc, - LHS, RHS, "cmp"); + Result = Builder.CreateFCmp(FCmpOpc, LHS, RHS, "cmp"); } else if (LHSTy->hasSignedIntegerRepresentation()) { - Result = Builder.CreateICmp((llvm::ICmpInst::Predicate)SICmpOpc, - LHS, RHS, "cmp"); + Result = Builder.CreateICmp(SICmpOpc, LHS, RHS, "cmp"); } else { // Unsigned integers and pointers. - Result = Builder.CreateICmp((llvm::ICmpInst::Predicate)UICmpOpc, - LHS, RHS, "cmp"); + Result = Builder.CreateICmp(UICmpOpc, LHS, RHS, "cmp"); } // If this is a vector comparison, sign extend the result to the appropriate @@ -2974,17 +2959,13 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc, Value *ResultR, *ResultI; if (CETy->isRealFloatingType()) { - ResultR = Builder.CreateFCmp((llvm::FCmpInst::Predicate)FCmpOpc, - LHS.first, RHS.first, "cmp.r"); - ResultI = Builder.CreateFCmp((llvm::FCmpInst::Predicate)FCmpOpc, - LHS.second, RHS.second, "cmp.i"); + ResultR = Builder.CreateFCmp(FCmpOpc, LHS.first, RHS.first, "cmp.r"); + ResultI = Builder.CreateFCmp(FCmpOpc, LHS.second, RHS.second, "cmp.i"); } else { // Complex comparisons can only be equality comparisons. As such, signed // and unsigned opcodes are the same. - ResultR = Builder.CreateICmp((llvm::ICmpInst::Predicate)UICmpOpc, - LHS.first, RHS.first, "cmp.r"); - ResultI = Builder.CreateICmp((llvm::ICmpInst::Predicate)UICmpOpc, - LHS.second, RHS.second, "cmp.i"); + ResultR = Builder.CreateICmp(UICmpOpc, LHS.first, RHS.first, "cmp.r"); + ResultI = Builder.CreateICmp(UICmpOpc, LHS.second, RHS.second, "cmp.i"); } if (E->getOpcode() == BO_EQ) { @@ -2996,7 +2977,8 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc, } } - return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType()); + return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType(), + E->getExprLoc()); } Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) { @@ -3382,13 +3364,14 @@ Value *ScalarExprEmitter::VisitVAArgExpr(VAArgExpr *VE) { if (Ty->isVariablyModifiedType()) CGF.EmitVariablyModifiedType(Ty); - llvm::Value *ArgValue = CGF.EmitVAListRef(VE->getSubExpr()); - llvm::Value *ArgPtr = CGF.EmitVAArg(ArgValue, VE->getType()); + Address ArgValue = Address::invalid(); + Address ArgPtr = CGF.EmitVAArg(VE, ArgValue); + llvm::Type *ArgTy = ConvertType(VE->getType()); // If EmitVAArg fails, we fall back to the LLVM instruction. - if (!ArgPtr) - return Builder.CreateVAArg(ArgValue, ArgTy); + if (!ArgPtr.isValid()) + return Builder.CreateVAArg(ArgValue.getPointer(), ArgTy); // FIXME Volatility. llvm::Value *Val = Builder.CreateLoad(ArgPtr); @@ -3465,8 +3448,8 @@ Value *ScalarExprEmitter::VisitAtomicExpr(AtomicExpr *E) { // Entry Point into this File //===----------------------------------------------------------------------===// -/// EmitScalarExpr - Emit the computation of the specified expression of scalar -/// type, ignoring the result. +/// Emit the computation of the specified expression of scalar type, ignoring +/// the result. Value *CodeGenFunction::EmitScalarExpr(const Expr *E, bool IgnoreResultAssign) { assert(E && hasScalarEvaluationKind(E->getType()) && "Invalid scalar expression to emit"); @@ -3475,25 +3458,26 @@ Value *CodeGenFunction::EmitScalarExpr(const Expr *E, bool IgnoreResultAssign) { .Visit(const_cast<Expr *>(E)); } -/// EmitScalarConversion - Emit a conversion from the specified type to the -/// specified destination type, both of which are LLVM scalar types. +/// Emit a conversion from the specified type to the specified destination type, +/// both of which are LLVM scalar types. Value *CodeGenFunction::EmitScalarConversion(Value *Src, QualType SrcTy, - QualType DstTy) { + QualType DstTy, + SourceLocation Loc) { assert(hasScalarEvaluationKind(SrcTy) && hasScalarEvaluationKind(DstTy) && "Invalid scalar expression to emit"); - return ScalarExprEmitter(*this).EmitScalarConversion(Src, SrcTy, DstTy); + return ScalarExprEmitter(*this).EmitScalarConversion(Src, SrcTy, DstTy, Loc); } -/// EmitComplexToScalarConversion - Emit a conversion from the specified complex -/// type to the specified destination type, where the destination type is an -/// LLVM scalar type. +/// Emit a conversion from the specified complex type to the specified +/// destination type, where the destination type is an LLVM scalar type. Value *CodeGenFunction::EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, - QualType DstTy) { + QualType DstTy, + SourceLocation Loc) { assert(SrcTy->isAnyComplexType() && hasScalarEvaluationKind(DstTy) && "Invalid complex -> scalar conversion"); - return ScalarExprEmitter(*this).EmitComplexToScalarConversion(Src, SrcTy, - DstTy); + return ScalarExprEmitter(*this) + .EmitComplexToScalarConversion(Src, SrcTy, DstTy, Loc); } @@ -3504,30 +3488,20 @@ EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, } LValue CodeGenFunction::EmitObjCIsaExpr(const ObjCIsaExpr *E) { - llvm::Value *V; // object->isa or (*object).isa // Generate code as for: *(Class*)object - // build Class* type - llvm::Type *ClassPtrTy = ConvertType(E->getType()); Expr *BaseExpr = E->getBase(); + Address Addr = Address::invalid(); if (BaseExpr->isRValue()) { - V = CreateMemTemp(E->getType(), "resval"); - llvm::Value *Src = EmitScalarExpr(BaseExpr); - Builder.CreateStore(Src, V); - V = ScalarExprEmitter(*this).EmitLoadOfLValue( - MakeNaturalAlignAddrLValue(V, E->getType()), E->getExprLoc()); + Addr = Address(EmitScalarExpr(BaseExpr), getPointerAlign()); } else { - if (E->isArrow()) - V = ScalarExprEmitter(*this).EmitLoadOfLValue(BaseExpr); - else - V = EmitLValue(BaseExpr).getAddress(); + Addr = EmitLValue(BaseExpr).getAddress(); } - // build Class* type - ClassPtrTy = ClassPtrTy->getPointerTo(); - V = Builder.CreateBitCast(V, ClassPtrTy); - return MakeNaturalAlignAddrLValue(V, E->getType()); + // Cast the address to Class*. + Addr = Builder.CreateElementBitCast(Addr, ConvertType(E->getType())); + return MakeAddrLValue(Addr, E->getType()); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp index 1163d63..0afe7db 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp @@ -8,6 +8,7 @@ //===----------------------------------------------------------------------===// #include "CGLoopInfo.h" +#include "clang/AST/ASTContext.h" #include "clang/AST/Attr.h" #include "clang/Sema/LoopHint.h" #include "llvm/IR/BasicBlock.h" @@ -20,9 +21,10 @@ using namespace llvm; static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) { - if (!Attrs.IsParallel && Attrs.VectorizerWidth == 0 && - Attrs.VectorizerUnroll == 0 && - Attrs.VectorizerEnable == LoopAttributes::VecUnspecified) + if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && + Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && + Attrs.VectorizeEnable == LoopAttributes::Unspecified && + Attrs.UnrollEnable == LoopAttributes::Unspecified) return nullptr; SmallVector<Metadata *, 4> Args; @@ -30,29 +32,49 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) { auto TempNode = MDNode::getTemporary(Ctx, None); Args.push_back(TempNode.get()); - // Setting vectorizer.width - if (Attrs.VectorizerWidth > 0) { + // Setting vectorize.width + if (Attrs.VectorizeWidth > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"), ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.VectorizerWidth))}; + Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))}; Args.push_back(MDNode::get(Ctx, Vals)); } - // Setting vectorizer.unroll - if (Attrs.VectorizerUnroll > 0) { + // Setting interleave.count + if (Attrs.InterleaveCount > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"), ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.VectorizerUnroll))}; + Type::getInt32Ty(Ctx), Attrs.InterleaveCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } - // Setting vectorizer.enable - if (Attrs.VectorizerEnable != LoopAttributes::VecUnspecified) { - Metadata *Vals[] = { - MDString::get(Ctx, "llvm.loop.vectorize.enable"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt1Ty(Ctx), - (Attrs.VectorizerEnable == LoopAttributes::VecEnable)))}; + // Setting interleave.count + if (Attrs.UnrollCount > 0) { + Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"), + ConstantAsMetadata::get(ConstantInt::get( + Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; + Args.push_back(MDNode::get(Ctx, Vals)); + } + + // Setting vectorize.enable + if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { + Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"), + ConstantAsMetadata::get(ConstantInt::get( + Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable == + LoopAttributes::Enable)))}; + Args.push_back(MDNode::get(Ctx, Vals)); + } + + // Setting unroll.full or unroll.disable + if (Attrs.UnrollEnable != LoopAttributes::Unspecified) { + std::string Name; + if (Attrs.UnrollEnable == LoopAttributes::Enable) + Name = "llvm.loop.unroll.enable"; + else if (Attrs.UnrollEnable == LoopAttributes::Full) + Name = "llvm.loop.unroll.full"; + else + Name = "llvm.loop.unroll.disable"; + Metadata *Vals[] = {MDString::get(Ctx, Name)}; Args.push_back(MDNode::get(Ctx, Vals)); } @@ -63,14 +85,17 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) { } LoopAttributes::LoopAttributes(bool IsParallel) - : IsParallel(IsParallel), VectorizerEnable(LoopAttributes::VecUnspecified), - VectorizerWidth(0), VectorizerUnroll(0) {} + : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), + UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0), + InterleaveCount(0), UnrollCount(0) {} void LoopAttributes::clear() { IsParallel = false; - VectorizerWidth = 0; - VectorizerUnroll = 0; - VectorizerEnable = LoopAttributes::VecUnspecified; + VectorizeWidth = 0; + InterleaveCount = 0; + UnrollCount = 0; + VectorizeEnable = LoopAttributes::Unspecified; + UnrollEnable = LoopAttributes::Unspecified; } LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs) @@ -78,8 +103,16 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs) LoopID = createMetadata(Header->getContext(), Attrs); } -void LoopInfoStack::push(BasicBlock *Header, +void LoopInfoStack::push(BasicBlock *Header) { + Active.push_back(LoopInfo(Header, StagedAttrs)); + // Clear the attributes so nested loops do not inherit them. + StagedAttrs.clear(); +} + +void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, ArrayRef<const clang::Attr *> Attrs) { + + // Identify loop hint attributes from Attrs. for (const auto *Attr : Attrs) { const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); @@ -87,28 +120,105 @@ void LoopInfoStack::push(BasicBlock *Header, if (!LH) continue; + auto *ValueExpr = LH->getValue(); + unsigned ValueInt = 1; + if (ValueExpr) { + llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx); + ValueInt = ValueAPS.getSExtValue(); + } + LoopHintAttr::OptionType Option = LH->getOption(); LoopHintAttr::LoopHintState State = LH->getState(); - switch (Option) { - case LoopHintAttr::Vectorize: - case LoopHintAttr::Interleave: - if (State == LoopHintAttr::AssumeSafety) { + switch (State) { + case LoopHintAttr::Disable: + switch (Option) { + case LoopHintAttr::Vectorize: + // Disable vectorization by specifying a width of 1. + setVectorizeWidth(1); + break; + case LoopHintAttr::Interleave: + // Disable interleaving by speciyfing a count of 1. + setInterleaveCount(1); + break; + case LoopHintAttr::Unroll: + setUnrollState(LoopAttributes::Disable); + break; + case LoopHintAttr::UnrollCount: + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::InterleaveCount: + llvm_unreachable("Options cannot be disabled."); + break; + } + break; + case LoopHintAttr::Enable: + switch (Option) { + case LoopHintAttr::Vectorize: + case LoopHintAttr::Interleave: + setVectorizeEnable(true); + break; + case LoopHintAttr::Unroll: + setUnrollState(LoopAttributes::Enable); + break; + case LoopHintAttr::UnrollCount: + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::InterleaveCount: + llvm_unreachable("Options cannot enabled."); + break; + } + break; + case LoopHintAttr::AssumeSafety: + switch (Option) { + case LoopHintAttr::Vectorize: + case LoopHintAttr::Interleave: // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. setParallel(true); + setVectorizeEnable(true); + break; + case LoopHintAttr::Unroll: + case LoopHintAttr::UnrollCount: + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::InterleaveCount: + llvm_unreachable("Options cannot be used to assume mem safety."); + break; + } + break; + case LoopHintAttr::Full: + switch (Option) { + case LoopHintAttr::Unroll: + setUnrollState(LoopAttributes::Full); + break; + case LoopHintAttr::Vectorize: + case LoopHintAttr::Interleave: + case LoopHintAttr::UnrollCount: + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::InterleaveCount: + llvm_unreachable("Options cannot be used with 'full' hint."); + break; } break; - case LoopHintAttr::VectorizeWidth: - case LoopHintAttr::InterleaveCount: - case LoopHintAttr::Unroll: - case LoopHintAttr::UnrollCount: - // Nothing to do here for these loop hints. + case LoopHintAttr::Numeric: + switch (Option) { + case LoopHintAttr::VectorizeWidth: + setVectorizeWidth(ValueInt); + break; + case LoopHintAttr::InterleaveCount: + setInterleaveCount(ValueInt); + break; + case LoopHintAttr::UnrollCount: + setUnrollCount(ValueInt); + break; + case LoopHintAttr::Unroll: + case LoopHintAttr::Vectorize: + case LoopHintAttr::Interleave: + llvm_unreachable("Options cannot be assigned a value."); + break; + } break; } } - Active.push_back(LoopInfo(Header, StagedAttrs)); - // Clear the attributes so nested loops do not inherit them. - StagedAttrs.clear(); + /// Stage the attributes. + push(Header); } void LoopInfoStack::pop() { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h index 2249937..ec33906 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h @@ -29,6 +29,7 @@ class MDNode; namespace clang { class Attr; +class ASTContext; namespace CodeGen { /// \brief Attributes that may be specified on loops. @@ -39,17 +40,23 @@ struct LoopAttributes { /// \brief Generate llvm.loop.parallel metadata for loads and stores. bool IsParallel; - /// \brief Values of llvm.loop.vectorize.enable metadata. - enum LVEnableState { VecUnspecified, VecEnable, VecDisable }; + /// \brief State of loop vectorization or unrolling. + enum LVEnableState { Unspecified, Enable, Disable, Full }; - /// \brief llvm.loop.vectorize.enable - LVEnableState VectorizerEnable; + /// \brief Value for llvm.loop.vectorize.enable metadata. + LVEnableState VectorizeEnable; - /// \brief llvm.loop.vectorize.width - unsigned VectorizerWidth; + /// \brief Value for llvm.loop.unroll.* metadata (enable, disable, or full). + LVEnableState UnrollEnable; - /// \brief llvm.loop.interleave.count - unsigned VectorizerUnroll; + /// \brief Value for llvm.loop.vectorize.width metadata. + unsigned VectorizeWidth; + + /// \brief Value for llvm.loop.interleave.count metadata. + unsigned InterleaveCount; + + /// \brief llvm.unroll. + unsigned UnrollCount; }; /// \brief Information used when generating a structured loop. @@ -88,8 +95,12 @@ public: /// \brief Begin a new structured loop. The set of staged attributes will be /// applied to the loop and then cleared. - void push(llvm::BasicBlock *Header, - llvm::ArrayRef<const Attr *> Attrs = llvm::None); + void push(llvm::BasicBlock *Header); + + /// \brief Begin a new structured loop. Stage attributes from the Attrs list. + /// The staged attributes are applied to the loop and then cleared. + void push(llvm::BasicBlock *Header, clang::ASTContext &Ctx, + llvm::ArrayRef<const Attr *> Attrs); /// \brief End the current loop. void pop(); @@ -109,17 +120,25 @@ public: /// \brief Set the next pushed loop as parallel. void setParallel(bool Enable = true) { StagedAttrs.IsParallel = Enable; } - /// \brief Set the next pushed loop 'vectorizer.enable' - void setVectorizerEnable(bool Enable = true) { - StagedAttrs.VectorizerEnable = - Enable ? LoopAttributes::VecEnable : LoopAttributes::VecDisable; + /// \brief Set the next pushed loop 'vectorize.enable' + void setVectorizeEnable(bool Enable = true) { + StagedAttrs.VectorizeEnable = + Enable ? LoopAttributes::Enable : LoopAttributes::Disable; } - /// \brief Set the vectorizer width for the next loop pushed. - void setVectorizerWidth(unsigned W) { StagedAttrs.VectorizerWidth = W; } + /// \brief Set the next pushed loop unroll state. + void setUnrollState(const LoopAttributes::LVEnableState &State) { + StagedAttrs.UnrollEnable = State; + } + + /// \brief Set the vectorize width for the next loop pushed. + void setVectorizeWidth(unsigned W) { StagedAttrs.VectorizeWidth = W; } + + /// \brief Set the interleave count for the next loop pushed. + void setInterleaveCount(unsigned C) { StagedAttrs.InterleaveCount = C; } - /// \brief Set the vectorizer unroll for the next loop pushed. - void setVectorizerUnroll(unsigned U) { StagedAttrs.VectorizerUnroll = U; } + /// \brief Set the unroll count for the next loop pushed. + void setUnrollCount(unsigned C) { StagedAttrs.UnrollCount = C; } private: /// \brief Returns true if there is LoopInfo on the stack. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp index eb76ad1..2d5991b 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp @@ -37,9 +37,8 @@ static RValue AdjustObjCObjectType(CodeGenFunction &CGF, /// Given the address of a variable of pointer type, find the correct /// null to store into it. -static llvm::Constant *getNullForVariable(llvm::Value *addr) { - llvm::Type *type = - cast<llvm::PointerType>(addr->getType())->getElementType(); +static llvm::Constant *getNullForVariable(Address addr) { + llvm::Type *type = addr.getElementType(); return llvm::ConstantPointerNull::get(cast<llvm::PointerType>(type)); } @@ -47,7 +46,7 @@ static llvm::Constant *getNullForVariable(llvm::Value *addr) { llvm::Value *CodeGenFunction::EmitObjCStringLiteral(const ObjCStringLiteral *E) { llvm::Constant *C = - CGM.getObjCRuntime().GenerateConstantString(E->getString()); + CGM.getObjCRuntime().GenerateConstantString(E->getString()).getPointer(); // FIXME: This bitcast should just be made an invariant on the Runtime. return llvm::ConstantExpr::getBitCast(C, ConvertType(E->getType())); } @@ -84,16 +83,15 @@ CodeGenFunction::EmitObjCBoxedExpr(const ObjCBoxedExpr *E) { if (ValueType->isObjCBoxableRecordType()) { // Emit CodeGen for first parameter // and cast value to correct type - llvm::Value *Temporary = CreateMemTemp(SubExpr->getType()); + Address Temporary = CreateMemTemp(SubExpr->getType()); EmitAnyExprToMem(SubExpr, Temporary, Qualifiers(), /*isInit*/ true); - llvm::Value *BitCast = Builder.CreateBitCast(Temporary, - ConvertType(ArgQT)); - Args.add(RValue::get(BitCast), ArgQT); + Address BitCast = Builder.CreateBitCast(Temporary, ConvertType(ArgQT)); + Args.add(RValue::get(BitCast.getPointer()), ArgQT); // Create char array to store type encoding std::string Str; getContext().getObjCEncodingForType(ValueType, Str); - llvm::GlobalVariable *GV = CGM.GetAddrOfConstantCString(Str); + llvm::Constant *GV = CGM.GetAddrOfConstantCString(Str).getPointer(); // Cast type encoding to correct type const ParmVarDecl *EncodingDecl = BoxingMethod->parameters()[1]; @@ -131,8 +129,8 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, ArrayType::Normal, /*IndexTypeQuals=*/0); // Allocate the temporary array(s). - llvm::AllocaInst *Objects = CreateMemTemp(ElementArrayType, "objects"); - llvm::AllocaInst *Keys = nullptr; + Address Objects = CreateMemTemp(ElementArrayType, "objects"); + Address Keys = Address::invalid(); if (DLE) Keys = CreateMemTemp(ElementArrayType, "keys"); @@ -148,9 +146,9 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, if (ALE) { // Emit the element and store it to the appropriate array slot. const Expr *Rhs = ALE->getElement(i); - LValue LV = LValue::MakeAddr( - Builder.CreateStructGEP(Objects->getAllocatedType(), Objects, i), - ElementType, Context.getTypeAlignInChars(Rhs->getType()), Context); + LValue LV = MakeAddrLValue( + Builder.CreateConstArrayGEP(Objects, i, getPointerSize()), + ElementType, AlignmentSource::Decl); llvm::Value *value = EmitScalarExpr(Rhs); EmitStoreThroughLValue(RValue::get(value), LV, true); @@ -160,17 +158,17 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, } else { // Emit the key and store it to the appropriate array slot. const Expr *Key = DLE->getKeyValueElement(i).Key; - LValue KeyLV = LValue::MakeAddr( - Builder.CreateStructGEP(Keys->getAllocatedType(), Keys, i), - ElementType, Context.getTypeAlignInChars(Key->getType()), Context); + LValue KeyLV = MakeAddrLValue( + Builder.CreateConstArrayGEP(Keys, i, getPointerSize()), + ElementType, AlignmentSource::Decl); llvm::Value *keyValue = EmitScalarExpr(Key); EmitStoreThroughLValue(RValue::get(keyValue), KeyLV, /*isInit=*/true); // Emit the value and store it to the appropriate array slot. const Expr *Value = DLE->getKeyValueElement(i).Value; - LValue ValueLV = LValue::MakeAddr( - Builder.CreateStructGEP(Objects->getAllocatedType(), Objects, i), - ElementType, Context.getTypeAlignInChars(Value->getType()), Context); + LValue ValueLV = MakeAddrLValue( + Builder.CreateConstArrayGEP(Objects, i, getPointerSize()), + ElementType, AlignmentSource::Decl); llvm::Value *valueValue = EmitScalarExpr(Value); EmitStoreThroughLValue(RValue::get(valueValue), ValueLV, /*isInit=*/true); if (TrackNeededObjects) { @@ -185,11 +183,11 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, ObjCMethodDecl::param_const_iterator PI = MethodWithObjects->param_begin(); const ParmVarDecl *argDecl = *PI++; QualType ArgQT = argDecl->getType().getUnqualifiedType(); - Args.add(RValue::get(Objects), ArgQT); + Args.add(RValue::get(Objects.getPointer()), ArgQT); if (DLE) { argDecl = *PI++; ArgQT = argDecl->getType().getUnqualifiedType(); - Args.add(RValue::get(Keys), ArgQT); + Args.add(RValue::get(Keys.getPointer()), ArgQT); } argDecl = *PI; ArgQT = argDecl->getType().getUnqualifiedType(); @@ -275,10 +273,23 @@ shouldExtendReceiverForInnerPointerMessage(const ObjCMessageExpr *message) { // receiver is loaded from a variable with precise lifetime. case ObjCMessageExpr::Instance: { const Expr *receiver = message->getInstanceReceiver(); + + // Look through OVEs. + if (auto opaque = dyn_cast<OpaqueValueExpr>(receiver)) { + if (opaque->getSourceExpr()) + receiver = opaque->getSourceExpr()->IgnoreParens(); + } + const ImplicitCastExpr *ice = dyn_cast<ImplicitCastExpr>(receiver); if (!ice || ice->getCastKind() != CK_LValueToRValue) return true; receiver = ice->getSubExpr()->IgnoreParens(); + // Look through OVEs. + if (auto opaque = dyn_cast<OpaqueValueExpr>(receiver)) { + if (opaque->getSourceExpr()) + receiver = opaque->getSourceExpr()->IgnoreParens(); + } + // Only __strong variables. if (receiver->getType().getObjCLifetime() != Qualifiers::OCL_Strong) return true; @@ -312,6 +323,21 @@ shouldExtendReceiverForInnerPointerMessage(const ObjCMessageExpr *message) { llvm_unreachable("invalid receiver kind"); } +/// Given an expression of ObjC pointer type, check whether it was +/// immediately loaded from an ARC __weak l-value. +static const Expr *findWeakLValue(const Expr *E) { + assert(E->getType()->isObjCRetainableType()); + E = E->IgnoreParens(); + if (auto CE = dyn_cast<CastExpr>(E)) { + if (CE->getCastKind() == CK_LValueToRValue) { + if (CE->getSubExpr()->getType().getObjCLifetime() == Qualifiers::OCL_Weak) + return CE->getSubExpr(); + } + } + + return nullptr; +} + RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E, ReturnValueSlot Return) { // Only the lookup mechanism and first two arguments of the method @@ -322,6 +348,17 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E, const ObjCMethodDecl *method = E->getMethodDecl(); + // If the method is -retain, and the receiver's being loaded from + // a __weak variable, peephole the entire operation to objc_loadWeakRetained. + if (method && E->getReceiverKind() == ObjCMessageExpr::Instance && + method->getMethodFamily() == OMF_retain) { + if (auto lvalueExpr = findWeakLValue(E->getInstanceReceiver())) { + LValue lvalue = EmitLValue(lvalueExpr); + llvm::Value *result = EmitARCLoadWeakRetained(lvalue.getAddress()); + return AdjustObjCObjectType(*this, E->getType(), RValue::get(result)); + } + } + // We don't retain the receiver in delegate init calls, and this is // safe because the receiver value is always loaded from 'self', // which we zero out. We don't want to Block_copy block receivers, @@ -390,7 +427,7 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E, QualType ResultType = method ? method->getReturnType() : E->getType(); CallArgList Args; - EmitCallArgs(Args, method, E->arg_begin(), E->arg_end()); + EmitCallArgs(Args, method, E->arguments()); // For delegate init calls in ARC, do an unsafe store of null into // self. This represents the call taking direct ownership of that @@ -404,10 +441,8 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E, "delegate init calls should only be marked in ARC"); // Do an unsafe store of null into self. - llvm::Value *selfAddr = - LocalDeclMap[cast<ObjCMethodDecl>(CurCodeDecl)->getSelfDecl()]; - assert(selfAddr && "no self entry for a delegate init call?"); - + Address selfAddr = + GetAddrOfLocalVar(cast<ObjCMethodDecl>(CurCodeDecl)->getSelfDecl()); Builder.CreateStore(getNullForVariable(selfAddr), selfAddr); } @@ -434,14 +469,13 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E, // For delegate init calls in ARC, implicitly store the result of // the call back into self. This takes ownership of the value. if (isDelegateInit) { - llvm::Value *selfAddr = - LocalDeclMap[cast<ObjCMethodDecl>(CurCodeDecl)->getSelfDecl()]; + Address selfAddr = + GetAddrOfLocalVar(cast<ObjCMethodDecl>(CurCodeDecl)->getSelfDecl()); llvm::Value *newSelf = result.getScalarVal(); // The delegate return type isn't necessarily a matching type; in // fact, it's quite likely to be 'id'. - llvm::Type *selfTy = - cast<llvm::PointerType>(selfAddr->getType())->getElementType(); + llvm::Type *selfTy = selfAddr.getElementType(); newSelf = Builder.CreateBitCast(newSelf, selfTy); Builder.CreateStore(newSelf, selfAddr); @@ -451,7 +485,7 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E, } namespace { -struct FinishARCDealloc : EHScopeStack::Cleanup { +struct FinishARCDealloc final : EHScopeStack::Cleanup { void Emit(CodeGenFunction &CGF, Flags flags) override { const ObjCMethodDecl *method = cast<ObjCMethodDecl>(CGF.CurCodeDecl); @@ -523,7 +557,7 @@ static llvm::Value *emitARCRetainLoadOfScalar(CodeGenFunction &CGF, /// its pointer, name, and types registered in the class struture. void CodeGenFunction::GenerateObjCMethod(const ObjCMethodDecl *OMD) { StartObjCMethod(OMD, OMD->getClassInterface()); - PGO.assignRegionCounters(OMD, CurFn); + PGO.assignRegionCounters(GlobalDecl(OMD), CurFn); assert(isa<CompoundStmt>(OMD->getBody())); incrementProfileCounter(OMD->getBody()); EmitCompoundStmtWithoutScope(*cast<CompoundStmt>(OMD->getBody())); @@ -536,19 +570,19 @@ static void emitStructGetterCall(CodeGenFunction &CGF, ObjCIvarDecl *ivar, bool isAtomic, bool hasStrong) { ASTContext &Context = CGF.getContext(); - llvm::Value *src = - CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(), CGF.LoadObjCSelf(), - ivar, 0).getAddress(); + Address src = + CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(), CGF.LoadObjCSelf(), ivar, 0) + .getAddress(); // objc_copyStruct (ReturnValue, &structIvar, // sizeof (Type of Ivar), isAtomic, false); CallArgList args; - llvm::Value *dest = CGF.Builder.CreateBitCast(CGF.ReturnValue, CGF.VoidPtrTy); - args.add(RValue::get(dest), Context.VoidPtrTy); + Address dest = CGF.Builder.CreateBitCast(CGF.ReturnValue, CGF.VoidPtrTy); + args.add(RValue::get(dest.getPointer()), Context.VoidPtrTy); src = CGF.Builder.CreateBitCast(src, CGF.VoidPtrTy); - args.add(RValue::get(src), Context.VoidPtrTy); + args.add(RValue::get(src.getPointer()), Context.VoidPtrTy); CharUnits size = CGF.getContext().getTypeSizeInChars(ivar->getType()); args.add(RValue::get(CGF.CGM.getSize(size)), Context.getSizeType()); @@ -812,8 +846,8 @@ static void emitCPPObjectAtomicGetterCall(CodeGenFunction &CGF, // The 2nd argument is the address of the ivar. llvm::Value *ivarAddr = - CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(), - CGF.LoadObjCSelf(), ivar, 0).getAddress(); + CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(), + CGF.LoadObjCSelf(), ivar, 0).getPointer(); ivarAddr = CGF.Builder.CreateBitCast(ivarAddr, CGF.Int8PtrTy); args.add(RValue::get(ivarAddr), CGF.getContext().VoidPtrTy); @@ -843,7 +877,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, } else { ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl(); - emitCPPObjectAtomicGetterCall(*this, ReturnValue, + emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(), ivar, AtomicHelperFn); } return; @@ -873,10 +907,9 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, bitcastType = bitcastType->getPointerTo(); // addrspace 0 okay // Perform an atomic load. This does not impose ordering constraints. - llvm::Value *ivarAddr = LV.getAddress(); + Address ivarAddr = LV.getAddress(); ivarAddr = Builder.CreateBitCast(ivarAddr, bitcastType); llvm::LoadInst *load = Builder.CreateLoad(ivarAddr, "load"); - load->setAlignment(strategy.getIvarAlignment().getQuantity()); load->setAtomic(llvm::Unordered); // Store that value into the return address. Doing this with a @@ -901,7 +934,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, // FIXME: Can't this be simpler? This might even be worse than the // corresponding gcc code. llvm::Value *cmd = - Builder.CreateLoad(LocalDeclMap[getterMethod->getCmdDecl()], "cmd"); + Builder.CreateLoad(GetAddrOfLocalVar(getterMethod->getCmdDecl()), "cmd"); llvm::Value *self = Builder.CreateBitCast(LoadObjCSelf(), VoidPtrTy); llvm::Value *ivarOffset = EmitIvarOffset(classImpl->getClassInterface(), ivar); @@ -916,11 +949,11 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, // FIXME: We shouldn't need to get the function info here, the // runtime already should have computed it to build the function. llvm::Instruction *CallInstruction; - RValue RV = EmitCall(getTypes().arrangeFreeFunctionCall(propType, args, - FunctionType::ExtInfo(), - RequiredArgs::All), - getPropertyFn, ReturnValueSlot(), args, nullptr, - &CallInstruction); + RValue RV = EmitCall( + getTypes().arrangeFreeFunctionCall( + propType, args, FunctionType::ExtInfo(), RequiredArgs::All), + getPropertyFn, ReturnValueSlot(), args, CGCalleeInfo(), + &CallInstruction); if (llvm::CallInst *call = dyn_cast<llvm::CallInst>(CallInstruction)) call->setTailCall(); @@ -952,8 +985,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, switch (getEvaluationKind(ivarType)) { case TEK_Complex: { ComplexPairTy pair = EmitLoadOfComplex(LV, SourceLocation()); - EmitStoreOfComplex(pair, - MakeNaturalAlignAddrLValue(ReturnValue, ivarType), + EmitStoreOfComplex(pair, MakeAddrLValue(ReturnValue, ivarType), /*init*/ true); return; } @@ -966,11 +998,15 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, case TEK_Scalar: { llvm::Value *value; if (propType->isReferenceType()) { - value = LV.getAddress(); + value = LV.getAddress().getPointer(); } else { // We want to load and autoreleaseReturnValue ARC __weak ivars. if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) { - value = emitARCRetainLoadOfScalar(*this, LV, ivarType); + if (getLangOpts().ObjCAutoRefCount) { + value = emitARCRetainLoadOfScalar(*this, LV, ivarType); + } else { + value = EmitARCLoadWeak(LV.getAddress()); + } // Otherwise we want to do a simple load, suppressing the // final autorelease. @@ -1006,7 +1042,7 @@ static void emitStructSetterCall(CodeGenFunction &CGF, ObjCMethodDecl *OMD, // The first argument is the address of the ivar. llvm::Value *ivarAddr = CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(), CGF.LoadObjCSelf(), ivar, 0) - .getAddress(); + .getPointer(); ivarAddr = CGF.Builder.CreateBitCast(ivarAddr, CGF.Int8PtrTy); args.add(RValue::get(ivarAddr), CGF.getContext().VoidPtrTy); @@ -1014,7 +1050,7 @@ static void emitStructSetterCall(CodeGenFunction &CGF, ObjCMethodDecl *OMD, ParmVarDecl *argVar = *OMD->param_begin(); DeclRefExpr argRef(argVar, false, argVar->getType().getNonReferenceType(), VK_LValue, SourceLocation()); - llvm::Value *argAddr = CGF.EmitLValue(&argRef).getAddress(); + llvm::Value *argAddr = CGF.EmitLValue(&argRef).getPointer(); argAddr = CGF.Builder.CreateBitCast(argAddr, CGF.Int8PtrTy); args.add(RValue::get(argAddr), CGF.getContext().VoidPtrTy); @@ -1052,7 +1088,7 @@ static void emitCPPObjectAtomicSetterCall(CodeGenFunction &CGF, // The first argument is the address of the ivar. llvm::Value *ivarAddr = CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(), - CGF.LoadObjCSelf(), ivar, 0).getAddress(); + CGF.LoadObjCSelf(), ivar, 0).getPointer(); ivarAddr = CGF.Builder.CreateBitCast(ivarAddr, CGF.Int8PtrTy); args.add(RValue::get(ivarAddr), CGF.getContext().VoidPtrTy); @@ -1060,7 +1096,7 @@ static void emitCPPObjectAtomicSetterCall(CodeGenFunction &CGF, ParmVarDecl *argVar = *OMD->param_begin(); DeclRefExpr argRef(argVar, false, argVar->getType().getNonReferenceType(), VK_LValue, SourceLocation()); - llvm::Value *argAddr = CGF.EmitLValue(&argRef).getAddress(); + llvm::Value *argAddr = CGF.EmitLValue(&argRef).getPointer(); argAddr = CGF.Builder.CreateBitCast(argAddr, CGF.Int8PtrTy); args.add(RValue::get(argAddr), CGF.getContext().VoidPtrTy); @@ -1135,29 +1171,27 @@ CodeGenFunction::generateObjCSetterBody(const ObjCImplementationDecl *classImpl, if (strategy.getIvarSize().isZero()) return; - llvm::Value *argAddr = LocalDeclMap[*setterMethod->param_begin()]; + Address argAddr = GetAddrOfLocalVar(*setterMethod->param_begin()); LValue ivarLValue = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), ivar, /*quals*/ 0); - llvm::Value *ivarAddr = ivarLValue.getAddress(); + Address ivarAddr = ivarLValue.getAddress(); // Currently, all atomic accesses have to be through integer // types, so there's no point in trying to pick a prettier type. llvm::Type *bitcastType = llvm::Type::getIntNTy(getLLVMContext(), getContext().toBits(strategy.getIvarSize())); - bitcastType = bitcastType->getPointerTo(); // addrspace 0 okay // Cast both arguments to the chosen operation type. - argAddr = Builder.CreateBitCast(argAddr, bitcastType); - ivarAddr = Builder.CreateBitCast(ivarAddr, bitcastType); + argAddr = Builder.CreateElementBitCast(argAddr, bitcastType); + ivarAddr = Builder.CreateElementBitCast(ivarAddr, bitcastType); // This bitcast load is likely to cause some nasty IR. llvm::Value *load = Builder.CreateLoad(argAddr); // Perform an atomic store. There are no memory ordering requirements. llvm::StoreInst *store = Builder.CreateStore(load, ivarAddr); - store->setAlignment(strategy.getIvarAlignment().getQuantity()); store->setAtomic(llvm::Unordered); return; } @@ -1189,13 +1223,14 @@ CodeGenFunction::generateObjCSetterBody(const ObjCImplementationDecl *classImpl, // Emit objc_setProperty((id) self, _cmd, offset, arg, // <is-atomic>, <is-copy>). llvm::Value *cmd = - Builder.CreateLoad(LocalDeclMap[setterMethod->getCmdDecl()]); + Builder.CreateLoad(GetAddrOfLocalVar(setterMethod->getCmdDecl())); llvm::Value *self = Builder.CreateBitCast(LoadObjCSelf(), VoidPtrTy); llvm::Value *ivarOffset = EmitIvarOffset(classImpl->getClassInterface(), ivar); - llvm::Value *arg = LocalDeclMap[*setterMethod->param_begin()]; - arg = Builder.CreateBitCast(Builder.CreateLoad(arg, "arg"), VoidPtrTy); + Address argAddr = GetAddrOfLocalVar(*setterMethod->param_begin()); + llvm::Value *arg = Builder.CreateLoad(argAddr, "arg"); + arg = Builder.CreateBitCast(arg, VoidPtrTy); CallArgList args; args.add(RValue::get(self), getContext().getObjCIdType()); @@ -1304,7 +1339,7 @@ void CodeGenFunction::GenerateObjCSetter(ObjCImplementationDecl *IMP, } namespace { - struct DestroyIvar : EHScopeStack::Cleanup { + struct DestroyIvar final : EHScopeStack::Cleanup { private: llvm::Value *addr; const ObjCIvarDecl *ivar; @@ -1328,7 +1363,7 @@ namespace { /// Like CodeGenFunction::destroyARCStrong, but do it with a call. static void destroyARCStrongWithStore(CodeGenFunction &CGF, - llvm::Value *addr, + Address addr, QualType type) { llvm::Value *null = getNullForVariable(addr); CGF.EmitARCStoreStrongCall(addr, null, /*ignored*/ true); @@ -1405,22 +1440,6 @@ void CodeGenFunction::GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP, FinishFunction(); } -bool CodeGenFunction::IndirectObjCSetterArg(const CGFunctionInfo &FI) { - CGFunctionInfo::const_arg_iterator it = FI.arg_begin(); - it++; it++; - const ABIArgInfo &AI = it->info; - // FIXME. Is this sufficient check? - return (AI.getKind() == ABIArgInfo::Indirect); -} - -bool CodeGenFunction::IvarTypeWithAggrGCObjects(QualType Ty) { - if (CGM.getLangOpts().getGC() == LangOptions::NonGC) - return false; - if (const RecordType *FDTTy = Ty.getTypePtr()->getAs<RecordType>()) - return FDTTy->getDecl()->hasObjectMember(); - return false; -} - llvm::Value *CodeGenFunction::LoadObjCSelf() { VarDecl *Self = cast<ObjCMethodDecl>(CurFuncDecl)->getSelfDecl(); DeclRefExpr DRE(Self, /*is enclosing local*/ (CurFuncDecl != CurCodeDecl), @@ -1458,7 +1477,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // Fast enumeration state. QualType StateTy = CGM.getObjCFastEnumerationStateType(); - llvm::AllocaInst *StatePtr = CreateMemTemp(StateTy, "state.ptr"); + Address StatePtr = CreateMemTemp(StateTy, "state.ptr"); EmitNullInitialization(StatePtr, StateTy); // Number of elements in the items array. @@ -1477,7 +1496,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ getContext().getConstantArrayType(getContext().getObjCIdType(), llvm::APInt(32, NumItems), ArrayType::Normal, 0); - llvm::Value *ItemsPtr = CreateMemTemp(ItemsTy, "items.ptr"); + Address ItemsPtr = CreateMemTemp(ItemsTy, "items.ptr"); // Emit the collection pointer. In ARC, we do a retain. llvm::Value *Collection; @@ -1498,14 +1517,16 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ CallArgList Args; // The first argument is a temporary of the enumeration-state type. - Args.add(RValue::get(StatePtr), getContext().getPointerType(StateTy)); + Args.add(RValue::get(StatePtr.getPointer()), + getContext().getPointerType(StateTy)); // The second argument is a temporary array with space for NumItems // pointers. We'll actually be loading elements from the array // pointer written into the control state; this buffer is so that // collections that *aren't* backed by arrays can still queue up // batches of elements. - Args.add(RValue::get(ItemsPtr), getContext().getPointerType(ItemsTy)); + Args.add(RValue::get(ItemsPtr.getPointer()), + getContext().getPointerType(ItemsTy)); // The third argument is the capacity of that temporary array. llvm::Type *UnsignedLongLTy = ConvertType(getContext().UnsignedLongTy); @@ -1542,13 +1563,14 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // Save the initial mutations value. This is the value at an // address that was written into the state object by // countByEnumeratingWithState:objects:count:. - llvm::Value *StateMutationsPtrPtr = Builder.CreateStructGEP( - StatePtr->getAllocatedType(), StatePtr, 2, "mutationsptr.ptr"); - llvm::Value *StateMutationsPtr = Builder.CreateLoad(StateMutationsPtrPtr, - "mutationsptr"); + Address StateMutationsPtrPtr = Builder.CreateStructGEP( + StatePtr, 2, 2 * getPointerSize(), "mutationsptr.ptr"); + llvm::Value *StateMutationsPtr + = Builder.CreateLoad(StateMutationsPtrPtr, "mutationsptr"); llvm::Value *initialMutations = - Builder.CreateLoad(StateMutationsPtr, "forcoll.initial-mutations"); + Builder.CreateAlignedLoad(StateMutationsPtr, getPointerAlign(), + "forcoll.initial-mutations"); // Start looping. This is the point we return to whenever we have a // fresh, non-empty batch of objects. @@ -1570,7 +1592,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // refreshes. StateMutationsPtr = Builder.CreateLoad(StateMutationsPtrPtr, "mutationsptr"); llvm::Value *currentMutations - = Builder.CreateLoad(StateMutationsPtr, "statemutations"); + = Builder.CreateAlignedLoad(StateMutationsPtr, getPointerAlign(), + "statemutations"); llvm::BasicBlock *WasMutatedBB = createBasicBlock("forcoll.mutated"); llvm::BasicBlock *WasNotMutatedBB = createBasicBlock("forcoll.notmutated"); @@ -1623,15 +1646,16 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // Fetch the buffer out of the enumeration state. // TODO: this pointer should actually be invariant between // refreshes, which would help us do certain loop optimizations. - llvm::Value *StateItemsPtr = Builder.CreateStructGEP( - StatePtr->getAllocatedType(), StatePtr, 1, "stateitems.ptr"); + Address StateItemsPtr = Builder.CreateStructGEP( + StatePtr, 1, getPointerSize(), "stateitems.ptr"); llvm::Value *EnumStateItems = Builder.CreateLoad(StateItemsPtr, "stateitems"); // Fetch the value at the current index from the buffer. llvm::Value *CurrentItemPtr = Builder.CreateGEP(EnumStateItems, index, "currentitem.ptr"); - llvm::Value *CurrentItem = Builder.CreateLoad(CurrentItemPtr); + llvm::Value *CurrentItem = + Builder.CreateAlignedLoad(CurrentItemPtr, getPointerAlign()); // Cast that value to the right type. CurrentItem = Builder.CreateBitCast(CurrentItem, convertedElementType, @@ -1735,15 +1759,8 @@ void CodeGenFunction::EmitObjCAtSynchronizedStmt( CGM.getObjCRuntime().EmitSynchronizedStmt(*this, S); } -/// Produce the code for a CK_ARCProduceObject. Just does a -/// primitive retain. -llvm::Value *CodeGenFunction::EmitObjCProduceObject(QualType type, - llvm::Value *value) { - return EmitARCRetain(type, value); -} - namespace { - struct CallObjCRelease : EHScopeStack::Cleanup { + struct CallObjCRelease final : EHScopeStack::Cleanup { CallObjCRelease(llvm::Value *object) : object(object) {} llvm::Value *object; @@ -1772,7 +1789,7 @@ llvm::Value *CodeGenFunction::EmitObjCExtendObjectLifetime(QualType type, /// Given a number of pointers, inform the optimizer that they're /// being intrinsically used up until this point in the program. void CodeGenFunction::EmitARCIntrinsicUse(ArrayRef<llvm::Value*> values) { - llvm::Constant *&fn = CGM.getARCEntrypoints().clang_arc_use; + llvm::Constant *&fn = CGM.getObjCEntrypoints().clang_arc_use; if (!fn) { llvm::FunctionType *fnType = llvm::FunctionType::get(CGM.VoidTy, None, true); @@ -1838,7 +1855,7 @@ static llvm::Value *emitARCValueOperation(CodeGenFunction &CGF, /// Perform an operation having the following signature: /// i8* (i8**) static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF, - llvm::Value *addr, + Address addr, llvm::Constant *&fn, StringRef fnName) { if (!fn) { @@ -1848,16 +1865,15 @@ static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF, } // Cast the argument to 'id*'. - llvm::Type *origType = addr->getType(); + llvm::Type *origType = addr.getElementType(); addr = CGF.Builder.CreateBitCast(addr, CGF.Int8PtrPtrTy); // Call the function. - llvm::Value *result = CGF.EmitNounwindRuntimeCall(fn, addr); + llvm::Value *result = CGF.EmitNounwindRuntimeCall(fn, addr.getPointer()); // Cast the result back to a dereference of the original type. - if (origType != CGF.Int8PtrPtrTy) - result = CGF.Builder.CreateBitCast(result, - cast<llvm::PointerType>(origType)->getElementType()); + if (origType != CGF.Int8PtrTy) + result = CGF.Builder.CreateBitCast(result, origType); return result; } @@ -1865,13 +1881,12 @@ static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF, /// Perform an operation having the following signature: /// i8* (i8**, i8*) static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF, - llvm::Value *addr, + Address addr, llvm::Value *value, llvm::Constant *&fn, StringRef fnName, bool ignored) { - assert(cast<llvm::PointerType>(addr->getType())->getElementType() - == value->getType()); + assert(addr.getElementType() == value->getType()); if (!fn) { llvm::Type *argTypes[] = { CGF.Int8PtrPtrTy, CGF.Int8PtrTy }; @@ -1884,7 +1899,7 @@ static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF, llvm::Type *origType = value->getType(); llvm::Value *args[] = { - CGF.Builder.CreateBitCast(addr, CGF.Int8PtrPtrTy), + CGF.Builder.CreateBitCast(addr.getPointer(), CGF.Int8PtrPtrTy), CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy) }; llvm::CallInst *result = CGF.EmitNounwindRuntimeCall(fn, args); @@ -1897,11 +1912,11 @@ static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF, /// Perform an operation having the following signature: /// void (i8**, i8**) static void emitARCCopyOperation(CodeGenFunction &CGF, - llvm::Value *dst, - llvm::Value *src, + Address dst, + Address src, llvm::Constant *&fn, StringRef fnName) { - assert(dst->getType() == src->getType()); + assert(dst.getType() == src.getType()); if (!fn) { llvm::Type *argTypes[] = { CGF.Int8PtrPtrTy, CGF.Int8PtrPtrTy }; @@ -1912,8 +1927,8 @@ static void emitARCCopyOperation(CodeGenFunction &CGF, } llvm::Value *args[] = { - CGF.Builder.CreateBitCast(dst, CGF.Int8PtrPtrTy), - CGF.Builder.CreateBitCast(src, CGF.Int8PtrPtrTy) + CGF.Builder.CreateBitCast(dst.getPointer(), CGF.Int8PtrPtrTy), + CGF.Builder.CreateBitCast(src.getPointer(), CGF.Int8PtrPtrTy) }; CGF.EmitNounwindRuntimeCall(fn, args); } @@ -1932,7 +1947,7 @@ llvm::Value *CodeGenFunction::EmitARCRetain(QualType type, llvm::Value *value) { /// call i8* \@objc_retain(i8* %value) llvm::Value *CodeGenFunction::EmitARCRetainNonBlock(llvm::Value *value) { return emitARCValueOperation(*this, value, - CGM.getARCEntrypoints().objc_retain, + CGM.getObjCEntrypoints().objc_retain, "objc_retain"); } @@ -1946,7 +1961,7 @@ llvm::Value *CodeGenFunction::EmitARCRetainBlock(llvm::Value *value, bool mandatory) { llvm::Value *result = emitARCValueOperation(*this, value, - CGM.getARCEntrypoints().objc_retainBlock, + CGM.getObjCEntrypoints().objc_retainBlock, "objc_retainBlock"); // If the copy isn't mandatory, add !clang.arc.copy_on_escape to @@ -1956,7 +1971,7 @@ llvm::Value *CodeGenFunction::EmitARCRetainBlock(llvm::Value *value, if (!mandatory && isa<llvm::Instruction>(result)) { llvm::CallInst *call = cast<llvm::CallInst>(result->stripPointerCasts()); - assert(call->getCalledValue() == CGM.getARCEntrypoints().objc_retainBlock); + assert(call->getCalledValue() == CGM.getObjCEntrypoints().objc_retainBlock); call->setMetadata("clang.arc.copy_on_escape", llvm::MDNode::get(Builder.getContext(), None)); @@ -1975,7 +1990,7 @@ CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) { // Fetch the void(void) inline asm which marks that we're going to // retain the autoreleased return value. llvm::InlineAsm *&marker - = CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker; + = CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker; if (!marker) { StringRef assembly = CGM.getTargetCodeGenInfo() @@ -2012,7 +2027,7 @@ CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) { Builder.CreateCall(marker); return emitARCValueOperation(*this, value, - CGM.getARCEntrypoints().objc_retainAutoreleasedReturnValue, + CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue, "objc_retainAutoreleasedReturnValue"); } @@ -2022,7 +2037,7 @@ void CodeGenFunction::EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise) { if (isa<llvm::ConstantPointerNull>(value)) return; - llvm::Constant *&fn = CGM.getARCEntrypoints().objc_release; + llvm::Constant *&fn = CGM.getObjCEntrypoints().objc_release; if (!fn) { llvm::FunctionType *fnType = llvm::FunctionType::get(Builder.getVoidTy(), Int8PtrTy, false); @@ -2050,12 +2065,10 @@ void CodeGenFunction::EmitARCRelease(llvm::Value *value, /// At -O1 and above, just load and call objc_release. /// /// call void \@objc_storeStrong(i8** %addr, i8* null) -void CodeGenFunction::EmitARCDestroyStrong(llvm::Value *addr, +void CodeGenFunction::EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise) { if (CGM.getCodeGenOpts().OptimizationLevel == 0) { - llvm::PointerType *addrTy = cast<llvm::PointerType>(addr->getType()); - llvm::Value *null = llvm::ConstantPointerNull::get( - cast<llvm::PointerType>(addrTy->getElementType())); + llvm::Value *null = getNullForVariable(addr); EmitARCStoreStrongCall(addr, null, /*ignored*/ true); return; } @@ -2066,13 +2079,12 @@ void CodeGenFunction::EmitARCDestroyStrong(llvm::Value *addr, /// Store into a strong object. Always calls this: /// call void \@objc_storeStrong(i8** %addr, i8* %value) -llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(llvm::Value *addr, +llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(Address addr, llvm::Value *value, bool ignored) { - assert(cast<llvm::PointerType>(addr->getType())->getElementType() - == value->getType()); + assert(addr.getElementType() == value->getType()); - llvm::Constant *&fn = CGM.getARCEntrypoints().objc_storeStrong; + llvm::Constant *&fn = CGM.getObjCEntrypoints().objc_storeStrong; if (!fn) { llvm::Type *argTypes[] = { Int8PtrPtrTy, Int8PtrTy }; llvm::FunctionType *fnType @@ -2081,7 +2093,7 @@ llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(llvm::Value *addr, } llvm::Value *args[] = { - Builder.CreateBitCast(addr, Int8PtrPtrTy), + Builder.CreateBitCast(addr.getPointer(), Int8PtrPtrTy), Builder.CreateBitCast(value, Int8PtrTy) }; EmitNounwindRuntimeCall(fn, args); @@ -2130,7 +2142,7 @@ llvm::Value *CodeGenFunction::EmitARCStoreStrong(LValue dst, /// call i8* \@objc_autorelease(i8* %value) llvm::Value *CodeGenFunction::EmitARCAutorelease(llvm::Value *value) { return emitARCValueOperation(*this, value, - CGM.getARCEntrypoints().objc_autorelease, + CGM.getObjCEntrypoints().objc_autorelease, "objc_autorelease"); } @@ -2139,7 +2151,7 @@ llvm::Value *CodeGenFunction::EmitARCAutorelease(llvm::Value *value) { llvm::Value * CodeGenFunction::EmitARCAutoreleaseReturnValue(llvm::Value *value) { return emitARCValueOperation(*this, value, - CGM.getARCEntrypoints().objc_autoreleaseReturnValue, + CGM.getObjCEntrypoints().objc_autoreleaseReturnValue, "objc_autoreleaseReturnValue", /*isTailCall*/ true); } @@ -2149,7 +2161,7 @@ CodeGenFunction::EmitARCAutoreleaseReturnValue(llvm::Value *value) { llvm::Value * CodeGenFunction::EmitARCRetainAutoreleaseReturnValue(llvm::Value *value) { return emitARCValueOperation(*this, value, - CGM.getARCEntrypoints().objc_retainAutoreleaseReturnValue, + CGM.getObjCEntrypoints().objc_retainAutoreleaseReturnValue, "objc_retainAutoreleaseReturnValue", /*isTailCall*/ true); } @@ -2178,32 +2190,32 @@ llvm::Value *CodeGenFunction::EmitARCRetainAutorelease(QualType type, llvm::Value * CodeGenFunction::EmitARCRetainAutoreleaseNonBlock(llvm::Value *value) { return emitARCValueOperation(*this, value, - CGM.getARCEntrypoints().objc_retainAutorelease, + CGM.getObjCEntrypoints().objc_retainAutorelease, "objc_retainAutorelease"); } /// i8* \@objc_loadWeak(i8** %addr) /// Essentially objc_autorelease(objc_loadWeakRetained(addr)). -llvm::Value *CodeGenFunction::EmitARCLoadWeak(llvm::Value *addr) { +llvm::Value *CodeGenFunction::EmitARCLoadWeak(Address addr) { return emitARCLoadOperation(*this, addr, - CGM.getARCEntrypoints().objc_loadWeak, + CGM.getObjCEntrypoints().objc_loadWeak, "objc_loadWeak"); } /// i8* \@objc_loadWeakRetained(i8** %addr) -llvm::Value *CodeGenFunction::EmitARCLoadWeakRetained(llvm::Value *addr) { +llvm::Value *CodeGenFunction::EmitARCLoadWeakRetained(Address addr) { return emitARCLoadOperation(*this, addr, - CGM.getARCEntrypoints().objc_loadWeakRetained, + CGM.getObjCEntrypoints().objc_loadWeakRetained, "objc_loadWeakRetained"); } /// i8* \@objc_storeWeak(i8** %addr, i8* %value) /// Returns %value. -llvm::Value *CodeGenFunction::EmitARCStoreWeak(llvm::Value *addr, +llvm::Value *CodeGenFunction::EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored) { return emitARCStoreOperation(*this, addr, value, - CGM.getARCEntrypoints().objc_storeWeak, + CGM.getObjCEntrypoints().objc_storeWeak, "objc_storeWeak", ignored); } @@ -2211,7 +2223,7 @@ llvm::Value *CodeGenFunction::EmitARCStoreWeak(llvm::Value *addr, /// Returns %value. %addr is known to not have a current weak entry. /// Essentially equivalent to: /// *addr = nil; objc_storeWeak(addr, value); -void CodeGenFunction::EmitARCInitWeak(llvm::Value *addr, llvm::Value *value) { +void CodeGenFunction::EmitARCInitWeak(Address addr, llvm::Value *value) { // If we're initializing to null, just write null to memory; no need // to get the runtime involved. But don't do this if optimization // is enabled, because accounting for this would make the optimizer @@ -2223,14 +2235,14 @@ void CodeGenFunction::EmitARCInitWeak(llvm::Value *addr, llvm::Value *value) { } emitARCStoreOperation(*this, addr, value, - CGM.getARCEntrypoints().objc_initWeak, + CGM.getObjCEntrypoints().objc_initWeak, "objc_initWeak", /*ignored*/ true); } /// void \@objc_destroyWeak(i8** %addr) /// Essentially objc_storeWeak(addr, nil). -void CodeGenFunction::EmitARCDestroyWeak(llvm::Value *addr) { - llvm::Constant *&fn = CGM.getARCEntrypoints().objc_destroyWeak; +void CodeGenFunction::EmitARCDestroyWeak(Address addr) { + llvm::Constant *&fn = CGM.getObjCEntrypoints().objc_destroyWeak; if (!fn) { llvm::FunctionType *fnType = llvm::FunctionType::get(Builder.getVoidTy(), Int8PtrPtrTy, false); @@ -2240,31 +2252,31 @@ void CodeGenFunction::EmitARCDestroyWeak(llvm::Value *addr) { // Cast the argument to 'id*'. addr = Builder.CreateBitCast(addr, Int8PtrPtrTy); - EmitNounwindRuntimeCall(fn, addr); + EmitNounwindRuntimeCall(fn, addr.getPointer()); } /// void \@objc_moveWeak(i8** %dest, i8** %src) /// Disregards the current value in %dest. Leaves %src pointing to nothing. /// Essentially (objc_copyWeak(dest, src), objc_destroyWeak(src)). -void CodeGenFunction::EmitARCMoveWeak(llvm::Value *dst, llvm::Value *src) { +void CodeGenFunction::EmitARCMoveWeak(Address dst, Address src) { emitARCCopyOperation(*this, dst, src, - CGM.getARCEntrypoints().objc_moveWeak, + CGM.getObjCEntrypoints().objc_moveWeak, "objc_moveWeak"); } /// void \@objc_copyWeak(i8** %dest, i8** %src) /// Disregards the current value in %dest. Essentially /// objc_release(objc_initWeak(dest, objc_readWeakRetained(src))) -void CodeGenFunction::EmitARCCopyWeak(llvm::Value *dst, llvm::Value *src) { +void CodeGenFunction::EmitARCCopyWeak(Address dst, Address src) { emitARCCopyOperation(*this, dst, src, - CGM.getARCEntrypoints().objc_copyWeak, + CGM.getObjCEntrypoints().objc_copyWeak, "objc_copyWeak"); } /// Produce the code to do a objc_autoreleasepool_push. /// call i8* \@objc_autoreleasePoolPush(void) llvm::Value *CodeGenFunction::EmitObjCAutoreleasePoolPush() { - llvm::Constant *&fn = CGM.getRREntrypoints().objc_autoreleasePoolPush; + llvm::Constant *&fn = CGM.getObjCEntrypoints().objc_autoreleasePoolPush; if (!fn) { llvm::FunctionType *fnType = llvm::FunctionType::get(Int8PtrTy, false); @@ -2279,7 +2291,7 @@ llvm::Value *CodeGenFunction::EmitObjCAutoreleasePoolPush() { void CodeGenFunction::EmitObjCAutoreleasePoolPop(llvm::Value *value) { assert(value->getType() == Int8PtrTy); - llvm::Constant *&fn = CGM.getRREntrypoints().objc_autoreleasePoolPop; + llvm::Constant *&fn = CGM.getObjCEntrypoints().objc_autoreleasePoolPop; if (!fn) { llvm::FunctionType *fnType = llvm::FunctionType::get(Builder.getVoidTy(), Int8PtrTy, false); @@ -2332,25 +2344,25 @@ void CodeGenFunction::EmitObjCMRRAutoreleasePoolPop(llvm::Value *Arg) { } void CodeGenFunction::destroyARCStrongPrecise(CodeGenFunction &CGF, - llvm::Value *addr, + Address addr, QualType type) { CGF.EmitARCDestroyStrong(addr, ARCPreciseLifetime); } void CodeGenFunction::destroyARCStrongImprecise(CodeGenFunction &CGF, - llvm::Value *addr, + Address addr, QualType type) { CGF.EmitARCDestroyStrong(addr, ARCImpreciseLifetime); } void CodeGenFunction::destroyARCWeak(CodeGenFunction &CGF, - llvm::Value *addr, + Address addr, QualType type) { CGF.EmitARCDestroyWeak(addr); } namespace { - struct CallObjCAutoreleasePoolObject : EHScopeStack::Cleanup { + struct CallObjCAutoreleasePoolObject final : EHScopeStack::Cleanup { llvm::Value *Token; CallObjCAutoreleasePoolObject(llvm::Value *token) : Token(token) {} @@ -2359,7 +2371,7 @@ namespace { CGF.EmitObjCAutoreleasePoolPop(Token); } }; - struct CallObjCMRRAutoreleasePoolObject : EHScopeStack::Cleanup { + struct CallObjCMRRAutoreleasePoolObject final : EHScopeStack::Cleanup { llvm::Value *Token; CallObjCMRRAutoreleasePoolObject(llvm::Value *token) : Token(token) {} @@ -2932,7 +2944,9 @@ CodeGenFunction::GenerateObjCAtomicSetterCopyHelperFunction( llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage, "__assign_helper_atomic_property_", &CGM.getModule()); - + + CGM.SetInternalFunctionAttributes(nullptr, Fn, FI); + StartFunction(FD, C.VoidTy, Fn, FI, args); DeclRefExpr DstExpr(&dstDecl, false, DestTy, @@ -3011,6 +3025,8 @@ CodeGenFunction::GenerateObjCAtomicGetterCopyHelperFunction( llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage, "__copy_helper_atomic_property_", &CGM.getModule()); + CGM.SetInternalFunctionAttributes(nullptr, Fn, FI); + StartFunction(FD, C.VoidTy, Fn, FI, args); DeclRefExpr SrcExpr(&srcDecl, false, SrcTy, @@ -3046,7 +3062,8 @@ CodeGenFunction::GenerateObjCAtomicGetterCopyHelperFunction( CharUnits Alignment = getContext().getTypeAlignInChars(TheCXXConstructExpr->getType()); EmitAggExpr(TheCXXConstructExpr, - AggValueSlot::forAddr(DV.getScalarVal(), Alignment, Qualifiers(), + AggValueSlot::forAddr(Address(DV.getScalarVal(), Alignment), + Qualifiers(), AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased)); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp index b52d623..f0af3e9 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp @@ -166,9 +166,9 @@ protected: /// where the C code specifies const char*. llvm::Constant *MakeConstantString(const std::string &Str, const std::string &Name="") { - auto *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str()); - return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), - ConstStr, Zeros); + ConstantAddress Array = CGM.GetAddrOfConstantCString(Str, Name.c_str()); + return llvm::ConstantExpr::getGetElementPtr(Array.getElementType(), + Array.getPointer(), Zeros); } /// Emits a linkonce_odr string, whose name is the prefix followed by the /// string value. This allows the linker to combine the strings between @@ -191,34 +191,41 @@ protected: /// first argument. llvm::GlobalVariable *MakeGlobal(llvm::StructType *Ty, ArrayRef<llvm::Constant *> V, + CharUnits Align, StringRef Name="", llvm::GlobalValue::LinkageTypes linkage =llvm::GlobalValue::InternalLinkage) { llvm::Constant *C = llvm::ConstantStruct::get(Ty, V); - return new llvm::GlobalVariable(TheModule, Ty, false, - linkage, C, Name); + auto GV = new llvm::GlobalVariable(TheModule, Ty, false, + linkage, C, Name); + GV->setAlignment(Align.getQuantity()); + return GV; } /// Generates a global array. The vector must contain the same number of /// elements that the array type declares, of the type specified as the array /// element type. llvm::GlobalVariable *MakeGlobal(llvm::ArrayType *Ty, ArrayRef<llvm::Constant *> V, + CharUnits Align, StringRef Name="", llvm::GlobalValue::LinkageTypes linkage =llvm::GlobalValue::InternalLinkage) { llvm::Constant *C = llvm::ConstantArray::get(Ty, V); - return new llvm::GlobalVariable(TheModule, Ty, false, - linkage, C, Name); + auto GV = new llvm::GlobalVariable(TheModule, Ty, false, + linkage, C, Name); + GV->setAlignment(Align.getQuantity()); + return GV; } /// Generates a global array, inferring the array type from the specified /// element type and the size of the initialiser. llvm::GlobalVariable *MakeGlobalArray(llvm::Type *Ty, ArrayRef<llvm::Constant *> V, + CharUnits Align, StringRef Name="", llvm::GlobalValue::LinkageTypes linkage =llvm::GlobalValue::InternalLinkage) { llvm::ArrayType *ArrayTy = llvm::ArrayType::get(Ty, V.size()); - return MakeGlobal(ArrayTy, V, Name, linkage); + return MakeGlobal(ArrayTy, V, Align, Name, linkage); } /// Returns a property name and encoding string. llvm::Constant *MakePropertyEncodingString(const ObjCPropertyDecl *PD, @@ -234,9 +241,7 @@ protected: NameAndAttributes += TypeStr; NameAndAttributes += '\0'; NameAndAttributes += PD->getNameAsString(); - auto *ConstStr = CGM.GetAddrOfConstantCString(NameAndAttributes); - return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), - ConstStr, Zeros); + return MakeConstantString(NameAndAttributes); } return MakeConstantString(PD->getNameAsString()); } @@ -275,6 +280,10 @@ protected: if (V->getType() == Ty) return V; return B.CreateBitCast(V, Ty); } + Address EnforceType(CGBuilderTy &B, Address V, llvm::Type *Ty) { + if (V.getType() == Ty) return V; + return B.CreateBitCast(V, Ty); + } // Some zeros used for GEPs in lots of places. llvm::Constant *Zeros[2]; /// Null pointer value. Mainly used as a terminator in various arrays. @@ -435,7 +444,7 @@ private: /// Returns a selector with the specified type encoding. An empty string is /// used to return an untyped selector (with the types field set to NULL). llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel, - const std::string &TypeEncoding, bool lval); + const std::string &TypeEncoding); /// Returns the variable used to store the offset of an instance variable. llvm::GlobalVariable *ObjCIvarOffsetVariable(const ObjCInterfaceDecl *ID, const ObjCIvarDecl *Ivar); @@ -458,7 +467,7 @@ protected: /// mechanism differs between the GCC and GNU runtimes, so this method must /// be overridden in subclasses. virtual llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, - llvm::Value *ObjCSuper, + Address ObjCSuper, llvm::Value *cmd, MessageSendInfo &MSI) = 0; /// Libobjc2 uses a bitfield representation where small(ish) bitfields are @@ -477,7 +486,7 @@ public: CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion, unsigned protocolClassVersion); - llvm::Constant *GenerateConstantString(const StringLiteral *) override; + ConstantAddress GenerateConstantString(const StringLiteral *) override; RValue GenerateMessageSend(CodeGenFunction &CGF, ReturnValueSlot Return, @@ -494,8 +503,8 @@ public: const ObjCMethodDecl *Method) override; llvm::Value *GetClass(CodeGenFunction &CGF, const ObjCInterfaceDecl *OID) override; - llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel, - bool lval = false) override; + llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel) override; + Address GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) override; llvm::Value *GetSelector(CodeGenFunction &CGF, const ObjCMethodDecl *Method) override; llvm::Constant *GetEHType(QualType T) override; @@ -527,18 +536,18 @@ public: const ObjCAtThrowStmt &S, bool ClearInsertionPoint=true) override; llvm::Value * EmitObjCWeakRead(CodeGenFunction &CGF, - llvm::Value *AddrWeakObj) override; + Address AddrWeakObj) override; void EmitObjCWeakAssign(CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst) override; + llvm::Value *src, Address dst) override; void EmitObjCGlobalAssign(CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest, + llvm::Value *src, Address dest, bool threadlocal=false) override; void EmitObjCIvarAssign(CodeGenFunction &CGF, llvm::Value *src, - llvm::Value *dest, llvm::Value *ivarOffset) override; + Address dest, llvm::Value *ivarOffset) override; void EmitObjCStrongCastAssign(CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest) override; - void EmitGCMemmoveCollectable(CodeGenFunction &CGF, llvm::Value *DestPtr, - llvm::Value *SrcPtr, + llvm::Value *src, Address dest) override; + void EmitGCMemmoveCollectable(CodeGenFunction &CGF, Address DestPtr, + Address SrcPtr, llvm::Value *Size) override; LValue EmitObjCValueForIvar(CodeGenFunction &CGF, QualType ObjectTy, llvm::Value *BaseValue, const ObjCIvarDecl *Ivar, @@ -593,11 +602,11 @@ protected: imp->setMetadata(msgSendMDKind, node); return imp.getInstruction(); } - llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, llvm::Value *ObjCSuper, + llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, Address ObjCSuper, llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; llvm::Value *lookupArgs[] = {EnforceType(Builder, ObjCSuper, - PtrToObjCSuperTy), cmd}; + PtrToObjCSuperTy).getPointer(), cmd}; return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs); } public: @@ -647,7 +656,8 @@ class CGObjCGNUstep : public CGObjCGNU { llvm::Function *LookupFn = SlotLookupFn; // Store the receiver on the stack so that we can reload it later - llvm::Value *ReceiverPtr = CGF.CreateTempAlloca(Receiver->getType()); + Address ReceiverPtr = + CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign()); Builder.CreateStore(Receiver, ReceiverPtr); llvm::Value *self; @@ -662,7 +672,7 @@ class CGObjCGNUstep : public CGObjCGNU { LookupFn->setDoesNotCapture(1); llvm::Value *args[] = { - EnforceType(Builder, ReceiverPtr, PtrToIdTy), + EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy), EnforceType(Builder, cmd, SelectorTy), EnforceType(Builder, self, IdTy) }; llvm::CallSite slot = CGF.EmitRuntimeCallOrInvoke(LookupFn, args); @@ -670,25 +680,27 @@ class CGObjCGNUstep : public CGObjCGNU { slot->setMetadata(msgSendMDKind, node); // Load the imp from the slot - llvm::Value *imp = Builder.CreateLoad( - Builder.CreateStructGEP(nullptr, slot.getInstruction(), 4)); + llvm::Value *imp = Builder.CreateAlignedLoad( + Builder.CreateStructGEP(nullptr, slot.getInstruction(), 4), + CGF.getPointerAlign()); // The lookup function may have changed the receiver, so make sure we use // the new one. Receiver = Builder.CreateLoad(ReceiverPtr, true); return imp; } - llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, llvm::Value *ObjCSuper, + llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, Address ObjCSuper, llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; - llvm::Value *lookupArgs[] = {ObjCSuper, cmd}; + llvm::Value *lookupArgs[] = {ObjCSuper.getPointer(), cmd}; llvm::CallInst *slot = CGF.EmitNounwindRuntimeCall(SlotLookupSuperFn, lookupArgs); slot->setOnlyReadsMemory(); - return Builder.CreateLoad(Builder.CreateStructGEP(nullptr, slot, 4)); + return Builder.CreateAlignedLoad(Builder.CreateStructGEP(nullptr, slot, 4), + CGF.getPointerAlign()); } public: CGObjCGNUstep(CodeGenModule &Mod) : CGObjCGNU(Mod, 9, 3) { @@ -807,10 +819,10 @@ protected: return imp.getInstruction(); } - llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, llvm::Value *ObjCSuper, + llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, Address ObjCSuper, llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; - llvm::Value *lookupArgs[] = {EnforceType(Builder, ObjCSuper, + llvm::Value *lookupArgs[] = {EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd}; if (CGM.ReturnTypeUsesSRet(MSI.CallInfo)) @@ -1011,7 +1023,7 @@ CGObjCGNU::CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion, llvm::Value *CGObjCGNU::GetClassNamed(CodeGenFunction &CGF, const std::string &Name, bool isWeak) { - llvm::GlobalVariable *ClassNameGV = CGM.GetAddrOfConstantCString(Name); + llvm::Constant *ClassName = MakeConstantString(Name); // With the incompatible ABI, this will need to be replaced with a direct // reference to the class symbol. For the compatible nonfragile ABI we are // still performing this lookup at run time but emitting the symbol for the @@ -1021,8 +1033,6 @@ llvm::Value *CGObjCGNU::GetClassNamed(CodeGenFunction &CGF, // with memoized versions or with static references if it's safe to do so. if (!isWeak) EmitClassRef(Name); - llvm::Value *ClassName = - CGF.Builder.CreateStructGEP(ClassNameGV->getValueType(), ClassNameGV, 0); llvm::Constant *ClassLookupFn = CGM.CreateRuntimeFunction(llvm::FunctionType::get(IdTy, PtrToInt8Ty, true), @@ -1041,7 +1051,7 @@ llvm::Value *CGObjCGNU::EmitNSAutoreleasePoolClassRef(CodeGenFunction &CGF) { } llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, Selector Sel, - const std::string &TypeEncoding, bool lval) { + const std::string &TypeEncoding) { SmallVectorImpl<TypedSelector> &Types = SelectorTable[Sel]; llvm::GlobalAlias *SelValue = nullptr; @@ -1055,29 +1065,34 @@ llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, Selector Sel, } if (!SelValue) { SelValue = llvm::GlobalAlias::create( - SelectorTy, llvm::GlobalValue::PrivateLinkage, + SelectorTy->getElementType(), 0, llvm::GlobalValue::PrivateLinkage, ".objc_selector_" + Sel.getAsString(), &TheModule); Types.emplace_back(TypeEncoding, SelValue); } - if (lval) { - llvm::Value *tmp = CGF.CreateTempAlloca(SelValue->getType()); - CGF.Builder.CreateStore(SelValue, tmp); - return tmp; - } return SelValue; } -llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, Selector Sel, - bool lval) { - return GetSelector(CGF, Sel, std::string(), lval); +Address CGObjCGNU::GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) { + llvm::Value *SelValue = GetSelector(CGF, Sel); + + // Store it to a temporary. Does this satisfy the semantics of + // GetAddrOfSelector? Hopefully. + Address tmp = CGF.CreateTempAlloca(SelValue->getType(), + CGF.getPointerAlign()); + CGF.Builder.CreateStore(SelValue, tmp); + return tmp; +} + +llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, Selector Sel) { + return GetSelector(CGF, Sel, std::string()); } llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, const ObjCMethodDecl *Method) { std::string SelTypes; CGM.getContext().getObjCEncodingForMethodDecl(Method, SelTypes); - return GetSelector(CGF, Method->getSelector(), SelTypes, false); + return GetSelector(CGF, Method->getSelector(), SelTypes); } llvm::Constant *CGObjCGNU::GetEHType(QualType T) { @@ -1160,21 +1175,23 @@ llvm::Constant *CGObjCGNUstep::GetEHType(QualType T) { fields.push_back(BVtable); fields.push_back(typeName); llvm::Constant *TI = - MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty, - nullptr), fields, "__objc_eh_typeinfo_" + className, + MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty, nullptr), + fields, CGM.getPointerAlign(), + "__objc_eh_typeinfo_" + className, llvm::GlobalValue::LinkOnceODRLinkage); return llvm::ConstantExpr::getBitCast(TI, PtrToInt8Ty); } /// Generate an NSConstantString object. -llvm::Constant *CGObjCGNU::GenerateConstantString(const StringLiteral *SL) { +ConstantAddress CGObjCGNU::GenerateConstantString(const StringLiteral *SL) { std::string Str = SL->getString().str(); + CharUnits Align = CGM.getPointerAlign(); // Look for an existing one llvm::StringMap<llvm::Constant*>::iterator old = ObjCStrings.find(Str); if (old != ObjCStrings.end()) - return old->getValue(); + return ConstantAddress(old->getValue(), Align); StringRef StringClass = CGM.getLangOpts().ObjCConstantStringClass; @@ -1197,11 +1214,11 @@ llvm::Constant *CGObjCGNU::GenerateConstantString(const StringLiteral *SL) { Ivars.push_back(llvm::ConstantInt::get(IntTy, Str.size())); llvm::Constant *ObjCStr = MakeGlobal( llvm::StructType::get(PtrToIdTy, PtrToInt8Ty, IntTy, nullptr), - Ivars, ".objc_str"); + Ivars, Align, ".objc_str"); ObjCStr = llvm::ConstantExpr::getBitCast(ObjCStr, PtrToInt8Ty); ObjCStrings[Str] = ObjCStr; ConstantStrings.push_back(ObjCStr); - return ObjCStr; + return ConstantAddress(ObjCStr, Align); } ///Generates a message send where the super is the receiver. This is a message @@ -1261,14 +1278,14 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF, if (IsClassMessage) { if (!MetaClassPtrAlias) { MetaClassPtrAlias = llvm::GlobalAlias::create( - IdTy, llvm::GlobalValue::InternalLinkage, + IdTy->getElementType(), 0, llvm::GlobalValue::InternalLinkage, ".objc_metaclass_ref" + Class->getNameAsString(), &TheModule); } ReceiverClass = MetaClassPtrAlias; } else { if (!ClassPtrAlias) { ClassPtrAlias = llvm::GlobalAlias::create( - IdTy, llvm::GlobalValue::InternalLinkage, + IdTy->getElementType(), 0, llvm::GlobalValue::InternalLinkage, ".objc_class_ref" + Class->getNameAsString(), &TheModule); } ReceiverClass = ClassPtrAlias; @@ -1281,16 +1298,20 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF, // Get the superclass pointer ReceiverClass = Builder.CreateStructGEP(CastTy, ReceiverClass, 1); // Load the superclass pointer - ReceiverClass = Builder.CreateLoad(ReceiverClass); + ReceiverClass = + Builder.CreateAlignedLoad(ReceiverClass, CGF.getPointerAlign()); // Construct the structure used to look up the IMP llvm::StructType *ObjCSuperTy = llvm::StructType::get( Receiver->getType(), IdTy, nullptr); - llvm::Value *ObjCSuper = Builder.CreateAlloca(ObjCSuperTy); + + // FIXME: Is this really supposed to be a dynamic alloca? + Address ObjCSuper = Address(Builder.CreateAlloca(ObjCSuperTy), + CGF.getPointerAlign()); Builder.CreateStore(Receiver, - Builder.CreateStructGEP(ObjCSuperTy, ObjCSuper, 0)); + Builder.CreateStructGEP(ObjCSuper, 0, CharUnits::Zero())); Builder.CreateStore(ReceiverClass, - Builder.CreateStructGEP(ObjCSuperTy, ObjCSuper, 1)); + Builder.CreateStructGEP(ObjCSuper, 1, CGF.getPointerSize())); ObjCSuper = EnforceType(Builder, ObjCSuper, PtrToObjCSuperTy); @@ -1306,8 +1327,8 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF, llvm::MDNode *node = llvm::MDNode::get(VMContext, impMD); llvm::Instruction *call; - RValue msgRet = CGF.EmitCall(MSI.CallInfo, imp, Return, ActualArgs, nullptr, - &call); + RValue msgRet = CGF.EmitCall(MSI.CallInfo, imp, Return, ActualArgs, + CGCalleeInfo(), &call); call->setMetadata(msgSendMDKind, node); return msgRet; } @@ -1419,8 +1440,8 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF, imp = EnforceType(Builder, imp, MSI.MessengerType); llvm::Instruction *call; - RValue msgRet = CGF.EmitCall(MSI.CallInfo, imp, Return, ActualArgs, nullptr, - &call); + RValue msgRet = CGF.EmitCall(MSI.CallInfo, imp, Return, ActualArgs, + CGCalleeInfo(), &call); call->setMetadata(msgSendMDKind, node); @@ -1435,16 +1456,14 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF, phi->addIncoming(llvm::Constant::getNullValue(v->getType()), startBB); msgRet = RValue::get(phi); } else if (msgRet.isAggregate()) { - llvm::Value *v = msgRet.getAggregateAddr(); - llvm::PHINode *phi = Builder.CreatePHI(v->getType(), 2); - llvm::PointerType *RetTy = cast<llvm::PointerType>(v->getType()); - llvm::AllocaInst *NullVal = - CGF.CreateTempAlloca(RetTy->getElementType(), "null"); - CGF.InitTempAlloca(NullVal, - llvm::Constant::getNullValue(RetTy->getElementType())); - phi->addIncoming(v, messageBB); - phi->addIncoming(NullVal, startBB); - msgRet = RValue::getAggregate(phi); + Address v = msgRet.getAggregateAddress(); + llvm::PHINode *phi = Builder.CreatePHI(v.getType(), 2); + llvm::Type *RetTy = v.getElementType(); + Address NullVal = CGF.CreateTempAlloca(RetTy, v.getAlignment(), "null"); + CGF.InitTempAlloca(NullVal, llvm::Constant::getNullValue(RetTy)); + phi->addIncoming(v.getPointer(), messageBB); + phi->addIncoming(NullVal.getPointer(), startBB); + msgRet = RValue::getAggregate(Address(phi, v.getAlignment())); } else /* isComplex() */ { std::pair<llvm::Value*,llvm::Value*> v = msgRet.getComplexVal(); llvm::PHINode *phi = Builder.CreatePHI(v.first->getType(), 2); @@ -1517,7 +1536,8 @@ GenerateMethodList(StringRef ClassName, Methods.push_back(MethodArray); // Create an instance of the structure - return MakeGlobal(ObjCMethodListTy, Methods, ".objc_method_list"); + return MakeGlobal(ObjCMethodListTy, Methods, CGM.getPointerAlign(), + ".objc_method_list"); } /// Generates an IvarList. Used in construction of a objc_class. @@ -1557,7 +1577,8 @@ GenerateIvarList(ArrayRef<llvm::Constant *> IvarNames, nullptr); // Create an instance of the structure - return MakeGlobal(ObjCIvarListTy, Elements, ".objc_ivar_list"); + return MakeGlobal(ObjCIvarListTy, Elements, CGM.getPointerAlign(), + ".objc_ivar_list"); } /// Generate a class structure @@ -1640,8 +1661,9 @@ llvm::Constant *CGObjCGNU::GenerateClassStructure( std::string ClassSym((isMeta ? "_OBJC_METACLASS_": "_OBJC_CLASS_") + std::string(Name)); llvm::GlobalVariable *ClassRef = TheModule.getNamedGlobal(ClassSym); - llvm::Constant *Class = MakeGlobal(ClassTy, Elements, ClassSym, - llvm::GlobalValue::ExternalLinkage); + llvm::Constant *Class = + MakeGlobal(ClassTy, Elements, CGM.getPointerAlign(), ClassSym, + llvm::GlobalValue::ExternalLinkage); if (ClassRef) { ClassRef->replaceAllUsesWith(llvm::ConstantExpr::getBitCast(Class, ClassRef->getType())); @@ -1676,7 +1698,8 @@ GenerateProtocolMethodList(ArrayRef<llvm::Constant *> MethodNames, Methods.clear(); Methods.push_back(llvm::ConstantInt::get(IntTy, MethodNames.size())); Methods.push_back(Array); - return MakeGlobal(ObjCMethodDescListTy, Methods, ".objc_method_list"); + return MakeGlobal(ObjCMethodDescListTy, Methods, CGM.getPointerAlign(), + ".objc_method_list"); } // Create the protocol list structure used in classes, categories and so on @@ -1709,7 +1732,8 @@ llvm::Constant *CGObjCGNU::GenerateProtocolList(ArrayRef<std::string>Protocols){ Elements.push_back(NULLPtr); Elements.push_back(llvm::ConstantInt::get(LongTy, Protocols.size())); Elements.push_back(ProtocolArray); - return MakeGlobal(ProtocolListTy, Elements, ".objc_protocol_list"); + return MakeGlobal(ProtocolListTy, Elements, CGM.getPointerAlign(), + ".objc_protocol_list"); } llvm::Value *CGObjCGNU::GenerateProtocolRef(CodeGenFunction &CGF, @@ -1749,7 +1773,8 @@ llvm::Constant *CGObjCGNU::GenerateEmptyProtocol( Elements.push_back(MethodList); Elements.push_back(MethodList); Elements.push_back(MethodList); - return MakeGlobal(ProtocolTy, Elements, ".objc_protocol"); + return MakeGlobal(ProtocolTy, Elements, CGM.getPointerAlign(), + ".objc_protocol"); } void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) { @@ -1910,7 +1935,7 @@ void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) { Elements.push_back(OptionalPropertyList); ExistingProtocols[ProtocolName] = llvm::ConstantExpr::getBitCast(MakeGlobal(ProtocolTy, Elements, - ".objc_protocol"), IdTy); + CGM.getPointerAlign(), ".objc_protocol"), IdTy); } void CGObjCGNU::GenerateProtocolHolderCategory() { // Collect information about instance methods @@ -1952,10 +1977,12 @@ void CGObjCGNU::GenerateProtocolHolderCategory() { ExistingProtocols.size())); ProtocolElements.push_back(ProtocolArray); Elements.push_back(llvm::ConstantExpr::getBitCast(MakeGlobal(ProtocolListTy, - ProtocolElements, ".objc_protocol_list"), PtrTy)); + ProtocolElements, CGM.getPointerAlign(), + ".objc_protocol_list"), PtrTy)); Categories.push_back(llvm::ConstantExpr::getBitCast( MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty, - PtrTy, PtrTy, PtrTy, nullptr), Elements), PtrTy)); + PtrTy, PtrTy, PtrTy, nullptr), Elements, CGM.getPointerAlign()), + PtrTy)); } /// Libobjc2 uses a bitfield representation where small(ish) bitfields are @@ -1995,7 +2022,7 @@ llvm::Constant *CGObjCGNU::MakeBitField(ArrayRef<bool> bits) { llvm::ConstantInt::get(Int32Ty, values.size()), array }; llvm::Constant *GS = MakeGlobal(llvm::StructType::get(Int32Ty, arrayTy, - nullptr), fields); + nullptr), fields, CharUnits::fromQuantity(4)); llvm::Constant *ptr = llvm::ConstantExpr::getPtrToInt(GS, IntPtrTy); return ptr; } @@ -2047,7 +2074,8 @@ void CGObjCGNU::GenerateCategory(const ObjCCategoryImplDecl *OCD) { GenerateProtocolList(Protocols), PtrTy)); Categories.push_back(llvm::ConstantExpr::getBitCast( MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty, - PtrTy, PtrTy, PtrTy, nullptr), Elements), PtrTy)); + PtrTy, PtrTy, PtrTy, nullptr), Elements, CGM.getPointerAlign()), + PtrTy)); } llvm::Constant *CGObjCGNU::GeneratePropertyList(const ObjCImplementationDecl *OID, @@ -2225,7 +2253,8 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) { llvm::Constant *StrongIvarBitmap = MakeBitField(StrongIvars); llvm::Constant *WeakIvarBitmap = MakeBitField(WeakIvars); llvm::GlobalVariable *IvarOffsetArray = - MakeGlobalArray(PtrToIntTy, IvarOffsetValues, ".ivar.offsets"); + MakeGlobalArray(PtrToIntTy, IvarOffsetValues, CGM.getPointerAlign(), + ".ivar.offsets"); // Collect information about instance methods @@ -2385,13 +2414,15 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { llvm::StructType::get(PtrToInt8Ty, StaticsArrayTy, nullptr); llvm::Type *StaticsListPtrTy = llvm::PointerType::getUnqual(StaticsListTy); - Statics = MakeGlobal(StaticsListTy, Elements, ".objc_statics"); + Statics = MakeGlobal(StaticsListTy, Elements, CGM.getPointerAlign(), + ".objc_statics"); llvm::ArrayType *StaticsListArrayTy = llvm::ArrayType::get(StaticsListPtrTy, 2); Elements.clear(); Elements.push_back(Statics); Elements.push_back(llvm::Constant::getNullValue(StaticsListPtrTy)); - Statics = MakeGlobal(StaticsListArrayTy, Elements, ".objc_statics_ptr"); + Statics = MakeGlobal(StaticsListArrayTy, Elements, + CGM.getPointerAlign(), ".objc_statics_ptr"); Statics = llvm::ConstantExpr::getBitCast(Statics, PtrTy); } // Array of classes, categories, and constant objects @@ -2442,7 +2473,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { // Number of static selectors Elements.push_back(llvm::ConstantInt::get(LongTy, SelectorCount)); llvm::GlobalVariable *SelectorList = - MakeGlobalArray(SelStructTy, Selectors, ".objc_selector_list"); + MakeGlobalArray(SelStructTy, Selectors, CGM.getPointerAlign(), + ".objc_selector_list"); Elements.push_back(llvm::ConstantExpr::getBitCast(SelectorList, SelStructPtrTy)); @@ -2475,7 +2507,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { llvm::Constant *ClassList = llvm::ConstantArray::get(ClassListTy, Classes); Elements.push_back(ClassList); // Construct the symbol table - llvm::Constant *SymTab= MakeGlobal(SymTabTy, Elements); + llvm::Constant *SymTab = + MakeGlobal(SymTabTy, Elements, CGM.getPointerAlign()); // The symbol table is contained in a module which has some version-checking // constants @@ -2516,7 +2549,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { break; } - llvm::Value *Module = MakeGlobal(ModuleTy, Elements); + llvm::Value *Module = MakeGlobal(ModuleTy, Elements, CGM.getPointerAlign()); // Create the load function calling the runtime entry point with the module // structure @@ -2526,7 +2559,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { &TheModule); llvm::BasicBlock *EntryBB = llvm::BasicBlock::Create(VMContext, "entry", LoadFunction); - CGBuilderTy Builder(VMContext); + CGBuilderTy Builder(CGM, VMContext); Builder.SetInsertPoint(EntryBB); llvm::FunctionType *FT = @@ -2678,57 +2711,63 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF, } llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF, - llvm::Value *AddrWeakObj) { + Address AddrWeakObj) { CGBuilderTy &B = CGF.Builder; AddrWeakObj = EnforceType(B, AddrWeakObj, PtrToIdTy); - return B.CreateCall(WeakReadFn.getType(), WeakReadFn, AddrWeakObj); + return B.CreateCall(WeakReadFn.getType(), WeakReadFn, + AddrWeakObj.getPointer()); } void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst) { + llvm::Value *src, Address dst) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, PtrToIdTy); - B.CreateCall(WeakAssignFn.getType(), WeakAssignFn, {src, dst}); + B.CreateCall(WeakAssignFn.getType(), WeakAssignFn, + {src, dst.getPointer()}); } void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst, + llvm::Value *src, Address dst, bool threadlocal) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, PtrToIdTy); // FIXME. Add threadloca assign API assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI"); - B.CreateCall(GlobalAssignFn.getType(), GlobalAssignFn, {src, dst}); + B.CreateCall(GlobalAssignFn.getType(), GlobalAssignFn, + {src, dst.getPointer()}); } void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst, + llvm::Value *src, Address dst, llvm::Value *ivarOffset) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, IdTy); - B.CreateCall(IvarAssignFn.getType(), IvarAssignFn, {src, dst, ivarOffset}); + B.CreateCall(IvarAssignFn.getType(), IvarAssignFn, + {src, dst.getPointer(), ivarOffset}); } void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst) { + llvm::Value *src, Address dst) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, PtrToIdTy); - B.CreateCall(StrongCastAssignFn.getType(), StrongCastAssignFn, {src, dst}); + B.CreateCall(StrongCastAssignFn.getType(), StrongCastAssignFn, + {src, dst.getPointer()}); } void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF, - llvm::Value *DestPtr, - llvm::Value *SrcPtr, + Address DestPtr, + Address SrcPtr, llvm::Value *Size) { CGBuilderTy &B = CGF.Builder; DestPtr = EnforceType(B, DestPtr, PtrTy); SrcPtr = EnforceType(B, SrcPtr, PtrTy); - B.CreateCall(MemMoveFn.getType(), MemMoveFn, {DestPtr, SrcPtr, Size}); + B.CreateCall(MemMoveFn.getType(), MemMoveFn, + {DestPtr.getPointer(), SrcPtr.getPointer(), Size}); } llvm::GlobalVariable *CGObjCGNU::ObjCIvarOffsetVariable( @@ -2811,17 +2850,22 @@ llvm::Value *CGObjCGNU::EmitIvarOffset(CodeGenFunction &CGF, Interface = FindIvarInterface(CGM.getContext(), Interface, Ivar); if (RuntimeVersion < 10) return CGF.Builder.CreateZExtOrBitCast( - CGF.Builder.CreateLoad(CGF.Builder.CreateLoad( - ObjCIvarOffsetVariable(Interface, Ivar), false, "ivar")), + CGF.Builder.CreateDefaultAlignedLoad(CGF.Builder.CreateAlignedLoad( + ObjCIvarOffsetVariable(Interface, Ivar), + CGF.getPointerAlign(), "ivar")), PtrDiffTy); std::string name = "__objc_ivar_offset_value_" + Interface->getNameAsString() +"." + Ivar->getNameAsString(); + CharUnits Align = CGM.getIntAlign(); llvm::Value *Offset = TheModule.getGlobalVariable(name); - if (!Offset) - Offset = new llvm::GlobalVariable(TheModule, IntTy, + if (!Offset) { + auto GV = new llvm::GlobalVariable(TheModule, IntTy, false, llvm::GlobalValue::LinkOnceAnyLinkage, llvm::Constant::getNullValue(IntTy), name); - Offset = CGF.Builder.CreateLoad(Offset); + GV->setAlignment(Align.getQuantity()); + Offset = GV; + } + Offset = CGF.Builder.CreateAlignedLoad(Offset, Align); if (Offset->getType() != PtrDiffTy) Offset = CGF.Builder.CreateZExtOrBitCast(Offset, PtrDiffTy); return Offset; @@ -2845,6 +2889,7 @@ clang::CodeGen::CreateGNUObjCRuntime(CodeGenModule &CGM) { case ObjCRuntime::FragileMacOSX: case ObjCRuntime::MacOSX: case ObjCRuntime::iOS: + case ObjCRuntime::WatchOS: llvm_unreachable("these runtimes are not GNU runtimes"); } llvm_unreachable("bad runtime"); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp index a45446a..5f3ebbd 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp @@ -659,9 +659,6 @@ public: // MessageRefCPtrTy - clang type for struct _message_ref_t* QualType MessageRefCPtrTy; - // MessengerTy - Type of the messenger (shown as IMP above) - llvm::FunctionType *MessengerTy; - // SuperMessageRefTy - LLVM for: // struct _super_message_ref_t { // SUPER_IMP messenger; @@ -735,20 +732,6 @@ public: class CGObjCCommonMac : public CodeGen::CGObjCRuntime { public: - // FIXME - accessibility - class GC_IVAR { - public: - unsigned ivar_bytepos; - unsigned ivar_size; - GC_IVAR(unsigned bytepos = 0, unsigned size = 0) - : ivar_bytepos(bytepos), ivar_size(size) {} - - // Allow sorting based on byte pos. - bool operator<(const GC_IVAR &b) const { - return ivar_bytepos < b.ivar_bytepos; - } - }; - class SKIP_SCAN { public: unsigned skip; @@ -830,10 +813,6 @@ protected: // FIXME! May not be needing this after all. unsigned ObjCABI; - // gc ivar layout bitmap calculation helper caches. - SmallVector<GC_IVAR, 16> SkipIvars; - SmallVector<GC_IVAR, 16> IvarsInfo; - // arc/mrr layout of captured block literal variables. SmallVector<RUN_SKIP, 16> RunSkipBlockVars; @@ -854,7 +833,7 @@ protected: llvm::DenseMap<Selector, llvm::GlobalVariable*> MethodVarNames; /// DefinedCategoryNames - list of category names in form Class_Category. - llvm::SetVector<std::string> DefinedCategoryNames; + llvm::SmallSetVector<std::string, 16> DefinedCategoryNames; /// MethodVarTypes - uniqued method type signatures. We have to use /// a StringMap here because have no other unique reference. @@ -934,20 +913,28 @@ protected: /// BuildIvarLayout - Builds ivar layout bitmap for the class /// implementation for the __strong or __weak case. /// + /// \param hasMRCWeakIvars - Whether we are compiling in MRC and there + /// are any weak ivars defined directly in the class. Meaningless unless + /// building a weak layout. Does not guarantee that the layout will + /// actually have any entries, because the ivar might be under-aligned. llvm::Constant *BuildIvarLayout(const ObjCImplementationDecl *OI, - bool ForStrongLayout); - - llvm::Constant *BuildIvarLayoutBitmap(std::string &BitMap); + CharUnits beginOffset, + CharUnits endOffset, + bool forStrongLayout, + bool hasMRCWeakIvars); - void BuildAggrIvarRecordLayout(const RecordType *RT, - unsigned int BytePos, bool ForStrongLayout, - bool &HasUnion); - void BuildAggrIvarLayout(const ObjCImplementationDecl *OI, - const llvm::StructLayout *Layout, - const RecordDecl *RD, - ArrayRef<const FieldDecl*> RecFields, - unsigned int BytePos, bool ForStrongLayout, - bool &HasUnion); + llvm::Constant *BuildStrongIvarLayout(const ObjCImplementationDecl *OI, + CharUnits beginOffset, + CharUnits endOffset) { + return BuildIvarLayout(OI, beginOffset, endOffset, true, false); + } + + llvm::Constant *BuildWeakIvarLayout(const ObjCImplementationDecl *OI, + CharUnits beginOffset, + CharUnits endOffset, + bool hasMRCWeakIvars) { + return BuildIvarLayout(OI, beginOffset, endOffset, false, hasMRCWeakIvars); + } Qualifiers::ObjCLifetime getBlockCaptureLifetime(QualType QT, bool ByrefLayout); @@ -970,7 +957,6 @@ protected: llvm::Constant *getBitmapBlockLayout(bool ComputeByrefLayout); - /// GetIvarLayoutName - Returns a unique constant for the given /// ivar layout bitmap. llvm::Constant *GetIvarLayoutName(IdentifierInfo *Ident, @@ -1002,6 +988,7 @@ protected: /// defined. The return value has type ProtocolPtrTy. llvm::Constant *GetProtocolRef(const ObjCProtocolDecl *PD); +public: /// CreateMetadataVar - Create a global variable with internal /// linkage for use by the Objective-C runtime. /// @@ -1017,9 +1004,10 @@ protected: /// \param AddToUsed - Whether the variable should be added to /// "llvm.used". llvm::GlobalVariable *CreateMetadataVar(Twine Name, llvm::Constant *Init, - StringRef Section, unsigned Align, + StringRef Section, CharUnits Align, bool AddToUsed); +protected: CodeGen::RValue EmitMessageSend(CodeGen::CodeGenFunction &CGF, ReturnValueSlot Return, QualType ResultType, @@ -1029,6 +1017,7 @@ protected: bool IsSuper, const CallArgList &CallArgs, const ObjCMethodDecl *OMD, + const ObjCInterfaceDecl *ClassReceiver, const ObjCCommonTypesHelper &ObjCTypes); /// EmitImageInfo - Emit the image info marker used to encode some module @@ -1039,7 +1028,11 @@ public: CGObjCCommonMac(CodeGen::CodeGenModule &cgm) : CGObjCRuntime(cgm), VMContext(cgm.getLLVMContext()) { } - llvm::Constant *GenerateConstantString(const StringLiteral *SL) override; + bool isNonFragileABI() const { + return ObjCABI == 2; + } + + ConstantAddress GenerateConstantString(const StringLiteral *SL) override; llvm::Function *GenerateMethod(const ObjCMethodDecl *OMD, const ObjCContainerDecl *CD=nullptr) override; @@ -1084,7 +1077,9 @@ private: /// EmitClassExtension - Generate the class extension structure used /// to store the weak ivar layout and properties. The return value /// has type ClassExtensionPtrTy. - llvm::Constant *EmitClassExtension(const ObjCImplementationDecl *ID); + llvm::Constant *EmitClassExtension(const ObjCImplementationDecl *ID, + CharUnits instanceSize, + bool hasMRCWeakIvars); /// EmitClassRef - Return a Value*, of type ObjCTypes.ClassPtrTy, /// for the given class. @@ -1172,8 +1167,8 @@ private: /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy, /// for the given selector. - llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel, - bool lval=false); + llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel); + Address EmitSelectorAddr(CodeGenFunction &CGF, Selector Sel); public: CGObjCMac(CodeGen::CodeGenModule &cgm); @@ -1199,8 +1194,8 @@ public: llvm::Value *GetClass(CodeGenFunction &CGF, const ObjCInterfaceDecl *ID) override; - llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel, - bool lval = false) override; + llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel) override; + Address GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) override; /// The NeXT/Apple runtimes do not support typed selectors; just emit an /// untyped one. @@ -1236,19 +1231,19 @@ public: void EmitThrowStmt(CodeGen::CodeGenFunction &CGF, const ObjCAtThrowStmt &S, bool ClearInsertionPoint=true) override; llvm::Value * EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF, - llvm::Value *AddrWeakObj) override; + Address AddrWeakObj) override; void EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst) override; + llvm::Value *src, Address dst) override; void EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest, + llvm::Value *src, Address dest, bool threadlocal = false) override; void EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest, + llvm::Value *src, Address dest, llvm::Value *ivarOffset) override; void EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest) override; + llvm::Value *src, Address dest) override; void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, - llvm::Value *dest, llvm::Value *src, + Address dest, Address src, llvm::Value *size) override; LValue EmitObjCValueForIvar(CodeGen::CodeGenFunction &CGF, QualType ObjectTy, @@ -1395,8 +1390,8 @@ private: /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy, /// for the given selector. - llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel, - bool lval=false); + llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel); + Address EmitSelectorAddr(CodeGenFunction &CGF, Selector Sel); /// GetInterfaceEHType - Get the cached ehtype for the given Objective-C /// interface. The return value has type EHTypePtrTy. @@ -1474,9 +1469,10 @@ public: llvm::Value *GetClass(CodeGenFunction &CGF, const ObjCInterfaceDecl *ID) override; - llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel, - bool lvalue = false) override - { return EmitSelector(CGF, Sel, lvalue); } + llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel) override + { return EmitSelector(CGF, Sel); } + Address GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) override + { return EmitSelectorAddr(CGF, Sel); } /// The NeXT/Apple runtimes do not support typed selectors; just emit an /// untyped one. @@ -1531,19 +1527,19 @@ public: void EmitThrowStmt(CodeGen::CodeGenFunction &CGF, const ObjCAtThrowStmt &S, bool ClearInsertionPoint=true) override; llvm::Value * EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF, - llvm::Value *AddrWeakObj) override; + Address AddrWeakObj) override; void EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst) override; + llvm::Value *src, Address edst) override; void EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest, + llvm::Value *src, Address dest, bool threadlocal = false) override; void EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest, + llvm::Value *src, Address dest, llvm::Value *ivarOffset) override; void EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest) override; + llvm::Value *src, Address dest) override; void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, - llvm::Value *dest, llvm::Value *src, + Address dest, Address src, llvm::Value *size) override; LValue EmitObjCValueForIvar(CodeGen::CodeGenFunction &CGF, QualType ObjectTy, llvm::Value *BaseValue, const ObjCIvarDecl *Ivar, @@ -1645,7 +1641,7 @@ struct NullReturnState { // memory or (2) agg values in registers. if (result.isAggregate()) { assert(result.isAggregate() && "null init of non-aggregate result?"); - CGF.EmitNullInitialization(result.getAggregateAddr(), resultType); + CGF.EmitNullInitialization(result.getAggregateAddress(), resultType); if (contBB) CGF.EmitBlock(contBB); return result; } @@ -1711,9 +1707,11 @@ llvm::Value *CGObjCMac::GetClass(CodeGenFunction &CGF, } /// GetSelector - Return the pointer to the unique'd string for this selector. -llvm::Value *CGObjCMac::GetSelector(CodeGenFunction &CGF, Selector Sel, - bool lval) { - return EmitSelector(CGF, Sel, lval); +llvm::Value *CGObjCMac::GetSelector(CodeGenFunction &CGF, Selector Sel) { + return EmitSelector(CGF, Sel); +} +Address CGObjCMac::GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) { + return EmitSelectorAddr(CGF, Sel); } llvm::Value *CGObjCMac::GetSelector(CodeGenFunction &CGF, const ObjCMethodDecl *Method) { @@ -1756,7 +1754,7 @@ llvm::Constant *CGObjCMac::GetEHType(QualType T) { }; */ -llvm::Constant *CGObjCCommonMac::GenerateConstantString( +ConstantAddress CGObjCCommonMac::GenerateConstantString( const StringLiteral *SL) { return (CGM.getLangOpts().NoConstantCFStrings == 0 ? CGM.GetAddrOfConstantCFString(SL) : @@ -1783,13 +1781,14 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, const ObjCMethodDecl *Method) { // Create and init a super structure; this is a (receiver, class) // pair we will pass to objc_msgSendSuper. - llvm::Value *ObjCSuper = - CGF.CreateTempAlloca(ObjCTypes.SuperTy, "objc_super"); + Address ObjCSuper = + CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(), + "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); CGF.Builder.CreateStore( ReceiverAsObject, - CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 0)); + CGF.Builder.CreateStructGEP(ObjCSuper, 0, CharUnits::Zero())); // If this is a class message the metaclass is passed as the target. llvm::Value *Target; @@ -1803,12 +1802,13 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, // isa" is the first ivar in a class (which it must be). Target = EmitClassRef(CGF, Class->getSuperClass()); Target = CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, Target, 0); - Target = CGF.Builder.CreateLoad(Target); + Target = CGF.Builder.CreateAlignedLoad(Target, CGF.getPointerAlign()); } else { llvm::Constant *MetaClassPtr = EmitMetaClassRef(Class); llvm::Value *SuperPtr = CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, MetaClassPtr, 1); - llvm::Value *Super = CGF.Builder.CreateLoad(SuperPtr); + llvm::Value *Super = + CGF.Builder.CreateAlignedLoad(SuperPtr, CGF.getPointerAlign()); Target = Super; } } else if (isCategoryImpl) @@ -1816,19 +1816,19 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, else { llvm::Value *ClassPtr = EmitSuperClassRef(Class); ClassPtr = CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, ClassPtr, 1); - Target = CGF.Builder.CreateLoad(ClassPtr); + Target = CGF.Builder.CreateAlignedLoad(ClassPtr, CGF.getPointerAlign()); } // FIXME: We shouldn't need to do this cast, rectify the ASTContext and // ObjCTypes types. llvm::Type *ClassTy = CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType()); Target = CGF.Builder.CreateBitCast(Target, ClassTy); - CGF.Builder.CreateStore( - Target, CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 1)); + CGF.Builder.CreateStore(Target, + CGF.Builder.CreateStructGEP(ObjCSuper, 1, CGF.getPointerSize())); return EmitMessageSend(CGF, Return, ResultType, EmitSelector(CGF, Sel), - ObjCSuper, ObjCTypes.SuperPtrCTy, - true, CallArgs, Method, ObjCTypes); + ObjCSuper.getPointer(), ObjCTypes.SuperPtrCTy, + true, CallArgs, Method, Class, ObjCTypes); } /// Generate code for a message send expression. @@ -1843,7 +1843,16 @@ CodeGen::RValue CGObjCMac::GenerateMessageSend(CodeGen::CodeGenFunction &CGF, return EmitMessageSend(CGF, Return, ResultType, EmitSelector(CGF, Sel), Receiver, CGF.getContext().getObjCIdType(), - false, CallArgs, Method, ObjCTypes); + false, CallArgs, Method, Class, ObjCTypes); +} + +static bool isWeakLinkedClass(const ObjCInterfaceDecl *ID) { + do { + if (ID->isWeakImported()) + return true; + } while ((ID = ID->getSuperClass())); + + return false; } CodeGen::RValue @@ -1856,6 +1865,7 @@ CGObjCCommonMac::EmitMessageSend(CodeGen::CodeGenFunction &CGF, bool IsSuper, const CallArgList &CallArgs, const ObjCMethodDecl *Method, + const ObjCInterfaceDecl *ClassReceiver, const ObjCCommonTypesHelper &ObjCTypes) { CallArgList ActualArgs; if (!IsSuper) @@ -1872,11 +1882,38 @@ CGObjCCommonMac::EmitMessageSend(CodeGen::CodeGenFunction &CGF, CGM.getContext().getCanonicalType(ResultType) && "Result type mismatch!"); + bool ReceiverCanBeNull = true; + + // Super dispatch assumes that self is non-null; even the messenger + // doesn't have a null check internally. + if (IsSuper) { + ReceiverCanBeNull = false; + + // If this is a direct dispatch of a class method, check whether the class, + // or anything in its hierarchy, was weak-linked. + } else if (ClassReceiver && Method && Method->isClassMethod()) { + ReceiverCanBeNull = isWeakLinkedClass(ClassReceiver); + + // If we're emitting a method, and self is const (meaning just ARC, for now), + // and the receiver is a load of self, then self is a valid object. + } else if (auto CurMethod = + dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl)) { + auto Self = CurMethod->getSelfDecl(); + if (Self->getType().isConstQualified()) { + if (auto LI = dyn_cast<llvm::LoadInst>(Arg0->stripPointerCasts())) { + llvm::Value *SelfAddr = CGF.GetAddrOfLocalVar(Self).getPointer(); + if (SelfAddr == LI->getPointerOperand()) { + ReceiverCanBeNull = false; + } + } + } + } + NullReturnState nullReturn; llvm::Constant *Fn = nullptr; if (CGM.ReturnSlotInterferesWithArgs(MSI.CallInfo)) { - if (!IsSuper) nullReturn.init(CGF, Arg0); + if (ReceiverCanBeNull) nullReturn.init(CGF, Arg0); Fn = (ObjCABI == 2) ? ObjCTypes.getSendStretFn2(IsSuper) : ObjCTypes.getSendStretFn(IsSuper); } else if (CGM.ReturnTypeUsesFPRet(ResultType)) { @@ -1888,76 +1925,182 @@ CGObjCCommonMac::EmitMessageSend(CodeGen::CodeGenFunction &CGF, } else { // arm64 uses objc_msgSend for stret methods and yet null receiver check // must be made for it. - if (!IsSuper && CGM.ReturnTypeUsesSRet(MSI.CallInfo)) + if (ReceiverCanBeNull && CGM.ReturnTypeUsesSRet(MSI.CallInfo)) nullReturn.init(CGF, Arg0); Fn = (ObjCABI == 2) ? ObjCTypes.getSendFn2(IsSuper) : ObjCTypes.getSendFn(IsSuper); } - - bool requiresnullCheck = false; - if (CGM.getLangOpts().ObjCAutoRefCount && Method) + + // Emit a null-check if there's a consumed argument other than the receiver. + bool RequiresNullCheck = false; + if (ReceiverCanBeNull && CGM.getLangOpts().ObjCAutoRefCount && Method) { for (const auto *ParamDecl : Method->params()) { if (ParamDecl->hasAttr<NSConsumedAttr>()) { if (!nullReturn.NullBB) nullReturn.init(CGF, Arg0); - requiresnullCheck = true; + RequiresNullCheck = true; break; } } + } + llvm::Instruction *CallSite; Fn = llvm::ConstantExpr::getBitCast(Fn, MSI.MessengerType); - RValue rvalue = CGF.EmitCall(MSI.CallInfo, Fn, Return, ActualArgs); + RValue rvalue = CGF.EmitCall(MSI.CallInfo, Fn, Return, ActualArgs, + CGCalleeInfo(), &CallSite); + + // Mark the call as noreturn if the method is marked noreturn and the + // receiver cannot be null. + if (Method && Method->hasAttr<NoReturnAttr>() && !ReceiverCanBeNull) { + llvm::CallSite(CallSite).setDoesNotReturn(); + } + return nullReturn.complete(CGF, rvalue, ResultType, CallArgs, - requiresnullCheck ? Method : nullptr); + RequiresNullCheck ? Method : nullptr); } -static Qualifiers::GC GetGCAttrTypeForType(ASTContext &Ctx, QualType FQT) { +static Qualifiers::GC GetGCAttrTypeForType(ASTContext &Ctx, QualType FQT, + bool pointee = false) { + // Note that GC qualification applies recursively to C pointer types + // that aren't otherwise decorated. This is weird, but it's probably + // an intentional workaround to the unreliable placement of GC qualifiers. if (FQT.isObjCGCStrong()) return Qualifiers::Strong; - - if (FQT.isObjCGCWeak() || FQT.getObjCLifetime() == Qualifiers::OCL_Weak) + + if (FQT.isObjCGCWeak()) return Qualifiers::Weak; + + if (auto ownership = FQT.getObjCLifetime()) { + // Ownership does not apply recursively to C pointer types. + if (pointee) return Qualifiers::GCNone; + switch (ownership) { + case Qualifiers::OCL_Weak: return Qualifiers::Weak; + case Qualifiers::OCL_Strong: return Qualifiers::Strong; + case Qualifiers::OCL_ExplicitNone: return Qualifiers::GCNone; + case Qualifiers::OCL_Autoreleasing: llvm_unreachable("autoreleasing ivar?"); + case Qualifiers::OCL_None: llvm_unreachable("known nonzero"); + } + llvm_unreachable("bad objc ownership"); + } - // check for __unsafe_unretained - if (FQT.getObjCLifetime() == Qualifiers::OCL_ExplicitNone) - return Qualifiers::GCNone; - + // Treat unqualified retainable pointers as strong. if (FQT->isObjCObjectPointerType() || FQT->isBlockPointerType()) return Qualifiers::Strong; - if (const PointerType *PT = FQT->getAs<PointerType>()) - return GetGCAttrTypeForType(Ctx, PT->getPointeeType()); + // Walk into C pointer types, but only in GC. + if (Ctx.getLangOpts().getGC() != LangOptions::NonGC) { + if (const PointerType *PT = FQT->getAs<PointerType>()) + return GetGCAttrTypeForType(Ctx, PT->getPointeeType(), /*pointee*/ true); + } return Qualifiers::GCNone; } +namespace { + struct IvarInfo { + CharUnits Offset; + uint64_t SizeInWords; + IvarInfo(CharUnits offset, uint64_t sizeInWords) + : Offset(offset), SizeInWords(sizeInWords) {} + + // Allow sorting based on byte pos. + bool operator<(const IvarInfo &other) const { + return Offset < other.Offset; + } + }; + + /// A helper class for building GC layout strings. + class IvarLayoutBuilder { + CodeGenModule &CGM; + + /// The start of the layout. Offsets will be relative to this value, + /// and entries less than this value will be silently discarded. + CharUnits InstanceBegin; + + /// The end of the layout. Offsets will never exceed this value. + CharUnits InstanceEnd; + + /// Whether we're generating the strong layout or the weak layout. + bool ForStrongLayout; + + /// Whether the offsets in IvarsInfo might be out-of-order. + bool IsDisordered = false; + + llvm::SmallVector<IvarInfo, 8> IvarsInfo; + public: + IvarLayoutBuilder(CodeGenModule &CGM, CharUnits instanceBegin, + CharUnits instanceEnd, bool forStrongLayout) + : CGM(CGM), InstanceBegin(instanceBegin), InstanceEnd(instanceEnd), + ForStrongLayout(forStrongLayout) { + } + + void visitRecord(const RecordType *RT, CharUnits offset); + + template <class Iterator, class GetOffsetFn> + void visitAggregate(Iterator begin, Iterator end, + CharUnits aggrOffset, + const GetOffsetFn &getOffset); + + void visitField(const FieldDecl *field, CharUnits offset); + + /// Add the layout of a block implementation. + void visitBlock(const CGBlockInfo &blockInfo); + + /// Is there any information for an interesting bitmap? + bool hasBitmapData() const { return !IvarsInfo.empty(); } + + llvm::Constant *buildBitmap(CGObjCCommonMac &CGObjC, + llvm::SmallVectorImpl<unsigned char> &buffer); + + static void dump(ArrayRef<unsigned char> buffer) { + const unsigned char *s = buffer.data(); + for (unsigned i = 0, e = buffer.size(); i < e; i++) + if (!(s[i] & 0xf0)) + printf("0x0%x%s", s[i], s[i] != 0 ? ", " : ""); + else + printf("0x%x%s", s[i], s[i] != 0 ? ", " : ""); + printf("\n"); + } + }; +} + llvm::Constant *CGObjCCommonMac::BuildGCBlockLayout(CodeGenModule &CGM, const CGBlockInfo &blockInfo) { llvm::Constant *nullPtr = llvm::Constant::getNullValue(CGM.Int8PtrTy); - if (CGM.getLangOpts().getGC() == LangOptions::NonGC && - !CGM.getLangOpts().ObjCAutoRefCount) + if (CGM.getLangOpts().getGC() == LangOptions::NonGC) return nullPtr; - bool hasUnion = false; - SkipIvars.clear(); - IvarsInfo.clear(); - unsigned WordSizeInBits = CGM.getTarget().getPointerWidth(0); - unsigned ByteSizeInBits = CGM.getTarget().getCharWidth(); + IvarLayoutBuilder builder(CGM, CharUnits::Zero(), blockInfo.BlockSize, + /*for strong layout*/ true); + + builder.visitBlock(blockInfo); + + if (!builder.hasBitmapData()) + return nullPtr; + + llvm::SmallVector<unsigned char, 32> buffer; + llvm::Constant *C = builder.buildBitmap(*this, buffer); + if (CGM.getLangOpts().ObjCGCBitmapPrint && !buffer.empty()) { + printf("\n block variable layout for block: "); + builder.dump(buffer); + } + return C; +} + +void IvarLayoutBuilder::visitBlock(const CGBlockInfo &blockInfo) { // __isa is the first field in block descriptor and must assume by runtime's // convention that it is GC'able. - IvarsInfo.push_back(GC_IVAR(0, 1)); + IvarsInfo.push_back(IvarInfo(CharUnits::Zero(), 1)); const BlockDecl *blockDecl = blockInfo.getBlockDecl(); - // Calculate the basic layout of the block structure. - const llvm::StructLayout *layout = - CGM.getDataLayout().getStructLayout(blockInfo.StructureType); - // Ignore the optional 'this' capture: C++ objects are not assumed // to be GC'ed. + CharUnits lastFieldOffset; + // Walk the captured variables. for (const auto &CI : blockDecl->captures()) { const VarDecl *variable = CI.getVariable(); @@ -1968,64 +2111,51 @@ llvm::Constant *CGObjCCommonMac::BuildGCBlockLayout(CodeGenModule &CGM, // Ignore constant captures. if (capture.isConstant()) continue; - uint64_t fieldOffset = layout->getElementOffset(capture.getIndex()); + CharUnits fieldOffset = capture.getOffset(); + + // Block fields are not necessarily ordered; if we detect that we're + // adding them out-of-order, make sure we sort later. + if (fieldOffset < lastFieldOffset) + IsDisordered = true; + lastFieldOffset = fieldOffset; // __block variables are passed by their descriptor address. if (CI.isByRef()) { - IvarsInfo.push_back(GC_IVAR(fieldOffset, /*size in words*/ 1)); + IvarsInfo.push_back(IvarInfo(fieldOffset, /*size in words*/ 1)); continue; } assert(!type->isArrayType() && "array variable should not be caught"); if (const RecordType *record = type->getAs<RecordType>()) { - BuildAggrIvarRecordLayout(record, fieldOffset, true, hasUnion); + visitRecord(record, fieldOffset); continue; } Qualifiers::GC GCAttr = GetGCAttrTypeForType(CGM.getContext(), type); - unsigned fieldSize = CGM.getContext().getTypeSize(type); - - if (GCAttr == Qualifiers::Strong) - IvarsInfo.push_back(GC_IVAR(fieldOffset, - fieldSize / WordSizeInBits)); - else if (GCAttr == Qualifiers::GCNone || GCAttr == Qualifiers::Weak) - SkipIvars.push_back(GC_IVAR(fieldOffset, - fieldSize / ByteSizeInBits)); - } - - if (IvarsInfo.empty()) - return nullPtr; - // Sort on byte position; captures might not be allocated in order, - // and unions can do funny things. - llvm::array_pod_sort(IvarsInfo.begin(), IvarsInfo.end()); - llvm::array_pod_sort(SkipIvars.begin(), SkipIvars.end()); - - std::string BitMap; - llvm::Constant *C = BuildIvarLayoutBitmap(BitMap); - if (CGM.getLangOpts().ObjCGCBitmapPrint) { - printf("\n block variable layout for block: "); - const unsigned char *s = (const unsigned char*)BitMap.c_str(); - for (unsigned i = 0, e = BitMap.size(); i < e; i++) - if (!(s[i] & 0xf0)) - printf("0x0%x%s", s[i], s[i] != 0 ? ", " : ""); - else - printf("0x%x%s", s[i], s[i] != 0 ? ", " : ""); - printf("\n"); + if (GCAttr == Qualifiers::Strong) { + assert(CGM.getContext().getTypeSize(type) + == CGM.getTarget().getPointerWidth(0)); + IvarsInfo.push_back(IvarInfo(fieldOffset, /*size in words*/ 1)); + } } - - return C; } + /// getBlockCaptureLifetime - This routine returns life time of the captured /// block variable for the purpose of block layout meta-data generation. FQT is /// the type of the variable captured in the block. Qualifiers::ObjCLifetime CGObjCCommonMac::getBlockCaptureLifetime(QualType FQT, bool ByrefLayout) { + // If it has an ownership qualifier, we're done. + if (auto lifetime = FQT.getObjCLifetime()) + return lifetime; + + // If it doesn't, and this is ARC, it has no ownership. if (CGM.getLangOpts().ObjCAutoRefCount) - return FQT.getObjCLifetime(); + return Qualifiers::OCL_None; - // MRR. + // In MRC, retainable pointers are owned by non-__block variables. if (FQT->isObjCObjectPointerType() || FQT->isBlockPointerType()) return ByrefLayout ? Qualifiers::OCL_ExplicitNone : Qualifiers::OCL_Strong; @@ -2361,9 +2491,8 @@ llvm::Constant *CGObjCCommonMac::getBitmapBlockLayout(bool ComputeByrefLayout) { } } - int e = Layout.size()-1; - while (e >= 0) { - unsigned char inst = Layout[e--]; + while (!Layout.empty()) { + unsigned char inst = Layout.back(); enum BLOCK_LAYOUT_OPCODE opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4); if (opcode == BLOCK_LAYOUT_NON_OBJECT_BYTES || opcode == BLOCK_LAYOUT_NON_OBJECT_WORDS) Layout.pop_back(); @@ -2376,19 +2505,19 @@ llvm::Constant *CGObjCCommonMac::getBitmapBlockLayout(bool ComputeByrefLayout) { // Block variable layout instruction has been inlined. if (CGM.getLangOpts().ObjCGCBitmapPrint) { if (ComputeByrefLayout) - printf("\n Inline instruction for BYREF variable layout: "); + printf("\n Inline BYREF variable layout: "); else - printf("\n Inline instruction for block variable layout: "); - printf("0x0%" PRIx64 "\n", Result); - } - if (WordSizeInBytes == 8) { - const llvm::APInt Instruction(64, Result); - return llvm::Constant::getIntegerValue(CGM.Int64Ty, Instruction); - } - else { - const llvm::APInt Instruction(32, Result); - return llvm::Constant::getIntegerValue(CGM.Int32Ty, Instruction); + printf("\n Inline block variable layout: "); + printf("0x0%" PRIx64 "", Result); + if (auto numStrong = (Result & 0xF00) >> 8) + printf(", BL_STRONG:%d", (int) numStrong); + if (auto numByref = (Result & 0x0F0) >> 4) + printf(", BL_BYREF:%d", (int) numByref); + if (auto numWeak = (Result & 0x00F) >> 0) + printf(", BL_WEAK:%d", (int) numWeak); + printf(", BL_OPERATOR:0\n"); } + return llvm::ConstantInt::get(CGM.IntPtrTy, Result); } unsigned char inst = (BLOCK_LAYOUT_OPERATOR << 4) | 0; @@ -2399,9 +2528,9 @@ llvm::Constant *CGObjCCommonMac::getBitmapBlockLayout(bool ComputeByrefLayout) { if (CGM.getLangOpts().ObjCGCBitmapPrint) { if (ComputeByrefLayout) - printf("\n BYREF variable layout: "); + printf("\n Byref variable layout: "); else - printf("\n block variable layout: "); + printf("\n Block variable layout: "); for (unsigned i = 0, e = BitMap.size(); i != e; i++) { unsigned char inst = BitMap[i]; enum BLOCK_LAYOUT_OPCODE opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4); @@ -2443,7 +2572,7 @@ llvm::Constant *CGObjCCommonMac::getBitmapBlockLayout(bool ComputeByrefLayout) { llvm::GlobalVariable *Entry = CreateMetadataVar( "OBJC_CLASS_NAME_", llvm::ConstantDataArray::getString(VMContext, BitMap, false), - "__TEXT,__objc_classname,cstring_literals", 1, true); + "__TEXT,__objc_classname,cstring_literals", CharUnits::One(), true); return getConstantGEP(VMContext, Entry, 0, 0); } @@ -2511,6 +2640,8 @@ llvm::Constant *CGObjCCommonMac::BuildByrefLayout(CodeGen::CodeGenModule &CGM, if (const RecordType *record = T->getAs<RecordType>()) { BuildRCBlockVarRecordLayout(record, fieldOffset, hasUnion, true /*ByrefLayout */); llvm::Constant *Result = getBitmapBlockLayout(true); + if (isa<llvm::ConstantInt>(Result)) + Result = llvm::ConstantExpr::getIntToPtr(Result, CGM.Int8PtrTy); return Result; } llvm::Constant *nullPtr = llvm::Constant::getNullValue(CGM.Int8PtrTy); @@ -2699,7 +2830,7 @@ CGObjCMac::EmitProtocolExtension(const ObjCProtocolDecl *PD, // No special section, but goes in llvm.used return CreateMetadataVar("\01l_OBJC_PROTOCOLEXT_" + PD->getName(), Init, - StringRef(), 0, true); + StringRef(), CGM.getPointerAlign(), true); } /* @@ -2738,7 +2869,7 @@ CGObjCMac::EmitProtocolList(Twine Name, llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); llvm::GlobalVariable *GV = CreateMetadataVar(Name, Init, "__OBJC,__cat_cls_meth,regular,no_dead_strip", - 4, false); + CGM.getPointerAlign(), false); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.ProtocolListPtrTy); } @@ -2779,15 +2910,26 @@ llvm::Constant *CGObjCCommonMac::EmitPropertyList(Twine Name, const ObjCCommonTypesHelper &ObjCTypes) { SmallVector<llvm::Constant *, 16> Properties; llvm::SmallPtrSet<const IdentifierInfo*, 16> PropertySet; + + auto AddProperty = [&](const ObjCPropertyDecl *PD) { + llvm::Constant *Prop[] = {GetPropertyName(PD->getIdentifier()), + GetPropertyTypeString(PD, Container)}; + Properties.push_back(llvm::ConstantStruct::get(ObjCTypes.PropertyTy, Prop)); + }; + if (const ObjCInterfaceDecl *OID = dyn_cast<ObjCInterfaceDecl>(OCD)) + for (const ObjCCategoryDecl *ClassExt : OID->known_extensions()) + for (auto *PD : ClassExt->properties()) { + PropertySet.insert(PD->getIdentifier()); + AddProperty(PD); + } for (const auto *PD : OCD->properties()) { - PropertySet.insert(PD->getIdentifier()); - llvm::Constant *Prop[] = { - GetPropertyName(PD->getIdentifier()), - GetPropertyTypeString(PD, Container) - }; - Properties.push_back(llvm::ConstantStruct::get(ObjCTypes.PropertyTy, - Prop)); + // Don't emit duplicate metadata for properties that were already in a + // class extension. + if (!PropertySet.insert(PD->getIdentifier()).second) + continue; + AddProperty(PD); } + if (const ObjCInterfaceDecl *OID = dyn_cast<ObjCInterfaceDecl>(OCD)) { for (const auto *P : OID->all_referenced_protocols()) PushProtocolProperties(PropertySet, Properties, Container, P, ObjCTypes); @@ -2815,7 +2957,7 @@ llvm::Constant *CGObjCCommonMac::EmitPropertyList(Twine Name, CreateMetadataVar(Name, Init, (ObjCABI == 2) ? "__DATA, __objc_const" : "__OBJC,__property,regular,no_dead_strip", - (ObjCABI == 2) ? 8 : 4, + CGM.getPointerAlign(), true); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.PropertyListPtrTy); } @@ -2834,7 +2976,7 @@ CGObjCCommonMac::EmitProtocolMethodTypes(Twine Name, llvm::GlobalVariable *GV = CreateMetadataVar( Name, Init, (ObjCABI == 2) ? "__DATA, __objc_const" : StringRef(), - (ObjCABI == 2) ? 8 : 4, true); + CGM.getPointerAlign(), true); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.Int8PtrPtrTy); } @@ -2872,7 +3014,8 @@ CGObjCMac::EmitMethodDescList(Twine Name, const char *Section, Values[1] = llvm::ConstantArray::get(AT, Methods); llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); - llvm::GlobalVariable *GV = CreateMetadataVar(Name, Init, Section, 4, true); + llvm::GlobalVariable *GV = + CreateMetadataVar(Name, Init, Section, CGM.getPointerAlign(), true); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.MethodDescriptionListPtrTy); } @@ -2944,7 +3087,8 @@ void CGObjCMac::GenerateCategory(const ObjCCategoryImplDecl *OCD) { llvm::GlobalVariable *GV = CreateMetadataVar("OBJC_CATEGORY_" + ExtName.str(), Init, - "__OBJC,__category,regular,no_dead_strip", 4, true); + "__OBJC,__category,regular,no_dead_strip", + CGM.getPointerAlign(), true); DefinedCategories.push_back(GV); DefinedCategoryNames.insert(ExtName.str()); // method definition entries must be clear for next implementation. @@ -2952,10 +3096,24 @@ void CGObjCMac::GenerateCategory(const ObjCCategoryImplDecl *OCD) { } enum FragileClassFlags { + /// Apparently: is not a meta-class. FragileABI_Class_Factory = 0x00001, + + /// Is a meta-class. FragileABI_Class_Meta = 0x00002, + + /// Has a non-trivial constructor or destructor. FragileABI_Class_HasCXXStructors = 0x02000, - FragileABI_Class_Hidden = 0x20000 + + /// Has hidden visibility. + FragileABI_Class_Hidden = 0x20000, + + /// Class implementation was compiled under ARC. + FragileABI_Class_CompiledByARC = 0x04000000, + + /// Class implementation was compiled under MRC and has MRC weak ivars. + /// Exclusive with CompiledByARC. + FragileABI_Class_HasMRCWeakIvars = 0x08000000, }; enum NonFragileClassFlags { @@ -2965,7 +3123,7 @@ enum NonFragileClassFlags { /// Is a root class. NonFragileABI_Class_Root = 0x00002, - /// Has a C++ constructor and destructor. + /// Has a non-trivial constructor or destructor. NonFragileABI_Class_HasCXXStructors = 0x00004, /// Has hidden visibility. @@ -2981,9 +3139,46 @@ enum NonFragileClassFlags { NonFragileABI_Class_CompiledByARC = 0x00080, /// Class has non-trivial destructors, but zero-initialization is okay. - NonFragileABI_Class_HasCXXDestructorOnly = 0x00100 + NonFragileABI_Class_HasCXXDestructorOnly = 0x00100, + + /// Class implementation was compiled under MRC and has MRC weak ivars. + /// Exclusive with CompiledByARC. + NonFragileABI_Class_HasMRCWeakIvars = 0x00200, }; +static bool hasWeakMember(QualType type) { + if (type.getObjCLifetime() == Qualifiers::OCL_Weak) { + return true; + } + + if (auto recType = type->getAs<RecordType>()) { + for (auto field : recType->getDecl()->fields()) { + if (hasWeakMember(field->getType())) + return true; + } + } + + return false; +} + +/// For compatibility, we only want to set the "HasMRCWeakIvars" flag +/// (and actually fill in a layout string) if we really do have any +/// __weak ivars. +static bool hasMRCWeakIvars(CodeGenModule &CGM, + const ObjCImplementationDecl *ID) { + if (!CGM.getLangOpts().ObjCWeak) return false; + assert(CGM.getLangOpts().getGC() == LangOptions::NonGC); + + for (const ObjCIvarDecl *ivar = + ID->getClassInterface()->all_declared_ivar_begin(); + ivar; ivar = ivar->getNextIvar()) { + if (hasWeakMember(ivar->getType())) + return true; + } + + return false; +} + /* struct _objc_class { Class isa; @@ -3017,8 +3212,16 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) { unsigned Flags = FragileABI_Class_Factory; if (ID->hasNonZeroConstructors() || ID->hasDestructors()) Flags |= FragileABI_Class_HasCXXStructors; - unsigned Size = - CGM.getContext().getASTObjCImplementationLayout(ID).getSize().getQuantity(); + + bool hasMRCWeak = false; + + if (CGM.getLangOpts().ObjCAutoRefCount) + Flags |= FragileABI_Class_CompiledByARC; + else if ((hasMRCWeak = hasMRCWeakIvars(CGM, ID))) + Flags |= FragileABI_Class_HasMRCWeakIvars; + + CharUnits Size = + CGM.getContext().getASTObjCImplementationLayout(ID).getSize(); // FIXME: Set CXX-structors flag. if (ID->getClassInterface()->getVisibility() == HiddenVisibility) @@ -3062,7 +3265,7 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) { // Version is always 0. Values[ 3] = llvm::ConstantInt::get(ObjCTypes.LongTy, 0); Values[ 4] = llvm::ConstantInt::get(ObjCTypes.LongTy, Flags); - Values[ 5] = llvm::ConstantInt::get(ObjCTypes.LongTy, Size); + Values[ 5] = llvm::ConstantInt::get(ObjCTypes.LongTy, Size.getQuantity()); Values[ 6] = EmitIvarList(ID, false); Values[7] = EmitMethodList("OBJC_INSTANCE_METHODS_" + ID->getName(), "__OBJC,__inst_meth,regular,no_dead_strip", @@ -3070,8 +3273,8 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) { // cache is always NULL. Values[ 8] = llvm::Constant::getNullValue(ObjCTypes.CachePtrTy); Values[ 9] = Protocols; - Values[10] = BuildIvarLayout(ID, true); - Values[11] = EmitClassExtension(ID); + Values[10] = BuildStrongIvarLayout(ID, CharUnits::Zero(), Size); + Values[11] = EmitClassExtension(ID, Size, hasMRCWeak); llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ClassTy, Values); std::string Name("OBJC_CLASS_"); @@ -3084,10 +3287,10 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) { "Forward metaclass reference has incorrect type."); GV->setInitializer(Init); GV->setSection(Section); - GV->setAlignment(4); + GV->setAlignment(CGM.getPointerAlign().getQuantity()); CGM.addCompilerUsedGlobal(GV); } else - GV = CreateMetadataVar(Name, Init, Section, 4, true); + GV = CreateMetadataVar(Name, Init, Section, CGM.getPointerAlign(), true); DefinedClasses.push_back(GV); ImplementedClasses.push_back(Interface); // method definition entries must be clear for next implementation. @@ -3198,6 +3401,10 @@ llvm::Value *CGObjCMac::EmitSuperClassRef(const ObjCInterfaceDecl *ID) { } /* + Emit a "class extension", which in this specific context means extra + data that doesn't fit in the normal fragile-ABI class structure, and + has nothing to do with the language concept of a class extension. + struct objc_class_ext { uint32_t size; const char *weak_ivar_layout; @@ -3205,13 +3412,15 @@ llvm::Value *CGObjCMac::EmitSuperClassRef(const ObjCInterfaceDecl *ID) { }; */ llvm::Constant * -CGObjCMac::EmitClassExtension(const ObjCImplementationDecl *ID) { +CGObjCMac::EmitClassExtension(const ObjCImplementationDecl *ID, + CharUnits InstanceSize, bool hasMRCWeakIvars) { uint64_t Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ClassExtensionTy); llvm::Constant *Values[3]; Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size); - Values[1] = BuildIvarLayout(ID, false); + Values[1] = BuildWeakIvarLayout(ID, CharUnits::Zero(), InstanceSize, + hasMRCWeakIvars); Values[2] = EmitPropertyList("\01l_OBJC_$_PROP_LIST_" + ID->getName(), ID, ID->getClassInterface(), ObjCTypes); @@ -3222,7 +3431,8 @@ CGObjCMac::EmitClassExtension(const ObjCImplementationDecl *ID) { llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ClassExtensionTy, Values); return CreateMetadataVar("OBJC_CLASSEXT_" + ID->getName(), Init, - "__OBJC,__class_ext,regular,no_dead_strip", 4, true); + "__OBJC,__class_ext,regular,no_dead_strip", + CGM.getPointerAlign(), true); } /* @@ -3280,11 +3490,12 @@ llvm::Constant *CGObjCMac::EmitIvarList(const ObjCImplementationDecl *ID, if (ForClass) GV = CreateMetadataVar("OBJC_CLASS_VARIABLES_" + ID->getName(), Init, - "__OBJC,__class_vars,regular,no_dead_strip", 4, true); + "__OBJC,__class_vars,regular,no_dead_strip", + CGM.getPointerAlign(), true); else GV = CreateMetadataVar("OBJC_INSTANCE_VARIABLES_" + ID->getName(), Init, - "__OBJC,__instance_vars,regular,no_dead_strip", 4, - true); + "__OBJC,__instance_vars,regular,no_dead_strip", + CGM.getPointerAlign(), true); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.IvarListPtrTy); } @@ -3334,7 +3545,8 @@ llvm::Constant *CGObjCMac::EmitMethodList(Twine Name, Values[2] = llvm::ConstantArray::get(AT, Methods); llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); - llvm::GlobalVariable *GV = CreateMetadataVar(Name, Init, Section, 4, true); + llvm::GlobalVariable *GV = + CreateMetadataVar(Name, Init, Section, CGM.getPointerAlign(), true); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.MethodListPtrTy); } @@ -3359,7 +3571,7 @@ llvm::Function *CGObjCCommonMac::GenerateMethod(const ObjCMethodDecl *OMD, llvm::GlobalVariable *CGObjCCommonMac::CreateMetadataVar(Twine Name, llvm::Constant *Init, StringRef Section, - unsigned Align, + CharUnits Align, bool AddToUsed) { llvm::Type *Ty = Init->getType(); llvm::GlobalVariable *GV = @@ -3367,8 +3579,7 @@ llvm::GlobalVariable *CGObjCCommonMac::CreateMetadataVar(Twine Name, llvm::GlobalValue::PrivateLinkage, Init, Name); if (!Section.empty()) GV->setSection(Section); - if (Align) - GV->setAlignment(Align); + GV->setAlignment(Align.getQuantity()); if (AddToUsed) CGM.addCompilerUsedGlobal(GV); return GV; @@ -3421,16 +3632,16 @@ void CGObjCMac::EmitSynchronizedStmt(CodeGenFunction &CGF, } namespace { - struct PerformFragileFinally : EHScopeStack::Cleanup { + struct PerformFragileFinally final : EHScopeStack::Cleanup { const Stmt &S; - llvm::Value *SyncArgSlot; - llvm::Value *CallTryExitVar; - llvm::Value *ExceptionData; + Address SyncArgSlot; + Address CallTryExitVar; + Address ExceptionData; ObjCTypesHelper &ObjCTypes; PerformFragileFinally(const Stmt *S, - llvm::Value *SyncArgSlot, - llvm::Value *CallTryExitVar, - llvm::Value *ExceptionData, + Address SyncArgSlot, + Address CallTryExitVar, + Address ExceptionData, ObjCTypesHelper *ObjCTypes) : S(*S), SyncArgSlot(SyncArgSlot), CallTryExitVar(CallTryExitVar), ExceptionData(ExceptionData), ObjCTypes(*ObjCTypes) {} @@ -3447,7 +3658,7 @@ namespace { CGF.EmitBlock(FinallyCallExit); CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryExitFn(), - ExceptionData); + ExceptionData.getPointer()); CGF.EmitBlock(FinallyNoCallExit); @@ -3568,7 +3779,7 @@ void FragileHazards::emitReadHazard(CGBuilderTy &Builder) { void FragileHazards::emitHazardsInNewBlocks() { if (Locals.empty()) return; - CGBuilderTy Builder(CGF.getLLVMContext()); + CGBuilderTy Builder(CGF, CGF.getLLVMContext()); // Iterate through all blocks, skipping those prior to the try. for (llvm::Function::iterator @@ -3607,6 +3818,10 @@ static void addIfPresent(llvm::DenseSet<llvm::Value*> &S, llvm::Value *V) { if (V) S.insert(V); } +static void addIfPresent(llvm::DenseSet<llvm::Value*> &S, Address V) { + if (V.isValid()) S.insert(V.getPointer()); +} + void FragileHazards::collectLocals() { // Compute a set of allocas to ignore. llvm::DenseSet<llvm::Value*> AllocasToIgnore; @@ -3760,21 +3975,23 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // @synchronized. We can't avoid a temp here because we need the // value to be preserved. If the backend ever does liveness // correctly after setjmp, this will be unnecessary. - llvm::Value *SyncArgSlot = nullptr; + Address SyncArgSlot = Address::invalid(); if (!isTry) { llvm::Value *SyncArg = CGF.EmitScalarExpr(cast<ObjCAtSynchronizedStmt>(S).getSynchExpr()); SyncArg = CGF.Builder.CreateBitCast(SyncArg, ObjCTypes.ObjectPtrTy); CGF.EmitNounwindRuntimeCall(ObjCTypes.getSyncEnterFn(), SyncArg); - SyncArgSlot = CGF.CreateTempAlloca(SyncArg->getType(), "sync.arg"); + SyncArgSlot = CGF.CreateTempAlloca(SyncArg->getType(), + CGF.getPointerAlign(), "sync.arg"); CGF.Builder.CreateStore(SyncArg, SyncArgSlot); } // Allocate memory for the setjmp buffer. This needs to be kept // live throughout the try and catch blocks. - llvm::Value *ExceptionData = CGF.CreateTempAlloca(ObjCTypes.ExceptionDataTy, - "exceptiondata.ptr"); + Address ExceptionData = CGF.CreateTempAlloca(ObjCTypes.ExceptionDataTy, + CGF.getPointerAlign(), + "exceptiondata.ptr"); // Create the fragile hazards. Note that this will not capture any // of the allocas required for exception processing, but will @@ -3790,12 +4007,13 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // The setjmp-safety rule here is that we should always store to this // variable in a place that dominates the branch through the cleanup // without passing through any setjmps. - llvm::Value *CallTryExitVar = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(), - "_call_try_exit"); + Address CallTryExitVar = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(), + CharUnits::One(), + "_call_try_exit"); // A slot containing the exception to rethrow. Only needed when we // have both a @catch and a @finally. - llvm::Value *PropagatingExnVar = nullptr; + Address PropagatingExnVar = Address::invalid(); // Push a normal cleanup to leave the try scope. CGF.EHStack.pushCleanup<PerformFragileFinally>(NormalAndEHCleanup, &S, @@ -3808,13 +4026,14 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // - Call objc_exception_try_enter to push ExceptionData on top of // the EH stack. CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), - ExceptionData); + ExceptionData.getPointer()); // - Call setjmp on the exception data buffer. llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0); llvm::Value *GEPIndexes[] = { Zero, Zero, Zero }; llvm::Value *SetJmpBuffer = CGF.Builder.CreateGEP( - ObjCTypes.ExceptionDataTy, ExceptionData, GEPIndexes, "setjmp_buffer"); + ObjCTypes.ExceptionDataTy, ExceptionData.getPointer(), GEPIndexes, + "setjmp_buffer"); llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall( ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result"); SetJmpResult->setCanReturnTwice(); @@ -3854,7 +4073,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // nothing can cross this so the value is already in SSA form. llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData, "caught"); + ExceptionData.getPointer(), "caught"); // Push the exception to rethrow onto the EH value stack for the // benefit of any @throws in the handlers. @@ -3870,13 +4089,14 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Save the currently-propagating exception before // objc_exception_try_enter clears the exception slot. PropagatingExnVar = CGF.CreateTempAlloca(Caught->getType(), + CGF.getPointerAlign(), "propagating_exception"); CGF.Builder.CreateStore(Caught, PropagatingExnVar); // Enter a new exception try block (in case a @catch block // throws an exception). CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), - ExceptionData); + ExceptionData.getPointer()); llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall(ObjCTypes.getSetJmpFn(), @@ -3928,7 +4148,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, assert(CGF.HaveInsertPoint() && "DeclStmt destroyed insert point?"); // These types work out because ConvertType(id) == i8*. - CGF.Builder.CreateStore(Caught, CGF.GetAddrOfLocalVar(CatchParam)); + EmitInitOfCatchParam(CGF, Caught, CatchParam); } CGF.EmitStmt(CatchStmt->getCatchBody()); @@ -3975,7 +4195,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, llvm::Value *Tmp = CGF.Builder.CreateBitCast(Caught, CGF.ConvertType(CatchParam->getType())); - CGF.Builder.CreateStore(Tmp, CGF.GetAddrOfLocalVar(CatchParam)); + EmitInitOfCatchParam(CGF, Tmp, CatchParam); CGF.EmitStmt(CatchStmt->getCatchBody()); @@ -4008,10 +4228,10 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Extract the new exception and save it to the // propagating-exception slot. - assert(PropagatingExnVar); + assert(PropagatingExnVar.isValid()); llvm::CallInst *NewCaught = CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData, "caught"); + ExceptionData.getPointer(), "caught"); CGF.Builder.CreateStore(NewCaught, PropagatingExnVar); // Don't pop the catch handler; the throw already did. @@ -4036,14 +4256,14 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, if (CGF.HaveInsertPoint()) { // If we have a propagating-exception variable, check it. llvm::Value *PropagatingExn; - if (PropagatingExnVar) { + if (PropagatingExnVar.isValid()) { PropagatingExn = CGF.Builder.CreateLoad(PropagatingExnVar); // Otherwise, just look in the buffer for the exception to throw. } else { llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData); + ExceptionData.getPointer()); PropagatingExn = Caught; } @@ -4083,14 +4303,13 @@ void CGObjCMac::EmitThrowStmt(CodeGen::CodeGenFunction &CGF, /// object: objc_read_weak (id *src) /// llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF, - llvm::Value *AddrWeakObj) { - llvm::Type* DestTy = - cast<llvm::PointerType>(AddrWeakObj->getType())->getElementType(); + Address AddrWeakObj) { + llvm::Type* DestTy = AddrWeakObj.getElementType(); AddrWeakObj = CGF.Builder.CreateBitCast(AddrWeakObj, ObjCTypes.PtrObjectPtrTy); llvm::Value *read_weak = CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(), - AddrWeakObj, "weakread"); + AddrWeakObj.getPointer(), "weakread"); read_weak = CGF.Builder.CreateBitCast(read_weak, DestTy); return read_weak; } @@ -4099,7 +4318,7 @@ llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF, /// objc_assign_weak (id src, id *dst) /// void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst) { + llvm::Value *src, Address dst) { llvm::Type * SrcTy = src->getType(); if (!isa<llvm::PointerType>(SrcTy)) { unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy); @@ -4110,7 +4329,7 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy); - llvm::Value *args[] = { src, dst }; + llvm::Value *args[] = { src, dst.getPointer() }; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(), args, "weakassign"); return; @@ -4120,7 +4339,7 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, /// objc_assign_global (id src, id *dst) /// void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst, + llvm::Value *src, Address dst, bool threadlocal) { llvm::Type * SrcTy = src->getType(); if (!isa<llvm::PointerType>(SrcTy)) { @@ -4132,7 +4351,7 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy); - llvm::Value *args[] = { src, dst }; + llvm::Value *args[] = { src, dst.getPointer() }; if (!threadlocal) CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(), args, "globalassign"); @@ -4146,7 +4365,7 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, /// objc_assign_ivar (id src, id *dst, ptrdiff_t ivaroffset) /// void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst, + llvm::Value *src, Address dst, llvm::Value *ivarOffset) { assert(ivarOffset && "EmitObjCIvarAssign - ivarOffset is NULL"); llvm::Type * SrcTy = src->getType(); @@ -4159,7 +4378,7 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy); - llvm::Value *args[] = { src, dst, ivarOffset }; + llvm::Value *args[] = { src, dst.getPointer(), ivarOffset }; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args); return; } @@ -4168,7 +4387,7 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, /// objc_assign_strongCast (id src, id *dst) /// void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst) { + llvm::Value *src, Address dst) { llvm::Type * SrcTy = src->getType(); if (!isa<llvm::PointerType>(SrcTy)) { unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy); @@ -4179,19 +4398,19 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy); - llvm::Value *args[] = { src, dst }; + llvm::Value *args[] = { src, dst.getPointer() }; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(), - args, "weakassign"); + args, "strongassign"); return; } void CGObjCMac::EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, - llvm::Value *DestPtr, - llvm::Value *SrcPtr, + Address DestPtr, + Address SrcPtr, llvm::Value *size) { SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, ObjCTypes.Int8PtrTy); DestPtr = CGF.Builder.CreateBitCast(DestPtr, ObjCTypes.Int8PtrTy); - llvm::Value *args[] = { DestPtr, SrcPtr, size }; + llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), size }; CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args); } @@ -4283,7 +4502,7 @@ void CGObjCCommonMac::EmitImageInfo() { // Indicate whether we're compiling this to run on a simulator. const llvm::Triple &Triple = CGM.getTarget().getTriple(); - if (Triple.isiOS() && + if ((Triple.isiOS() || Triple.isWatchOS()) && (Triple.getArch() == llvm::Triple::x86 || Triple.getArch() == llvm::Triple::x86_64)) Mod.addModuleFlag(llvm::Module::Error, "Objective-C Is Simulated", @@ -4312,7 +4531,8 @@ void CGObjCMac::EmitModuleInfo() { }; CreateMetadataVar("OBJC_MODULES", llvm::ConstantStruct::get(ObjCTypes.ModuleTy, Values), - "__OBJC,__module_info,regular,no_dead_strip", 4, true); + "__OBJC,__module_info,regular,no_dead_strip", + CGM.getPointerAlign(), true); } llvm::Constant *CGObjCMac::EmitModuleSymbols() { @@ -4356,7 +4576,8 @@ llvm::Constant *CGObjCMac::EmitModuleSymbols() { llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values); llvm::GlobalVariable *GV = CreateMetadataVar( - "OBJC_SYMBOLS", Init, "__OBJC,__symbols,regular,no_dead_strip", 4, true); + "OBJC_SYMBOLS", Init, "__OBJC,__symbols,regular,no_dead_strip", + CGM.getPointerAlign(), true); return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.SymtabPtrTy); } @@ -4372,10 +4593,11 @@ llvm::Value *CGObjCMac::EmitClassRefFromId(CodeGenFunction &CGF, ObjCTypes.ClassPtrTy); Entry = CreateMetadataVar( "OBJC_CLASS_REFERENCES_", Casted, - "__OBJC,__cls_refs,literal_pointers,no_dead_strip", 4, true); + "__OBJC,__cls_refs,literal_pointers,no_dead_strip", + CGM.getPointerAlign(), true); } - return CGF.Builder.CreateLoad(Entry); + return CGF.Builder.CreateAlignedLoad(Entry, CGF.getPointerAlign()); } llvm::Value *CGObjCMac::EmitClassRef(CodeGenFunction &CGF, @@ -4388,23 +4610,25 @@ llvm::Value *CGObjCMac::EmitNSAutoreleasePoolClassRef(CodeGenFunction &CGF) { return EmitClassRefFromId(CGF, II); } -llvm::Value *CGObjCMac::EmitSelector(CodeGenFunction &CGF, Selector Sel, - bool lvalue) { - llvm::GlobalVariable *&Entry = SelectorReferences[Sel]; +llvm::Value *CGObjCMac::EmitSelector(CodeGenFunction &CGF, Selector Sel) { + return CGF.Builder.CreateLoad(EmitSelectorAddr(CGF, Sel)); +} + +Address CGObjCMac::EmitSelectorAddr(CodeGenFunction &CGF, Selector Sel) { + CharUnits Align = CGF.getPointerAlign(); + llvm::GlobalVariable *&Entry = SelectorReferences[Sel]; if (!Entry) { llvm::Constant *Casted = llvm::ConstantExpr::getBitCast(GetMethodVarName(Sel), ObjCTypes.SelectorPtrTy); Entry = CreateMetadataVar( "OBJC_SELECTOR_REFERENCES_", Casted, - "__OBJC,__message_refs,literal_pointers,no_dead_strip", 4, true); + "__OBJC,__message_refs,literal_pointers,no_dead_strip", Align, true); Entry->setExternallyInitialized(true); } - if (lvalue) - return Entry; - return CGF.Builder.CreateLoad(Entry); + return Address(Entry, Align); } llvm::Constant *CGObjCCommonMac::GetClassName(StringRef RuntimeName) { @@ -4415,7 +4639,7 @@ llvm::Constant *CGObjCCommonMac::GetClassName(StringRef RuntimeName) { llvm::ConstantDataArray::getString(VMContext, RuntimeName), ((ObjCABI == 2) ? "__TEXT,__objc_classname,cstring_literals" : "__TEXT,__cstring,cstring_literals"), - 1, true); + CharUnits::One(), true); return getConstantGEP(VMContext, Entry, 0, 0); } @@ -4435,308 +4659,247 @@ llvm::Constant *CGObjCCommonMac::GetIvarLayoutName(IdentifierInfo *Ident, return llvm::Constant::getNullValue(ObjCTypes.Int8PtrTy); } -void CGObjCCommonMac::BuildAggrIvarRecordLayout(const RecordType *RT, - unsigned int BytePos, - bool ForStrongLayout, - bool &HasUnion) { +void IvarLayoutBuilder::visitRecord(const RecordType *RT, + CharUnits offset) { const RecordDecl *RD = RT->getDecl(); - // FIXME - Use iterator. - SmallVector<const FieldDecl*, 16> Fields(RD->fields()); - llvm::Type *Ty = CGM.getTypes().ConvertType(QualType(RT, 0)); - const llvm::StructLayout *RecLayout = - CGM.getDataLayout().getStructLayout(cast<llvm::StructType>(Ty)); - BuildAggrIvarLayout(nullptr, RecLayout, RD, Fields, BytePos, ForStrongLayout, - HasUnion); -} + // If this is a union, remember that we had one, because it might mess + // up the ordering of layout entries. + if (RD->isUnion()) + IsDisordered = true; + + const ASTRecordLayout *recLayout = nullptr; + visitAggregate(RD->field_begin(), RD->field_end(), offset, + [&](const FieldDecl *field) -> CharUnits { + if (!recLayout) + recLayout = &CGM.getContext().getASTRecordLayout(RD); + auto offsetInBits = recLayout->getFieldOffset(field->getFieldIndex()); + return CGM.getContext().toCharUnitsFromBits(offsetInBits); + }); +} + +template <class Iterator, class GetOffsetFn> +void IvarLayoutBuilder::visitAggregate(Iterator begin, Iterator end, + CharUnits aggregateOffset, + const GetOffsetFn &getOffset) { + for (; begin != end; ++begin) { + auto field = *begin; + + // Skip over bitfields. + if (field->isBitField()) { + continue; + } -void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI, - const llvm::StructLayout *Layout, - const RecordDecl *RD, - ArrayRef<const FieldDecl*> RecFields, - unsigned int BytePos, bool ForStrongLayout, - bool &HasUnion) { - bool IsUnion = (RD && RD->isUnion()); - uint64_t MaxUnionIvarSize = 0; - uint64_t MaxSkippedUnionIvarSize = 0; - const FieldDecl *MaxField = nullptr; - const FieldDecl *MaxSkippedField = nullptr; - const FieldDecl *LastFieldBitfieldOrUnnamed = nullptr; - uint64_t MaxFieldOffset = 0; - uint64_t MaxSkippedFieldOffset = 0; - uint64_t LastBitfieldOrUnnamedOffset = 0; - uint64_t FirstFieldDelta = 0; + // Compute the offset of the field within the aggregate. + CharUnits fieldOffset = aggregateOffset + getOffset(field); - if (RecFields.empty()) - return; - unsigned WordSizeInBits = CGM.getTarget().getPointerWidth(0); - unsigned ByteSizeInBits = CGM.getTarget().getCharWidth(); - if (!RD && CGM.getLangOpts().ObjCAutoRefCount) { - const FieldDecl *FirstField = RecFields[0]; - FirstFieldDelta = - ComputeIvarBaseOffset(CGM, OI, cast<ObjCIvarDecl>(FirstField)); + visitField(field, fieldOffset); } - - for (unsigned i = 0, e = RecFields.size(); i != e; ++i) { - const FieldDecl *Field = RecFields[i]; - uint64_t FieldOffset; - if (RD) { - // Note that 'i' here is actually the field index inside RD of Field, - // although this dependency is hidden. - const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD); - FieldOffset = (RL.getFieldOffset(i) / ByteSizeInBits) - FirstFieldDelta; - } else - FieldOffset = - ComputeIvarBaseOffset(CGM, OI, cast<ObjCIvarDecl>(Field)) - FirstFieldDelta; +} - // Skip over unnamed or bitfields - if (!Field->getIdentifier() || Field->isBitField()) { - LastFieldBitfieldOrUnnamed = Field; - LastBitfieldOrUnnamedOffset = FieldOffset; - continue; - } +/// Collect layout information for the given fields into IvarsInfo. +void IvarLayoutBuilder::visitField(const FieldDecl *field, + CharUnits fieldOffset) { + QualType fieldType = field->getType(); - LastFieldBitfieldOrUnnamed = nullptr; - QualType FQT = Field->getType(); - if (FQT->isRecordType() || FQT->isUnionType()) { - if (FQT->isUnionType()) - HasUnion = true; + // Drill down into arrays. + uint64_t numElts = 1; + while (auto arrayType = CGM.getContext().getAsConstantArrayType(fieldType)) { + numElts *= arrayType->getSize().getZExtValue(); + fieldType = arrayType->getElementType(); + } - BuildAggrIvarRecordLayout(FQT->getAs<RecordType>(), - BytePos + FieldOffset, - ForStrongLayout, HasUnion); - continue; - } + assert(!fieldType->isArrayType() && "ivar of non-constant array type?"); - if (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) { - const ConstantArrayType *CArray = - dyn_cast_or_null<ConstantArrayType>(Array); - uint64_t ElCount = CArray->getSize().getZExtValue(); - assert(CArray && "only array with known element size is supported"); - FQT = CArray->getElementType(); - while (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) { - const ConstantArrayType *CArray = - dyn_cast_or_null<ConstantArrayType>(Array); - ElCount *= CArray->getSize().getZExtValue(); - FQT = CArray->getElementType(); - } - if (FQT->isRecordType() && ElCount) { - int OldIndex = IvarsInfo.size() - 1; - int OldSkIndex = SkipIvars.size() -1; + // If we ended up with a zero-sized array, we've done what we can do within + // the limits of this layout encoding. + if (numElts == 0) return; - const RecordType *RT = FQT->getAs<RecordType>(); - BuildAggrIvarRecordLayout(RT, BytePos + FieldOffset, - ForStrongLayout, HasUnion); + // Recurse if the base element type is a record type. + if (auto recType = fieldType->getAs<RecordType>()) { + size_t oldEnd = IvarsInfo.size(); - // Replicate layout information for each array element. Note that - // one element is already done. - uint64_t ElIx = 1; - for (int FirstIndex = IvarsInfo.size() - 1, - FirstSkIndex = SkipIvars.size() - 1 ;ElIx < ElCount; ElIx++) { - uint64_t Size = CGM.getContext().getTypeSize(RT)/ByteSizeInBits; - for (int i = OldIndex+1; i <= FirstIndex; ++i) - IvarsInfo.push_back(GC_IVAR(IvarsInfo[i].ivar_bytepos + Size*ElIx, - IvarsInfo[i].ivar_size)); - for (int i = OldSkIndex+1; i <= FirstSkIndex; ++i) - SkipIvars.push_back(GC_IVAR(SkipIvars[i].ivar_bytepos + Size*ElIx, - SkipIvars[i].ivar_size)); - } - continue; - } - } - // At this point, we are done with Record/Union and array there of. - // For other arrays we are down to its element type. - Qualifiers::GC GCAttr = GetGCAttrTypeForType(CGM.getContext(), FQT); - - unsigned FieldSize = CGM.getContext().getTypeSize(Field->getType()); - if ((ForStrongLayout && GCAttr == Qualifiers::Strong) - || (!ForStrongLayout && GCAttr == Qualifiers::Weak)) { - if (IsUnion) { - uint64_t UnionIvarSize = FieldSize / WordSizeInBits; - if (UnionIvarSize > MaxUnionIvarSize) { - MaxUnionIvarSize = UnionIvarSize; - MaxField = Field; - MaxFieldOffset = FieldOffset; - } - } else { - IvarsInfo.push_back(GC_IVAR(BytePos + FieldOffset, - FieldSize / WordSizeInBits)); - } - } else if ((ForStrongLayout && - (GCAttr == Qualifiers::GCNone || GCAttr == Qualifiers::Weak)) - || (!ForStrongLayout && GCAttr != Qualifiers::Weak)) { - if (IsUnion) { - // FIXME: Why the asymmetry? We divide by word size in bits on other - // side. - uint64_t UnionIvarSize = FieldSize / ByteSizeInBits; - if (UnionIvarSize > MaxSkippedUnionIvarSize) { - MaxSkippedUnionIvarSize = UnionIvarSize; - MaxSkippedField = Field; - MaxSkippedFieldOffset = FieldOffset; + visitRecord(recType, fieldOffset); + + // If we have an array, replicate the first entry's layout information. + auto numEltEntries = IvarsInfo.size() - oldEnd; + if (numElts != 1 && numEltEntries != 0) { + CharUnits eltSize = CGM.getContext().getTypeSizeInChars(recType); + for (uint64_t eltIndex = 1; eltIndex != numElts; ++eltIndex) { + // Copy the last numEltEntries onto the end of the array, adjusting + // each for the element size. + for (size_t i = 0; i != numEltEntries; ++i) { + auto firstEntry = IvarsInfo[oldEnd + i]; + IvarsInfo.push_back(IvarInfo(firstEntry.Offset + eltIndex * eltSize, + firstEntry.SizeInWords)); } - } else { - // FIXME: Why the asymmetry, we divide by byte size in bits here? - SkipIvars.push_back(GC_IVAR(BytePos + FieldOffset, - FieldSize / ByteSizeInBits)); } } + + return; } - if (LastFieldBitfieldOrUnnamed) { - if (LastFieldBitfieldOrUnnamed->isBitField()) { - // Last field was a bitfield. Must update skip info. - uint64_t BitFieldSize - = LastFieldBitfieldOrUnnamed->getBitWidthValue(CGM.getContext()); - GC_IVAR skivar; - skivar.ivar_bytepos = BytePos + LastBitfieldOrUnnamedOffset; - skivar.ivar_size = (BitFieldSize / ByteSizeInBits) - + ((BitFieldSize % ByteSizeInBits) != 0); - SkipIvars.push_back(skivar); - } else { - assert(!LastFieldBitfieldOrUnnamed->getIdentifier() &&"Expected unnamed"); - // Last field was unnamed. Must update skip info. - unsigned FieldSize - = CGM.getContext().getTypeSize(LastFieldBitfieldOrUnnamed->getType()); - SkipIvars.push_back(GC_IVAR(BytePos + LastBitfieldOrUnnamedOffset, - FieldSize / ByteSizeInBits)); - } + // Classify the element type. + Qualifiers::GC GCAttr = GetGCAttrTypeForType(CGM.getContext(), fieldType); + + // If it matches what we're looking for, add an entry. + if ((ForStrongLayout && GCAttr == Qualifiers::Strong) + || (!ForStrongLayout && GCAttr == Qualifiers::Weak)) { + assert(CGM.getContext().getTypeSizeInChars(fieldType) + == CGM.getPointerSize()); + IvarsInfo.push_back(IvarInfo(fieldOffset, numElts)); } +} - if (MaxField) - IvarsInfo.push_back(GC_IVAR(BytePos + MaxFieldOffset, - MaxUnionIvarSize)); - if (MaxSkippedField) - SkipIvars.push_back(GC_IVAR(BytePos + MaxSkippedFieldOffset, - MaxSkippedUnionIvarSize)); -} - -/// BuildIvarLayoutBitmap - This routine is the horsework for doing all -/// the computations and returning the layout bitmap (for ivar or blocks) in -/// the given argument BitMap string container. Routine reads -/// two containers, IvarsInfo and SkipIvars which are assumed to be -/// filled already by the caller. -llvm::Constant *CGObjCCommonMac::BuildIvarLayoutBitmap(std::string &BitMap) { - unsigned int WordsToScan, WordsToSkip; - llvm::Type *PtrTy = CGM.Int8PtrTy; - - // Build the string of skip/scan nibbles - SmallVector<SKIP_SCAN, 32> SkipScanIvars; - unsigned int WordSize = - CGM.getTypes().getDataLayout().getTypeAllocSize(PtrTy); - if (IvarsInfo[0].ivar_bytepos == 0) { - WordsToSkip = 0; - WordsToScan = IvarsInfo[0].ivar_size; +/// buildBitmap - This routine does the horsework of taking the offsets of +/// strong/weak references and creating a bitmap. The bitmap is also +/// returned in the given buffer, suitable for being passed to \c dump(). +llvm::Constant *IvarLayoutBuilder::buildBitmap(CGObjCCommonMac &CGObjC, + llvm::SmallVectorImpl<unsigned char> &buffer) { + // The bitmap is a series of skip/scan instructions, aligned to word + // boundaries. The skip is performed first. + const unsigned char MaxNibble = 0xF; + const unsigned char SkipMask = 0xF0, SkipShift = 4; + const unsigned char ScanMask = 0x0F, ScanShift = 0; + + assert(!IvarsInfo.empty() && "generating bitmap for no data"); + + // Sort the ivar info on byte position in case we encounterred a + // union nested in the ivar list. + if (IsDisordered) { + // This isn't a stable sort, but our algorithm should handle it fine. + llvm::array_pod_sort(IvarsInfo.begin(), IvarsInfo.end()); } else { - WordsToSkip = IvarsInfo[0].ivar_bytepos/WordSize; - WordsToScan = IvarsInfo[0].ivar_size; - } - for (unsigned int i=1, Last=IvarsInfo.size(); i != Last; i++) { - unsigned int TailPrevGCObjC = - IvarsInfo[i-1].ivar_bytepos + IvarsInfo[i-1].ivar_size * WordSize; - if (IvarsInfo[i].ivar_bytepos == TailPrevGCObjC) { - // consecutive 'scanned' object pointers. - WordsToScan += IvarsInfo[i].ivar_size; - } else { - // Skip over 'gc'able object pointer which lay over each other. - if (TailPrevGCObjC > IvarsInfo[i].ivar_bytepos) - continue; - // Must skip over 1 or more words. We save current skip/scan values - // and start a new pair. - SKIP_SCAN SkScan; - SkScan.skip = WordsToSkip; - SkScan.scan = WordsToScan; - SkipScanIvars.push_back(SkScan); - - // Skip the hole. - SkScan.skip = (IvarsInfo[i].ivar_bytepos - TailPrevGCObjC) / WordSize; - SkScan.scan = 0; - SkipScanIvars.push_back(SkScan); - WordsToSkip = 0; - WordsToScan = IvarsInfo[i].ivar_size; +#ifndef NDEBUG + for (unsigned i = 1; i != IvarsInfo.size(); ++i) { + assert(IvarsInfo[i - 1].Offset <= IvarsInfo[i].Offset); } +#endif } - if (WordsToScan > 0) { - SKIP_SCAN SkScan; - SkScan.skip = WordsToSkip; - SkScan.scan = WordsToScan; - SkipScanIvars.push_back(SkScan); - } - - if (!SkipIvars.empty()) { - unsigned int LastIndex = SkipIvars.size()-1; - int LastByteSkipped = - SkipIvars[LastIndex].ivar_bytepos + SkipIvars[LastIndex].ivar_size; - LastIndex = IvarsInfo.size()-1; - int LastByteScanned = - IvarsInfo[LastIndex].ivar_bytepos + - IvarsInfo[LastIndex].ivar_size * WordSize; - // Compute number of bytes to skip at the tail end of the last ivar scanned. - if (LastByteSkipped > LastByteScanned) { - unsigned int TotalWords = (LastByteSkipped + (WordSize -1)) / WordSize; - SKIP_SCAN SkScan; - SkScan.skip = TotalWords - (LastByteScanned/WordSize); - SkScan.scan = 0; - SkipScanIvars.push_back(SkScan); + assert(IvarsInfo.back().Offset < InstanceEnd); + + assert(buffer.empty()); + + // Skip the next N words. + auto skip = [&](unsigned numWords) { + assert(numWords > 0); + + // Try to merge into the previous byte. Since scans happen second, we + // can't do this if it includes a scan. + if (!buffer.empty() && !(buffer.back() & ScanMask)) { + unsigned lastSkip = buffer.back() >> SkipShift; + if (lastSkip < MaxNibble) { + unsigned claimed = std::min(MaxNibble - lastSkip, numWords); + numWords -= claimed; + lastSkip += claimed; + buffer.back() = (lastSkip << SkipShift); + } } - } - // Mini optimization of nibbles such that an 0xM0 followed by 0x0N is produced - // as 0xMN. - int SkipScan = SkipScanIvars.size()-1; - for (int i = 0; i <= SkipScan; i++) { - if ((i < SkipScan) && SkipScanIvars[i].skip && SkipScanIvars[i].scan == 0 - && SkipScanIvars[i+1].skip == 0 && SkipScanIvars[i+1].scan) { - // 0xM0 followed by 0x0N detected. - SkipScanIvars[i].scan = SkipScanIvars[i+1].scan; - for (int j = i+1; j < SkipScan; j++) - SkipScanIvars[j] = SkipScanIvars[j+1]; - --SkipScan; + + while (numWords >= MaxNibble) { + buffer.push_back(MaxNibble << SkipShift); + numWords -= MaxNibble; } - } - - // Generate the string. - for (int i = 0; i <= SkipScan; i++) { - unsigned char byte; - unsigned int skip_small = SkipScanIvars[i].skip % 0xf; - unsigned int scan_small = SkipScanIvars[i].scan % 0xf; - unsigned int skip_big = SkipScanIvars[i].skip / 0xf; - unsigned int scan_big = SkipScanIvars[i].scan / 0xf; - - // first skip big. - for (unsigned int ix = 0; ix < skip_big; ix++) - BitMap += (unsigned char)(0xf0); - - // next (skip small, scan) - if (skip_small) { - byte = skip_small << 4; - if (scan_big > 0) { - byte |= 0xf; - --scan_big; - } else if (scan_small) { - byte |= scan_small; - scan_small = 0; + if (numWords) { + buffer.push_back(numWords << SkipShift); + } + }; + + // Scan the next N words. + auto scan = [&](unsigned numWords) { + assert(numWords > 0); + + // Try to merge into the previous byte. Since scans happen second, we can + // do this even if it includes a skip. + if (!buffer.empty()) { + unsigned lastScan = (buffer.back() & ScanMask) >> ScanShift; + if (lastScan < MaxNibble) { + unsigned claimed = std::min(MaxNibble - lastScan, numWords); + numWords -= claimed; + lastScan += claimed; + buffer.back() = (buffer.back() & SkipMask) | (lastScan << ScanShift); } - BitMap += byte; } - // next scan big - for (unsigned int ix = 0; ix < scan_big; ix++) - BitMap += (unsigned char)(0x0f); - // last scan small - if (scan_small) { - byte = scan_small; - BitMap += byte; + + while (numWords >= MaxNibble) { + buffer.push_back(MaxNibble << ScanShift); + numWords -= MaxNibble; + } + if (numWords) { + buffer.push_back(numWords << ScanShift); + } + }; + + // One past the end of the last scan. + unsigned endOfLastScanInWords = 0; + const CharUnits WordSize = CGM.getPointerSize(); + + // Consider all the scan requests. + for (auto &request : IvarsInfo) { + CharUnits beginOfScan = request.Offset - InstanceBegin; + + // Ignore scan requests that don't start at an even multiple of the + // word size. We can't encode them. + if ((beginOfScan % WordSize) != 0) continue; + + // Ignore scan requests that start before the instance start. + // This assumes that scans never span that boundary. The boundary + // isn't the true start of the ivars, because in the fragile-ARC case + // it's rounded up to word alignment, but the test above should leave + // us ignoring that possibility. + if (beginOfScan.isNegative()) { + assert(request.Offset + request.SizeInWords * WordSize <= InstanceBegin); + continue; } + + unsigned beginOfScanInWords = beginOfScan / WordSize; + unsigned endOfScanInWords = beginOfScanInWords + request.SizeInWords; + + // If the scan starts some number of words after the last one ended, + // skip forward. + if (beginOfScanInWords > endOfLastScanInWords) { + skip(beginOfScanInWords - endOfLastScanInWords); + + // Otherwise, start scanning where the last left off. + } else { + beginOfScanInWords = endOfLastScanInWords; + + // If that leaves us with nothing to scan, ignore this request. + if (beginOfScanInWords >= endOfScanInWords) continue; + } + + // Scan to the end of the request. + assert(beginOfScanInWords < endOfScanInWords); + scan(endOfScanInWords - beginOfScanInWords); + endOfLastScanInWords = endOfScanInWords; } - // null terminate string. - unsigned char zero = 0; - BitMap += zero; - llvm::GlobalVariable *Entry = CreateMetadataVar( + if (buffer.empty()) + return llvm::ConstantPointerNull::get(CGM.Int8PtrTy); + + // For GC layouts, emit a skip to the end of the allocation so that we + // have precise information about the entire thing. This isn't useful + // or necessary for the ARC-style layout strings. + if (CGM.getLangOpts().getGC() != LangOptions::NonGC) { + unsigned lastOffsetInWords = + (InstanceEnd - InstanceBegin + WordSize - CharUnits::One()) / WordSize; + if (lastOffsetInWords > endOfLastScanInWords) { + skip(lastOffsetInWords - endOfLastScanInWords); + } + } + + // Null terminate the string. + buffer.push_back(0); + + bool isNonFragileABI = CGObjC.isNonFragileABI(); + + llvm::GlobalVariable *Entry = CGObjC.CreateMetadataVar( "OBJC_CLASS_NAME_", - llvm::ConstantDataArray::getString(VMContext, BitMap, false), - ((ObjCABI == 2) ? "__TEXT,__objc_classname,cstring_literals" - : "__TEXT,__cstring,cstring_literals"), - 1, true); - return getConstantGEP(VMContext, Entry, 0, 0); + llvm::ConstantDataArray::get(CGM.getLLVMContext(), buffer), + (isNonFragileABI ? "__TEXT,__objc_classname,cstring_literals" + : "__TEXT,__cstring,cstring_literals"), + CharUnits::One(), true); + return getConstantGEP(CGM.getLLVMContext(), Entry, 0, 0); } /// BuildIvarLayout - Builds ivar layout bitmap for the class @@ -4755,62 +4918,75 @@ llvm::Constant *CGObjCCommonMac::BuildIvarLayoutBitmap(std::string &BitMap) { /// 2. When ForStrongLayout is false, following ivars are scanned: /// - __weak anything /// -llvm::Constant *CGObjCCommonMac::BuildIvarLayout( - const ObjCImplementationDecl *OMD, - bool ForStrongLayout) { - bool hasUnion = false; - +llvm::Constant * +CGObjCCommonMac::BuildIvarLayout(const ObjCImplementationDecl *OMD, + CharUnits beginOffset, CharUnits endOffset, + bool ForStrongLayout, bool HasMRCWeakIvars) { + // If this is MRC, and we're either building a strong layout or there + // are no weak ivars, bail out early. llvm::Type *PtrTy = CGM.Int8PtrTy; if (CGM.getLangOpts().getGC() == LangOptions::NonGC && - !CGM.getLangOpts().ObjCAutoRefCount) + !CGM.getLangOpts().ObjCAutoRefCount && + (ForStrongLayout || !HasMRCWeakIvars)) return llvm::Constant::getNullValue(PtrTy); const ObjCInterfaceDecl *OI = OMD->getClassInterface(); - SmallVector<const FieldDecl*, 32> RecFields; - if (CGM.getLangOpts().ObjCAutoRefCount) { + SmallVector<const ObjCIvarDecl*, 32> ivars; + + // GC layout strings include the complete object layout, possibly + // inaccurately in the non-fragile ABI; the runtime knows how to fix this + // up. + // + // ARC layout strings only include the class's ivars. In non-fragile + // runtimes, that means starting at InstanceStart, rounded up to word + // alignment. In fragile runtimes, there's no InstanceStart, so it means + // starting at the offset of the first ivar, rounded up to word alignment. + // + // MRC weak layout strings follow the ARC style. + CharUnits baseOffset; + if (CGM.getLangOpts().getGC() == LangOptions::NonGC) { for (const ObjCIvarDecl *IVD = OI->all_declared_ivar_begin(); IVD; IVD = IVD->getNextIvar()) - RecFields.push_back(cast<FieldDecl>(IVD)); + ivars.push_back(IVD); + + if (isNonFragileABI()) { + baseOffset = beginOffset; // InstanceStart + } else if (!ivars.empty()) { + baseOffset = + CharUnits::fromQuantity(ComputeIvarBaseOffset(CGM, OMD, ivars[0])); + } else { + baseOffset = CharUnits::Zero(); + } + + baseOffset = baseOffset.RoundUpToAlignment(CGM.getPointerAlign()); } else { - SmallVector<const ObjCIvarDecl*, 32> Ivars; - CGM.getContext().DeepCollectObjCIvars(OI, true, Ivars); + CGM.getContext().DeepCollectObjCIvars(OI, true, ivars); - // FIXME: This is not ideal; we shouldn't have to do this copy. - RecFields.append(Ivars.begin(), Ivars.end()); + baseOffset = CharUnits::Zero(); } - if (RecFields.empty()) + if (ivars.empty()) return llvm::Constant::getNullValue(PtrTy); - SkipIvars.clear(); - IvarsInfo.clear(); + IvarLayoutBuilder builder(CGM, baseOffset, endOffset, ForStrongLayout); + + builder.visitAggregate(ivars.begin(), ivars.end(), CharUnits::Zero(), + [&](const ObjCIvarDecl *ivar) -> CharUnits { + return CharUnits::fromQuantity(ComputeIvarBaseOffset(CGM, OMD, ivar)); + }); - BuildAggrIvarLayout(OMD, nullptr, nullptr, RecFields, 0, ForStrongLayout, - hasUnion); - if (IvarsInfo.empty()) + if (!builder.hasBitmapData()) return llvm::Constant::getNullValue(PtrTy); - // Sort on byte position in case we encounterred a union nested in - // the ivar list. - if (hasUnion && !IvarsInfo.empty()) - std::sort(IvarsInfo.begin(), IvarsInfo.end()); - if (hasUnion && !SkipIvars.empty()) - std::sort(SkipIvars.begin(), SkipIvars.end()); - - std::string BitMap; - llvm::Constant *C = BuildIvarLayoutBitmap(BitMap); + + llvm::SmallVector<unsigned char, 4> buffer; + llvm::Constant *C = builder.buildBitmap(*this, buffer); - if (CGM.getLangOpts().ObjCGCBitmapPrint) { + if (CGM.getLangOpts().ObjCGCBitmapPrint && !buffer.empty()) { printf("\n%s ivar layout for class '%s': ", ForStrongLayout ? "strong" : "weak", OMD->getClassInterface()->getName().str().c_str()); - const unsigned char *s = (const unsigned char*)BitMap.c_str(); - for (unsigned i = 0, e = BitMap.size(); i < e; i++) - if (!(s[i] & 0xf0)) - printf("0x0%x%s", s[i], s[i] != 0 ? ", " : ""); - else - printf("0x%x%s", s[i], s[i] != 0 ? ", " : ""); - printf("\n"); + builder.dump(buffer); } return C; } @@ -4825,7 +5001,7 @@ llvm::Constant *CGObjCCommonMac::GetMethodVarName(Selector Sel) { llvm::ConstantDataArray::getString(VMContext, Sel.getAsString()), ((ObjCABI == 2) ? "__TEXT,__objc_methname,cstring_literals" : "__TEXT,__cstring,cstring_literals"), - 1, true); + CharUnits::One(), true); return getConstantGEP(VMContext, Entry, 0, 0); } @@ -4847,7 +5023,7 @@ llvm::Constant *CGObjCCommonMac::GetMethodVarType(const FieldDecl *Field) { llvm::ConstantDataArray::getString(VMContext, TypeStr), ((ObjCABI == 2) ? "__TEXT,__objc_methtype,cstring_literals" : "__TEXT,__cstring,cstring_literals"), - 1, true); + CharUnits::One(), true); return getConstantGEP(VMContext, Entry, 0, 0); } @@ -4866,7 +5042,7 @@ llvm::Constant *CGObjCCommonMac::GetMethodVarType(const ObjCMethodDecl *D, llvm::ConstantDataArray::getString(VMContext, TypeStr), ((ObjCABI == 2) ? "__TEXT,__objc_methtype,cstring_literals" : "__TEXT,__cstring,cstring_literals"), - 1, true); + CharUnits::One(), true); return getConstantGEP(VMContext, Entry, 0, 0); } @@ -4879,7 +5055,7 @@ llvm::Constant *CGObjCCommonMac::GetPropertyName(IdentifierInfo *Ident) { Entry = CreateMetadataVar( "OBJC_PROP_NAME_ATTR_", llvm::ConstantDataArray::getString(VMContext, Ident->getName()), - "__TEXT,__cstring,cstring_literals", 1, true); + "__TEXT,__cstring,cstring_literals", CharUnits::One(), true); return getConstantGEP(VMContext, Entry, 0, 0); } @@ -5583,8 +5759,14 @@ llvm::GlobalVariable * CGObjCNonFragileABIMac::BuildClassRoTInitializer( std::string ClassName = ID->getObjCRuntimeNameAsString(); llvm::Constant *Values[10]; // 11 for 64bit targets! + CharUnits beginInstance = CharUnits::fromQuantity(InstanceStart); + CharUnits endInstance = CharUnits::fromQuantity(InstanceSize); + + bool hasMRCWeak = false; if (CGM.getLangOpts().ObjCAutoRefCount) flags |= NonFragileABI_Class_CompiledByARC; + else if ((hasMRCWeak = hasMRCWeakIvars(CGM, ID))) + flags |= NonFragileABI_Class_HasMRCWeakIvars; Values[ 0] = llvm::ConstantInt::get(ObjCTypes.IntTy, flags); Values[ 1] = llvm::ConstantInt::get(ObjCTypes.IntTy, InstanceStart); @@ -5592,7 +5774,7 @@ llvm::GlobalVariable * CGObjCNonFragileABIMac::BuildClassRoTInitializer( // FIXME. For 64bit targets add 0 here. Values[ 3] = (flags & NonFragileABI_Class_Meta) ? GetIvarLayoutName(nullptr, ObjCTypes) - : BuildIvarLayout(ID, true); + : BuildStrongIvarLayout(ID, beginInstance, endInstance); Values[ 4] = GetClassName(ID->getObjCRuntimeNameAsString()); // const struct _method_list_t * const baseMethods; std::vector<llvm::Constant*> Methods; @@ -5639,7 +5821,8 @@ llvm::GlobalVariable * CGObjCNonFragileABIMac::BuildClassRoTInitializer( Values[ 9] = llvm::Constant::getNullValue(ObjCTypes.PropertyListPtrTy); } else { Values[ 7] = EmitIvarList(ID); - Values[ 8] = BuildIvarLayout(ID, false); + Values[ 8] = BuildWeakIvarLayout(ID, beginInstance, endInstance, + hasMRCWeak); Values[ 9] = EmitPropertyList("\01l_OBJC_$_PROP_LIST_" + ID->getObjCRuntimeNameAsString(), ID, ID->getClassInterface(), ObjCTypes); } @@ -5731,7 +5914,8 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) { // Make this entry NULL for any iOS device target, any iOS simulator target, // OS X with deployment target 10.9 or later. const llvm::Triple &Triple = CGM.getTarget().getTriple(); - if (Triple.isiOS() || (Triple.isMacOSX() && !Triple.isMacOSXVersionLT(10, 9))) + if (Triple.isiOS() || Triple.isWatchOS() || + (Triple.isMacOSX() && !Triple.isMacOSXVersionLT(10, 9))) // This entry will be null. ObjCEmptyVtableVar = nullptr; else @@ -5887,9 +6071,11 @@ llvm::Value *CGObjCNonFragileABIMac::GenerateProtocolRef(CodeGenFunction &CGF, std::string ProtocolName("\01l_OBJC_PROTOCOL_REFERENCE_$_"); ProtocolName += PD->getObjCRuntimeNameAsString(); + CharUnits Align = CGF.getPointerAlign(); + llvm::GlobalVariable *PTGV = CGM.getModule().getGlobalVariable(ProtocolName); if (PTGV) - return CGF.Builder.CreateLoad(PTGV); + return CGF.Builder.CreateAlignedLoad(PTGV, Align); PTGV = new llvm::GlobalVariable( CGM.getModule(), Init->getType(), false, @@ -5898,8 +6084,9 @@ llvm::Value *CGObjCNonFragileABIMac::GenerateProtocolRef(CodeGenFunction &CGF, ProtocolName); PTGV->setSection("__DATA, __objc_protorefs, coalesced, no_dead_strip"); PTGV->setVisibility(llvm::GlobalValue::HiddenVisibility); + PTGV->setAlignment(Align.getQuantity()); CGM.addCompilerUsedGlobal(PTGV); - return CGF.Builder.CreateLoad(PTGV); + return CGF.Builder.CreateAlignedLoad(PTGV, Align); } /// GenerateCategory - Build metadata for a category implementation. @@ -6428,7 +6615,8 @@ llvm::Value *CGObjCNonFragileABIMac::EmitIvarOffset( const ObjCInterfaceDecl *Interface, const ObjCIvarDecl *Ivar) { llvm::Value *IvarOffsetValue = ObjCIvarOffsetVariable(Interface, Ivar); - IvarOffsetValue = CGF.Builder.CreateLoad(IvarOffsetValue, "ivar"); + IvarOffsetValue = CGF.Builder.CreateAlignedLoad(IvarOffsetValue, + CGF.getSizeAlign(), "ivar"); if (IsIvarOffsetKnownIdempotent(CGF, Ivar)) cast<llvm::LoadInst>(IvarOffsetValue) ->setMetadata(CGM.getModule().getMDKindID("invariant.load"), @@ -6559,16 +6747,17 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF, } } - llvm::Value *mref = - CGF.Builder.CreateBitCast(messageRef, ObjCTypes.MessageRefPtrTy); + Address mref = + Address(CGF.Builder.CreateBitCast(messageRef, ObjCTypes.MessageRefPtrTy), + CGF.getPointerAlign()); // Update the message ref argument. - args[1].RV = RValue::get(mref); + args[1].RV = RValue::get(mref.getPointer()); // Load the function to call from the message ref table. - llvm::Value *callee = - CGF.Builder.CreateStructGEP(ObjCTypes.MessageRefTy, mref, 0); - callee = CGF.Builder.CreateLoad(callee, "msgSend_fn"); + Address calleeAddr = + CGF.Builder.CreateStructGEP(mref, 0, CharUnits::Zero()); + llvm::Value *callee = CGF.Builder.CreateLoad(calleeAddr, "msgSend_fn"); callee = CGF.Builder.CreateBitCast(callee, MSI.MessengerType); @@ -6594,7 +6783,7 @@ CGObjCNonFragileABIMac::GenerateMessageSend(CodeGen::CodeGenFunction &CGF, : EmitMessageSend(CGF, Return, ResultType, EmitSelector(CGF, Sel), Receiver, CGF.getContext().getObjCIdType(), - false, CallArgs, Method, ObjCTypes); + false, CallArgs, Method, Class, ObjCTypes); } llvm::GlobalVariable * @@ -6617,6 +6806,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitClassRefFromId(CodeGenFunction &CGF, IdentifierInfo *II, bool Weak, const ObjCInterfaceDecl *ID) { + CharUnits Align = CGF.getPointerAlign(); llvm::GlobalVariable *&Entry = ClassReferences[II]; if (!Entry) { @@ -6627,13 +6817,11 @@ llvm::Value *CGObjCNonFragileABIMac::EmitClassRefFromId(CodeGenFunction &CGF, Entry = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassnfABIPtrTy, false, llvm::GlobalValue::PrivateLinkage, ClassGV, "OBJC_CLASSLIST_REFERENCES_$_"); - Entry->setAlignment( - CGM.getDataLayout().getABITypeAlignment( - ObjCTypes.ClassnfABIPtrTy)); + Entry->setAlignment(Align.getQuantity()); Entry->setSection("__DATA, __objc_classrefs, regular, no_dead_strip"); CGM.addCompilerUsedGlobal(Entry); } - return CGF.Builder.CreateLoad(Entry); + return CGF.Builder.CreateAlignedLoad(Entry, Align); } llvm::Value *CGObjCNonFragileABIMac::EmitClassRef(CodeGenFunction &CGF, @@ -6644,12 +6832,13 @@ llvm::Value *CGObjCNonFragileABIMac::EmitClassRef(CodeGenFunction &CGF, llvm::Value *CGObjCNonFragileABIMac::EmitNSAutoreleasePoolClassRef( CodeGenFunction &CGF) { IdentifierInfo *II = &CGM.getContext().Idents.get("NSAutoreleasePool"); - return EmitClassRefFromId(CGF, II, false, 0); + return EmitClassRefFromId(CGF, II, false, nullptr); } llvm::Value * CGObjCNonFragileABIMac::EmitSuperClassRef(CodeGenFunction &CGF, const ObjCInterfaceDecl *ID) { + CharUnits Align = CGF.getPointerAlign(); llvm::GlobalVariable *&Entry = SuperClassReferences[ID->getIdentifier()]; if (!Entry) { @@ -6660,13 +6849,11 @@ CGObjCNonFragileABIMac::EmitSuperClassRef(CodeGenFunction &CGF, Entry = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassnfABIPtrTy, false, llvm::GlobalValue::PrivateLinkage, ClassGV, "OBJC_CLASSLIST_SUP_REFS_$_"); - Entry->setAlignment( - CGM.getDataLayout().getABITypeAlignment( - ObjCTypes.ClassnfABIPtrTy)); + Entry->setAlignment(Align.getQuantity()); Entry->setSection("__DATA, __objc_superrefs, regular, no_dead_strip"); CGM.addCompilerUsedGlobal(Entry); } - return CGF.Builder.CreateLoad(Entry); + return CGF.Builder.CreateAlignedLoad(Entry, Align); } /// EmitMetaClassRef - Return a Value * of the address of _class_t @@ -6675,6 +6862,7 @@ CGObjCNonFragileABIMac::EmitSuperClassRef(CodeGenFunction &CGF, llvm::Value *CGObjCNonFragileABIMac::EmitMetaClassRef(CodeGenFunction &CGF, const ObjCInterfaceDecl *ID, bool Weak) { + CharUnits Align = CGF.getPointerAlign(); llvm::GlobalVariable * &Entry = MetaClassReferences[ID->getIdentifier()]; if (!Entry) { llvm::SmallString<64> MetaClassName(getMetaclassSymbolPrefix()); @@ -6685,14 +6873,13 @@ llvm::Value *CGObjCNonFragileABIMac::EmitMetaClassRef(CodeGenFunction &CGF, Entry = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassnfABIPtrTy, false, llvm::GlobalValue::PrivateLinkage, MetaClassGV, "OBJC_CLASSLIST_SUP_REFS_$_"); - Entry->setAlignment( - CGM.getDataLayout().getABITypeAlignment(ObjCTypes.ClassnfABIPtrTy)); + Entry->setAlignment(Align.getQuantity()); Entry->setSection("__DATA, __objc_superrefs, regular, no_dead_strip"); CGM.addCompilerUsedGlobal(Entry); } - return CGF.Builder.CreateLoad(Entry); + return CGF.Builder.CreateAlignedLoad(Entry, Align); } /// GetClass - Return a reference to the class for the given interface @@ -6727,14 +6914,15 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, // ... // Create and init a super structure; this is a (receiver, class) // pair we will pass to objc_msgSendSuper. - llvm::Value *ObjCSuper = - CGF.CreateTempAlloca(ObjCTypes.SuperTy, "objc_super"); + Address ObjCSuper = + CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(), + "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); CGF.Builder.CreateStore( ReceiverAsObject, - CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 0)); + CGF.Builder.CreateStructGEP(ObjCSuper, 0, CharUnits::Zero())); // If this is a class message the metaclass is passed as the target. llvm::Value *Target; @@ -6749,22 +6937,33 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType()); Target = CGF.Builder.CreateBitCast(Target, ClassTy); CGF.Builder.CreateStore( - Target, CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 1)); + Target, CGF.Builder.CreateStructGEP(ObjCSuper, 1, CGF.getPointerSize())); return (isVTableDispatchedSelector(Sel)) ? EmitVTableMessageSend(CGF, Return, ResultType, Sel, - ObjCSuper, ObjCTypes.SuperPtrCTy, + ObjCSuper.getPointer(), ObjCTypes.SuperPtrCTy, true, CallArgs, Method) : EmitMessageSend(CGF, Return, ResultType, EmitSelector(CGF, Sel), - ObjCSuper, ObjCTypes.SuperPtrCTy, - true, CallArgs, Method, ObjCTypes); + ObjCSuper.getPointer(), ObjCTypes.SuperPtrCTy, + true, CallArgs, Method, Class, ObjCTypes); } llvm::Value *CGObjCNonFragileABIMac::EmitSelector(CodeGenFunction &CGF, - Selector Sel, bool lval) { + Selector Sel) { + Address Addr = EmitSelectorAddr(CGF, Sel); + + llvm::LoadInst* LI = CGF.Builder.CreateLoad(Addr); + LI->setMetadata(CGM.getModule().getMDKindID("invariant.load"), + llvm::MDNode::get(VMContext, None)); + return LI; +} + +Address CGObjCNonFragileABIMac::EmitSelectorAddr(CodeGenFunction &CGF, + Selector Sel) { llvm::GlobalVariable *&Entry = SelectorReferences[Sel]; + CharUnits Align = CGF.getPointerAlign(); if (!Entry) { llvm::Constant *Casted = llvm::ConstantExpr::getBitCast(GetMethodVarName(Sel), @@ -6774,23 +6973,19 @@ llvm::Value *CGObjCNonFragileABIMac::EmitSelector(CodeGenFunction &CGF, Casted, "OBJC_SELECTOR_REFERENCES_"); Entry->setExternallyInitialized(true); Entry->setSection("__DATA, __objc_selrefs, literal_pointers, no_dead_strip"); + Entry->setAlignment(Align.getQuantity()); CGM.addCompilerUsedGlobal(Entry); } - if (lval) - return Entry; - llvm::LoadInst* LI = CGF.Builder.CreateLoad(Entry); - - LI->setMetadata(CGM.getModule().getMDKindID("invariant.load"), - llvm::MDNode::get(VMContext, None)); - return LI; + return Address(Entry, Align); } + /// EmitObjCIvarAssign - Code gen for assigning to a __strong object. /// objc_assign_ivar (id src, id *dst, ptrdiff_t) /// void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, llvm::Value *src, - llvm::Value *dst, + Address dst, llvm::Value *ivarOffset) { llvm::Type * SrcTy = src->getType(); if (!isa<llvm::PointerType>(SrcTy)) { @@ -6802,7 +6997,7 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy); - llvm::Value *args[] = { src, dst, ivarOffset }; + llvm::Value *args[] = { src, dst.getPointer(), ivarOffset }; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args); } @@ -6811,7 +7006,7 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, /// void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign( CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst) { + llvm::Value *src, Address dst) { llvm::Type * SrcTy = src->getType(); if (!isa<llvm::PointerType>(SrcTy)) { unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy); @@ -6822,19 +7017,19 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign( } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy); - llvm::Value *args[] = { src, dst }; + llvm::Value *args[] = { src, dst.getPointer() }; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(), args, "weakassign"); } void CGObjCNonFragileABIMac::EmitGCMemmoveCollectable( CodeGen::CodeGenFunction &CGF, - llvm::Value *DestPtr, - llvm::Value *SrcPtr, + Address DestPtr, + Address SrcPtr, llvm::Value *Size) { SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, ObjCTypes.Int8PtrTy); DestPtr = CGF.Builder.CreateBitCast(DestPtr, ObjCTypes.Int8PtrTy); - llvm::Value *args[] = { DestPtr, SrcPtr, Size }; + llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), Size }; CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args); } @@ -6843,13 +7038,12 @@ void CGObjCNonFragileABIMac::EmitGCMemmoveCollectable( /// llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead( CodeGen::CodeGenFunction &CGF, - llvm::Value *AddrWeakObj) { - llvm::Type* DestTy = - cast<llvm::PointerType>(AddrWeakObj->getType())->getElementType(); + Address AddrWeakObj) { + llvm::Type *DestTy = AddrWeakObj.getElementType(); AddrWeakObj = CGF.Builder.CreateBitCast(AddrWeakObj, ObjCTypes.PtrObjectPtrTy); llvm::Value *read_weak = CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(), - AddrWeakObj, "weakread"); + AddrWeakObj.getPointer(), "weakread"); read_weak = CGF.Builder.CreateBitCast(read_weak, DestTy); return read_weak; } @@ -6858,7 +7052,7 @@ llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead( /// objc_assign_weak (id src, id *dst) /// void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst) { + llvm::Value *src, Address dst) { llvm::Type * SrcTy = src->getType(); if (!isa<llvm::PointerType>(SrcTy)) { unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy); @@ -6869,7 +7063,7 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy); - llvm::Value *args[] = { src, dst }; + llvm::Value *args[] = { src, dst.getPointer() }; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(), args, "weakassign"); } @@ -6878,7 +7072,7 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, /// objc_assign_global (id src, id *dst) /// void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dst, + llvm::Value *src, Address dst, bool threadlocal) { llvm::Type * SrcTy = src->getType(); if (!isa<llvm::PointerType>(SrcTy)) { @@ -6890,7 +7084,7 @@ void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy); - llvm::Value *args[] = { src, dst }; + llvm::Value *args[] = { src, dst.getPointer() }; if (!threadlocal) CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(), args, "globalassign"); @@ -7043,6 +7237,7 @@ CodeGen::CreateMacObjCRuntime(CodeGen::CodeGenModule &CGM) { case ObjCRuntime::MacOSX: case ObjCRuntime::iOS: + case ObjCRuntime::WatchOS: return new CGObjCNonFragileABIMac(CGM); case ObjCRuntime::GNUstep: diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp index 2ac6bb2..7be9ae9 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp @@ -136,12 +136,13 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF, CGF.CGM.getContext().toBits(StorageSize), CharUnits::fromQuantity(0))); - V = CGF.Builder.CreateBitCast(V, - llvm::Type::getIntNPtrTy(CGF.getLLVMContext(), + Address Addr(V, Alignment); + Addr = CGF.Builder.CreateElementBitCast(Addr, + llvm::Type::getIntNTy(CGF.getLLVMContext(), Info->StorageSize)); - return LValue::MakeBitfield(V, *Info, + return LValue::MakeBitfield(Addr, *Info, IvarTy.withCVRQualifiers(CVRQualifiers), - Alignment); + AlignmentSource::Decl); } namespace { @@ -152,7 +153,7 @@ namespace { llvm::Constant *TypeInfo; }; - struct CallObjCEndCatch : EHScopeStack::Cleanup { + struct CallObjCEndCatch final : EHScopeStack::Cleanup { CallObjCEndCatch(bool MightThrow, llvm::Value *Fn) : MightThrow(MightThrow), Fn(Fn) {} bool MightThrow; @@ -255,24 +256,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF, llvm::Value *CastExn = CGF.Builder.CreateBitCast(Exn, CatchType); CGF.EmitAutoVarDecl(*CatchParam); - - llvm::Value *CatchParamAddr = CGF.GetAddrOfLocalVar(CatchParam); - - switch (CatchParam->getType().getQualifiers().getObjCLifetime()) { - case Qualifiers::OCL_Strong: - CastExn = CGF.EmitARCRetainNonBlock(CastExn); - // fallthrough - - case Qualifiers::OCL_None: - case Qualifiers::OCL_ExplicitNone: - case Qualifiers::OCL_Autoreleasing: - CGF.Builder.CreateStore(CastExn, CatchParamAddr); - break; - - case Qualifiers::OCL_Weak: - CGF.EmitARCInitWeak(CatchParamAddr, CastExn); - break; - } + EmitInitOfCatchParam(CGF, CastExn, CatchParam); } CGF.ObjCEHValueStack.push_back(Exn); @@ -296,8 +280,32 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF, CGF.EmitBlock(Cont.getBlock()); } +void CGObjCRuntime::EmitInitOfCatchParam(CodeGenFunction &CGF, + llvm::Value *exn, + const VarDecl *paramDecl) { + + Address paramAddr = CGF.GetAddrOfLocalVar(paramDecl); + + switch (paramDecl->getType().getQualifiers().getObjCLifetime()) { + case Qualifiers::OCL_Strong: + exn = CGF.EmitARCRetainNonBlock(exn); + // fallthrough + + case Qualifiers::OCL_None: + case Qualifiers::OCL_ExplicitNone: + case Qualifiers::OCL_Autoreleasing: + CGF.Builder.CreateStore(exn, paramAddr); + return; + + case Qualifiers::OCL_Weak: + CGF.EmitARCInitWeak(paramAddr, exn); + return; + } + llvm_unreachable("invalid ownership qualifier"); +} + namespace { - struct CallSyncExit : EHScopeStack::Cleanup { + struct CallSyncExit final : EHScopeStack::Cleanup { llvm::Value *SyncExitFn; llvm::Value *SyncArg; CallSyncExit(llvm::Value *SyncExitFn, llvm::Value *SyncArg) diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h index 4752546..28d88dd 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.h @@ -100,6 +100,10 @@ protected: llvm::Constant *beginCatchFn, llvm::Constant *endCatchFn, llvm::Constant *exceptionRethrowFn); + + void EmitInitOfCatchParam(CodeGenFunction &CGF, llvm::Value *exn, + const VarDecl *paramDecl); + /// Emits an \@synchronize() statement, using the \p syncEnterFn and /// \p syncExitFn arguments as the functions called to lock and unlock /// the object. This function can be called by subclasses that use @@ -116,11 +120,16 @@ public: /// this compilation unit with the runtime library. virtual llvm::Function *ModuleInitFunction() = 0; - /// Get a selector for the specified name and type values. The - /// return value should have the LLVM type for pointer-to + /// Get a selector for the specified name and type values. + /// The result should have the LLVM type for ASTContext::getObjCSelType(). + virtual llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel) = 0; + + /// Get the address of a selector for the specified name and type values. + /// This is a rarely-used language extension, but sadly it exists. + /// + /// The result should have the LLVM type for a pointer to /// ASTContext::getObjCSelType(). - virtual llvm::Value *GetSelector(CodeGenFunction &CGF, - Selector Sel, bool lval=false) = 0; + virtual Address GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) = 0; /// Get a typed selector. virtual llvm::Value *GetSelector(CodeGenFunction &CGF, @@ -133,7 +142,7 @@ public: virtual llvm::Constant *GetEHType(QualType T) = 0; /// Generate a constant string object. - virtual llvm::Constant *GenerateConstantString(const StringLiteral *) = 0; + virtual ConstantAddress GenerateConstantString(const StringLiteral *) = 0; /// Generate a category. A category contains a list of methods (and /// accompanying metadata) and a list of protocols. @@ -238,17 +247,17 @@ public: const ObjCAtThrowStmt &S, bool ClearInsertionPoint=true) = 0; virtual llvm::Value *EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF, - llvm::Value *AddrWeakObj) = 0; + Address AddrWeakObj) = 0; virtual void EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest) = 0; + llvm::Value *src, Address dest) = 0; virtual void EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest, + llvm::Value *src, Address dest, bool threadlocal=false) = 0; virtual void EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest, + llvm::Value *src, Address dest, llvm::Value *ivarOffset) = 0; virtual void EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, - llvm::Value *src, llvm::Value *dest) = 0; + llvm::Value *src, Address dest) = 0; virtual LValue EmitObjCValueForIvar(CodeGen::CodeGenFunction &CGF, QualType ObjectTy, @@ -259,15 +268,18 @@ public: const ObjCInterfaceDecl *Interface, const ObjCIvarDecl *Ivar) = 0; virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, - llvm::Value *DestPtr, - llvm::Value *SrcPtr, + Address DestPtr, + Address SrcPtr, llvm::Value *Size) = 0; virtual llvm::Constant *BuildGCBlockLayout(CodeGen::CodeGenModule &CGM, const CodeGen::CGBlockInfo &blockInfo) = 0; virtual llvm::Constant *BuildRCBlockLayout(CodeGen::CodeGenModule &CGM, const CodeGen::CGBlockInfo &blockInfo) = 0; + + /// Returns an i8* which points to the byref layout information. virtual llvm::Constant *BuildByrefLayout(CodeGen::CodeGenModule &CGM, QualType T) = 0; + virtual llvm::GlobalVariable *GetClassGlobal(const std::string &Name, bool Weak = false) = 0; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.cpp index 079ef72..8af39ce 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenCLRuntime.cpp @@ -55,6 +55,28 @@ llvm::Type *CGOpenCLRuntime::convertOpenCLSpecificType(const Type *T) { case BuiltinType::OCLImage2dArray: return llvm::PointerType::get(llvm::StructType::create( Ctx, "opencl.image2d_array_t"), ImgAddrSpc); + case BuiltinType::OCLImage2dDepth: + return llvm::PointerType::get( + llvm::StructType::create(Ctx, "opencl.image2d_depth_t"), ImgAddrSpc); + case BuiltinType::OCLImage2dArrayDepth: + return llvm::PointerType::get( + llvm::StructType::create(Ctx, "opencl.image2d_array_depth_t"), + ImgAddrSpc); + case BuiltinType::OCLImage2dMSAA: + return llvm::PointerType::get( + llvm::StructType::create(Ctx, "opencl.image2d_msaa_t"), ImgAddrSpc); + case BuiltinType::OCLImage2dArrayMSAA: + return llvm::PointerType::get( + llvm::StructType::create(Ctx, "opencl.image2d_array_msaa_t"), + ImgAddrSpc); + case BuiltinType::OCLImage2dMSAADepth: + return llvm::PointerType::get( + llvm::StructType::create(Ctx, "opencl.image2d_msaa_depth_t"), + ImgAddrSpc); + case BuiltinType::OCLImage2dArrayMSAADepth: + return llvm::PointerType::get( + llvm::StructType::create(Ctx, "opencl.image2d_array_msaa_depth_t"), + ImgAddrSpc); case BuiltinType::OCLImage3d: return llvm::PointerType::get(llvm::StructType::create( Ctx, "opencl.image3d_t"), ImgAddrSpc); @@ -63,5 +85,17 @@ llvm::Type *CGOpenCLRuntime::convertOpenCLSpecificType(const Type *T) { case BuiltinType::OCLEvent: return llvm::PointerType::get(llvm::StructType::create( Ctx, "opencl.event_t"), 0); + case BuiltinType::OCLClkEvent: + return llvm::PointerType::get( + llvm::StructType::create(Ctx, "opencl.clk_event_t"), 0); + case BuiltinType::OCLQueue: + return llvm::PointerType::get( + llvm::StructType::create(Ctx, "opencl.queue_t"), 0); + case BuiltinType::OCLNDRange: + return llvm::PointerType::get( + llvm::StructType::create(Ctx, "opencl.ndrange_t"), 0); + case BuiltinType::OCLReserveID: + return llvm::PointerType::get( + llvm::StructType::create(Ctx, "opencl.reserve_id_t"), 0); } } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 8148839..0ba7e06 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -41,25 +41,29 @@ public: /// \brief Region for constructs that do not require function outlining, /// like 'for', 'sections', 'atomic' etc. directives. InlinedRegion, + /// \brief Region with outlined function for standalone 'target' directive. + TargetRegion, }; CGOpenMPRegionInfo(const CapturedStmt &CS, const CGOpenMPRegionKind RegionKind, - const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind) + const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind, + bool HasCancel) : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind), - CodeGen(CodeGen), Kind(Kind) {} + CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {} CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind, - const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind) + const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind, + bool HasCancel) : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen), - Kind(Kind) {} + Kind(Kind), HasCancel(HasCancel) {} /// \brief Get a variable or parameter for storing global thread id /// inside OpenMP construct. virtual const VarDecl *getThreadIDVariable() const = 0; /// \brief Emit the captured statement body. - virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) override; + void EmitBody(CodeGenFunction &CGF, const Stmt *S) override; /// \brief Get an LValue for the current ThreadID variable. /// \return LValue for thread id variable. This LValue always has type int32*. @@ -69,6 +73,8 @@ public: OpenMPDirectiveKind getDirectiveKind() const { return Kind; } + bool hasCancel() const { return HasCancel; } + static bool classof(const CGCapturedStmtInfo *Info) { return Info->getKind() == CR_OpenMP; } @@ -77,6 +83,7 @@ protected: CGOpenMPRegionKind RegionKind; const RegionCodeGenTy &CodeGen; OpenMPDirectiveKind Kind; + bool HasCancel; }; /// \brief API for captured statement code generation in OpenMP constructs. @@ -84,8 +91,9 @@ class CGOpenMPOutlinedRegionInfo : public CGOpenMPRegionInfo { public: CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar, const RegionCodeGenTy &CodeGen, - OpenMPDirectiveKind Kind) - : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind), + OpenMPDirectiveKind Kind, bool HasCancel) + : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind, + HasCancel), ThreadIDVar(ThreadIDVar) { assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); } @@ -114,8 +122,8 @@ public: CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar, const RegionCodeGenTy &CodeGen, - OpenMPDirectiveKind Kind) - : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind), + OpenMPDirectiveKind Kind, bool HasCancel) + : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel), ThreadIDVar(ThreadIDVar) { assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); } @@ -147,8 +155,9 @@ class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo { public: CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI, const RegionCodeGenTy &CodeGen, - OpenMPDirectiveKind Kind) - : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind), OldCSI(OldCSI), + OpenMPDirectiveKind Kind, bool HasCancel) + : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel), + OldCSI(OldCSI), OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {} // \brief Retrieve the value of the context parameter. llvm::Value *getContextValue() const override { @@ -156,7 +165,7 @@ public: return OuterRegionInfo->getContextValue(); llvm_unreachable("No context value for inlined OpenMP region"); } - virtual void setContextValue(llvm::Value *V) override { + void setContextValue(llvm::Value *V) override { if (OuterRegionInfo) { OuterRegionInfo->setContextValue(V); return; @@ -204,6 +213,29 @@ private: CGOpenMPRegionInfo *OuterRegionInfo; }; +/// \brief API for captured statement code generation in OpenMP target +/// constructs. For this captures, implicit parameters are used instead of the +/// captured fields. +class CGOpenMPTargetRegionInfo : public CGOpenMPRegionInfo { +public: + CGOpenMPTargetRegionInfo(const CapturedStmt &CS, + const RegionCodeGenTy &CodeGen) + : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target, + /*HasCancel = */ false) {} + + /// \brief This is unused for target regions because each starts executing + /// with a single thread. + const VarDecl *getThreadIDVariable() const override { return nullptr; } + + /// \brief Get the name of the capture helper. + StringRef getHelperName() const override { return ".omp_offloading."; } + + static bool classof(const CGCapturedStmtInfo *Info) { + return CGOpenMPRegionInfo::classof(Info) && + cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion; + } +}; + /// \brief RAII for emitting code of OpenMP constructs. class InlinedOpenMPRegionRAII { CodeGenFunction &CGF; @@ -214,11 +246,11 @@ public: /// a list of functions used for code generation of implicitly inlined /// regions. InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, - OpenMPDirectiveKind Kind) + OpenMPDirectiveKind Kind, bool HasCancel) : CGF(CGF) { // Start emission for the construct. - CGF.CapturedStmtInfo = - new CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, CodeGen, Kind); + CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo( + CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel); } ~InlinedOpenMPRegionRAII() { // Restore original CapturedStmtInfo only if we're done with code emission. @@ -229,20 +261,25 @@ public: } }; -} // namespace +} // anonymous namespace + +static LValue emitLoadOfPointerLValue(CodeGenFunction &CGF, Address PtrAddr, + QualType Ty) { + AlignmentSource Source; + CharUnits Align = CGF.getNaturalPointeeTypeAlignment(Ty, &Source); + return CGF.MakeAddrLValue(Address(CGF.Builder.CreateLoad(PtrAddr), Align), + Ty->getPointeeType(), Source); +} LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { - return CGF.MakeNaturalAlignAddrLValue( - CGF.Builder.CreateAlignedLoad( - CGF.GetAddrOfLocalVar(getThreadIDVariable()), - CGF.PointerAlignInBytes), - getThreadIDVariable() - ->getType() - ->castAs<PointerType>() - ->getPointeeType()); + return emitLoadOfPointerLValue(CGF, + CGF.GetAddrOfLocalVar(getThreadIDVariable()), + getThreadIDVariable()->getType()); } void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) { + if (!CGF.HaveInsertPoint()) + return; // 1.2.2 OpenMP Language Terminology // Structured block - An executable statement with a single entry at the // top and a single exit at the bottom. @@ -258,9 +295,9 @@ void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) { LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue( CodeGenFunction &CGF) { - return CGF.MakeNaturalAlignAddrLValue( - CGF.GetAddrOfLocalVar(getThreadIDVariable()), - getThreadIDVariable()->getType()); + return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()), + getThreadIDVariable()->getType(), + AlignmentSource::Decl); } CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM) @@ -280,6 +317,25 @@ void CGOpenMPRuntime::clear() { InternalVars.clear(); } +// Layout information for ident_t. +static CharUnits getIdentAlign(CodeGenModule &CGM) { + return CGM.getPointerAlign(); +} +static CharUnits getIdentSize(CodeGenModule &CGM) { + assert((4 * CGM.getPointerSize()).isMultipleOf(CGM.getPointerAlign())); + return CharUnits::fromQuantity(16) + CGM.getPointerSize(); +} +static CharUnits getOffsetOfIdentField(CGOpenMPRuntime::IdentFieldIndex Field) { + // All the fields except the last are i32, so this works beautifully. + return unsigned(Field) * CharUnits::fromQuantity(4); +} +static Address createIdentFieldGEP(CodeGenFunction &CGF, Address Addr, + CGOpenMPRuntime::IdentFieldIndex Field, + const llvm::Twine &Name = "") { + auto Offset = getOffsetOfIdentField(Field); + return CGF.Builder.CreateStructGEP(Addr, Field, Offset, Name); +} + llvm::Value *CGOpenMPRuntime::emitParallelOutlinedFunction( const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { @@ -287,9 +343,17 @@ llvm::Value *CGOpenMPRuntime::emitParallelOutlinedFunction( "thread id variable must be of type kmp_int32 *"); const CapturedStmt *CS = cast<CapturedStmt>(D.getAssociatedStmt()); CodeGenFunction CGF(CGM, true); - CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind); + bool HasCancel = false; + if (auto *OPD = dyn_cast<OMPParallelDirective>(&D)) + HasCancel = OPD->hasCancel(); + else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D)) + HasCancel = OPSD->hasCancel(); + else if (auto *OPFD = dyn_cast<OMPParallelForDirective>(&D)) + HasCancel = OPFD->hasCancel(); + CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind, + HasCancel); CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); - return CGF.GenerateCapturedStmtFunction(*CS); + return CGF.GenerateOpenMPCapturedStmtFunction(*CS); } llvm::Value *CGOpenMPRuntime::emitTaskOutlinedFunction( @@ -300,13 +364,14 @@ llvm::Value *CGOpenMPRuntime::emitTaskOutlinedFunction( auto *CS = cast<CapturedStmt>(D.getAssociatedStmt()); CodeGenFunction CGF(CGM, true); CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, - InnermostKind); + InnermostKind, + cast<OMPTaskDirective>(D).hasCancel()); CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); return CGF.GenerateCapturedStmtFunction(*CS); } -llvm::Value * -CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) { +Address CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) { + CharUnits Align = getIdentAlign(CGM); llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags); if (!Entry) { if (!DefaultOpenMPPSource) { @@ -315,7 +380,7 @@ CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) { // Taken from // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp_str.c DefaultOpenMPPSource = - CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;"); + CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer(); DefaultOpenMPPSource = llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy); } @@ -323,6 +388,7 @@ CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) { CGM.getModule(), IdentTy, /*isConstant*/ true, llvm::GlobalValue::PrivateLinkage, /*Initializer*/ nullptr); DefaultOpenMPLocation->setUnnamedAddr(true); + DefaultOpenMPLocation->setAlignment(Align.getQuantity()); llvm::Constant *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0, true); llvm::Constant *Values[] = {Zero, @@ -330,10 +396,9 @@ CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) { Zero, Zero, DefaultOpenMPPSource}; llvm::Constant *Init = llvm::ConstantStruct::get(IdentTy, Values); DefaultOpenMPLocation->setInitializer(Init); - OpenMPDefaultLocMap[Flags] = DefaultOpenMPLocation; - return DefaultOpenMPLocation; + OpenMPDefaultLocMap[Flags] = Entry = DefaultOpenMPLocation; } - return Entry; + return Address(Entry, Align); } llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, @@ -342,34 +407,33 @@ llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, // If no debug info is generated - return global default location. if (CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::NoDebugInfo || Loc.isInvalid()) - return getOrCreateDefaultLocation(Flags); + return getOrCreateDefaultLocation(Flags).getPointer(); assert(CGF.CurFn && "No function in current CodeGenFunction."); - llvm::Value *LocValue = nullptr; + Address LocValue = Address::invalid(); auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); if (I != OpenMPLocThreadIDMap.end()) - LocValue = I->second.DebugLoc; + LocValue = Address(I->second.DebugLoc, getIdentAlign(CGF.CGM)); + // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if // GetOpenMPThreadID was called before this routine. - if (LocValue == nullptr) { + if (!LocValue.isValid()) { // Generate "ident_t .kmpc_loc.addr;" - llvm::AllocaInst *AI = CGF.CreateTempAlloca(IdentTy, ".kmpc_loc.addr"); - AI->setAlignment(CGM.getDataLayout().getPrefTypeAlignment(IdentTy)); + Address AI = CGF.CreateTempAlloca(IdentTy, getIdentAlign(CGF.CGM), + ".kmpc_loc.addr"); auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); - Elem.second.DebugLoc = AI; + Elem.second.DebugLoc = AI.getPointer(); LocValue = AI; CGBuilderTy::InsertPointGuard IPG(CGF.Builder); CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt); CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags), - llvm::ConstantExpr::getSizeOf(IdentTy), - CGM.PointerAlignInBytes); + CGM.getSize(getIdentSize(CGF.CGM))); } // char **psource = &.kmpc_loc_<flags>.addr.psource; - auto *PSource = CGF.Builder.CreateConstInBoundsGEP2_32(IdentTy, LocValue, 0, - IdentField_PSource); + Address PSource = createIdentFieldGEP(CGF, LocValue, IdentField_PSource); auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding()); if (OMPDebugLoc == nullptr) { @@ -389,7 +453,9 @@ llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, // *psource = ";<File>;<Function>;<Line>;<Column>;;"; CGF.Builder.CreateStore(OMPDebugLoc, PSource); - return LocValue; + // Our callers always pass this to a runtime function, so for + // convenience, go ahead and return a naked pointer. + return LocValue.getPointer(); } llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, @@ -493,6 +559,17 @@ CGOpenMPRuntime::createRuntimeFunction(OpenMPRTLFunction Function) { RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical"); break; } + case OMPRTL__kmpc_critical_with_hint: { + // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid, + // kmp_critical_name *crit, uintptr_t hint); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, + llvm::PointerType::getUnqual(KmpCriticalNameTy), + CGM.IntPtrTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint"); + break; + } case OMPRTL__kmpc_threadprivate_register: { // Build void __kmpc_threadprivate_register(ident_t *, void *data, // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor); @@ -838,10 +915,46 @@ CGOpenMPRuntime::createRuntimeFunction(OpenMPRTLFunction Function) { RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel"); break; } + case OMPRTL__tgt_target: { + // Build int32_t __tgt_target(int32_t device_id, void *host_ptr, int32_t + // arg_num, void** args_base, void **args, size_t *arg_sizes, int32_t + // *arg_types); + llvm::Type *TypeParams[] = {CGM.Int32Ty, + CGM.VoidPtrTy, + CGM.Int32Ty, + CGM.VoidPtrPtrTy, + CGM.VoidPtrPtrTy, + CGM.SizeTy->getPointerTo(), + CGM.Int32Ty->getPointerTo()}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target"); + break; + } } return RTLFn; } +static llvm::Value *getTypeSize(CodeGenFunction &CGF, QualType Ty) { + auto &C = CGF.getContext(); + llvm::Value *Size = nullptr; + auto SizeInChars = C.getTypeSizeInChars(Ty); + if (SizeInChars.isZero()) { + // getTypeSizeInChars() returns 0 for a VLA. + while (auto *VAT = C.getAsVariableArrayType(Ty)) { + llvm::Value *ArraySize; + std::tie(ArraySize, Ty) = CGF.getVLASize(VAT); + Size = Size ? CGF.Builder.CreateNUWMul(Size, ArraySize) : ArraySize; + } + SizeInChars = C.getTypeSizeInChars(Ty); + assert(!SizeInChars.isZero()); + Size = CGF.Builder.CreateNUWMul( + Size, llvm::ConstantInt::get(CGF.SizeTy, SizeInChars.getQuantity())); + } else + Size = llvm::ConstantInt::get(CGF.SizeTy, SizeInChars.getQuantity()); + return Size; +} + llvm::Constant *CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) { assert((IVSize == 32 || IVSize == 64) && @@ -939,25 +1052,27 @@ CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) { Twine(CGM.getMangledName(VD)) + ".cache."); } -llvm::Value *CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, - const VarDecl *VD, - llvm::Value *VDAddr, - SourceLocation Loc) { +Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, + const VarDecl *VD, + Address VDAddr, + SourceLocation Loc) { if (CGM.getLangOpts().OpenMPUseTLS && CGM.getContext().getTargetInfo().isTLSSupported()) return VDAddr; - auto VarTy = VDAddr->getType()->getPointerElementType(); + auto VarTy = VDAddr.getElementType(); llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), - CGF.Builder.CreatePointerCast(VDAddr, CGM.Int8PtrTy), + CGF.Builder.CreatePointerCast(VDAddr.getPointer(), + CGM.Int8PtrTy), CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)), getOrCreateThreadPrivateCache(VD)}; - return CGF.EmitRuntimeCall( - createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args); + return Address(CGF.EmitRuntimeCall( + createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args), + VDAddr.getAlignment()); } void CGOpenMPRuntime::emitThreadPrivateVarInit( - CodeGenFunction &CGF, llvm::Value *VDAddr, llvm::Value *Ctor, + CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) { // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime // library. @@ -967,14 +1082,15 @@ void CGOpenMPRuntime::emitThreadPrivateVarInit( // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor) // to register constructor/destructor for variable. llvm::Value *Args[] = {OMPLoc, - CGF.Builder.CreatePointerCast(VDAddr, CGM.VoidPtrTy), + CGF.Builder.CreatePointerCast(VDAddr.getPointer(), + CGM.VoidPtrTy), Ctor, CopyCtor, Dtor}; CGF.EmitRuntimeCall( createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args); } llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition( - const VarDecl *VD, llvm::Value *VDAddr, SourceLocation Loc, + const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF) { if (CGM.getLangOpts().OpenMPUseTLS && CGM.getContext().getTargetInfo().isTLSSupported()) @@ -1001,21 +1117,19 @@ llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition( /*isVariadic=*/false); auto FTy = CGM.getTypes().GetFunctionType(FI); auto Fn = CGM.CreateGlobalInitOrDestructFunction( - FTy, ".__kmpc_global_ctor_.", Loc); + FTy, ".__kmpc_global_ctor_.", FI, Loc); CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI, Args, SourceLocation()); auto ArgVal = CtorCGF.EmitLoadOfScalar( - CtorCGF.GetAddrOfLocalVar(&Dst), - /*Volatile=*/false, CGM.PointerAlignInBytes, + CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation()); - auto Arg = CtorCGF.Builder.CreatePointerCast( - ArgVal, - CtorCGF.ConvertTypeForMem(CGM.getContext().getPointerType(ASTTy))); + Address Arg = Address(ArgVal, VDAddr.getAlignment()); + Arg = CtorCGF.Builder.CreateElementBitCast(Arg, + CtorCGF.ConvertTypeForMem(ASTTy)); CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(), /*IsInitializer=*/true); ArgVal = CtorCGF.EmitLoadOfScalar( - CtorCGF.GetAddrOfLocalVar(&Dst), - /*Volatile=*/false, CGM.PointerAlignInBytes, + CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation()); CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue); CtorCGF.FinishFunction(); @@ -1035,14 +1149,13 @@ llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition( /*isVariadic=*/false); auto FTy = CGM.getTypes().GetFunctionType(FI); auto Fn = CGM.CreateGlobalInitOrDestructFunction( - FTy, ".__kmpc_global_dtor_.", Loc); + FTy, ".__kmpc_global_dtor_.", FI, Loc); DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args, SourceLocation()); auto ArgVal = DtorCGF.EmitLoadOfScalar( DtorCGF.GetAddrOfLocalVar(&Dst), - /*Volatile=*/false, CGM.PointerAlignInBytes, - CGM.getContext().VoidPtrTy, Dst.getLocation()); - DtorCGF.emitDestroy(ArgVal, ASTTy, + /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation()); + DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()), DtorCGF.needsEHCleanup(ASTTy.isDestructedType())); DtorCGF.FinishFunction(); @@ -1074,7 +1187,8 @@ llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition( auto InitFunctionTy = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false); auto InitFunction = CGM.CreateGlobalInitOrDestructFunction( - InitFunctionTy, ".__omp_threadprivate_init_."); + InitFunctionTy, ".__omp_threadprivate_init_.", + CGM.getTypes().arrangeNullaryFunction()); CodeGenFunction InitCGF(CGM); FunctionArgList ArgList; InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction, @@ -1149,25 +1263,27 @@ static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct, + ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond) { + if (!CGF.HaveInsertPoint()) + return; auto *RTLoc = emitUpdateLocation(CGF, Loc); - auto &&ThenGen = - [this, OutlinedFn, CapturedStruct, RTLoc](CodeGenFunction &CGF) { - // Build call __kmpc_fork_call(loc, 1, microtask, - // captured_struct/*context*/) - llvm::Value *Args[] = { - RTLoc, - CGF.Builder.getInt32( - 1), // Number of arguments after 'microtask' argument - // (there is only one additional argument - 'context') - CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()), - CGF.EmitCastToVoidPtr(CapturedStruct)}; - auto RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_call); - CGF.EmitRuntimeCall(RTLFn, Args); - }; - auto &&ElseGen = [this, OutlinedFn, CapturedStruct, RTLoc, Loc]( - CodeGenFunction &CGF) { + auto &&ThenGen = [this, OutlinedFn, CapturedVars, + RTLoc](CodeGenFunction &CGF) { + // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn); + llvm::Value *Args[] = { + RTLoc, + CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars + CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())}; + llvm::SmallVector<llvm::Value *, 16> RealArgs; + RealArgs.append(std::begin(Args), std::end(Args)); + RealArgs.append(CapturedVars.begin(), CapturedVars.end()); + + auto RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_call); + CGF.EmitRuntimeCall(RTLFn, RealArgs); + }; + auto &&ElseGen = [this, OutlinedFn, CapturedVars, RTLoc, + Loc](CodeGenFunction &CGF) { auto ThreadID = getThreadID(CGF, Loc); // Build calls: // __kmpc_serialized_parallel(&Loc, GTid); @@ -1177,11 +1293,14 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, // OutlinedFn(>id, &zero, CapturedStruct); auto ThreadIDAddr = emitThreadIDAddress(CGF, Loc); - auto Int32Ty = CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, - /*Signed*/ true); - auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr"); + Address ZeroAddr = + CGF.CreateTempAlloca(CGF.Int32Ty, CharUnits::fromQuantity(4), + /*Name*/ ".zero.addr"); CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); - llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct}; + llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs; + OutlinedFnArgs.push_back(ThreadIDAddr.getPointer()); + OutlinedFnArgs.push_back(ZeroAddr.getPointer()); + OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs); // __kmpc_end_serialized_parallel(&Loc, GTid); @@ -1203,8 +1322,8 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, // regular serial code region, get thread ID by calling kmp_int32 // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and // return the address of that temp. -llvm::Value *CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, - SourceLocation Loc) { +Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, + SourceLocation Loc) { if (auto OMPRegionInfo = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) if (OMPRegionInfo->getThreadIDVariable()) @@ -1215,7 +1334,7 @@ llvm::Value *CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); CGF.EmitStoreOfScalar(ThreadID, - CGF.MakeNaturalAlignAddrLValue(ThreadIDTemp, Int32Ty)); + CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty)); return ThreadIDTemp; } @@ -1246,7 +1365,7 @@ llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) { } namespace { -template <size_t N> class CallEndCleanup : public EHScopeStack::Cleanup { +template <size_t N> class CallEndCleanup final : public EHScopeStack::Cleanup { llvm::Value *Callee; llvm::Value *Args[N]; @@ -1257,39 +1376,50 @@ public: std::copy(CleanupArgs.begin(), CleanupArgs.end(), std::begin(Args)); } void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { + if (!CGF.HaveInsertPoint()) + return; CGF.EmitRuntimeCall(Callee, Args); } }; -} // namespace +} // anonymous namespace void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, - SourceLocation Loc) { - // __kmpc_critical(ident_t *, gtid, Lock); + SourceLocation Loc, const Expr *Hint) { + // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]); // CriticalOpGen(); // __kmpc_end_critical(ident_t *, gtid, Lock); // Prepare arguments and build a call to __kmpc_critical - { - CodeGenFunction::RunCleanupsScope Scope(CGF); - llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), - getCriticalRegionLock(CriticalName)}; + if (!CGF.HaveInsertPoint()) + return; + CodeGenFunction::RunCleanupsScope Scope(CGF); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + getCriticalRegionLock(CriticalName)}; + if (Hint) { + llvm::SmallVector<llvm::Value *, 8> ArgsWithHint(std::begin(Args), + std::end(Args)); + auto *HintVal = CGF.EmitScalarExpr(Hint); + ArgsWithHint.push_back( + CGF.Builder.CreateIntCast(HintVal, CGM.IntPtrTy, /*isSigned=*/false)); + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical_with_hint), + ArgsWithHint); + } else CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical), Args); - // Build a call to __kmpc_end_critical - CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( - NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_critical), - llvm::makeArrayRef(Args)); - emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); - } + // Build a call to __kmpc_end_critical + CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_critical), + llvm::makeArrayRef(Args)); + emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); } static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond, - OpenMPDirectiveKind Kind, + OpenMPDirectiveKind Kind, SourceLocation Loc, const RegionCodeGenTy &BodyOpGen) { llvm::Value *CallBool = CGF.EmitScalarConversion( IfCond, CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true), - CGF.getContext().BoolTy); + CGF.getContext().BoolTy, Loc); auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); auto *ContBlock = CGF.createBasicBlock("omp_if.end"); @@ -1305,6 +1435,8 @@ static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond, void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) { + if (!CGF.HaveInsertPoint()) + return; // if(__kmpc_master(ident_t *, gtid)) { // MasterOpGen(); // __kmpc_end_master(ident_t *, gtid); @@ -1315,17 +1447,20 @@ void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_master), Args); typedef CallEndCleanup<std::extent<decltype(Args)>::value> MasterCallEndCleanup; - emitIfStmt(CGF, IsMaster, OMPD_master, [&](CodeGenFunction &CGF) -> void { - CodeGenFunction::RunCleanupsScope Scope(CGF); - CGF.EHStack.pushCleanup<MasterCallEndCleanup>( - NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_master), - llvm::makeArrayRef(Args)); - MasterOpGen(CGF); - }); + emitIfStmt( + CGF, IsMaster, OMPD_master, Loc, [&](CodeGenFunction &CGF) -> void { + CodeGenFunction::RunCleanupsScope Scope(CGF); + CGF.EHStack.pushCleanup<MasterCallEndCleanup>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_master), + llvm::makeArrayRef(Args)); + MasterOpGen(CGF); + }); } void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) { + if (!CGF.HaveInsertPoint()) + return; // Build call __kmpc_omp_taskyield(loc, thread_id, 0); llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), @@ -1336,6 +1471,8 @@ void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) { + if (!CGF.HaveInsertPoint()) + return; // __kmpc_taskgroup(ident_t *, gtid); // TaskgroupOpGen(); // __kmpc_end_taskgroup(ident_t *, gtid); @@ -1352,6 +1489,21 @@ void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, } } +/// Given an array of pointers to variables, project the address of a +/// given variable. +static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, + unsigned Index, const VarDecl *Var) { + // Pull out the pointer to the variable. + Address PtrAddr = + CGF.Builder.CreateConstArrayGEP(Array, Index, CGF.getPointerSize()); + llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr); + + Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var)); + Addr = CGF.Builder.CreateElementBitCast( + Addr, CGF.ConvertTypeForMem(Var->getType())); + return Addr; +} + static llvm::Value *emitCopyprivateCopyFunction( CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs, @@ -1371,40 +1523,31 @@ static llvm::Value *emitCopyprivateCopyFunction( auto *Fn = llvm::Function::Create( CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage, ".omp.copyprivate.copy_func", &CGM.getModule()); - CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn); + CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, CGFI); CodeGenFunction CGF(CGM); CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args); // Dest = (void*[n])(LHSArg); // Src = (void*[n])(RHSArg); - auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg), - CGF.PointerAlignInBytes), - ArgsType); - auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg), - CGF.PointerAlignInBytes), - ArgsType); + Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), + ArgsType), CGF.getPointerAlign()); + Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), + ArgsType), CGF.getPointerAlign()); // *(Type0*)Dst[0] = *(Type0*)Src[0]; // *(Type1*)Dst[1] = *(Type1*)Src[1]; // ... // *(Typen*)Dst[n] = *(Typen*)Src[n]; for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) { - auto *DestAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateAlignedLoad( - CGF.Builder.CreateStructGEP(nullptr, LHS, I), - CGM.PointerAlignInBytes), - CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType()))); - auto *SrcAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateAlignedLoad( - CGF.Builder.CreateStructGEP(nullptr, RHS, I), - CGM.PointerAlignInBytes), - CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType()))); + auto DestVar = cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl()); + Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar); + + auto SrcVar = cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl()); + Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar); + auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl(); QualType Type = VD->getType(); - CGF.EmitOMPCopy(CGF, Type, DestAddr, SrcAddr, - cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl()), - cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl()), - AssignmentOps[I]); + CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]); } CGF.FinishFunction(); return Fn; @@ -1417,6 +1560,8 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> DstExprs, ArrayRef<const Expr *> AssignmentOps) { + if (!CGF.HaveInsertPoint()) + return; assert(CopyprivateVars.size() == SrcExprs.size() && CopyprivateVars.size() == DstExprs.size() && CopyprivateVars.size() == AssignmentOps.size()); @@ -1430,13 +1575,12 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>, // <copy_func>, did_it); - llvm::AllocaInst *DidIt = nullptr; + Address DidIt = Address::invalid(); if (!CopyprivateVars.empty()) { // int32 did_it = 0; auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it"); - CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(0), DidIt, - DidIt->getAlignment()); + CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt); } // Prepare arguments and build a call to __kmpc_single llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; @@ -1444,52 +1588,51 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_single), Args); typedef CallEndCleanup<std::extent<decltype(Args)>::value> SingleCallEndCleanup; - emitIfStmt(CGF, IsSingle, OMPD_single, [&](CodeGenFunction &CGF) -> void { - CodeGenFunction::RunCleanupsScope Scope(CGF); - CGF.EHStack.pushCleanup<SingleCallEndCleanup>( - NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_single), - llvm::makeArrayRef(Args)); - SingleOpGen(CGF); - if (DidIt) { - // did_it = 1; - CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(1), DidIt, - DidIt->getAlignment()); - } - }); + emitIfStmt( + CGF, IsSingle, OMPD_single, Loc, [&](CodeGenFunction &CGF) -> void { + CodeGenFunction::RunCleanupsScope Scope(CGF); + CGF.EHStack.pushCleanup<SingleCallEndCleanup>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_single), + llvm::makeArrayRef(Args)); + SingleOpGen(CGF); + if (DidIt.isValid()) { + // did_it = 1; + CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); + } + }); // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>, // <copy_func>, did_it); - if (DidIt) { + if (DidIt.isValid()) { llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size()); auto CopyprivateArrayTy = C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal, /*IndexTypeQuals=*/0); // Create a list of all private variables for copyprivate. - auto *CopyprivateList = + Address CopyprivateList = CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) { - auto *Elem = CGF.Builder.CreateStructGEP( - CopyprivateList->getAllocatedType(), CopyprivateList, I); - CGF.Builder.CreateAlignedStore( + Address Elem = CGF.Builder.CreateConstArrayGEP( + CopyprivateList, I, CGF.getPointerSize()); + CGF.Builder.CreateStore( CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.EmitLValue(CopyprivateVars[I]).getAddress(), CGF.VoidPtrTy), - Elem, CGM.PointerAlignInBytes); + CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy), + Elem); } // Build function that copies private values from single region to all other // threads in the corresponding parallel region. auto *CpyFn = emitCopyprivateCopyFunction( CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(), CopyprivateVars, SrcExprs, DstExprs, AssignmentOps); - auto *BufSize = llvm::ConstantInt::get( - CGM.SizeTy, C.getTypeSizeInChars(CopyprivateArrayTy).getQuantity()); - auto *CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList, - CGF.VoidPtrTy); - auto *DidItVal = - CGF.Builder.CreateAlignedLoad(DidIt, CGF.PointerAlignInBytes); + auto *BufSize = getTypeSize(CGF, CopyprivateArrayTy); + Address CL = + CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList, + CGF.VoidPtrTy); + auto *DidItVal = CGF.Builder.CreateLoad(DidIt); llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc), // ident_t *<loc> getThreadID(CGF, Loc), // i32 <gtid> BufSize, // size_t <buf_size> - CL, // void *<copyprivate list> + CL.getPointer(), // void *<copyprivate list> CpyFn, // void (*) (void *, void *) <copy_func> DidItVal // i32 did_it }; @@ -1499,26 +1642,30 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, - SourceLocation Loc) { + SourceLocation Loc, bool IsThreads) { + if (!CGF.HaveInsertPoint()) + return; // __kmpc_ordered(ident_t *, gtid); // OrderedOpGen(); // __kmpc_end_ordered(ident_t *, gtid); // Prepare arguments and build a call to __kmpc_ordered - { - CodeGenFunction::RunCleanupsScope Scope(CGF); + CodeGenFunction::RunCleanupsScope Scope(CGF); + if (IsThreads) { llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_ordered), Args); // Build a call to __kmpc_end_ordered CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_ordered), llvm::makeArrayRef(Args)); - emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); } + emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); } void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPDirectiveKind Kind, - bool CheckForCancel) { + OpenMPDirectiveKind Kind, bool EmitChecks, + bool ForceSimpleCall) { + if (!CGF.HaveInsertPoint()) + return; // Build call __kmpc_cancel_barrier(loc, thread_id); // Build call __kmpc_barrier(loc, thread_id); OpenMPLocationFlags Flags = OMP_IDENT_KMPC; @@ -1538,16 +1685,19 @@ void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, } // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc, // thread_id); + auto *OMPRegionInfo = + dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo); + // Do not emit barrier call in the single directive emitted in some rare cases + // for sections directives. + if (OMPRegionInfo && OMPRegionInfo->getDirectiveKind() == OMPD_single) + return; llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), getThreadID(CGF, Loc)}; - if (auto *OMPRegionInfo = - dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { - auto CancelDestination = - CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); - if (CancelDestination.isValid()) { + if (OMPRegionInfo) { + if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) { auto *Result = CGF.EmitRuntimeCall( createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args); - if (CheckForCancel) { + if (EmitChecks) { // if (__kmpc_cancel_barrier()) { // exit from construct; // } @@ -1557,6 +1707,8 @@ void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); CGF.EmitBlock(ExitBB); // exit from construct; + auto CancelDestination = + CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); CGF.EmitBranchThroughCleanup(CancelDestination); CGF.EmitBlock(ContBB, /*IsFinished=*/true); } @@ -1623,65 +1775,87 @@ bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const { return Schedule != OMP_sch_static; } -void CGOpenMPRuntime::emitForInit(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPScheduleClauseKind ScheduleKind, - unsigned IVSize, bool IVSigned, bool Ordered, - llvm::Value *IL, llvm::Value *LB, - llvm::Value *UB, llvm::Value *ST, - llvm::Value *Chunk) { +void CGOpenMPRuntime::emitForDispatchInit(CodeGenFunction &CGF, + SourceLocation Loc, + OpenMPScheduleClauseKind ScheduleKind, + unsigned IVSize, bool IVSigned, + bool Ordered, llvm::Value *UB, + llvm::Value *Chunk) { + if (!CGF.HaveInsertPoint()) + return; OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunk != nullptr, Ordered); - if (Ordered || - (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && - Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked)) { - // Call __kmpc_dispatch_init( - // ident_t *loc, kmp_int32 tid, kmp_int32 schedule, - // kmp_int[32|64] lower, kmp_int[32|64] upper, - // kmp_int[32|64] stride, kmp_int[32|64] chunk); + assert(Ordered || + (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && + Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked)); + // Call __kmpc_dispatch_init( + // ident_t *loc, kmp_int32 tid, kmp_int32 schedule, + // kmp_int[32|64] lower, kmp_int[32|64] upper, + // kmp_int[32|64] stride, kmp_int[32|64] chunk); + + // If the Chunk was not specified in the clause - use default value 1. + if (Chunk == nullptr) + Chunk = CGF.Builder.getIntN(IVSize, 1); + llvm::Value *Args[] = { + emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), + getThreadID(CGF, Loc), + CGF.Builder.getInt32(Schedule), // Schedule type + CGF.Builder.getIntN(IVSize, 0), // Lower + UB, // Upper + CGF.Builder.getIntN(IVSize, 1), // Stride + Chunk // Chunk + }; + CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args); +} +void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF, + SourceLocation Loc, + OpenMPScheduleClauseKind ScheduleKind, + unsigned IVSize, bool IVSigned, + bool Ordered, Address IL, Address LB, + Address UB, Address ST, + llvm::Value *Chunk) { + if (!CGF.HaveInsertPoint()) + return; + OpenMPSchedType Schedule = + getRuntimeSchedule(ScheduleKind, Chunk != nullptr, Ordered); + assert(!Ordered); + assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || + Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked); + + // Call __kmpc_for_static_init( + // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype, + // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower, + // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride, + // kmp_int[32|64] incr, kmp_int[32|64] chunk); + if (Chunk == nullptr) { + assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static) && + "expected static non-chunked schedule"); // If the Chunk was not specified in the clause - use default value 1. - if (Chunk == nullptr) Chunk = CGF.Builder.getIntN(IVSize, 1); - llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), - getThreadID(CGF, Loc), - CGF.Builder.getInt32(Schedule), // Schedule type - CGF.Builder.getIntN(IVSize, 0), // Lower - UB, // Upper - CGF.Builder.getIntN(IVSize, 1), // Stride - Chunk // Chunk - }; - CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args); } else { - // Call __kmpc_for_static_init( - // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype, - // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower, - // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride, - // kmp_int[32|64] incr, kmp_int[32|64] chunk); - if (Chunk == nullptr) { - assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static) && - "expected static non-chunked schedule"); - // If the Chunk was not specified in the clause - use default value 1. - Chunk = CGF.Builder.getIntN(IVSize, 1); - } else - assert((Schedule == OMP_sch_static_chunked || - Schedule == OMP_ord_static_chunked) && - "expected static chunked schedule"); - llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), - getThreadID(CGF, Loc), - CGF.Builder.getInt32(Schedule), // Schedule type - IL, // &isLastIter - LB, // &LB - UB, // &UB - ST, // &Stride - CGF.Builder.getIntN(IVSize, 1), // Incr - Chunk // Chunk - }; - CGF.EmitRuntimeCall(createForStaticInitFunction(IVSize, IVSigned), Args); + assert((Schedule == OMP_sch_static_chunked || + Schedule == OMP_ord_static_chunked) && + "expected static chunked schedule"); } + llvm::Value *Args[] = { + emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), + getThreadID(CGF, Loc), + CGF.Builder.getInt32(Schedule), // Schedule type + IL.getPointer(), // &isLastIter + LB.getPointer(), // &LB + UB.getPointer(), // &UB + ST.getPointer(), // &Stride + CGF.Builder.getIntN(IVSize, 1), // Incr + Chunk // Chunk + }; + CGF.EmitRuntimeCall(createForStaticInitFunction(IVSize, IVSigned), Args); } void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc) { + if (!CGF.HaveInsertPoint()) + return; // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid); llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), getThreadID(CGF, Loc)}; @@ -1693,6 +1867,8 @@ void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) { + if (!CGF.HaveInsertPoint()) + return; // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid); llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), getThreadID(CGF, Loc)}; @@ -1701,30 +1877,32 @@ void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, - bool IVSigned, llvm::Value *IL, - llvm::Value *LB, llvm::Value *UB, - llvm::Value *ST) { + bool IVSigned, Address IL, + Address LB, Address UB, + Address ST) { // Call __kmpc_dispatch_next( // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, // kmp_int[32|64] *p_stride); llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), getThreadID(CGF, Loc), - IL, // &isLastIter - LB, // &Lower - UB, // &Upper - ST // &Stride + IL.getPointer(), // &isLastIter + LB.getPointer(), // &Lower + UB.getPointer(), // &Upper + ST.getPointer() // &Stride }; llvm::Value *Call = CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args); return CGF.EmitScalarConversion( Call, CGF.getContext().getIntTypeForBitwidth(32, /* Signed */ true), - CGF.getContext().BoolTy); + CGF.getContext().BoolTy, Loc); } void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc) { + if (!CGF.HaveInsertPoint()) + return; // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads) llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), @@ -1736,6 +1914,8 @@ void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, OpenMPProcBindClauseKind ProcBind, SourceLocation Loc) { + if (!CGF.HaveInsertPoint()) + return; // Constants for proc bind value accepted by the runtime. enum ProcBindTy { ProcBindFalse = 0, @@ -1768,6 +1948,8 @@ void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, SourceLocation Loc) { + if (!CGF.HaveInsertPoint()) + return; // Build call void __kmpc_flush(ident_t *loc) CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush), emitUpdateLocation(CGF, Loc)); @@ -1785,7 +1967,7 @@ enum KmpTaskTFields { /// \brief Function with call of destructors for private variables. KmpTaskTDestructors, }; -} // namespace +} // anonymous namespace void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) { if (!KmpRoutineEntryPtrTy) { @@ -1799,14 +1981,15 @@ void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) { } } -static void addFieldToRecordDecl(ASTContext &C, DeclContext *DC, - QualType FieldTy) { +static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC, + QualType FieldTy) { auto *Field = FieldDecl::Create( C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy, C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()), /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit); Field->setAccess(AS_public); DC->addDecl(Field); + return Field; } namespace { @@ -1820,11 +2003,10 @@ struct PrivateHelpersTy { const VarDecl *PrivateElemInit; }; typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy; -} // namespace +} // anonymous namespace static RecordDecl * -createPrivatesRecordDecl(CodeGenModule &CGM, - const ArrayRef<PrivateDataTy> Privates) { +createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) { if (!Privates.empty()) { auto &C = CGM.getContext(); // Build struct .kmp_privates_t. { @@ -1833,9 +2015,16 @@ createPrivatesRecordDecl(CodeGenModule &CGM, auto *RD = C.buildImplicitRecord(".kmp_privates.t"); RD->startDefinition(); for (auto &&Pair : Privates) { - auto Type = Pair.second.Original->getType(); + auto *VD = Pair.second.Original; + auto Type = VD->getType(); Type = Type.getNonReferenceType(); - addFieldToRecordDecl(C, RD, Type); + auto *FD = addFieldToRecordDecl(C, RD, Type); + if (VD->hasAttrs()) { + for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()), + E(VD->getAttrs().end()); + I != E; ++I) + FD->addAttr(*I); + } } RD->completeDefinition(); return RD; @@ -1865,7 +2054,7 @@ createKmpTaskTRecordDecl(CodeGenModule &CGM, QualType KmpInt32Ty, static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, - const ArrayRef<PrivateDataTy> Privates) { + ArrayRef<PrivateDataTy> Privates) { auto &C = CGM.getContext(); // Build struct kmp_task_t_with_privates { // kmp_task_t task_data; @@ -1900,7 +2089,8 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, FunctionArgList Args; ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty); ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, - /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy); + /*Id=*/nullptr, + KmpTaskTWithPrivatesPtrQTy.withRestrict()); Args.push_back(&GtidArg); Args.push_back(&TaskTypeArg); FunctionType::ExtInfo Info; @@ -1911,7 +2101,7 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, auto *TaskEntry = llvm::Function::Create(TaskEntryTy, llvm::GlobalValue::InternalLinkage, ".omp_task_entry.", &CGM.getModule()); - CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskEntryFnInfo, TaskEntry); + CGM.SetInternalFunctionAttributes(/*D=*/nullptr, TaskEntry, TaskEntryFnInfo); CodeGenFunction CGF(CGM); CGF.disableDebugInfo(); CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args); @@ -1919,12 +2109,9 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map, // tt->task_data.shareds); auto *GtidParam = CGF.EmitLoadOfScalar( - CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, - C.getTypeAlignInChars(KmpInt32Ty).getQuantity(), KmpInt32Ty, Loc); - auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad( - CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes); - LValue TDBase = - CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy); + CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc); + LValue TDBase = emitLoadOfPointerLValue( + CGF, CGF.GetAddrOfLocalVar(&TaskTypeArg), KmpTaskTWithPrivatesPtrQTy); auto *KmpTaskTWithPrivatesQTyRD = cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl()); LValue Base = @@ -1945,7 +2132,7 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) { auto PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - PrivatesLVal.getAddress(), CGF.VoidPtrTy); + PrivatesLVal.getPointer(), CGF.VoidPtrTy); } else { PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); } @@ -1955,7 +2142,7 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, CGF.EmitCallOrInvoke(TaskFunction, CallArgs); CGF.EmitStoreThroughLValue( RValue::get(CGF.Builder.getInt32(/*C=*/0)), - CGF.MakeNaturalAlignAddrLValue(CGF.ReturnValue, KmpInt32Ty)); + CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty)); CGF.FinishFunction(); return TaskEntry; } @@ -1969,7 +2156,8 @@ static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM, FunctionArgList Args; ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty); ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, - /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy); + /*Id=*/nullptr, + KmpTaskTWithPrivatesPtrQTy.withRestrict()); Args.push_back(&GtidArg); Args.push_back(&TaskTypeArg); FunctionType::ExtInfo Info; @@ -1980,16 +2168,15 @@ static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM, auto *DestructorFn = llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage, ".omp_task_destructor.", &CGM.getModule()); - CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, DestructorFnInfo, DestructorFn); + CGM.SetInternalFunctionAttributes(/*D=*/nullptr, DestructorFn, + DestructorFnInfo); CodeGenFunction CGF(CGM); CGF.disableDebugInfo(); CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, Args); - auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad( - CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes); - LValue Base = - CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy); + LValue Base = emitLoadOfPointerLValue( + CGF, CGF.GetAddrOfLocalVar(&TaskTypeArg), KmpTaskTWithPrivatesPtrQTy); auto *KmpTaskTWithPrivatesQTyRD = cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl()); auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); @@ -2017,10 +2204,10 @@ static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM, /// \endcode static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, - const ArrayRef<const Expr *> PrivateVars, - const ArrayRef<const Expr *> FirstprivateVars, + ArrayRef<const Expr *> PrivateVars, + ArrayRef<const Expr *> FirstprivateVars, QualType PrivatesQTy, - const ArrayRef<PrivateDataTy> Privates) { + ArrayRef<PrivateDataTy> Privates) { auto &C = CGM.getContext(); FunctionArgList Args; ImplicitParamDecl TaskPrivatesArg( @@ -2058,8 +2245,8 @@ emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, auto *TaskPrivatesMap = llvm::Function::Create( TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, ".omp_task_privates_map.", &CGM.getModule()); - CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskPrivatesMapFnInfo, - TaskPrivatesMap); + CGM.SetInternalFunctionAttributes(/*D=*/nullptr, TaskPrivatesMap, + TaskPrivatesMapFnInfo); TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline); CodeGenFunction CGF(CGM); CGF.disableDebugInfo(); @@ -2067,22 +2254,17 @@ emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, TaskPrivatesMapFnInfo, Args); // *privi = &.privates.privi; - auto *TaskPrivatesArgAddr = CGF.Builder.CreateAlignedLoad( - CGF.GetAddrOfLocalVar(&TaskPrivatesArg), CGM.PointerAlignInBytes); - LValue Base = - CGF.MakeNaturalAlignAddrLValue(TaskPrivatesArgAddr, PrivatesQTy); + LValue Base = emitLoadOfPointerLValue( + CGF, CGF.GetAddrOfLocalVar(&TaskPrivatesArg), TaskPrivatesArg.getType()); auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl()); Counter = 0; for (auto *Field : PrivatesQTyRD->fields()) { auto FieldLVal = CGF.EmitLValueForField(Base, Field); auto *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]]; - auto RefLVal = CGF.MakeNaturalAlignAddrLValue(CGF.GetAddrOfLocalVar(VD), - VD->getType()); - auto RefLoadRVal = CGF.EmitLoadOfLValue(RefLVal, Loc); - CGF.EmitStoreOfScalar( - FieldLVal.getAddress(), - CGF.MakeNaturalAlignAddrLValue(RefLoadRVal.getScalarVal(), - RefLVal.getType()->getPointeeType())); + auto RefLVal = CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); + auto RefLoadLVal = + emitLoadOfPointerLValue(CGF, RefLVal.getAddress(), RefLVal.getType()); + CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal); ++Counter; } CGF.FinishFunction(); @@ -2097,13 +2279,15 @@ static int array_pod_sort_comparator(const PrivateDataTy *P1, void CGOpenMPRuntime::emitTaskCall( CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, bool Tied, llvm::PointerIntPair<llvm::Value *, 1, bool> Final, - llvm::Value *TaskFunction, QualType SharedsTy, llvm::Value *Shareds, + llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, ArrayRef<const Expr *> PrivateVars, ArrayRef<const Expr *> PrivateCopies, ArrayRef<const Expr *> FirstprivateVars, ArrayRef<const Expr *> FirstprivateCopies, ArrayRef<const Expr *> FirstprivateInits, ArrayRef<std::pair<OpenMPDependClauseKind, const Expr *>> Dependences) { + if (!CGF.HaveInsertPoint()) + return; auto &C = CGM.getContext(); llvm::SmallVector<PrivateDataTy, 8> Privates; // Aggregate privates and sort them by the alignment. @@ -2111,7 +2295,7 @@ void CGOpenMPRuntime::emitTaskCall( for (auto *E : PrivateVars) { auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); Privates.push_back(std::make_pair( - C.getTypeAlignInChars(VD->getType()), + C.getDeclAlign(VD), PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()), /*PrivateElemInit=*/nullptr))); ++I; @@ -2121,7 +2305,7 @@ void CGOpenMPRuntime::emitTaskCall( for (auto *E : FirstprivateVars) { auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); Privates.push_back(std::make_pair( - C.getTypeAlignInChars(VD->getType()), + C.getDeclAlign(VD), PrivateHelpersTy( VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()), cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())))); @@ -2146,8 +2330,7 @@ void CGOpenMPRuntime::emitTaskCall( C.getPointerType(KmpTaskTWithPrivatesQTy); auto *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy); auto *KmpTaskTWithPrivatesPtrTy = KmpTaskTWithPrivatesTy->getPointerTo(); - auto KmpTaskTWithPrivatesTySize = - CGM.getSize(C.getTypeSizeInChars(KmpTaskTWithPrivatesQTy)); + auto *KmpTaskTWithPrivatesTySize = getTypeSize(CGF, KmpTaskTWithPrivatesQTy); QualType SharedsPtrTy = C.getPointerType(SharedsTy); // Emit initial values for private copies (if any). @@ -2188,12 +2371,12 @@ void CGOpenMPRuntime::emitTaskCall( CGF.Builder.getInt32(/*C=*/0)) : CGF.Builder.getInt32(Final.getInt() ? FinalFlag : 0); TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); - auto SharedsSize = C.getTypeSizeInChars(SharedsTy); - llvm::Value *AllocArgs[] = { - emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), TaskFlags, - KmpTaskTWithPrivatesTySize, CGM.getSize(SharedsSize), - CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskEntry, - KmpRoutineEntryPtrTy)}; + auto *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy)); + llvm::Value *AllocArgs[] = {emitUpdateLocation(CGF, Loc), + getThreadID(CGF, Loc), TaskFlags, + KmpTaskTWithPrivatesTySize, SharedsSize, + CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + TaskEntry, KmpRoutineEntryPtrTy)}; auto *NewTask = CGF.EmitRuntimeCall( createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs); auto *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( @@ -2204,12 +2387,15 @@ void CGOpenMPRuntime::emitTaskCall( CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); // Fill the data in the resulting kmp_task_t record. // Copy shareds if there are any. - llvm::Value *KmpTaskSharedsPtr = nullptr; + Address KmpTaskSharedsPtr = Address::invalid(); if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) { - KmpTaskSharedsPtr = CGF.EmitLoadOfScalar( - CGF.EmitLValueForField( - TDBase, *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)), - Loc); + KmpTaskSharedsPtr = + Address(CGF.EmitLoadOfScalar( + CGF.EmitLValueForField( + TDBase, *std::next(KmpTaskTQTyRD->field_begin(), + KmpTaskTShareds)), + Loc), + CGF.getNaturalTypeAlignment(SharedsTy)); CGF.EmitAggregateCopy(KmpTaskSharedsPtr, Shareds, SharedsTy); } // Emit initial values for private copies (if any). @@ -2220,7 +2406,7 @@ void CGOpenMPRuntime::emitTaskCall( FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin(); LValue SharedsBase; if (!FirstprivateVars.empty()) { - SharedsBase = CGF.MakeNaturalAlignAddrLValue( + SharedsBase = CGF.MakeAddrLValue( CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)), SharedsTy); @@ -2237,6 +2423,9 @@ void CGOpenMPRuntime::emitTaskCall( auto *SharedField = CapturesInfo.lookup(OriginalVD); auto SharedRefLValue = CGF.EmitLValueForField(SharedsBase, SharedField); + SharedRefLValue = CGF.MakeAddrLValue( + Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)), + SharedRefLValue.getType(), AlignmentSource::Decl); QualType Type = OriginalVD->getType(); if (Type->isArrayType()) { // Initialize firstprivate array. @@ -2251,10 +2440,10 @@ void CGOpenMPRuntime::emitTaskCall( CGF.EmitOMPAggregateAssign( PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type, [&CGF, Elem, Init, &CapturesInfo]( - llvm::Value *DestElement, llvm::Value *SrcElement) { + Address DestElement, Address SrcElement) { // Clean up any temporaries needed by the initialization. CodeGenFunction::OMPPrivateScope InitScope(CGF); - InitScope.addPrivate(Elem, [SrcElement]() -> llvm::Value *{ + InitScope.addPrivate(Elem, [SrcElement]() -> Address { return SrcElement; }); (void)InitScope.Privatize(); @@ -2268,7 +2457,7 @@ void CGOpenMPRuntime::emitTaskCall( } } else { CodeGenFunction::OMPPrivateScope InitScope(CGF); - InitScope.addPrivate(Elem, [SharedRefLValue]() -> llvm::Value *{ + InitScope.addPrivate(Elem, [SharedRefLValue]() -> Address { return SharedRefLValue.getAddress(); }); (void)InitScope.Privatize(); @@ -2298,15 +2487,15 @@ void CGOpenMPRuntime::emitTaskCall( Destructor); // Process list of dependences. - llvm::Value *DependInfo = nullptr; - unsigned DependencesNumber = Dependences.size(); - if (!Dependences.empty()) { + Address DependenciesArray = Address::invalid(); + unsigned NumDependencies = Dependences.size(); + if (NumDependencies) { // Dependence kind for RTL. - enum RTLDependenceKindTy { DepIn = 1, DepOut = 2, DepInOut = 3 }; + enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3 }; enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags }; RecordDecl *KmpDependInfoRD; - QualType FlagsTy = C.getIntTypeForBitwidth( - C.toBits(C.getTypeSizeInChars(C.BoolTy)), /*Signed=*/false); + QualType FlagsTy = + C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false); llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); if (KmpDependInfoTy.isNull()) { KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info"); @@ -2319,25 +2508,37 @@ void CGOpenMPRuntime::emitTaskCall( } else { KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl()); } + CharUnits DependencySize = C.getTypeSizeInChars(KmpDependInfoTy); // Define type kmp_depend_info[<Dependences.size()>]; QualType KmpDependInfoArrayTy = C.getConstantArrayType( - KmpDependInfoTy, llvm::APInt(/*numBits=*/64, Dependences.size()), + KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies), ArrayType::Normal, /*IndexTypeQuals=*/0); // kmp_depend_info[<Dependences.size()>] deps; - DependInfo = CGF.CreateMemTemp(KmpDependInfoArrayTy); - for (unsigned i = 0; i < DependencesNumber; ++i) { - auto Addr = CGF.EmitLValue(Dependences[i].second); - auto *Size = llvm::ConstantInt::get( - CGF.SizeTy, - C.getTypeSizeInChars(Dependences[i].second->getType()).getQuantity()); - auto Base = CGF.MakeNaturalAlignAddrLValue( - CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, DependInfo, i), + DependenciesArray = CGF.CreateMemTemp(KmpDependInfoArrayTy); + for (unsigned i = 0; i < NumDependencies; ++i) { + const Expr *E = Dependences[i].second; + auto Addr = CGF.EmitLValue(E); + llvm::Value *Size; + QualType Ty = E->getType(); + if (auto *ASE = dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) { + LValue UpAddrLVal = + CGF.EmitOMPArraySectionExpr(ASE, /*LowerBound=*/false); + llvm::Value *UpAddr = + CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1); + llvm::Value *LowIntPtr = + CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy); + llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy); + Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); + } else + Size = getTypeSize(CGF, Ty); + auto Base = CGF.MakeAddrLValue( + CGF.Builder.CreateConstArrayGEP(DependenciesArray, i, DependencySize), KmpDependInfoTy); // deps[i].base_addr = &<Dependences[i].second>; auto BaseAddrLVal = CGF.EmitLValueForField( Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr)); CGF.EmitStoreOfScalar( - CGF.Builder.CreatePtrToInt(Addr.getAddress(), CGF.IntPtrTy), + CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy), BaseAddrLVal); // deps[i].len = sizeof(<Dependences[i].second>); auto LenLVal = CGF.EmitLValueForField( @@ -2349,12 +2550,13 @@ void CGOpenMPRuntime::emitTaskCall( case OMPC_DEPEND_in: DepKind = DepIn; break; + // Out and InOut dependencies must use the same code. case OMPC_DEPEND_out: - DepKind = DepOut; - break; case OMPC_DEPEND_inout: DepKind = DepInOut; break; + case OMPC_DEPEND_source: + case OMPC_DEPEND_sink: case OMPC_DEPEND_unknown: llvm_unreachable("Unknown task dependence type"); } @@ -2363,8 +2565,8 @@ void CGOpenMPRuntime::emitTaskCall( CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind), FlagsLVal); } - DependInfo = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, DependInfo, 0), + DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateStructGEP(DependenciesArray, 0, CharUnits::Zero()), CGF.VoidPtrTy); } @@ -2378,40 +2580,48 @@ void CGOpenMPRuntime::emitTaskCall( // list is not empty auto *ThreadID = getThreadID(CGF, Loc); auto *UpLoc = emitUpdateLocation(CGF, Loc); - llvm::Value *TaskArgs[] = {UpLoc, ThreadID, NewTask}; - llvm::Value *DepTaskArgs[] = { - UpLoc, - ThreadID, - NewTask, - DependInfo ? CGF.Builder.getInt32(DependencesNumber) : nullptr, - DependInfo, - DependInfo ? CGF.Builder.getInt32(0) : nullptr, - DependInfo ? llvm::ConstantPointerNull::get(CGF.VoidPtrTy) : nullptr}; - auto &&ThenCodeGen = [this, DependInfo, &TaskArgs, - &DepTaskArgs](CodeGenFunction &CGF) { - // TODO: add check for untied tasks. - CGF.EmitRuntimeCall( - createRuntimeFunction(DependInfo ? OMPRTL__kmpc_omp_task_with_deps - : OMPRTL__kmpc_omp_task), - DependInfo ? makeArrayRef(DepTaskArgs) : makeArrayRef(TaskArgs)); + llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask }; + llvm::Value *DepTaskArgs[7]; + if (NumDependencies) { + DepTaskArgs[0] = UpLoc; + DepTaskArgs[1] = ThreadID; + DepTaskArgs[2] = NewTask; + DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies); + DepTaskArgs[4] = DependenciesArray.getPointer(); + DepTaskArgs[5] = CGF.Builder.getInt32(0); + DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); + } + auto &&ThenCodeGen = [this, NumDependencies, + &TaskArgs, &DepTaskArgs](CodeGenFunction &CGF) { + // TODO: add check for untied tasks. + if (NumDependencies) { + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), + DepTaskArgs); + } else { + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), + TaskArgs); + } }; typedef CallEndCleanup<std::extent<decltype(TaskArgs)>::value> IfCallEndCleanup; - llvm::Value *DepWaitTaskArgs[] = { - UpLoc, - ThreadID, - DependInfo ? CGF.Builder.getInt32(DependencesNumber) : nullptr, - DependInfo, - DependInfo ? CGF.Builder.getInt32(0) : nullptr, - DependInfo ? llvm::ConstantPointerNull::get(CGF.VoidPtrTy) : nullptr}; + + llvm::Value *DepWaitTaskArgs[6]; + if (NumDependencies) { + DepWaitTaskArgs[0] = UpLoc; + DepWaitTaskArgs[1] = ThreadID; + DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies); + DepWaitTaskArgs[3] = DependenciesArray.getPointer(); + DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); + DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); + } auto &&ElseCodeGen = [this, &TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry, - DependInfo, &DepWaitTaskArgs](CodeGenFunction &CGF) { + NumDependencies, &DepWaitTaskArgs](CodeGenFunction &CGF) { CodeGenFunction::RunCleanupsScope LocalScope(CGF); // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid, // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info // is specified. - if (DependInfo) + if (NumDependencies) CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps), DepWaitTaskArgs); // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid, @@ -2429,6 +2639,7 @@ void CGOpenMPRuntime::emitTaskCall( llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy}; CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs); }; + if (IfCond) { emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); } else { @@ -2437,8 +2648,89 @@ void CGOpenMPRuntime::emitTaskCall( } } +/// \brief Emit reduction operation for each element of array (required for +/// array sections) LHS op = RHS. +/// \param Type Type of array. +/// \param LHSVar Variable on the left side of the reduction operation +/// (references element of array in original variable). +/// \param RHSVar Variable on the right side of the reduction operation +/// (references element of array in original variable). +/// \param RedOpGen Generator of reduction operation with use of LHSVar and +/// RHSVar. +static void EmitOMPAggregateReduction( + CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, + const VarDecl *RHSVar, + const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *, + const Expr *, const Expr *)> &RedOpGen, + const Expr *XExpr = nullptr, const Expr *EExpr = nullptr, + const Expr *UpExpr = nullptr) { + // Perform element-by-element initialization. + QualType ElementTy; + Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar); + Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar); + + // Drill down to the base element type on both arrays. + auto ArrayTy = Type->getAsArrayTypeUnsafe(); + auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); + + auto RHSBegin = RHSAddr.getPointer(); + auto LHSBegin = LHSAddr.getPointer(); + // Cast from pointer to array type to pointer to single element. + auto LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements); + // The basic structure here is a while-do loop. + auto BodyBB = CGF.createBasicBlock("omp.arraycpy.body"); + auto DoneBB = CGF.createBasicBlock("omp.arraycpy.done"); + auto IsEmpty = + CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty"); + CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); + + // Enter the loop body, making that address the current address. + auto EntryBB = CGF.Builder.GetInsertBlock(); + CGF.EmitBlock(BodyBB); + + CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); + + llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI( + RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast"); + RHSElementPHI->addIncoming(RHSBegin, EntryBB); + Address RHSElementCurrent = + Address(RHSElementPHI, + RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize)); + + llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI( + LHSBegin->getType(), 2, "omp.arraycpy.destElementPast"); + LHSElementPHI->addIncoming(LHSBegin, EntryBB); + Address LHSElementCurrent = + Address(LHSElementPHI, + LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize)); + + // Emit copy. + CodeGenFunction::OMPPrivateScope Scope(CGF); + Scope.addPrivate(LHSVar, [=]() -> Address { return LHSElementCurrent; }); + Scope.addPrivate(RHSVar, [=]() -> Address { return RHSElementCurrent; }); + Scope.Privatize(); + RedOpGen(CGF, XExpr, EExpr, UpExpr); + Scope.ForceCleanup(); + + // Shift the address forward by one element. + auto LHSElementNext = CGF.Builder.CreateConstGEP1_32( + LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); + auto RHSElementNext = CGF.Builder.CreateConstGEP1_32( + RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element"); + // Check whether we've reached the end. + auto Done = + CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done"); + CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); + LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock()); + RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock()); + + // Done. + CGF.EmitBlock(DoneBB, /*IsFinished=*/true); +} + static llvm::Value *emitReductionFunction(CodeGenModule &CGM, llvm::Type *ArgsType, + ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) { @@ -2458,48 +2750,66 @@ static llvm::Value *emitReductionFunction(CodeGenModule &CGM, auto *Fn = llvm::Function::Create( CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage, ".omp.reduction.reduction_func", &CGM.getModule()); - CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn); + CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, CGFI); CodeGenFunction CGF(CGM); CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args); // Dst = (void*[n])(LHSArg); // Src = (void*[n])(RHSArg); - auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg), - CGF.PointerAlignInBytes), - ArgsType); - auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg), - CGF.PointerAlignInBytes), - ArgsType); + Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), + ArgsType), CGF.getPointerAlign()); + Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), + ArgsType), CGF.getPointerAlign()); // ... // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]); // ... CodeGenFunction::OMPPrivateScope Scope(CGF); - for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I) { - Scope.addPrivate( - cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl()), - [&]() -> llvm::Value *{ - return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateAlignedLoad( - CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, RHS, I), - CGM.PointerAlignInBytes), - CGF.ConvertTypeForMem(C.getPointerType(RHSExprs[I]->getType()))); - }); - Scope.addPrivate( - cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl()), - [&]() -> llvm::Value *{ - return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateAlignedLoad( - CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, LHS, I), - CGM.PointerAlignInBytes), - CGF.ConvertTypeForMem(C.getPointerType(LHSExprs[I]->getType()))); - }); + auto IPriv = Privates.begin(); + unsigned Idx = 0; + for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) { + auto RHSVar = cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl()); + Scope.addPrivate(RHSVar, [&]() -> Address { + return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar); + }); + auto LHSVar = cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl()); + Scope.addPrivate(LHSVar, [&]() -> Address { + return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar); + }); + QualType PrivTy = (*IPriv)->getType(); + if (PrivTy->isArrayType()) { + // Get array size and emit VLA type. + ++Idx; + Address Elem = + CGF.Builder.CreateConstArrayGEP(LHS, Idx, CGF.getPointerSize()); + llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem); + CodeGenFunction::OpaqueValueMapping OpaqueMap( + CGF, + cast<OpaqueValueExpr>( + CGF.getContext().getAsVariableArrayType(PrivTy)->getSizeExpr()), + RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy))); + CGF.EmitVariablyModifiedType(PrivTy); + } } Scope.Privatize(); + IPriv = Privates.begin(); + auto ILHS = LHSExprs.begin(); + auto IRHS = RHSExprs.begin(); for (auto *E : ReductionOps) { - CGF.EmitIgnoredExpr(E); + if ((*IPriv)->getType()->isArrayType()) { + // Emit reduction for array section. + auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); + auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, + [=](CodeGenFunction &CGF, const Expr *, + const Expr *, + const Expr *) { CGF.EmitIgnoredExpr(E); }); + } else + // Emit reduction for array subscript or single variable. + CGF.EmitIgnoredExpr(E); + ++IPriv, ++ILHS, ++IRHS; } Scope.ForceCleanup(); CGF.FinishFunction(); @@ -2507,10 +2817,13 @@ static llvm::Value *emitReductionFunction(CodeGenModule &CGM, } void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, + ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps, bool WithNowait, bool SimpleReduction) { + if (!CGF.HaveInsertPoint()) + return; // Next code should be emitted for reduction: // // static kmp_critical_name lock = { 0 }; @@ -2550,32 +2863,68 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, if (SimpleReduction) { CodeGenFunction::RunCleanupsScope Scope(CGF); + auto IPriv = Privates.begin(); + auto ILHS = LHSExprs.begin(); + auto IRHS = RHSExprs.begin(); for (auto *E : ReductionOps) { - CGF.EmitIgnoredExpr(E); + if ((*IPriv)->getType()->isArrayType()) { + auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); + auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + EmitOMPAggregateReduction( + CGF, (*IPriv)->getType(), LHSVar, RHSVar, + [=](CodeGenFunction &CGF, const Expr *, const Expr *, + const Expr *) { CGF.EmitIgnoredExpr(E); }); + } else + CGF.EmitIgnoredExpr(E); + ++IPriv, ++ILHS, ++IRHS; } return; } // 1. Build a list of reduction variables. // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; - llvm::APInt ArraySize(/*unsigned int numBits=*/32, RHSExprs.size()); + auto Size = RHSExprs.size(); + for (auto *E : Privates) { + if (E->getType()->isArrayType()) + // Reserve place for array size. + ++Size; + } + llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size); QualType ReductionArrayTy = C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal, /*IndexTypeQuals=*/0); - auto *ReductionList = + Address ReductionList = CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); - for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I) { - auto *Elem = CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, ReductionList, I); - CGF.Builder.CreateAlignedStore( + auto IPriv = Privates.begin(); + unsigned Idx = 0; + for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) { + Address Elem = + CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, CGF.getPointerSize()); + CGF.Builder.CreateStore( CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.EmitLValue(RHSExprs[I]).getAddress(), CGF.VoidPtrTy), - Elem, CGM.PointerAlignInBytes); + CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy), + Elem); + if ((*IPriv)->getType()->isArrayType()) { + // Store array size. + ++Idx; + Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, + CGF.getPointerSize()); + CGF.Builder.CreateStore( + CGF.Builder.CreateIntToPtr( + CGF.Builder.CreateIntCast( + CGF.getVLASize(CGF.getContext().getAsVariableArrayType( + (*IPriv)->getType())) + .first, + CGF.SizeTy, /*isSigned=*/false), + CGF.VoidPtrTy), + Elem); + } } // 2. Emit reduce_func(). auto *ReductionFn = emitReductionFunction( - CGM, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), LHSExprs, - RHSExprs, ReductionOps); + CGM, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates, + LHSExprs, RHSExprs, ReductionOps); // 3. Create static kmp_critical_name lock = { 0 }; auto *Lock = getCriticalRegionLock(".reduction"); @@ -2586,10 +2935,10 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, CGF, Loc, static_cast<OpenMPLocationFlags>(OMP_IDENT_KMPC | OMP_ATOMIC_REDUCE)); auto *ThreadId = getThreadID(CGF, Loc); - auto *ReductionArrayTySize = llvm::ConstantInt::get( - CGM.SizeTy, C.getTypeSizeInChars(ReductionArrayTy).getQuantity()); - auto *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, - CGF.VoidPtrTy); + auto *ReductionArrayTySize = getTypeSize(CGF, ReductionArrayTy); + auto *RL = + CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList.getPointer(), + CGF.VoidPtrTy); llvm::Value *Args[] = { IdentTLoc, // ident_t *<loc> ThreadId, // i32 <gtid> @@ -2632,8 +2981,22 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait : OMPRTL__kmpc_end_reduce), llvm::makeArrayRef(EndArgs)); + auto IPriv = Privates.begin(); + auto ILHS = LHSExprs.begin(); + auto IRHS = RHSExprs.begin(); for (auto *E : ReductionOps) { - CGF.EmitIgnoredExpr(E); + if ((*IPriv)->getType()->isArrayType()) { + // Emit reduction for array section. + auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); + auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + EmitOMPAggregateReduction( + CGF, (*IPriv)->getType(), LHSVar, RHSVar, + [=](CodeGenFunction &CGF, const Expr *, const Expr *, + const Expr *) { CGF.EmitIgnoredExpr(E); }); + } else + // Emit reduction for array subscript or single variable. + CGF.EmitIgnoredExpr(E); + ++IPriv, ++ILHS, ++IRHS; } } @@ -2663,62 +3026,84 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, createRuntimeFunction(OMPRTL__kmpc_end_reduce), llvm::makeArrayRef(EndArgs)); } - auto I = LHSExprs.begin(); + auto ILHS = LHSExprs.begin(); + auto IRHS = RHSExprs.begin(); + auto IPriv = Privates.begin(); for (auto *E : ReductionOps) { - const Expr *XExpr = nullptr; - const Expr *EExpr = nullptr; - const Expr *UpExpr = nullptr; - BinaryOperatorKind BO = BO_Comma; - if (auto *BO = dyn_cast<BinaryOperator>(E)) { - if (BO->getOpcode() == BO_Assign) { - XExpr = BO->getLHS(); - UpExpr = BO->getRHS(); - } - } - // Try to emit update expression as a simple atomic. - auto *RHSExpr = UpExpr; - if (RHSExpr) { - // Analyze RHS part of the whole expression. - if (auto *ACO = dyn_cast<AbstractConditionalOperator>( - RHSExpr->IgnoreParenImpCasts())) { - // If this is a conditional operator, analyze its condition for - // min/max reduction operator. - RHSExpr = ACO->getCond(); + const Expr *XExpr = nullptr; + const Expr *EExpr = nullptr; + const Expr *UpExpr = nullptr; + BinaryOperatorKind BO = BO_Comma; + if (auto *BO = dyn_cast<BinaryOperator>(E)) { + if (BO->getOpcode() == BO_Assign) { + XExpr = BO->getLHS(); + UpExpr = BO->getRHS(); + } } - if (auto *BORHS = - dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) { - EExpr = BORHS->getRHS(); - BO = BORHS->getOpcode(); + // Try to emit update expression as a simple atomic. + auto *RHSExpr = UpExpr; + if (RHSExpr) { + // Analyze RHS part of the whole expression. + if (auto *ACO = dyn_cast<AbstractConditionalOperator>( + RHSExpr->IgnoreParenImpCasts())) { + // If this is a conditional operator, analyze its condition for + // min/max reduction operator. + RHSExpr = ACO->getCond(); + } + if (auto *BORHS = + dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) { + EExpr = BORHS->getRHS(); + BO = BORHS->getOpcode(); + } } - } - if (XExpr) { - auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()); - LValue X = CGF.EmitLValue(XExpr); - RValue E; - if (EExpr) - E = CGF.EmitAnyExpr(EExpr); - CGF.EmitOMPAtomicSimpleUpdateExpr( - X, E, BO, /*IsXLHSInRHSPart=*/true, llvm::Monotonic, Loc, - [&CGF, UpExpr, VD](RValue XRValue) { - CodeGenFunction::OMPPrivateScope PrivateScope(CGF); - PrivateScope.addPrivate( - VD, [&CGF, VD, XRValue]() -> llvm::Value *{ - auto *LHSTemp = CGF.CreateMemTemp(VD->getType()); + if (XExpr) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); + auto &&AtomicRedGen = [this, BO, VD, IPriv, + Loc](CodeGenFunction &CGF, const Expr *XExpr, + const Expr *EExpr, const Expr *UpExpr) { + LValue X = CGF.EmitLValue(XExpr); + RValue E; + if (EExpr) + E = CGF.EmitAnyExpr(EExpr); + CGF.EmitOMPAtomicSimpleUpdateExpr( + X, E, BO, /*IsXLHSInRHSPart=*/true, llvm::Monotonic, Loc, + [&CGF, UpExpr, VD, IPriv](RValue XRValue) { + CodeGenFunction::OMPPrivateScope PrivateScope(CGF); + PrivateScope.addPrivate(VD, [&CGF, VD, XRValue]() -> Address { + Address LHSTemp = CGF.CreateMemTemp(VD->getType()); CGF.EmitStoreThroughLValue( - XRValue, - CGF.MakeNaturalAlignAddrLValue(LHSTemp, VD->getType())); + XRValue, CGF.MakeAddrLValue(LHSTemp, VD->getType())); return LHSTemp; }); - (void)PrivateScope.Privatize(); - return CGF.EmitAnyExpr(UpExpr); - }); - } else { - // Emit as a critical region. - emitCriticalRegion(CGF, ".atomic_reduction", [E](CodeGenFunction &CGF) { - CGF.EmitIgnoredExpr(E); - }, Loc); - } - ++I; + (void)PrivateScope.Privatize(); + return CGF.EmitAnyExpr(UpExpr); + }); + }; + if ((*IPriv)->getType()->isArrayType()) { + // Emit atomic reduction for array section. + auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, + AtomicRedGen, XExpr, EExpr, UpExpr); + } else + // Emit atomic reduction for array subscript or single variable. + AtomicRedGen(CGF, XExpr, EExpr, UpExpr); + } else { + // Emit as a critical region. + auto &&CritRedGen = [this, E, Loc](CodeGenFunction &CGF, const Expr *, + const Expr *, const Expr *) { + emitCriticalRegion( + CGF, ".atomic_reduction", + [E](CodeGenFunction &CGF) { CGF.EmitIgnoredExpr(E); }, Loc); + }; + if ((*IPriv)->getType()->isArrayType()) { + auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); + auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, + CritRedGen); + } else + CritRedGen(CGF, nullptr, nullptr, nullptr); + } + ++ILHS, ++IRHS, ++IPriv; } } @@ -2728,6 +3113,8 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc) { + if (!CGF.HaveInsertPoint()) + return; // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 // global_tid); llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; @@ -2737,8 +3124,11 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnerKind, - const RegionCodeGenTy &CodeGen) { - InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind); + const RegionCodeGenTy &CodeGen, + bool HasCancel) { + if (!CGF.HaveInsertPoint()) + return; + InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel); CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); } @@ -2770,13 +3160,15 @@ static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) { void CGOpenMPRuntime::emitCancellationPointCall( CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) { + if (!CGF.HaveInsertPoint()) + return; // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32 // global_tid, kmp_int32 cncl_kind); if (auto *OMPRegionInfo = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { - auto CancelDest = - CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); - if (CancelDest.isValid()) { + if (OMPRegionInfo->getDirectiveKind() == OMPD_single) + return; + if (OMPRegionInfo->hasCancel()) { llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; @@ -2793,8 +3185,10 @@ void CGOpenMPRuntime::emitCancellationPointCall( CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); CGF.EmitBlock(ExitBB); // __kmpc_cancel_barrier(); - emitBarrierCall(CGF, Loc, OMPD_unknown, /*CheckForCancel=*/false); + emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); // exit from construct; + auto CancelDest = + CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); CGF.EmitBranchThroughCleanup(CancelDest); CGF.EmitBlock(ContBB, /*IsFinished=*/true); } @@ -2802,14 +3196,18 @@ void CGOpenMPRuntime::emitCancellationPointCall( } void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, + const Expr *IfCond, OpenMPDirectiveKind CancelRegion) { + if (!CGF.HaveInsertPoint()) + return; // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid, // kmp_int32 cncl_kind); if (auto *OMPRegionInfo = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { - auto CancelDest = - CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); - if (CancelDest.isValid()) { + if (OMPRegionInfo->getDirectiveKind() == OMPD_single) + return; + auto &&ThenGen = [this, Loc, CancelRegion, + OMPRegionInfo](CodeGenFunction &CGF) { llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; @@ -2826,11 +3224,332 @@ void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); CGF.EmitBlock(ExitBB); // __kmpc_cancel_barrier(); - emitBarrierCall(CGF, Loc, OMPD_unknown, /*CheckForCancel=*/false); + emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); // exit from construct; + auto CancelDest = + CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); CGF.EmitBranchThroughCleanup(CancelDest); CGF.EmitBlock(ContBB, /*IsFinished=*/true); - } + }; + if (IfCond) + emitOMPIfClause(CGF, IfCond, ThenGen, [](CodeGenFunction &) {}); + else + ThenGen(CGF); } } +llvm::Value * +CGOpenMPRuntime::emitTargetOutlinedFunction(const OMPExecutableDirective &D, + const RegionCodeGenTy &CodeGen) { + const CapturedStmt &CS = *cast<CapturedStmt>(D.getAssociatedStmt()); + + CodeGenFunction CGF(CGM, true); + CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen); + CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); + return CGF.GenerateOpenMPCapturedStmtFunction(CS); +} + +void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF, + const OMPExecutableDirective &D, + llvm::Value *OutlinedFn, + const Expr *IfCond, const Expr *Device, + ArrayRef<llvm::Value *> CapturedVars) { + if (!CGF.HaveInsertPoint()) + return; + /// \brief Values for bit flags used to specify the mapping type for + /// offloading. + enum OpenMPOffloadMappingFlags { + /// \brief Allocate memory on the device and move data from host to device. + OMP_MAP_TO = 0x01, + /// \brief Allocate memory on the device and move data from device to host. + OMP_MAP_FROM = 0x02, + /// \brief The element passed to the device is a pointer. + OMP_MAP_PTR = 0x20, + /// \brief Pass the element to the device by value. + OMP_MAP_BYCOPY = 0x80, + }; + + enum OpenMPOffloadingReservedDeviceIDs { + /// \brief Device ID if the device was not defined, runtime should get it + /// from environment variables in the spec. + OMP_DEVICEID_UNDEF = -1, + }; + + auto &Ctx = CGF.getContext(); + + // Fill up the arrays with the all the captured variables. + SmallVector<llvm::Value *, 16> BasePointers; + SmallVector<llvm::Value *, 16> Pointers; + SmallVector<llvm::Value *, 16> Sizes; + SmallVector<unsigned, 16> MapTypes; + + bool hasVLACaptures = false; + + const CapturedStmt &CS = *cast<CapturedStmt>(D.getAssociatedStmt()); + auto RI = CS.getCapturedRecordDecl()->field_begin(); + // auto II = CS.capture_init_begin(); + auto CV = CapturedVars.begin(); + for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(), + CE = CS.capture_end(); + CI != CE; ++CI, ++RI, ++CV) { + StringRef Name; + QualType Ty; + llvm::Value *BasePointer; + llvm::Value *Pointer; + llvm::Value *Size; + unsigned MapType; + + // VLA sizes are passed to the outlined region by copy. + if (CI->capturesVariableArrayType()) { + BasePointer = Pointer = *CV; + Size = getTypeSize(CGF, RI->getType()); + // Copy to the device as an argument. No need to retrieve it. + MapType = OMP_MAP_BYCOPY; + hasVLACaptures = true; + } else if (CI->capturesThis()) { + BasePointer = Pointer = *CV; + const PointerType *PtrTy = cast<PointerType>(RI->getType().getTypePtr()); + Size = getTypeSize(CGF, PtrTy->getPointeeType()); + // Default map type. + MapType = OMP_MAP_TO | OMP_MAP_FROM; + } else if (CI->capturesVariableByCopy()) { + MapType = OMP_MAP_BYCOPY; + if (!RI->getType()->isAnyPointerType()) { + // If the field is not a pointer, we need to save the actual value and + // load it as a void pointer. + auto DstAddr = CGF.CreateMemTemp( + Ctx.getUIntPtrType(), + Twine(CI->getCapturedVar()->getName()) + ".casted"); + LValue DstLV = CGF.MakeAddrLValue(DstAddr, Ctx.getUIntPtrType()); + + auto *SrcAddrVal = CGF.EmitScalarConversion( + DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()), + Ctx.getPointerType(RI->getType()), SourceLocation()); + LValue SrcLV = + CGF.MakeNaturalAlignAddrLValue(SrcAddrVal, RI->getType()); + + // Store the value using the source type pointer. + CGF.EmitStoreThroughLValue(RValue::get(*CV), SrcLV); + + // Load the value using the destination type pointer. + BasePointer = Pointer = + CGF.EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal(); + } else { + MapType |= OMP_MAP_PTR; + BasePointer = Pointer = *CV; + } + Size = getTypeSize(CGF, RI->getType()); + } else { + assert(CI->capturesVariable() && "Expected captured reference."); + BasePointer = Pointer = *CV; + + const ReferenceType *PtrTy = + cast<ReferenceType>(RI->getType().getTypePtr()); + QualType ElementType = PtrTy->getPointeeType(); + Size = getTypeSize(CGF, ElementType); + // The default map type for a scalar/complex type is 'to' because by + // default the value doesn't have to be retrieved. For an aggregate type, + // the default is 'tofrom'. + MapType = ElementType->isAggregateType() ? (OMP_MAP_TO | OMP_MAP_FROM) + : OMP_MAP_TO; + if (ElementType->isAnyPointerType()) + MapType |= OMP_MAP_PTR; + } + + BasePointers.push_back(BasePointer); + Pointers.push_back(Pointer); + Sizes.push_back(Size); + MapTypes.push_back(MapType); + } + + // Keep track on whether the host function has to be executed. + auto OffloadErrorQType = + Ctx.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true); + auto OffloadError = CGF.MakeAddrLValue( + CGF.CreateMemTemp(OffloadErrorQType, ".run_host_version"), + OffloadErrorQType); + CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty), + OffloadError); + + // Fill up the pointer arrays and transfer execution to the device. + auto &&ThenGen = [this, &Ctx, &BasePointers, &Pointers, &Sizes, &MapTypes, + hasVLACaptures, Device, OffloadError, + OffloadErrorQType](CodeGenFunction &CGF) { + unsigned PointerNumVal = BasePointers.size(); + llvm::Value *PointerNum = CGF.Builder.getInt32(PointerNumVal); + llvm::Value *BasePointersArray; + llvm::Value *PointersArray; + llvm::Value *SizesArray; + llvm::Value *MapTypesArray; + + if (PointerNumVal) { + llvm::APInt PointerNumAP(32, PointerNumVal, /*isSigned=*/true); + QualType PointerArrayType = Ctx.getConstantArrayType( + Ctx.VoidPtrTy, PointerNumAP, ArrayType::Normal, + /*IndexTypeQuals=*/0); + + BasePointersArray = + CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer(); + PointersArray = + CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer(); + + // If we don't have any VLA types, we can use a constant array for the map + // sizes, otherwise we need to fill up the arrays as we do for the + // pointers. + if (hasVLACaptures) { + QualType SizeArrayType = Ctx.getConstantArrayType( + Ctx.getSizeType(), PointerNumAP, ArrayType::Normal, + /*IndexTypeQuals=*/0); + SizesArray = + CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer(); + } else { + // We expect all the sizes to be constant, so we collect them to create + // a constant array. + SmallVector<llvm::Constant *, 16> ConstSizes; + for (auto S : Sizes) + ConstSizes.push_back(cast<llvm::Constant>(S)); + + auto *SizesArrayInit = llvm::ConstantArray::get( + llvm::ArrayType::get(CGM.SizeTy, ConstSizes.size()), ConstSizes); + auto *SizesArrayGbl = new llvm::GlobalVariable( + CGM.getModule(), SizesArrayInit->getType(), + /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, + SizesArrayInit, ".offload_sizes"); + SizesArrayGbl->setUnnamedAddr(true); + SizesArray = SizesArrayGbl; + } + + // The map types are always constant so we don't need to generate code to + // fill arrays. Instead, we create an array constant. + llvm::Constant *MapTypesArrayInit = + llvm::ConstantDataArray::get(CGF.Builder.getContext(), MapTypes); + auto *MapTypesArrayGbl = new llvm::GlobalVariable( + CGM.getModule(), MapTypesArrayInit->getType(), + /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, + MapTypesArrayInit, ".offload_maptypes"); + MapTypesArrayGbl->setUnnamedAddr(true); + MapTypesArray = MapTypesArrayGbl; + + for (unsigned i = 0; i < PointerNumVal; ++i) { + + llvm::Value *BPVal = BasePointers[i]; + if (BPVal->getType()->isPointerTy()) + BPVal = CGF.Builder.CreateBitCast(BPVal, CGM.VoidPtrTy); + else { + assert(BPVal->getType()->isIntegerTy() && + "If not a pointer, the value type must be an integer."); + BPVal = CGF.Builder.CreateIntToPtr(BPVal, CGM.VoidPtrTy); + } + llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32( + llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal), + BasePointersArray, 0, i); + Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy)); + CGF.Builder.CreateStore(BPVal, BPAddr); + + llvm::Value *PVal = Pointers[i]; + if (PVal->getType()->isPointerTy()) + PVal = CGF.Builder.CreateBitCast(PVal, CGM.VoidPtrTy); + else { + assert(PVal->getType()->isIntegerTy() && + "If not a pointer, the value type must be an integer."); + PVal = CGF.Builder.CreateIntToPtr(PVal, CGM.VoidPtrTy); + } + llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( + llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal), PointersArray, + 0, i); + Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy)); + CGF.Builder.CreateStore(PVal, PAddr); + + if (hasVLACaptures) { + llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32( + llvm::ArrayType::get(CGM.SizeTy, PointerNumVal), SizesArray, + /*Idx0=*/0, + /*Idx1=*/i); + Address SAddr(S, Ctx.getTypeAlignInChars(Ctx.getSizeType())); + CGF.Builder.CreateStore(CGF.Builder.CreateIntCast( + Sizes[i], CGM.SizeTy, /*isSigned=*/true), + SAddr); + } + } + + BasePointersArray = CGF.Builder.CreateConstInBoundsGEP2_32( + llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal), BasePointersArray, + /*Idx0=*/0, /*Idx1=*/0); + PointersArray = CGF.Builder.CreateConstInBoundsGEP2_32( + llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal), PointersArray, + /*Idx0=*/0, + /*Idx1=*/0); + SizesArray = CGF.Builder.CreateConstInBoundsGEP2_32( + llvm::ArrayType::get(CGM.SizeTy, PointerNumVal), SizesArray, + /*Idx0=*/0, /*Idx1=*/0); + MapTypesArray = CGF.Builder.CreateConstInBoundsGEP2_32( + llvm::ArrayType::get(CGM.Int32Ty, PointerNumVal), MapTypesArray, + /*Idx0=*/0, + /*Idx1=*/0); + + } else { + BasePointersArray = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy); + PointersArray = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy); + SizesArray = llvm::ConstantPointerNull::get(CGM.SizeTy->getPointerTo()); + MapTypesArray = + llvm::ConstantPointerNull::get(CGM.Int32Ty->getPointerTo()); + } + + // On top of the arrays that were filled up, the target offloading call + // takes as arguments the device id as well as the host pointer. The host + // pointer is used by the runtime library to identify the current target + // region, so it only has to be unique and not necessarily point to + // anything. It could be the pointer to the outlined function that + // implements the target region, but we aren't using that so that the + // compiler doesn't need to keep that, and could therefore inline the host + // function if proven worthwhile during optimization. + + llvm::Value *HostPtr = new llvm::GlobalVariable( + CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true, + llvm::GlobalValue::PrivateLinkage, + llvm::Constant::getNullValue(CGM.Int8Ty), ".offload_hstptr"); + + // Emit device ID if any. + llvm::Value *DeviceID; + if (Device) + DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), + CGM.Int32Ty, /*isSigned=*/true); + else + DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF); + + llvm::Value *OffloadingArgs[] = { + DeviceID, HostPtr, PointerNum, BasePointersArray, + PointersArray, SizesArray, MapTypesArray}; + auto Return = CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target), + OffloadingArgs); + + CGF.EmitStoreOfScalar(Return, OffloadError); + }; + + if (IfCond) { + // Notify that the host version must be executed. + auto &&ElseGen = [this, OffloadError, + OffloadErrorQType](CodeGenFunction &CGF) { + CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/-1u), + OffloadError); + }; + emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen); + } else { + CodeGenFunction::RunCleanupsScope Scope(CGF); + ThenGen(CGF); + } + + // Check the error code and execute the host version if required. + auto OffloadFailedBlock = CGF.createBasicBlock("omp_offload.failed"); + auto OffloadContBlock = CGF.createBasicBlock("omp_offload.cont"); + auto OffloadErrorVal = CGF.EmitLoadOfScalar(OffloadError, SourceLocation()); + auto Failed = CGF.Builder.CreateIsNotNull(OffloadErrorVal); + CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock); + + CGF.EmitBlock(OffloadFailedBlock); + CGF.Builder.CreateCall(OutlinedFn, BasePointers); + CGF.EmitBranch(OffloadContBlock); + + CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true); + return; +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h index 44bc8a1..992f9a8 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h @@ -39,7 +39,7 @@ class OMPExecutableDirective; class VarDecl; namespace CodeGen { - +class Address; class CodeGenFunction; class CodeGenModule; @@ -62,6 +62,9 @@ private: // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, // kmp_critical_name *crit); OMPRTL__kmpc_critical, + // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 + // global_tid, kmp_critical_name *crit, uintptr_t hint); + OMPRTL__kmpc_critical_with_hint, // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, // kmp_critical_name *crit); OMPRTL__kmpc_end_critical, @@ -154,6 +157,14 @@ private: // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid, // kmp_int32 cncl_kind); OMPRTL__kmpc_cancel, + + // + // Offloading related calls + // + // Call to int32_t __tgt_target(int32_t device_id, void *host_ptr, int32_t + // arg_num, void** args_base, void **args, size_t *arg_sizes, int32_t + // *arg_types); + OMPRTL__tgt_target, }; /// \brief Values for bit flags used in the ident_t to describe the fields. @@ -184,7 +195,9 @@ private: /// \brief Map of flags and corresponding default locations. typedef llvm::DenseMap<unsigned, llvm::Value *> OpenMPDefaultLocMapTy; OpenMPDefaultLocMapTy OpenMPDefaultLocMap; - llvm::Value *getOrCreateDefaultLocation(OpenMPLocationFlags Flags); + Address getOrCreateDefaultLocation(OpenMPLocationFlags Flags); + +public: /// \brief Describes ident structure that describes a source location. /// All descriptions are taken from /// http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h @@ -225,6 +238,7 @@ private: /// and a pair of line numbers that delimit the construct. IdentField_PSource }; +private: llvm::StructType *IdentTy; /// \brief Map for SourceLocation and OpenMP runtime library debug locations. typedef llvm::DenseMap<unsigned, llvm::Value *> OpenMPDebugLocMapTy; @@ -321,8 +335,7 @@ private: /// \brief Emits address of the word in a memory where current thread id is /// stored. - virtual llvm::Value *emitThreadIDAddress(CodeGenFunction &CGF, - SourceLocation Loc); + virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc); /// \brief Gets thread id value for the current thread. /// @@ -346,7 +359,7 @@ private: /// \param CopyCtor Pointer to a global copy function for \a VD. /// \param Dtor Pointer to a global destructor function for \a VD. /// \param Loc Location of threadprivate declaration. - void emitThreadPrivateVarInit(CodeGenFunction &CGF, llvm::Value *VDAddr, + void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc); @@ -396,23 +409,25 @@ public: /// CapturedStruct. /// \param OutlinedFn Outlined function to be run in parallel threads. Type of /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). - /// \param CapturedStruct A pointer to the record with the references to + /// \param CapturedVars A pointer to the record with the references to /// variables used in \a OutlinedFn function. /// \param IfCond Condition in the associated 'if' clause, if it was /// specified, nullptr otherwise. /// virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct, + ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond); /// \brief Emits a critical region. /// \param CriticalName Name of the critical region. /// \param CriticalOpGen Generator for the statement associated with the given /// critical region. + /// \param Hint Value of the 'hint' clause (optional). virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, - SourceLocation Loc); + SourceLocation Loc, + const Expr *Hint = nullptr); /// \brief Emits a master region. /// \param MasterOpGen Generator for the statement associated with the given @@ -447,17 +462,20 @@ public: /// ordered region. virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, - SourceLocation Loc); + SourceLocation Loc, bool IsThreads); /// \brief Emit an implicit/explicit barrier for OpenMP threads. /// \param Kind Directive for which this implicit barrier call must be /// generated. Must be OMPD_barrier for explicit barrier generation. - /// \param CheckForCancel true if check for possible cancellation must be - /// performed, false otherwise. + /// \param EmitChecks true if need to emit checks for cancellation barriers. + /// \param ForceSimpleCall true simple barrier call must be emitted, false if + /// runtime class decides which one to emit (simple or with cancellation + /// checks). /// virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, - bool CheckForCancel = true); + bool EmitChecks = true, + bool ForceSimpleCall = false); /// \brief Check if the specified \a ScheduleKind is static non-chunked. /// This kind of worksharing directive is emitted without outer loop. @@ -473,6 +491,12 @@ public: /// virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const; + virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPScheduleClauseKind SchedKind, + unsigned IVSize, bool IVSigned, + bool Ordered, llvm::Value *UB, + llvm::Value *Chunk = nullptr); + /// \brief Call the appropriate runtime routine to initialize it before start /// of loop. /// @@ -497,11 +521,12 @@ public: /// \param Chunk Value of the chunk for the static_chunked scheduled loop. /// For the default (nullptr) value, the chunk 1 will be used. /// - virtual void emitForInit(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPScheduleClauseKind SchedKind, unsigned IVSize, - bool IVSigned, bool Ordered, llvm::Value *IL, - llvm::Value *LB, llvm::Value *UB, llvm::Value *ST, - llvm::Value *Chunk = nullptr); + virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPScheduleClauseKind SchedKind, + unsigned IVSize, bool IVSigned, bool Ordered, + Address IL, Address LB, + Address UB, Address ST, + llvm::Value *Chunk = nullptr); /// \brief Call the appropriate runtime routine to notify that we finished /// iteration of the ordered loop with the dynamic scheduling. @@ -539,8 +564,8 @@ public: /// returned. virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, - llvm::Value *IL, llvm::Value *LB, - llvm::Value *UB, llvm::Value *ST); + Address IL, Address LB, + Address UB, Address ST); /// \brief Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads' @@ -562,10 +587,10 @@ public: /// \param VDAddr Address of the global variable \a VD. /// \param Loc Location of the reference to threadprivate var. /// \return Address of the threadprivate variable for the current thread. - virtual llvm::Value *getAddrOfThreadPrivate(CodeGenFunction &CGF, - const VarDecl *VD, - llvm::Value *VDAddr, - SourceLocation Loc); + virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, + const VarDecl *VD, + Address VDAddr, + SourceLocation Loc); /// \brief Emit a code for initialization of threadprivate variable. It emits /// a call to runtime library which adds initial value to the newly created @@ -576,7 +601,7 @@ public: /// \param Loc Location of threadprivate declaration. /// \param PerformInit true if initialization expression is not constant. virtual llvm::Function * - emitThreadPrivateVarDefinition(const VarDecl *VD, llvm::Value *VDAddr, + emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF = nullptr); @@ -632,7 +657,7 @@ public: virtual void emitTaskCall( CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, bool Tied, llvm::PointerIntPair<llvm::Value *, 1, bool> Final, - llvm::Value *TaskFunction, QualType SharedsTy, llvm::Value *Shareds, + llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, ArrayRef<const Expr *> PrivateVars, ArrayRef<const Expr *> PrivateCopies, ArrayRef<const Expr *> FirstprivateVars, @@ -645,9 +670,12 @@ public: /// \param InnermostKind Kind of innermost directive (for simple directives it /// is a directive itself, for combined - its innermost directive). /// \param CodeGen Code generation sequence for the \a D directive. + /// \param HasCancel true if region has inner cancel directive, false + /// otherwise. virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, - const RegionCodeGenTy &CodeGen); + const RegionCodeGenTy &CodeGen, + bool HasCancel = false); /// \brief Emit a code for reduction clause. Next code should be emitted for /// reduction: /// \code @@ -679,6 +707,7 @@ public: /// } /// \endcode /// + /// \param Privates List of private copies for original reduction arguments. /// \param LHSExprs List of LHS in \a ReductionOps reduction operations. /// \param RHSExprs List of RHS in \a ReductionOps reduction operations. /// \param ReductionOps List of reduction operations in form 'LHS binop RHS' @@ -686,6 +715,7 @@ public: /// \param WithNowait true if parent directive has also nowait clause, false /// otherwise. virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, + ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps, @@ -703,10 +733,36 @@ public: OpenMPDirectiveKind CancelRegion); /// \brief Emit code for 'cancel' construct. + /// \param IfCond Condition in the associated 'if' clause, if it was + /// specified, nullptr otherwise. /// \param CancelRegion Region kind for which the cancel must be emitted. /// virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, + const Expr *IfCond, OpenMPDirectiveKind CancelRegion); + + /// \brief Emit outilined function for 'target' directive. + /// \param D Directive to emit. + /// \param CodeGen Code generation sequence for the \a D directive. + virtual llvm::Value * + emitTargetOutlinedFunction(const OMPExecutableDirective &D, + const RegionCodeGenTy &CodeGen); + + /// \brief Emit the target offloading code associated with \a D. The emitted + /// code attempts offloading the execution to the device, an the event of + /// a failure it executes the host version outlined in \a OutlinedFn. + /// \param D Directive to emit. + /// \param OutlinedFn Host version of the code to be offloaded. + /// \param IfCond Expression evaluated in if clause associated with the target + /// directive, or null if no if clause is used. + /// \param Device Expression evaluated in device clause associated with the + /// target directive, or null if no device clause is used. + /// \param CapturedVars Values captured in the current region. + virtual void emitTargetCall(CodeGenFunction &CGF, + const OMPExecutableDirective &D, + llvm::Value *OutlinedFn, const Expr *IfCond, + const Expr *Device, + ArrayRef<llvm::Value *> CapturedVars); }; } // namespace CodeGen diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp index f91eceb..375b59c 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp @@ -454,7 +454,7 @@ void CGRecordLowering::accumulateBases() { // contain only a trailing array member. const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); if (!BaseDecl->isEmpty() && - !Context.getASTRecordLayout(BaseDecl).getSize().isZero()) + !Context.getASTRecordLayout(BaseDecl).getNonVirtualSize().isZero()) Members.push_back(MemberInfo(Layout.getBaseClassOffset(BaseDecl), MemberInfo::Base, getStorageType(BaseDecl), BaseDecl)); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp index 7a0b8a3..cc4fa2e 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp @@ -16,6 +16,7 @@ #include "CodeGenModule.h" #include "TargetInfo.h" #include "clang/AST/StmtVisitor.h" +#include "clang/Basic/Builtins.h" #include "clang/Basic/PrettyStackTrace.h" #include "clang/Basic/TargetInfo.h" #include "clang/Sema/LoopHint.h" @@ -25,6 +26,8 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/MDBuilder.h" + using namespace clang; using namespace CodeGen; @@ -138,6 +141,10 @@ void CodeGenFunction::EmitStmt(const Stmt *S) { case Stmt::SwitchStmtClass: EmitSwitchStmt(cast<SwitchStmt>(*S)); break; case Stmt::GCCAsmStmtClass: // Intentional fall-through. case Stmt::MSAsmStmtClass: EmitAsmStmt(cast<AsmStmt>(*S)); break; + case Stmt::CoroutineBodyStmtClass: + case Stmt::CoreturnStmtClass: + CGM.ErrorUnsupported(S, "coroutine"); + break; case Stmt::CapturedStmtClass: { const CapturedStmt *CS = cast<CapturedStmt>(S); EmitCapturedStmt(*CS, CS->getCapturedRegionKind()); @@ -246,6 +253,18 @@ void CodeGenFunction::EmitStmt(const Stmt *S) { case Stmt::OMPCancelDirectiveClass: EmitOMPCancelDirective(cast<OMPCancelDirective>(*S)); break; + case Stmt::OMPTargetDataDirectiveClass: + EmitOMPTargetDataDirective(cast<OMPTargetDataDirective>(*S)); + break; + case Stmt::OMPTaskLoopDirectiveClass: + EmitOMPTaskLoopDirective(cast<OMPTaskLoopDirective>(*S)); + break; + case Stmt::OMPTaskLoopSimdDirectiveClass: + EmitOMPTaskLoopSimdDirective(cast<OMPTaskLoopSimdDirective>(*S)); + break; +case Stmt::OMPDistributeDirectiveClass: + EmitOMPDistributeDirective(cast<OMPDistributeDirective>(*S)); + break; } } @@ -272,8 +291,8 @@ bool CodeGenFunction::EmitSimpleStmt(const Stmt *S) { /// EmitCompoundStmt - Emit a compound statement {..} node. If GetLast is true, /// this captures the expression result of the last sub-statement and returns it /// (for use by the statement expression extension). -llvm::Value* CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast, - AggValueSlot AggSlot) { +Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast, + AggValueSlot AggSlot) { PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(), "LLVM IR generation of compound statement ('{}')"); @@ -283,7 +302,7 @@ llvm::Value* CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLa return EmitCompoundStmtWithoutScope(S, GetLast, AggSlot); } -llvm::Value* +Address CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S, bool GetLast, AggValueSlot AggSlot) { @@ -292,7 +311,7 @@ CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S, E = S.body_end()-GetLast; I != E; ++I) EmitStmt(*I); - llvm::Value *RetAlloca = nullptr; + Address RetAlloca = Address::invalid(); if (GetLast) { // We have to special case labels here. They are statements, but when put // at the end of a statement expression, they yield the value of their @@ -337,7 +356,7 @@ void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) { return; // Can only simplify empty blocks. - if (BI != BB->begin()) + if (BI->getIterator() != BB->begin()) return; BB->replaceAllUsesWith(BI->getSuccessor(0)); @@ -359,7 +378,7 @@ void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) { // Place the block after the current block, if possible, or else at // the end of the function. if (CurBB && CurBB->getParent()) - CurFn->getBasicBlockList().insertAfter(CurBB, BB); + CurFn->getBasicBlockList().insertAfter(CurBB->getIterator(), BB); else CurFn->getBasicBlockList().push_back(BB); Builder.SetInsertPoint(BB); @@ -386,7 +405,8 @@ void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) { bool inserted = false; for (llvm::User *u : block->users()) { if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(u)) { - CurFn->getBasicBlockList().insertAfter(insn->getParent(), block); + CurFn->getBasicBlockList().insertAfter(insn->getParent()->getIterator(), + block); inserted = true; break; } @@ -590,100 +610,6 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { EmitBlock(ContBlock, true); } -void CodeGenFunction::EmitCondBrHints(llvm::LLVMContext &Context, - llvm::BranchInst *CondBr, - ArrayRef<const Attr *> Attrs) { - // Return if there are no hints. - if (Attrs.empty()) - return; - - // Add vectorize and unroll hints to the metadata on the conditional branch. - // - // FIXME: Should this really start with a size of 1? - SmallVector<llvm::Metadata *, 2> Metadata(1); - for (const auto *Attr : Attrs) { - const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); - - // Skip non loop hint attributes - if (!LH) - continue; - - LoopHintAttr::OptionType Option = LH->getOption(); - LoopHintAttr::LoopHintState State = LH->getState(); - const char *MetadataName; - switch (Option) { - case LoopHintAttr::Vectorize: - case LoopHintAttr::VectorizeWidth: - MetadataName = "llvm.loop.vectorize.width"; - break; - case LoopHintAttr::Interleave: - case LoopHintAttr::InterleaveCount: - MetadataName = "llvm.loop.interleave.count"; - break; - case LoopHintAttr::Unroll: - // With the unroll loop hint, a non-zero value indicates full unrolling. - MetadataName = State == LoopHintAttr::Disable ? "llvm.loop.unroll.disable" - : "llvm.loop.unroll.full"; - break; - case LoopHintAttr::UnrollCount: - MetadataName = "llvm.loop.unroll.count"; - break; - } - - Expr *ValueExpr = LH->getValue(); - int ValueInt = 1; - if (ValueExpr) { - llvm::APSInt ValueAPS = - ValueExpr->EvaluateKnownConstInt(CGM.getContext()); - ValueInt = static_cast<int>(ValueAPS.getSExtValue()); - } - - llvm::Constant *Value; - llvm::MDString *Name; - switch (Option) { - case LoopHintAttr::Vectorize: - case LoopHintAttr::Interleave: - if (State != LoopHintAttr::Disable) { - // FIXME: In the future I will modifiy the behavior of the metadata - // so we can enable/disable vectorization and interleaving separately. - Name = llvm::MDString::get(Context, "llvm.loop.vectorize.enable"); - Value = Builder.getTrue(); - break; - } - // Vectorization/interleaving is disabled, set width/count to 1. - ValueInt = 1; - // Fallthrough. - case LoopHintAttr::VectorizeWidth: - case LoopHintAttr::InterleaveCount: - case LoopHintAttr::UnrollCount: - Name = llvm::MDString::get(Context, MetadataName); - Value = llvm::ConstantInt::get(Int32Ty, ValueInt); - break; - case LoopHintAttr::Unroll: - Name = llvm::MDString::get(Context, MetadataName); - Value = nullptr; - break; - } - - SmallVector<llvm::Metadata *, 2> OpValues; - OpValues.push_back(Name); - if (Value) - OpValues.push_back(llvm::ConstantAsMetadata::get(Value)); - - // Set or overwrite metadata indicated by Name. - Metadata.push_back(llvm::MDNode::get(Context, OpValues)); - } - - // FIXME: This condition is never false. Should it be an assert? - if (!Metadata.empty()) { - // Add llvm.loop MDNode to CondBr. - llvm::MDNode *LoopID = llvm::MDNode::get(Context, Metadata); - LoopID->replaceOperandWith(0, LoopID); // First op points to itself. - - CondBr->setMetadata("llvm.loop", LoopID); - } -} - void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> WhileAttrs) { // Emit the header for the loop, which will also become @@ -691,7 +617,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond"); EmitBlock(LoopHeader.getBlock()); - LoopStack.push(LoopHeader.getBlock(), WhileAttrs); + LoopStack.push(LoopHeader.getBlock(), CGM.getContext(), WhileAttrs); // Create an exit block for when the condition fails, which will // also become the break target. @@ -730,7 +656,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); if (ConditionScope.requiresCleanups()) ExitBlock = createBasicBlock("while.exit"); - llvm::BranchInst *CondBr = Builder.CreateCondBr( + Builder.CreateCondBr( BoolCondVal, LoopBody, ExitBlock, createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); @@ -738,9 +664,6 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, EmitBlock(ExitBlock); EmitBranchThroughCleanup(LoopExit); } - - // Attach metadata to loop body conditional branch. - EmitCondBrHints(LoopBody->getContext(), CondBr, WhileAttrs); } // Emit the loop body. We have to emit this in a cleanup scope @@ -785,7 +708,7 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, // Emit the body of the loop. llvm::BasicBlock *LoopBody = createBasicBlock("do.body"); - LoopStack.push(LoopBody, DoAttrs); + LoopStack.push(LoopBody, CGM.getContext(), DoAttrs); EmitBlockWithFallThrough(LoopBody, &S); { @@ -815,12 +738,9 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, // As long as the condition is true, iterate the loop. if (EmitBoolCondBranch) { uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount; - llvm::BranchInst *CondBr = Builder.CreateCondBr( + Builder.CreateCondBr( BoolCondVal, LoopBody, LoopExit.getBlock(), createProfileWeightsForLoop(S.getCond(), BackedgeCount)); - - // Attach metadata to loop body conditional branch. - EmitCondBrHints(LoopBody->getContext(), CondBr, DoAttrs); } LoopStack.pop(); @@ -851,7 +771,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, llvm::BasicBlock *CondBlock = Continue.getBlock(); EmitBlock(CondBlock); - LoopStack.push(CondBlock, ForAttrs); + LoopStack.push(CondBlock, CGM.getContext(), ForAttrs); // If the for loop doesn't have an increment we can just use the // condition as the continue block. Otherwise we'll need to create @@ -885,13 +805,10 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, // C99 6.8.5p2/p4: The first substatement is executed if the expression // compares unequal to 0. The condition must be a scalar type. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - llvm::BranchInst *CondBr = Builder.CreateCondBr( + Builder.CreateCondBr( BoolCondVal, ForBody, ExitBlock, createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); - // Attach metadata to loop body conditional branch. - EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs); - if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); EmitBranchThroughCleanup(LoopExit); @@ -949,7 +866,7 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, llvm::BasicBlock *CondBlock = createBasicBlock("for.cond"); EmitBlock(CondBlock); - LoopStack.push(CondBlock, ForAttrs); + LoopStack.push(CondBlock, CGM.getContext(), ForAttrs); // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. @@ -963,13 +880,10 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, // The body is executed if the expression, contextually converted // to bool, is true. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - llvm::BranchInst *CondBr = Builder.CreateCondBr( + Builder.CreateCondBr( BoolCondVal, ForBody, ExitBlock, createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); - // Attach metadata to loop body conditional branch. - EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs); - if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); EmitBranchThroughCleanup(LoopExit); @@ -1012,10 +926,9 @@ void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) { if (RV.isScalar()) { Builder.CreateStore(RV.getScalarVal(), ReturnValue); } else if (RV.isAggregate()) { - EmitAggregateCopy(ReturnValue, RV.getAggregateAddr(), Ty); + EmitAggregateCopy(ReturnValue, RV.getAggregateAddress(), Ty); } else { - EmitStoreOfComplex(RV.getComplexVal(), - MakeNaturalAlignAddrLValue(ReturnValue, Ty), + EmitStoreOfComplex(RV.getComplexVal(), MakeAddrLValue(ReturnValue, Ty), /*init*/ true); } EmitBranchThroughCleanup(ReturnBlock); @@ -1056,8 +969,8 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) { // If there is an NRVO flag for this variable, set it to 1 into indicate // that the cleanup code should not destroy the variable. if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()]) - Builder.CreateStore(Builder.getTrue(), NRVOFlag); - } else if (!ReturnValue || (RV && RV->getType()->isVoidType())) { + Builder.CreateFlagStore(Builder.getTrue(), NRVOFlag); + } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) { // Make sure not to return anything, but evaluate the expression // for side effects. if (RV) @@ -1075,20 +988,17 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) { Builder.CreateStore(EmitScalarExpr(RV), ReturnValue); break; case TEK_Complex: - EmitComplexExprIntoLValue(RV, - MakeNaturalAlignAddrLValue(ReturnValue, RV->getType()), + EmitComplexExprIntoLValue(RV, MakeAddrLValue(ReturnValue, RV->getType()), /*isInit*/ true); break; - case TEK_Aggregate: { - CharUnits Alignment = getContext().getTypeAlignInChars(RV->getType()); - EmitAggExpr(RV, AggValueSlot::forAddr(ReturnValue, Alignment, + case TEK_Aggregate: + EmitAggExpr(RV, AggValueSlot::forAddr(ReturnValue, Qualifiers(), AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased)); break; } - } } ++NumReturnExprs; @@ -1624,6 +1534,22 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { EmitBlock(SwitchExit.getBlock(), true); incrementProfileCounter(&S); + // If the switch has a condition wrapped by __builtin_unpredictable, + // create metadata that specifies that the switch is unpredictable. + // Don't bother if not optimizing because that metadata would not be used. + if (CGM.getCodeGenOpts().OptimizationLevel != 0) { + if (const CallExpr *Call = dyn_cast<CallExpr>(S.getCond())) { + const Decl *TargetDecl = Call->getCalleeDecl(); + if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) { + if (FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) { + llvm::MDBuilder MDHelper(getLLVMContext()); + SwitchInsn->setMetadata(llvm::LLVMContext::MD_unpredictable, + MDHelper.createUnpredictable()); + } + } + } + } + if (SwitchWeights) { assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() && "switch weights do not match switch cases"); @@ -1675,9 +1601,7 @@ SimplifyConstraint(const char *Constraint, const TargetInfo &Target, assert(OutCons && "Must pass output names to constraints with a symbolic name"); unsigned Index; - bool result = Target.resolveSymbolicName(Constraint, - &(*OutCons)[0], - OutCons->size(), Index); + bool result = Target.resolveSymbolicName(Constraint, *OutCons, Index); assert(result && "Could not resolve symbolic name"); (void)result; Result += llvm::utostr(Index); break; @@ -1743,12 +1667,12 @@ CodeGenFunction::EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, Arg = Builder.CreateLoad(Builder.CreateBitCast(InputValue.getAddress(), Ty)); } else { - Arg = InputValue.getAddress(); + Arg = InputValue.getPointer(); ConstraintStr += '*'; } } } else { - Arg = InputValue.getAddress(); + Arg = InputValue.getPointer(); ConstraintStr += '*'; } @@ -1772,7 +1696,8 @@ llvm::Value* CodeGenFunction::EmitAsmInput( if (Info.allowsRegister() || !Info.allowsMemory()) if (CodeGenFunction::hasScalarEvaluationKind(InputExpr->getType())) return EmitScalarExpr(InputExpr); - + if (InputExpr->getStmtClass() == Expr::CXXThisExprClass) + return EmitScalarExpr(InputExpr); InputExpr = InputExpr->IgnoreParenNoopCasts(getContext()); LValue Dest = EmitLValue(InputExpr); return EmitAsmInputLValue(Info, Dest, InputExpr->getType(), ConstraintStr, @@ -1793,13 +1718,15 @@ static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str, if (!StrVal.empty()) { const SourceManager &SM = CGF.CGM.getContext().getSourceManager(); const LangOptions &LangOpts = CGF.CGM.getLangOpts(); + unsigned StartToken = 0; + unsigned ByteOffset = 0; // Add the location of the start of each subsequent line of the asm to the // MDNode. - for (unsigned i = 0, e = StrVal.size()-1; i != e; ++i) { + for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) { if (StrVal[i] != '\n') continue; - SourceLocation LineLoc = Str->getLocationOfByte(i+1, SM, LangOpts, - CGF.getTarget()); + SourceLocation LineLoc = Str->getLocationOfByte( + i + 1, SM, LangOpts, CGF.getTarget(), &StartToken, &ByteOffset); Locs.push_back(llvm::ConstantAsMetadata::get( llvm::ConstantInt::get(CGF.Int32Ty, LineLoc.getRawEncoding()))); } @@ -1832,8 +1759,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { Name = GAS->getInputName(i); TargetInfo::ConstraintInfo Info(S.getInputConstraint(i), Name); bool IsValid = - getTarget().validateInputConstraint(OutputConstraintInfos.data(), - S.getNumOutputs(), Info); + getTarget().validateInputConstraint(OutputConstraintInfos, Info); assert(IsValid && "Failed to parse input constraint"); (void)IsValid; InputConstraintInfos.push_back(Info); } @@ -1919,8 +1845,8 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { << OutExpr->getType() << OutputConstraint; } } else { - ArgTypes.push_back(Dest.getAddress()->getType()); - Args.push_back(Dest.getAddress()); + ArgTypes.push_back(Dest.getAddress().getType()); + Args.push_back(Dest.getPointer()); Constraints += "=*"; Constraints += OutputConstraint; ReadOnly = ReadNone = false; @@ -2077,6 +2003,15 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { Result->addAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::NoUnwind); + if (isa<MSAsmStmt>(&S)) { + // If the assembly contains any labels, mark the call noduplicate to prevent + // defining the same ASM label twice (PR23715). This is pretty hacky, but it + // works. + if (AsmString.find("__MSASMLABEL_") != std::string::npos) + Result->addAttribute(llvm::AttributeSet::FunctionIndex, + llvm::Attribute::NoDuplicate); + } + // Attach readnone and readonly attributes. if (!HasSideEffect) { if (ReadNone) @@ -2152,12 +2087,12 @@ LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) { QualType RecordTy = getContext().getRecordType(RD); // Initialize the captured struct. - LValue SlotLV = MakeNaturalAlignAddrLValue( - CreateMemTemp(RecordTy, "agg.captured"), RecordTy); + LValue SlotLV = + MakeAddrLValue(CreateMemTemp(RecordTy, "agg.captured"), RecordTy); RecordDecl::field_iterator CurField = RD->field_begin(); - for (CapturedStmt::capture_init_iterator I = S.capture_init_begin(), - E = S.capture_init_end(); + for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(), + E = S.capture_init_end(); I != E; ++I, ++CurField) { LValue LV = EmitLValueForFieldInitialization(SlotLV, *CurField); if (CurField->hasCapturedVLAType()) { @@ -2184,13 +2119,12 @@ CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) { delete CGF.CapturedStmtInfo; // Emit call to the helper function. - EmitCallOrInvoke(F, CapStruct.getAddress()); + EmitCallOrInvoke(F, CapStruct.getPointer()); return F; } -llvm::Value * -CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) { +Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) { LValue CapStruct = InitCapturedStruct(S); return CapStruct.getAddress(); } @@ -2229,8 +2163,7 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) { CD->getLocation(), CD->getBody()->getLocStart()); // Set the context parameter in CapturedStmtInfo. - llvm::Value *DeclPtr = LocalDeclMap[CD->getContextParam()]; - assert(DeclPtr && "missing context parameter for CapturedStmt"); + Address DeclPtr = GetAddrOfLocalVar(CD->getContextParam()); CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr)); // Initialize variable-length arrays. @@ -2252,7 +2185,7 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) { CXXThisValue = EmitLoadOfLValue(ThisLValue, Loc).getScalarVal(); } - PGO.assignRegionCounters(CD, F); + PGO.assignRegionCounters(GlobalDecl(CD), F); CapturedStmtInfo->EmitBody(*this, CD->getBody()); FinishFunction(CD->getBodyRBrace()); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp index e5f507a..bcd2ac5 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -20,21 +20,195 @@ using namespace clang; using namespace CodeGen; +void CodeGenFunction::GenerateOpenMPCapturedVars( + const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) { + const RecordDecl *RD = S.getCapturedRecordDecl(); + auto CurField = RD->field_begin(); + auto CurCap = S.captures().begin(); + for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(), + E = S.capture_init_end(); + I != E; ++I, ++CurField, ++CurCap) { + if (CurField->hasCapturedVLAType()) { + auto VAT = CurField->getCapturedVLAType(); + auto *Val = VLASizeMap[VAT->getSizeExpr()]; + CapturedVars.push_back(Val); + } else if (CurCap->capturesThis()) + CapturedVars.push_back(CXXThisValue); + else if (CurCap->capturesVariableByCopy()) + CapturedVars.push_back( + EmitLoadOfLValue(EmitLValue(*I), SourceLocation()).getScalarVal()); + else { + assert(CurCap->capturesVariable() && "Expected capture by reference."); + CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer()); + } + } +} + +static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType, + StringRef Name, LValue AddrLV, + bool isReferenceType = false) { + ASTContext &Ctx = CGF.getContext(); + + auto *CastedPtr = CGF.EmitScalarConversion( + AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(), + Ctx.getPointerType(DstType), SourceLocation()); + auto TmpAddr = + CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType)) + .getAddress(); + + // If we are dealing with references we need to return the address of the + // reference instead of the reference of the value. + if (isReferenceType) { + QualType RefType = Ctx.getLValueReferenceType(DstType); + auto *RefVal = TmpAddr.getPointer(); + TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name) + ".ref"); + auto TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType); + CGF.EmitScalarInit(RefVal, TmpLVal); + } + + return TmpAddr; +} + +llvm::Function * +CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) { + assert( + CapturedStmtInfo && + "CapturedStmtInfo should be set when generating the captured function"); + const CapturedDecl *CD = S.getCapturedDecl(); + const RecordDecl *RD = S.getCapturedRecordDecl(); + assert(CD->hasBody() && "missing CapturedDecl body"); + + // Build the argument list. + ASTContext &Ctx = CGM.getContext(); + FunctionArgList Args; + Args.append(CD->param_begin(), + std::next(CD->param_begin(), CD->getContextParamPosition())); + auto I = S.captures().begin(); + for (auto *FD : RD->fields()) { + QualType ArgType = FD->getType(); + IdentifierInfo *II = nullptr; + VarDecl *CapVar = nullptr; + + // If this is a capture by copy and the type is not a pointer, the outlined + // function argument type should be uintptr and the value properly casted to + // uintptr. This is necessary given that the runtime library is only able to + // deal with pointers. We can pass in the same way the VLA type sizes to the + // outlined function. + if ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) || + I->capturesVariableArrayType()) + ArgType = Ctx.getUIntPtrType(); + + if (I->capturesVariable() || I->capturesVariableByCopy()) { + CapVar = I->getCapturedVar(); + II = CapVar->getIdentifier(); + } else if (I->capturesThis()) + II = &getContext().Idents.get("this"); + else { + assert(I->capturesVariableArrayType()); + II = &getContext().Idents.get("vla"); + } + if (ArgType->isVariablyModifiedType()) + ArgType = getContext().getVariableArrayDecayedType(ArgType); + Args.push_back(ImplicitParamDecl::Create(getContext(), nullptr, + FD->getLocation(), II, ArgType)); + ++I; + } + Args.append( + std::next(CD->param_begin(), CD->getContextParamPosition() + 1), + CD->param_end()); + + // Create the function declaration. + FunctionType::ExtInfo ExtInfo; + const CGFunctionInfo &FuncInfo = + CGM.getTypes().arrangeFreeFunctionDeclaration(Ctx.VoidTy, Args, ExtInfo, + /*IsVariadic=*/false); + llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo); + + llvm::Function *F = llvm::Function::Create( + FuncLLVMTy, llvm::GlobalValue::InternalLinkage, + CapturedStmtInfo->getHelperName(), &CGM.getModule()); + CGM.SetInternalFunctionAttributes(CD, F, FuncInfo); + if (CD->isNothrow()) + F->addFnAttr(llvm::Attribute::NoUnwind); + + // Generate the function. + StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(), + CD->getBody()->getLocStart()); + unsigned Cnt = CD->getContextParamPosition(); + I = S.captures().begin(); + for (auto *FD : RD->fields()) { + // If we are capturing a pointer by copy we don't need to do anything, just + // use the value that we get from the arguments. + if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) { + setAddrOfLocalVar(I->getCapturedVar(), GetAddrOfLocalVar(Args[Cnt])); + ++Cnt, ++I; + continue; + } + + LValue ArgLVal = + MakeAddrLValue(GetAddrOfLocalVar(Args[Cnt]), Args[Cnt]->getType(), + AlignmentSource::Decl); + if (FD->hasCapturedVLAType()) { + LValue CastedArgLVal = + MakeAddrLValue(castValueFromUintptr(*this, FD->getType(), + Args[Cnt]->getName(), ArgLVal), + FD->getType(), AlignmentSource::Decl); + auto *ExprArg = + EmitLoadOfLValue(CastedArgLVal, SourceLocation()).getScalarVal(); + auto VAT = FD->getCapturedVLAType(); + VLASizeMap[VAT->getSizeExpr()] = ExprArg; + } else if (I->capturesVariable()) { + auto *Var = I->getCapturedVar(); + QualType VarTy = Var->getType(); + Address ArgAddr = ArgLVal.getAddress(); + if (!VarTy->isReferenceType()) { + ArgAddr = EmitLoadOfReference( + ArgAddr, ArgLVal.getType()->castAs<ReferenceType>()); + } + setAddrOfLocalVar( + Var, Address(ArgAddr.getPointer(), getContext().getDeclAlign(Var))); + } else if (I->capturesVariableByCopy()) { + assert(!FD->getType()->isAnyPointerType() && + "Not expecting a captured pointer."); + auto *Var = I->getCapturedVar(); + QualType VarTy = Var->getType(); + setAddrOfLocalVar(I->getCapturedVar(), + castValueFromUintptr(*this, FD->getType(), + Args[Cnt]->getName(), ArgLVal, + VarTy->isReferenceType())); + } else { + // If 'this' is captured, load it into CXXThisValue. + assert(I->capturesThis()); + CXXThisValue = + EmitLoadOfLValue(ArgLVal, Args[Cnt]->getLocation()).getScalarVal(); + } + ++Cnt, ++I; + } + + PGO.assignRegionCounters(GlobalDecl(CD), F); + CapturedStmtInfo->EmitBody(*this, CD->getBody()); + FinishFunction(CD->getBodyRBrace()); + + return F; +} + //===----------------------------------------------------------------------===// // OpenMP Directive Emission //===----------------------------------------------------------------------===// void CodeGenFunction::EmitOMPAggregateAssign( - llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType, - const llvm::function_ref<void(llvm::Value *, llvm::Value *)> &CopyGen) { + Address DestAddr, Address SrcAddr, QualType OriginalType, + const llvm::function_ref<void(Address, Address)> &CopyGen) { // Perform element-by-element initialization. QualType ElementTy; - auto SrcBegin = SrcAddr; - auto DestBegin = DestAddr; + + // Drill down to the base element type on both arrays. auto ArrayTy = OriginalType->getAsArrayTypeUnsafe(); - auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin); + auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr); + SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); + + auto SrcBegin = SrcAddr.getPointer(); + auto DestBegin = DestAddr.getPointer(); // Cast from pointer to array type to pointer to single element. - SrcBegin = Builder.CreatePointerBitCastOrAddrSpaceCast(SrcBegin, - DestBegin->getType()); auto DestEnd = Builder.CreateGEP(DestBegin, NumElements); // The basic structure here is a while-do loop. auto BodyBB = createBasicBlock("omp.arraycpy.body"); @@ -46,77 +220,144 @@ void CodeGenFunction::EmitOMPAggregateAssign( // Enter the loop body, making that address the current address. auto EntryBB = Builder.GetInsertBlock(); EmitBlock(BodyBB); - auto SrcElementCurrent = - Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); - SrcElementCurrent->addIncoming(SrcBegin, EntryBB); - auto DestElementCurrent = Builder.CreatePHI(DestBegin->getType(), 2, - "omp.arraycpy.destElementPast"); - DestElementCurrent->addIncoming(DestBegin, EntryBB); + + CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy); + + llvm::PHINode *SrcElementPHI = + Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); + SrcElementPHI->addIncoming(SrcBegin, EntryBB); + Address SrcElementCurrent = + Address(SrcElementPHI, + SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize)); + + llvm::PHINode *DestElementPHI = + Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast"); + DestElementPHI->addIncoming(DestBegin, EntryBB); + Address DestElementCurrent = + Address(DestElementPHI, + DestAddr.getAlignment().alignmentOfArrayElement(ElementSize)); // Emit copy. CopyGen(DestElementCurrent, SrcElementCurrent); // Shift the address forward by one element. auto DestElementNext = Builder.CreateConstGEP1_32( - DestElementCurrent, /*Idx0=*/1, "omp.arraycpy.dest.element"); + DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); auto SrcElementNext = Builder.CreateConstGEP1_32( - SrcElementCurrent, /*Idx0=*/1, "omp.arraycpy.src.element"); + SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element"); // Check whether we've reached the end. auto Done = Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); Builder.CreateCondBr(Done, DoneBB, BodyBB); - DestElementCurrent->addIncoming(DestElementNext, Builder.GetInsertBlock()); - SrcElementCurrent->addIncoming(SrcElementNext, Builder.GetInsertBlock()); + DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock()); + SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock()); // Done. EmitBlock(DoneBB, /*IsFinished=*/true); } -void CodeGenFunction::EmitOMPCopy(CodeGenFunction &CGF, - QualType OriginalType, llvm::Value *DestAddr, - llvm::Value *SrcAddr, const VarDecl *DestVD, +/// \brief Emit initialization of arrays of complex types. +/// \param DestAddr Address of the array. +/// \param Type Type of array. +/// \param Init Initial expression of array. +static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, + QualType Type, const Expr *Init) { + // Perform element-by-element initialization. + QualType ElementTy; + + // Drill down to the base element type on both arrays. + auto ArrayTy = Type->getAsArrayTypeUnsafe(); + auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr); + DestAddr = + CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType()); + + auto DestBegin = DestAddr.getPointer(); + // Cast from pointer to array type to pointer to single element. + auto DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements); + // The basic structure here is a while-do loop. + auto BodyBB = CGF.createBasicBlock("omp.arrayinit.body"); + auto DoneBB = CGF.createBasicBlock("omp.arrayinit.done"); + auto IsEmpty = + CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty"); + CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); + + // Enter the loop body, making that address the current address. + auto EntryBB = CGF.Builder.GetInsertBlock(); + CGF.EmitBlock(BodyBB); + + CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); + + llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI( + DestBegin->getType(), 2, "omp.arraycpy.destElementPast"); + DestElementPHI->addIncoming(DestBegin, EntryBB); + Address DestElementCurrent = + Address(DestElementPHI, + DestAddr.getAlignment().alignmentOfArrayElement(ElementSize)); + + // Emit copy. + { + CodeGenFunction::RunCleanupsScope InitScope(CGF); + CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), + /*IsInitializer=*/false); + } + + // Shift the address forward by one element. + auto DestElementNext = CGF.Builder.CreateConstGEP1_32( + DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); + // Check whether we've reached the end. + auto Done = + CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); + CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); + DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock()); + + // Done. + CGF.EmitBlock(DoneBB, /*IsFinished=*/true); +} + +void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr, + Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy) { if (OriginalType->isArrayType()) { auto *BO = dyn_cast<BinaryOperator>(Copy); if (BO && BO->getOpcode() == BO_Assign) { // Perform simple memcpy for simple copying. - CGF.EmitAggregateAssign(DestAddr, SrcAddr, OriginalType); + EmitAggregateAssign(DestAddr, SrcAddr, OriginalType); } else { // For arrays with complex element types perform element by element // copying. - CGF.EmitOMPAggregateAssign( + EmitOMPAggregateAssign( DestAddr, SrcAddr, OriginalType, - [&CGF, Copy, SrcVD, DestVD](llvm::Value *DestElement, - llvm::Value *SrcElement) { + [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) { // Working with the single array element, so have to remap // destination and source variables to corresponding array // elements. - CodeGenFunction::OMPPrivateScope Remap(CGF); - Remap.addPrivate(DestVD, [DestElement]() -> llvm::Value *{ + CodeGenFunction::OMPPrivateScope Remap(*this); + Remap.addPrivate(DestVD, [DestElement]() -> Address { return DestElement; }); Remap.addPrivate( - SrcVD, [SrcElement]() -> llvm::Value *{ return SrcElement; }); + SrcVD, [SrcElement]() -> Address { return SrcElement; }); (void)Remap.Privatize(); - CGF.EmitIgnoredExpr(Copy); + EmitIgnoredExpr(Copy); }); } } else { // Remap pseudo source variable to private copy. - CodeGenFunction::OMPPrivateScope Remap(CGF); - Remap.addPrivate(SrcVD, [SrcAddr]() -> llvm::Value *{ return SrcAddr; }); - Remap.addPrivate(DestVD, [DestAddr]() -> llvm::Value *{ return DestAddr; }); + CodeGenFunction::OMPPrivateScope Remap(*this); + Remap.addPrivate(SrcVD, [SrcAddr]() -> Address { return SrcAddr; }); + Remap.addPrivate(DestVD, [DestAddr]() -> Address { return DestAddr; }); (void)Remap.Privatize(); // Emit copying of the whole variable. - CGF.EmitIgnoredExpr(Copy); + EmitIgnoredExpr(Copy); } } bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) { + if (!HaveInsertPoint()) + return false; llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate; - for (auto &&I = D.getClausesOfKind(OMPC_firstprivate); I; ++I) { - auto *C = cast<OMPFirstprivateClause>(*I); + for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) { auto IRef = C->varlist_begin(); auto InitsRef = C->inits().begin(); for (auto IInit : C->private_copies()) { @@ -131,13 +372,13 @@ bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup( OrigVD) != nullptr, (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); - auto *OriginalAddr = EmitLValue(&DRE).getAddress(); + Address OriginalAddr = EmitLValue(&DRE).getAddress(); QualType Type = OrigVD->getType(); if (Type->isArrayType()) { // Emit VarDecl with copy init for arrays. // Get the address of the original variable captured in current // captured region. - IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { auto Emission = EmitAutoVarAlloca(*VD); auto *Init = VD->getInit(); if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) { @@ -147,12 +388,12 @@ bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, } else { EmitOMPAggregateAssign( Emission.getAllocatedAddress(), OriginalAddr, Type, - [this, VDInit, Init](llvm::Value *DestElement, - llvm::Value *SrcElement) { + [this, VDInit, Init](Address DestElement, + Address SrcElement) { // Clean up any temporaries needed by the initialization. RunCleanupsScope InitScope(*this); // Emit initialization for single element. - LocalDeclMap[VDInit] = SrcElement; + setAddrOfLocalVar(VDInit, SrcElement); EmitAnyExprToMem(Init, DestElement, Init->getType().getQualifiers(), /*IsInitializer*/ false); @@ -163,12 +404,12 @@ bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, return Emission.getAllocatedAddress(); }); } else { - IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { // Emit private VarDecl with copy init. // Remap temp VDInit variable to the address of the original // variable // (for proper handling of captured global variables). - LocalDeclMap[VDInit] = OriginalAddr; + setAddrOfLocalVar(VDInit, OriginalAddr); EmitDecl(*VD); LocalDeclMap.erase(VDInit); return GetAddrOfLocalVar(VD); @@ -188,16 +429,17 @@ bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, void CodeGenFunction::EmitOMPPrivateClause( const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { + if (!HaveInsertPoint()) + return; llvm::DenseSet<const VarDecl *> EmittedAsPrivate; - for (auto &&I = D.getClausesOfKind(OMPC_private); I; ++I) { - auto *C = cast<OMPPrivateClause>(*I); + for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) { auto IRef = C->varlist_begin(); for (auto IInit : C->private_copies()) { auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); bool IsRegistered = - PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + PrivateScope.addPrivate(OrigVD, [&]() -> Address { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); @@ -212,14 +454,15 @@ void CodeGenFunction::EmitOMPPrivateClause( } bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { + if (!HaveInsertPoint()) + return false; // threadprivate_var1 = master_threadprivate_var1; // operator=(threadprivate_var2, master_threadprivate_var2); // ... // __kmpc_barrier(&loc, global_tid); llvm::DenseSet<const VarDecl *> CopiedVars; llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr; - for (auto &&I = D.getClausesOfKind(OMPC_copyin); I; ++I) { - auto *C = cast<OMPCopyinClause>(*I); + for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) { auto IRef = C->varlist_begin(); auto ISrcRef = C->source_exprs().begin(); auto IDestRef = C->destination_exprs().begin(); @@ -231,7 +474,7 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { // Get the address of the master variable. If we are emitting code with // TLS support, the address is passed from the master as field in the // captured declaration. - llvm::Value *MasterAddr; + Address MasterAddr = Address::invalid(); if (getLangOpts().OpenMPUseTLS && getContext().getTargetInfo().isTLSSupported()) { assert(CapturedStmtInfo->lookup(VD) && @@ -239,12 +482,15 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); MasterAddr = EmitLValue(&DRE).getAddress(); + LocalDeclMap.erase(VD); } else { - MasterAddr = VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD) - : CGM.GetAddrOfGlobal(VD); + MasterAddr = + Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD) + : CGM.GetAddrOfGlobal(VD), + getContext().getDeclAlign(VD)); } // Get the address of the threadprivate variable. - auto *PrivateAddr = EmitLValue(*IRef).getAddress(); + Address PrivateAddr = EmitLValue(*IRef).getAddress(); if (CopiedVars.size() == 1) { // At first check if current thread is a master thread. If it is, no // need to copy data. @@ -252,15 +498,14 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { CopyEnd = createBasicBlock("copyin.not.master.end"); Builder.CreateCondBr( Builder.CreateICmpNE( - Builder.CreatePtrToInt(MasterAddr, CGM.IntPtrTy), - Builder.CreatePtrToInt(PrivateAddr, CGM.IntPtrTy)), + Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy), + Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)), CopyBegin, CopyEnd); EmitBlock(CopyBegin); } auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); - EmitOMPCopy(*this, Type, PrivateAddr, MasterAddr, DestVD, SrcVD, - AssignOp); + EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp); } ++IRef; ++ISrcRef; @@ -277,11 +522,12 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { bool CodeGenFunction::EmitOMPLastprivateClauseInit( const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) { + if (!HaveInsertPoint()) + return false; bool HasAtLeastOneLastprivate = false; llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; - for (auto &&I = D.getClausesOfKind(OMPC_lastprivate); I; ++I) { + for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) { HasAtLeastOneLastprivate = true; - auto *C = cast<OMPLastprivateClause>(*I); auto IRef = C->varlist_begin(); auto IDestRef = C->destination_exprs().begin(); for (auto *IInit : C->private_copies()) { @@ -290,7 +536,7 @@ bool CodeGenFunction::EmitOMPLastprivateClauseInit( auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) { auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); - PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> llvm::Value *{ + PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> Address { DeclRefExpr DRE( const_cast<VarDecl *>(OrigVD), /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup( @@ -304,7 +550,7 @@ bool CodeGenFunction::EmitOMPLastprivateClauseInit( if (IInit) { auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); bool IsRegistered = - PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + PrivateScope.addPrivate(OrigVD, [&]() -> Address { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); @@ -322,6 +568,8 @@ bool CodeGenFunction::EmitOMPLastprivateClauseInit( void CodeGenFunction::EmitOMPLastprivateClauseFinal( const OMPExecutableDirective &D, llvm::Value *IsLastIterCond) { + if (!HaveInsertPoint()) + return; // Emit following code: // if (<IsLastIterCond>) { // orig_var1 = private_orig_var1; @@ -359,8 +607,7 @@ void CodeGenFunction::EmitOMPLastprivateClauseFinal( { llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; bool FirstLCV = true; - for (auto &&I = D.getClausesOfKind(OMPC_lastprivate); I; ++I) { - auto *C = cast<OMPLastprivateClause>(*I); + for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) { auto IRef = C->varlist_begin(); auto ISrcRef = C->source_exprs().begin(); auto IDestRef = C->destination_exprs().begin(); @@ -385,11 +632,14 @@ void CodeGenFunction::EmitOMPLastprivateClauseFinal( auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); // Get the address of the original variable. - auto *OriginalAddr = GetAddrOfLocalVar(DestVD); + Address OriginalAddr = GetAddrOfLocalVar(DestVD); // Get the address of the private variable. - auto *PrivateAddr = GetAddrOfLocalVar(PrivateVD); - EmitOMPCopy(*this, Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, - AssignOp); + Address PrivateAddr = GetAddrOfLocalVar(PrivateVD); + if (auto RefTy = PrivateVD->getType()->getAs<ReferenceType>()) + PrivateAddr = + Address(Builder.CreateLoad(PrivateAddr), + getNaturalTypeAlignment(RefTy->getPointeeType())); + EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp); } ++IRef; ++ISrcRef; @@ -405,46 +655,174 @@ void CodeGenFunction::EmitOMPLastprivateClauseFinal( void CodeGenFunction::EmitOMPReductionClauseInit( const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { - for (auto &&I = D.getClausesOfKind(OMPC_reduction); I; ++I) { - auto *C = cast<OMPReductionClause>(*I); + if (!HaveInsertPoint()) + return; + for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) { auto ILHS = C->lhs_exprs().begin(); auto IRHS = C->rhs_exprs().begin(); + auto IPriv = C->privates().begin(); for (auto IRef : C->varlists()) { - auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl()); auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); - auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); - // Store the address of the original variable associated with the LHS - // implicit variable. - PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef]() -> llvm::Value *{ - DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), - CapturedStmtInfo->lookup(OrigVD) != nullptr, - IRef->getType(), VK_LValue, IRef->getExprLoc()); - return EmitLValue(&DRE).getAddress(); - }); - // Emit reduction copy. - bool IsRegistered = - PrivateScope.addPrivate(OrigVD, [this, PrivateVD]() -> llvm::Value *{ - // Emit private VarDecl with reduction init. - EmitDecl(*PrivateVD); - return GetAddrOfLocalVar(PrivateVD); - }); - assert(IsRegistered && "private var already registered as private"); - // Silence the warning about unused variable. - (void)IsRegistered; - ++ILHS, ++IRHS; + auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl()); + if (auto *OASE = dyn_cast<OMPArraySectionExpr>(IRef)) { + auto *Base = OASE->getBase()->IgnoreParenImpCasts(); + while (auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base)) + Base = TempOASE->getBase()->IgnoreParenImpCasts(); + while (auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base)) + Base = TempASE->getBase()->IgnoreParenImpCasts(); + auto *DE = cast<DeclRefExpr>(Base); + auto *OrigVD = cast<VarDecl>(DE->getDecl()); + auto OASELValueLB = EmitOMPArraySectionExpr(OASE); + auto OASELValueUB = + EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false); + auto OriginalBaseLValue = EmitLValue(DE); + auto BaseLValue = OriginalBaseLValue; + auto *Zero = Builder.getInt64(/*C=*/0); + llvm::SmallVector<llvm::Value *, 4> Indexes; + Indexes.push_back(Zero); + auto *ItemTy = + OASELValueLB.getPointer()->getType()->getPointerElementType(); + auto *Ty = BaseLValue.getPointer()->getType()->getPointerElementType(); + while (Ty != ItemTy) { + Indexes.push_back(Zero); + Ty = Ty->getPointerElementType(); + } + BaseLValue = MakeAddrLValue( + Address(Builder.CreateInBoundsGEP(BaseLValue.getPointer(), Indexes), + OASELValueLB.getAlignment()), + OASELValueLB.getType(), OASELValueLB.getAlignmentSource()); + // Store the address of the original variable associated with the LHS + // implicit variable. + PrivateScope.addPrivate(LHSVD, [this, OASELValueLB]() -> Address { + return OASELValueLB.getAddress(); + }); + // Emit reduction copy. + bool IsRegistered = PrivateScope.addPrivate( + OrigVD, [this, PrivateVD, BaseLValue, OASELValueLB, OASELValueUB, + OriginalBaseLValue]() -> Address { + // Emit VarDecl with copy init for arrays. + // Get the address of the original variable captured in current + // captured region. + auto *Size = Builder.CreatePtrDiff(OASELValueUB.getPointer(), + OASELValueLB.getPointer()); + Size = Builder.CreateNUWAdd( + Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1)); + CodeGenFunction::OpaqueValueMapping OpaqueMap( + *this, cast<OpaqueValueExpr>( + getContext() + .getAsVariableArrayType(PrivateVD->getType()) + ->getSizeExpr()), + RValue::get(Size)); + EmitVariablyModifiedType(PrivateVD->getType()); + auto Emission = EmitAutoVarAlloca(*PrivateVD); + auto Addr = Emission.getAllocatedAddress(); + auto *Init = PrivateVD->getInit(); + EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(), Init); + EmitAutoVarCleanups(Emission); + // Emit private VarDecl with reduction init. + auto *Offset = Builder.CreatePtrDiff(BaseLValue.getPointer(), + OASELValueLB.getPointer()); + auto *Ptr = Builder.CreateGEP(Addr.getPointer(), Offset); + Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast( + Ptr, OriginalBaseLValue.getPointer()->getType()); + return Address(Ptr, OriginalBaseLValue.getAlignment()); + }); + assert(IsRegistered && "private var already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address { + return GetAddrOfLocalVar(PrivateVD); + }); + } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(IRef)) { + auto *Base = ASE->getBase()->IgnoreParenImpCasts(); + while (auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base)) + Base = TempASE->getBase()->IgnoreParenImpCasts(); + auto *DE = cast<DeclRefExpr>(Base); + auto *OrigVD = cast<VarDecl>(DE->getDecl()); + auto ASELValue = EmitLValue(ASE); + auto OriginalBaseLValue = EmitLValue(DE); + auto BaseLValue = OriginalBaseLValue; + auto *Zero = Builder.getInt64(/*C=*/0); + llvm::SmallVector<llvm::Value *, 4> Indexes; + Indexes.push_back(Zero); + auto *ItemTy = + ASELValue.getPointer()->getType()->getPointerElementType(); + auto *Ty = BaseLValue.getPointer()->getType()->getPointerElementType(); + while (Ty != ItemTy) { + Indexes.push_back(Zero); + Ty = Ty->getPointerElementType(); + } + BaseLValue = MakeAddrLValue( + Address(Builder.CreateInBoundsGEP(BaseLValue.getPointer(), Indexes), + ASELValue.getAlignment()), + ASELValue.getType(), ASELValue.getAlignmentSource()); + // Store the address of the original variable associated with the LHS + // implicit variable. + PrivateScope.addPrivate(LHSVD, [this, ASELValue]() -> Address { + return ASELValue.getAddress(); + }); + // Emit reduction copy. + bool IsRegistered = PrivateScope.addPrivate( + OrigVD, [this, PrivateVD, BaseLValue, ASELValue, + OriginalBaseLValue]() -> Address { + // Emit private VarDecl with reduction init. + EmitDecl(*PrivateVD); + auto Addr = GetAddrOfLocalVar(PrivateVD); + auto *Offset = Builder.CreatePtrDiff(BaseLValue.getPointer(), + ASELValue.getPointer()); + auto *Ptr = Builder.CreateGEP(Addr.getPointer(), Offset); + Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast( + Ptr, OriginalBaseLValue.getPointer()->getType()); + return Address(Ptr, OriginalBaseLValue.getAlignment()); + }); + assert(IsRegistered && "private var already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address { + return GetAddrOfLocalVar(PrivateVD); + }); + } else { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl()); + // Store the address of the original variable associated with the LHS + // implicit variable. + PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef]() -> Address { + DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), + CapturedStmtInfo->lookup(OrigVD) != nullptr, + IRef->getType(), VK_LValue, IRef->getExprLoc()); + return EmitLValue(&DRE).getAddress(); + }); + // Emit reduction copy. + bool IsRegistered = + PrivateScope.addPrivate(OrigVD, [this, PrivateVD]() -> Address { + // Emit private VarDecl with reduction init. + EmitDecl(*PrivateVD); + return GetAddrOfLocalVar(PrivateVD); + }); + assert(IsRegistered && "private var already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address { + return GetAddrOfLocalVar(PrivateVD); + }); + } + ++ILHS, ++IRHS, ++IPriv; } } } void CodeGenFunction::EmitOMPReductionClauseFinal( const OMPExecutableDirective &D) { + if (!HaveInsertPoint()) + return; + llvm::SmallVector<const Expr *, 8> Privates; llvm::SmallVector<const Expr *, 8> LHSExprs; llvm::SmallVector<const Expr *, 8> RHSExprs; llvm::SmallVector<const Expr *, 8> ReductionOps; bool HasAtLeastOneReduction = false; - for (auto &&I = D.getClausesOfKind(OMPC_reduction); I; ++I) { + for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) { HasAtLeastOneReduction = true; - auto *C = cast<OMPReductionClause>(*I); + Privates.append(C->privates().begin(), C->privates().end()); LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end()); RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end()); ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end()); @@ -453,8 +831,8 @@ void CodeGenFunction::EmitOMPReductionClauseFinal( // Emit nowait reduction if nowait clause is present or directive is a // parallel directive (it always has implicit barrier). CGM.getOpenMPRuntime().emitReduction( - *this, D.getLocEnd(), LHSExprs, RHSExprs, ReductionOps, - D.getSingleClause(OMPC_nowait) || + *this, D.getLocEnd(), Privates, LHSExprs, RHSExprs, ReductionOps, + D.getSingleClause<OMPNowaitClause>() || isOpenMPParallelDirective(D.getDirectiveKind()) || D.getDirectiveKind() == OMPD_simd, D.getDirectiveKind() == OMPD_simd); @@ -466,29 +844,32 @@ static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); - auto CapturedStruct = CGF.GenerateCapturedStmtArgument(*CS); + llvm::SmallVector<llvm::Value *, 16> CapturedVars; + CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); - if (auto C = S.getSingleClause(OMPC_num_threads)) { + if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) { CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); - auto NumThreadsClause = cast<OMPNumThreadsClause>(C); auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), /*IgnoreResultAssign*/ true); CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( CGF, NumThreads, NumThreadsClause->getLocStart()); } - if (auto *C = S.getSingleClause(OMPC_proc_bind)) { + if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) { CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); - auto *ProcBindClause = cast<OMPProcBindClause>(C); CGF.CGM.getOpenMPRuntime().emitProcBindClause( CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart()); } const Expr *IfCond = nullptr; - if (auto C = S.getSingleClause(OMPC_if)) { - IfCond = cast<OMPIfClause>(C)->getCondition(); + for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { + if (C->getNameModifier() == OMPD_unknown || + C->getNameModifier() == OMPD_parallel) { + IfCond = C->getCondition(); + break; + } } CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn, - CapturedStruct, IfCond); + CapturedVars, IfCond); } void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { @@ -503,17 +884,15 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { // initialization of firstprivate variables or propagation master's thread // values of threadprivate variables to local instances of that variables // of all other implicit threads. - CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), - OMPD_unknown); + CGF.CGM.getOpenMPRuntime().emitBarrierCall( + CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, + /*ForceSimpleCall=*/true); } CGF.EmitOMPPrivateClause(S, PrivateScope); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); CGF.EmitOMPReductionClauseFinal(S); - // Emit implicit barrier at the end of the 'parallel' directive. - CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), - OMPD_unknown); }; emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen); } @@ -526,8 +905,7 @@ void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D, EmitIgnoredExpr(I); } // Update the linear variables. - for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { - auto *C = cast<OMPLinearClause>(*I); + for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { for (auto U : C->updates()) { EmitIgnoredExpr(U); } @@ -595,9 +973,10 @@ void CodeGenFunction::EmitOMPInnerLoop( } void CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) { + if (!HaveInsertPoint()) + return; // Emit inits for the linear variables. - for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { - auto *C = cast<OMPLinearClause>(*I); + for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { for (auto Init : C->inits()) { auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl()); auto *OrigVD = cast<VarDecl>( @@ -608,8 +987,7 @@ void CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) { VD->getInit()->getExprLoc()); AutoVarEmission Emission = EmitAutoVarAlloca(*VD); EmitExprAsInit(&DRE, VD, - MakeAddrLValue(Emission.getAllocatedAddress(), - VD->getType(), Emission.Alignment), + MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()), /*capturedByInit=*/false); EmitAutoVarCleanups(Emission); } @@ -626,19 +1004,20 @@ void CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) { static void emitLinearClauseFinal(CodeGenFunction &CGF, const OMPLoopDirective &D) { + if (!CGF.HaveInsertPoint()) + return; // Emit the final values of the linear variables. - for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { - auto *C = cast<OMPLinearClause>(*I); + for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { auto IC = C->varlist_begin(); for (auto F : C->finals()) { auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl()); DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); - auto *OrigAddr = CGF.EmitLValue(&DRE).getAddress(); + Address OrigAddr = CGF.EmitLValue(&DRE).getAddress(); CodeGenFunction::OMPPrivateScope VarScope(CGF); VarScope.addPrivate(OrigVD, - [OrigAddr]() -> llvm::Value *{ return OrigAddr; }); + [OrigAddr]() -> Address { return OrigAddr; }); (void)VarScope.Privatize(); CGF.EmitIgnoredExpr(F); ++IC; @@ -648,8 +1027,9 @@ static void emitLinearClauseFinal(CodeGenFunction &CGF, static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D) { - for (auto &&I = D.getClausesOfKind(OMPC_aligned); I; ++I) { - auto *Clause = cast<OMPAlignedClause>(*I); + if (!CGF.HaveInsertPoint()) + return; + for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) { unsigned ClauseAlignment = 0; if (auto AlignmentExpr = Clause->getAlignment()) { auto AlignmentCI = @@ -680,24 +1060,36 @@ static void emitAlignedClause(CodeGenFunction &CGF, static void emitPrivateLoopCounters(CodeGenFunction &CGF, CodeGenFunction::OMPPrivateScope &LoopScope, - ArrayRef<Expr *> Counters) { + ArrayRef<Expr *> Counters, + ArrayRef<Expr *> PrivateCounters) { + if (!CGF.HaveInsertPoint()) + return; + auto I = PrivateCounters.begin(); for (auto *E : Counters) { - auto VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - (void)LoopScope.addPrivate(VD, [&]() -> llvm::Value *{ + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()); + Address Addr = Address::invalid(); + (void)LoopScope.addPrivate(PrivateVD, [&]() -> Address { // Emit var without initialization. - auto VarEmission = CGF.EmitAutoVarAlloca(*VD); + auto VarEmission = CGF.EmitAutoVarAlloca(*PrivateVD); CGF.EmitAutoVarCleanups(VarEmission); - return VarEmission.getAllocatedAddress(); + Addr = VarEmission.getAllocatedAddress(); + return Addr; }); + (void)LoopScope.addPrivate(VD, [&]() -> Address { return Addr; }); + ++I; } } static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount) { + if (!CGF.HaveInsertPoint()) + return; { CodeGenFunction::OMPPrivateScope PreCondScope(CGF); - emitPrivateLoopCounters(CGF, PreCondScope, S.counters()); + emitPrivateLoopCounters(CGF, PreCondScope, S.counters(), + S.private_counters()); (void)PreCondScope.Privatize(); // Get initial values of real counters. for (auto I : S.inits()) { @@ -711,31 +1103,45 @@ static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, static void emitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { - for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { - auto *C = cast<OMPLinearClause>(*I); + if (!CGF.HaveInsertPoint()) + return; + for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { + auto CurPrivate = C->privates().begin(); for (auto *E : C->varlists()) { - auto VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - bool IsRegistered = PrivateScope.addPrivate(VD, [&]()->llvm::Value * { - // Emit var without initialization. - auto VarEmission = CGF.EmitAutoVarAlloca(*VD); - CGF.EmitAutoVarCleanups(VarEmission); - return VarEmission.getAllocatedAddress(); + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + auto *PrivateVD = + cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl()); + bool IsRegistered = PrivateScope.addPrivate(VD, [&]() -> Address { + // Emit private VarDecl with copy init. + CGF.EmitVarDecl(*PrivateVD); + return CGF.GetAddrOfLocalVar(PrivateVD); }); assert(IsRegistered && "linear var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; + ++CurPrivate; } } } -static void emitSafelenClause(CodeGenFunction &CGF, - const OMPExecutableDirective &D) { - if (auto *C = - cast_or_null<OMPSafelenClause>(D.getSingleClause(OMPC_safelen))) { +static void emitSimdlenSafelenClause(CodeGenFunction &CGF, + const OMPExecutableDirective &D) { + if (!CGF.HaveInsertPoint()) + return; + if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) { + RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(), + /*ignoreResult=*/true); + llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); + CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); + // In presence of finite 'safelen', it may be unsafe to mark all + // the memory instructions parallel, because loop-carried + // dependences of 'safelen' iterations are possible. + CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>()); + } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) { RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), /*ignoreResult=*/true); llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); - CGF.LoopStack.setVectorizerWidth(Val->getZExtValue()); + CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); // In presence of finite 'safelen', it may be unsafe to mark all // the memory instructions parallel, because loop-carried // dependences of 'safelen' iterations are possible. @@ -746,22 +1152,24 @@ static void emitSafelenClause(CodeGenFunction &CGF, void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D) { // Walk clauses and process safelen/lastprivate. LoopStack.setParallel(); - LoopStack.setVectorizerEnable(true); - emitSafelenClause(*this, D); + LoopStack.setVectorizeEnable(true); + emitSimdlenSafelenClause(*this, D); } void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &D) { + if (!HaveInsertPoint()) + return; auto IC = D.counters().begin(); for (auto F : D.finals()) { auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl()); - if (LocalDeclMap.lookup(OrigVD) || CapturedStmtInfo->lookup(OrigVD)) { + if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD)) { DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), CapturedStmtInfo->lookup(OrigVD) != nullptr, (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); - auto *OrigAddr = EmitLValue(&DRE).getAddress(); + Address OrigAddr = EmitLValue(&DRE).getAddress(); OMPPrivateScope VarScope(*this); VarScope.addPrivate(OrigVD, - [OrigAddr]() -> llvm::Value *{ return OrigAddr; }); + [OrigAddr]() -> Address { return OrigAddr; }); (void)VarScope.Privatize(); EmitIgnoredExpr(F); } @@ -817,7 +1225,8 @@ void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { bool HasLastprivateClause; { OMPPrivateScope LoopScope(CGF); - emitPrivateLoopCounters(CGF, LoopScope, S.counters()); + emitPrivateLoopCounters(CGF, LoopScope, S.counters(), + S.private_counters()); emitPrivateLinearVars(CGF, S, LoopScope); CGF.EmitOMPPrivateClause(S, LoopScope); CGF.EmitOMPReductionClauseInit(S, LoopScope); @@ -849,9 +1258,9 @@ void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { void CodeGenFunction::EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind, const OMPLoopDirective &S, OMPPrivateScope &LoopScope, - bool Ordered, llvm::Value *LB, - llvm::Value *UB, llvm::Value *ST, - llvm::Value *IL, llvm::Value *Chunk) { + bool Ordered, Address LB, + Address UB, Address ST, + Address IL, llvm::Value *Chunk) { auto &RT = CGM.getOpenMPRuntime(); // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime). @@ -915,11 +1324,14 @@ void CodeGenFunction::EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind, const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); - RT.emitForInit( - *this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, Ordered, IL, LB, - (DynamicOrOrdered ? EmitAnyExpr(S.getLastIteration()).getScalarVal() - : UB), - ST, Chunk); + if (DynamicOrOrdered) { + llvm::Value *UBVal = EmitScalarExpr(S.getLastIteration()); + RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind, + IVSize, IVSigned, Ordered, UBVal, Chunk); + } else { + RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind, + IVSize, IVSigned, Ordered, IL, LB, UB, ST, Chunk); + } auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end"); @@ -1019,8 +1431,7 @@ emitScheduleClause(CodeGenFunction &CGF, const OMPLoopDirective &S, // Detect the loop schedule kind and chunk. auto ScheduleKind = OMPC_SCHEDULE_unknown; llvm::Value *Chunk = nullptr; - if (auto *C = - cast_or_null<OMPScheduleClause>(S.getSingleClause(OMPC_schedule))) { + if (const auto *C = S.getSingleClause<OMPScheduleClause>()) { ScheduleKind = C->getScheduleKind(); if (const auto *Ch = C->getChunkSize()) { if (auto *ImpRef = cast_or_null<DeclRefExpr>(C->getHelperChunkSize())) { @@ -1029,8 +1440,8 @@ emitScheduleClause(CodeGenFunction &CGF, const OMPLoopDirective &S, CGF.EmitVarDecl(*ImpVar); CGF.EmitStoreThroughLValue( CGF.EmitAnyExpr(Ch), - CGF.MakeNaturalAlignAddrLValue(CGF.GetAddrOfLocalVar(ImpVar), - ImpVar->getType())); + CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(ImpVar), + ImpVar->getType())); } else { Ch = ImpRef; } @@ -1038,7 +1449,8 @@ emitScheduleClause(CodeGenFunction &CGF, const OMPLoopDirective &S, if (!C->getHelperChunkSize() || !OuterRegion) { Chunk = CGF.EmitScalarExpr(Ch); Chunk = CGF.EmitScalarConversion(Chunk, Ch->getType(), - S.getIterationVariable()->getType()); + S.getIterationVariable()->getType(), + S.getLocStart()); } } } @@ -1100,13 +1512,15 @@ bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { if (EmitOMPFirstprivateClause(S, LoopScope)) { // Emit implicit barrier to synchronize threads and avoid data races on // initialization of firstprivate variables. - CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), - OMPD_unknown); + CGM.getOpenMPRuntime().emitBarrierCall( + *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, + /*ForceSimpleCall=*/true); } EmitOMPPrivateClause(S, LoopScope); HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); EmitOMPReductionClauseInit(S, LoopScope); - emitPrivateLoopCounters(*this, LoopScope, S.counters()); + emitPrivateLoopCounters(*this, LoopScope, S.counters(), + S.private_counters()); emitPrivateLinearVars(*this, S, LoopScope); (void)LoopScope.Privatize(); @@ -1119,7 +1533,7 @@ bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { ScheduleKind = ScheduleInfo.second; const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); - const bool Ordered = S.getSingleClause(OMPC_ordered) != nullptr; + const bool Ordered = S.getSingleClause<OMPOrderedClause>() != nullptr; if (RT.isStaticNonchunked(ScheduleKind, /* Chunked */ Chunk != nullptr) && !Ordered) { @@ -1131,9 +1545,10 @@ bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { // chunks that are approximately equal in size, and at most one chunk is // distributed to each thread. Note that the size of the chunks is // unspecified in this case. - RT.emitForInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, - Ordered, IL.getAddress(), LB.getAddress(), - UB.getAddress(), ST.getAddress()); + RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind, + IVSize, IVSigned, Ordered, + IL.getAddress(), LB.getAddress(), + UB.getAddress(), ST.getAddress()); auto LoopExit = getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit")); // UB = min(UB, GlobalUB); EmitIgnoredExpr(S.getEnsureUpperBound()); @@ -1181,10 +1596,11 @@ void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) { auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) { HasLastprivates = CGF.EmitOMPWorksharingLoop(S); }; - CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen); + CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen, + S.hasCancel()); // Emit an implicit barrier at the end. - if (!S.getSingleClause(OMPC_nowait) || HasLastprivates) { + if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) { CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for); } } @@ -1198,7 +1614,7 @@ void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); // Emit an implicit barrier at the end. - if (!S.getSingleClause(OMPC_nowait) || HasLastprivates) { + if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) { CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for); } } @@ -1206,7 +1622,7 @@ void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) { static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init = nullptr) { - auto LVal = CGF.MakeNaturalAlignAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); + auto LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); if (Init) CGF.EmitScalarInit(Init, LVal); return LVal; @@ -1276,8 +1692,9 @@ CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { // Emit implicit barrier to synchronize threads and avoid data races on // initialization of firstprivate variables. - CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), - OMPD_unknown); + CGF.CGM.getOpenMPRuntime().emitBarrierCall( + CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, + /*ForceSimpleCall=*/true); } CGF.EmitOMPPrivateClause(S, LoopScope); HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); @@ -1285,7 +1702,7 @@ CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { (void)LoopScope.Privatize(); // Emit static non-chunked loop. - CGF.CGM.getOpenMPRuntime().emitForInit( + CGF.CGM.getOpenMPRuntime().emitForStaticInit( CGF, S.getLocStart(), OMPC_SCHEDULE_static, /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); @@ -1310,11 +1727,17 @@ CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { CGF.EmitLoadOfScalar(IL, S.getLocStart()))); }; - CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen); + bool HasCancel = false; + if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S)) + HasCancel = OSD->hasCancel(); + else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S)) + HasCancel = OPSD->hasCancel(); + CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen, + HasCancel); // Emit barrier for lastprivates only if 'sections' directive has 'nowait' // clause. Otherwise the barrier will be generated by the codegen for the // directive. - if (HasLastprivates && S.getSingleClause(OMPC_nowait)) { + if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) { // Emit implicit barrier to synchronize threads and avoid data races on // initialization of firstprivate variables. CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), @@ -1327,11 +1750,11 @@ CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { bool HasFirstprivates; // No need to generate reductions for sections with single section region, we // can use original shared variables for all operations. - bool HasReductions = !S.getClausesOfKind(OMPC_reduction).empty(); + bool HasReductions = S.hasClausesOfKind<OMPReductionClause>(); // No need to generate lastprivates for sections with single section region, // we can use original shared variable for all calculations with barrier at // the end of the sections. - bool HasLastprivates = !S.getClausesOfKind(OMPC_lastprivate).empty(); + bool HasLastprivates = S.hasClausesOfKind<OMPLastprivateClause>(); auto &&CodeGen = [Stmt, &S, &HasFirstprivates](CodeGenFunction &CGF) { CodeGenFunction::OMPPrivateScope SingleScope(CGF); HasFirstprivates = CGF.EmitOMPFirstprivateClause(S, SingleScope); @@ -1347,10 +1770,12 @@ CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { // 'sections' directive has 'nowait' clause. Otherwise the barrier will be // generated by the codegen for the directive. if ((HasFirstprivates || HasLastprivates || HasReductions) && - S.getSingleClause(OMPC_nowait)) { + S.getSingleClause<OMPNowaitClause>()) { // Emit implicit barrier to synchronize threads and avoid data races on // initialization of firstprivate variables. - CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_unknown); + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_unknown, + /*EmitChecks=*/false, + /*ForceSimpleCall=*/true); } return OMPD_single; } @@ -1359,7 +1784,7 @@ void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) { LexicalScope Scope(*this, S.getSourceRange()); OpenMPDirectiveKind EmittedAs = EmitSections(S); // Emit an implicit barrier at the end. - if (!S.getSingleClause(OMPC_nowait)) { + if (!S.getSingleClause<OMPNowaitClause>()) { CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), EmittedAs); } } @@ -1368,9 +1793,9 @@ void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) { LexicalScope Scope(*this, S.getSourceRange()); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); - CGF.EnsureInsertPoint(); }; - CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen); + CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen, + S.hasCancel()); } void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) { @@ -1383,8 +1808,7 @@ void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) { // construct. // Build a list of copyprivate variables along with helper expressions // (<source>, <destination>, <destination>=<source> expressions) - for (auto &&I = S.getClausesOfKind(OMPC_copyprivate); I; ++I) { - auto *C = cast<OMPCopyprivateClause>(*I); + for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) { CopyprivateVars.append(C->varlists().begin(), C->varlists().end()); DestExprs.append(C->destination_exprs().begin(), C->destination_exprs().end()); @@ -1402,18 +1826,17 @@ void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) { (void)SingleScope.Privatize(); CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); - CGF.EnsureInsertPoint(); }; CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(), CopyprivateVars, DestExprs, SrcExprs, AssignmentOps); // Emit an implicit barrier at the end (to avoid data race on firstprivate // init or if no 'nowait' clause was specified and no 'copyprivate' clause). - if ((!S.getSingleClause(OMPC_nowait) || HasFirstprivates) && + if ((!S.getSingleClause<OMPNowaitClause>() || HasFirstprivates) && CopyprivateVars.empty()) { CGM.getOpenMPRuntime().emitBarrierCall( *this, S.getLocStart(), - S.getSingleClause(OMPC_nowait) ? OMPD_unknown : OMPD_single); + S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single); } } @@ -1421,7 +1844,6 @@ void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) { LexicalScope Scope(*this, S.getSourceRange()); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); - CGF.EnsureInsertPoint(); }; CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart()); } @@ -1430,10 +1852,13 @@ void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) { LexicalScope Scope(*this, S.getSourceRange()); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); - CGF.EnsureInsertPoint(); }; - CGM.getOpenMPRuntime().emitCriticalRegion( - *this, S.getDirectiveName().getAsString(), CodeGen, S.getLocStart()); + Expr *Hint = nullptr; + if (auto *HintClause = S.getSingleClause<OMPHintClause>()) + Hint = HintClause->getHint(); + CGM.getOpenMPRuntime().emitCriticalRegion(*this, + S.getDirectiveName().getAsString(), + CodeGen, S.getLocStart(), Hint); } void CodeGenFunction::EmitOMPParallelForDirective( @@ -1444,11 +1869,6 @@ void CodeGenFunction::EmitOMPParallelForDirective( (void)emitScheduleClause(*this, S, /*OuterRegion=*/true); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitOMPWorksharingLoop(S); - // Emit implicit barrier at the end of parallel region, but this barrier - // is at the end of 'for' directive, so emit it as the implicit barrier for - // this 'for' directive. - CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), - OMPD_parallel); }; emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen); } @@ -1461,11 +1881,6 @@ void CodeGenFunction::EmitOMPParallelForSimdDirective( (void)emitScheduleClause(*this, S, /*OuterRegion=*/true); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitOMPWorksharingLoop(S); - // Emit implicit barrier at the end of parallel region, but this barrier - // is at the end of 'for' directive, so emit it as the implicit barrier for - // this 'for' directive. - CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), - OMPD_parallel); }; emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen); } @@ -1477,9 +1892,6 @@ void CodeGenFunction::EmitOMPParallelSectionsDirective( LexicalScope Scope(*this, S.getSourceRange()); auto &&CodeGen = [&S](CodeGenFunction &CGF) { (void)CGF.EmitSections(S); - // Emit implicit barrier at the end of parallel region. - CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), - OMPD_parallel); }; emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen); } @@ -1497,8 +1909,7 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { // Get list of private variables. llvm::SmallVector<const Expr *, 8> PrivateVars; llvm::SmallVector<const Expr *, 8> PrivateCopies; - for (auto &&I = S.getClausesOfKind(OMPC_private); I; ++I) { - auto *C = cast<OMPPrivateClause>(*I); + for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) { auto IRef = C->varlist_begin(); for (auto *IInit : C->private_copies()) { auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); @@ -1514,8 +1925,7 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { llvm::SmallVector<const Expr *, 8> FirstprivateVars; llvm::SmallVector<const Expr *, 8> FirstprivateCopies; llvm::SmallVector<const Expr *, 8> FirstprivateInits; - for (auto &&I = S.getClausesOfKind(OMPC_firstprivate); I; ++I) { - auto *C = cast<OMPFirstprivateClause>(*I); + for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) { auto IRef = C->varlist_begin(); auto IElemInitRef = C->inits().begin(); for (auto *IInit : C->private_copies()) { @@ -1531,8 +1941,7 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { // Build list of dependences. llvm::SmallVector<std::pair<OpenMPDependClauseKind, const Expr *>, 8> Dependences; - for (auto &&I = S.getClausesOfKind(OMPC_depend); I; ++I) { - auto *C = cast<OMPDependClause>(*I); + for (const auto *C : S.getClausesOfKind<OMPDependClause>()) { for (auto *IRef : C->varlists()) { Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef)); } @@ -1543,35 +1952,33 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { auto *CS = cast<CapturedStmt>(S.getAssociatedStmt()); OMPPrivateScope Scope(CGF); if (!PrivateVars.empty() || !FirstprivateVars.empty()) { - auto *CopyFn = CGF.Builder.CreateAlignedLoad( - CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)), - CGF.PointerAlignInBytes); - auto *PrivatesPtr = CGF.Builder.CreateAlignedLoad( - CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)), - CGF.PointerAlignInBytes); + auto *CopyFn = CGF.Builder.CreateLoad( + CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3))); + auto *PrivatesPtr = CGF.Builder.CreateLoad( + CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2))); // Map privates. - llvm::SmallVector<std::pair<const VarDecl *, llvm::Value *>, 16> + llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs; llvm::SmallVector<llvm::Value *, 16> CallArgs; CallArgs.push_back(PrivatesPtr); for (auto *E : PrivateVars) { auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - auto *PrivatePtr = + Address PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType())); PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr)); - CallArgs.push_back(PrivatePtr); + CallArgs.push_back(PrivatePtr.getPointer()); } for (auto *E : FirstprivateVars) { auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - auto *PrivatePtr = + Address PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType())); PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr)); - CallArgs.push_back(PrivatePtr); + CallArgs.push_back(PrivatePtr.getPointer()); } CGF.EmitRuntimeCall(CopyFn, CallArgs); for (auto &&Pair : PrivatePtrs) { - auto *Replacement = - CGF.Builder.CreateAlignedLoad(Pair.second, CGF.PointerAlignInBytes); + Address Replacement(CGF.Builder.CreateLoad(Pair.second), + CGF.getContext().getDeclAlign(Pair.first)); Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; }); } } @@ -1584,13 +1991,13 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { auto OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( S, *I, OMPD_task, CodeGen); // Check if we should emit tied or untied task. - bool Tied = !S.getSingleClause(OMPC_untied); + bool Tied = !S.getSingleClause<OMPUntiedClause>(); // Check if the task is final llvm::PointerIntPair<llvm::Value *, 1, bool> Final; - if (auto *Clause = S.getSingleClause(OMPC_final)) { + if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) { // If the condition constant folds and can be elided, try to avoid emitting // the condition and the dead arm of the if/else. - auto *Cond = cast<OMPFinalClause>(Clause)->getCondition(); + auto *Cond = Clause->getCondition(); bool CondConstant; if (ConstantFoldsToSimpleInteger(Cond, CondConstant)) Final.setInt(CondConstant); @@ -1602,8 +2009,12 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { } auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); const Expr *IfCond = nullptr; - if (auto C = S.getSingleClause(OMPC_if)) { - IfCond = cast<OMPIfClause>(C)->getCondition(); + for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { + if (C->getNameModifier() == OMPD_unknown || + C->getNameModifier() == OMPD_task) { + IfCond = C->getCondition(); + break; + } } CGM.getOpenMPRuntime().emitTaskCall( *this, S.getLocStart(), S, Tied, Final, OutlinedFn, SharedsTy, @@ -1629,15 +2040,13 @@ void CodeGenFunction::EmitOMPTaskgroupDirective( LexicalScope Scope(*this, S.getSourceRange()); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); - CGF.EnsureInsertPoint(); }; CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart()); } void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> { - if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) { - auto FlushClause = cast<OMPFlushClause>(C); + if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>()) { return llvm::makeArrayRef(FlushClause->varlist_begin(), FlushClause->varlist_end()); } @@ -1645,37 +2054,65 @@ void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { }(), S.getLocStart()); } +void CodeGenFunction::EmitOMPDistributeDirective( + const OMPDistributeDirective &S) { + llvm_unreachable("CodeGen for 'omp distribute' is not supported yet."); +} + +static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM, + const CapturedStmt *S) { + CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); + CodeGenFunction::CGCapturedStmtInfo CapStmtInfo; + CGF.CapturedStmtInfo = &CapStmtInfo; + auto *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S); + Fn->addFnAttr(llvm::Attribute::NoInline); + return Fn; +} + void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) { + if (!S.getAssociatedStmt()) + return; LexicalScope Scope(*this, S.getSourceRange()); - auto &&CodeGen = [&S](CodeGenFunction &CGF) { - CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); - CGF.EnsureInsertPoint(); + auto *C = S.getSingleClause<OMPSIMDClause>(); + auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF) { + if (C) { + auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); + llvm::SmallVector<llvm::Value *, 16> CapturedVars; + CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); + auto *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS); + CGF.EmitNounwindRuntimeCall(OutlinedFn, CapturedVars); + } else { + CGF.EmitStmt( + cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + } }; - CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart()); + CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart(), !C); } static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, - QualType SrcType, QualType DestType) { + QualType SrcType, QualType DestType, + SourceLocation Loc) { assert(CGF.hasScalarEvaluationKind(DestType) && "DestType must have scalar evaluation kind."); assert(!Val.isAggregate() && "Must be a scalar or complex."); return Val.isScalar() - ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType) + ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType, + Loc) : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType, - DestType); + DestType, Loc); } static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, - QualType DestType) { + QualType DestType, SourceLocation Loc) { assert(CGF.getEvaluationKind(DestType) == TEK_Complex && "DestType must have complex evaluation kind."); CodeGenFunction::ComplexPairTy ComplexVal; if (Val.isScalar()) { // Convert the input element to the element type of the complex. auto DestElementType = DestType->castAs<ComplexType>()->getElementType(); - auto ScalarVal = - CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestElementType); + auto ScalarVal = CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, + DestElementType, Loc); ComplexVal = CodeGenFunction::ComplexPairTy( ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType())); } else { @@ -1683,9 +2120,9 @@ convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType(); auto DestElementType = DestType->castAs<ComplexType>()->getElementType(); ComplexVal.first = CGF.EmitScalarConversion( - Val.getComplexVal().first, SrcElementType, DestElementType); + Val.getComplexVal().first, SrcElementType, DestElementType, Loc); ComplexVal.second = CGF.EmitScalarConversion( - Val.getComplexVal().second, SrcElementType, DestElementType); + Val.getComplexVal().second, SrcElementType, DestElementType, Loc); } return ComplexVal; } @@ -1702,16 +2139,16 @@ static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, } static void emitSimpleStore(CodeGenFunction &CGF, LValue LVal, RValue RVal, - QualType RValTy) { + QualType RValTy, SourceLocation Loc) { switch (CGF.getEvaluationKind(LVal.getType())) { case TEK_Scalar: - CGF.EmitStoreThroughLValue( - RValue::get(convertToScalarValue(CGF, RVal, RValTy, LVal.getType())), - LVal); + CGF.EmitStoreThroughLValue(RValue::get(convertToScalarValue( + CGF, RVal, RValTy, LVal.getType(), Loc)), + LVal); break; case TEK_Complex: CGF.EmitStoreOfComplex( - convertToComplexValue(CGF, RVal, RValTy, LVal.getType()), LVal, + convertToComplexValue(CGF, RVal, RValTy, LVal.getType(), Loc), LVal, /*isInit=*/false); break; case TEK_Aggregate: @@ -1739,7 +2176,7 @@ static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, // list. if (IsSeqCst) CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); - emitSimpleStore(CGF,VLValue, Res, X->getType().getNonReferenceType()); + emitSimpleStore(CGF, VLValue, Res, X->getType().getNonReferenceType(), Loc); } static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, @@ -1769,8 +2206,8 @@ static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, !Update.getScalarVal()->getType()->isIntegerTy() || !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) && (Update.getScalarVal()->getType() != - X.getAddress()->getType()->getPointerElementType())) || - !X.getAddress()->getType()->getPointerElementType()->isIntegerTy() || + X.getAddress().getElementType())) || + !X.getAddress().getElementType()->isIntegerTy() || !Context.getTargetInfo().hasBuiltinAtomic( Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment()))) return std::make_pair(false, RValue::get(nullptr)); @@ -1841,10 +2278,10 @@ static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, auto *UpdateVal = Update.getScalarVal(); if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) { UpdateVal = CGF.Builder.CreateIntCast( - IC, X.getAddress()->getType()->getPointerElementType(), + IC, X.getAddress().getElementType(), X.getType()->hasSignedIntegerRepresentation()); } - auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getAddress(), UpdateVal, AO); + auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO); return std::make_pair(true, RValue::get(Res)); } @@ -1910,12 +2347,14 @@ static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, } static RValue convertToType(CodeGenFunction &CGF, RValue Value, - QualType SourceType, QualType ResType) { + QualType SourceType, QualType ResType, + SourceLocation Loc) { switch (CGF.getEvaluationKind(ResType)) { case TEK_Scalar: - return RValue::get(convertToScalarValue(CGF, Value, SourceType, ResType)); + return RValue::get( + convertToScalarValue(CGF, Value, SourceType, ResType, Loc)); case TEK_Complex: { - auto Res = convertToComplexValue(CGF, Value, SourceType, ResType); + auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc); return RValue::getComplex(Res.first, Res.second); } case TEK_Aggregate: @@ -1980,7 +2419,7 @@ static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, // 'x' is simply rewritten with some 'expr'. NewVValType = X->getType().getNonReferenceType(); ExprRValue = convertToType(CGF, ExprRValue, E->getType(), - X->getType().getNonReferenceType()); + X->getType().getNonReferenceType(), Loc); auto &&Gen = [&CGF, &NewVVal, ExprRValue](RValue XRValue) -> RValue { NewVVal = XRValue; return ExprRValue; @@ -1995,7 +2434,7 @@ static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, } } // Emit post-update store to 'v' of old/new 'x' value. - emitSimpleStore(CGF, VLValue, NewVVal, NewVValType); + emitSimpleStore(CGF, VLValue, NewVVal, NewVValType, Loc); // OpenMP, 2.12.6, atomic Construct // Any atomic construct with a seq_cst clause forces the atomically // performed operation to include an implicit flush operation without a @@ -2032,6 +2471,7 @@ static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, case OMPC_lastprivate: case OMPC_reduction: case OMPC_safelen: + case OMPC_simdlen: case OMPC_collapse: case OMPC_default: case OMPC_seq_cst: @@ -2049,12 +2489,23 @@ static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, case OMPC_threadprivate: case OMPC_depend: case OMPC_mergeable: + case OMPC_device: + case OMPC_threads: + case OMPC_simd: + case OMPC_map: + case OMPC_num_teams: + case OMPC_thread_limit: + case OMPC_priority: + case OMPC_grainsize: + case OMPC_nogroup: + case OMPC_num_tasks: + case OMPC_hint: llvm_unreachable("Clause is not allowed in 'omp atomic'."); } } void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) { - bool IsSeqCst = S.getSingleClause(/*K=*/OMPC_seq_cst); + bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>(); OpenMPClauseKind Kind = OMPC_unknown; for (auto *C : S.clauses()) { // Find first clause (skip seq_cst clause, if it is first). @@ -2079,7 +2530,8 @@ void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) { } LexicalScope Scope(*this, S.getSourceRange()); - auto &&CodeGen = [&S, Kind, IsSeqCst](CodeGenFunction &CGF) { + auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF) { + CGF.EmitStopPoint(CS); EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(), S.getV(), S.getExpr(), S.getUpdateExpr(), S.isXLHSInRHSPart(), S.getLocStart()); @@ -2087,8 +2539,37 @@ void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen); } -void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) { - llvm_unreachable("CodeGen for 'omp target' is not supported yet."); +void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) { + LexicalScope Scope(*this, S.getSourceRange()); + const CapturedStmt &CS = *cast<CapturedStmt>(S.getAssociatedStmt()); + + llvm::SmallVector<llvm::Value *, 16> CapturedVars; + GenerateOpenMPCapturedVars(CS, CapturedVars); + + // Emit target region as a standalone region. + auto &&CodeGen = [&CS](CodeGenFunction &CGF) { + CGF.EmitStmt(CS.getCapturedStmt()); + }; + + // Obtain the target region outlined function. + llvm::Value *Fn = + CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, CodeGen); + + // Check if we have any if clause associated with the directive. + const Expr *IfCond = nullptr; + + if (auto *C = S.getSingleClause<OMPIfClause>()) { + IfCond = C->getCondition(); + } + + // Check if we have any device clause associated with the directive. + const Expr *Device = nullptr; + if (auto *C = S.getSingleClause<OMPDeviceClause>()) { + Device = C->getDevice(); + } + + CGM.getOpenMPRuntime().emitTargetCall(*this, S, Fn, IfCond, Device, + CapturedVars); } void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &) { @@ -2102,7 +2583,15 @@ void CodeGenFunction::EmitOMPCancellationPointDirective( } void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) { - CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), + const Expr *IfCond = nullptr; + for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { + if (C->getNameModifier() == OMPD_unknown || + C->getNameModifier() == OMPD_cancel) { + IfCond = C->getCondition(); + break; + } + } + CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), IfCond, S.getCancelRegion()); } @@ -2110,8 +2599,35 @@ CodeGenFunction::JumpDest CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) { if (Kind == OMPD_parallel || Kind == OMPD_task) return ReturnBlock; - else if (Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections) - return BreakContinueStack.empty() ? JumpDest() - : BreakContinueStack.back().BreakBlock; - return JumpDest(); + assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || + Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for); + return BreakContinueStack.back().BreakBlock; +} + +// Generate the instructions for '#pragma omp target data' directive. +void CodeGenFunction::EmitOMPTargetDataDirective( + const OMPTargetDataDirective &S) { + // emit the code inside the construct for now + auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_target_data, + [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); }); } + +void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) { + // emit the code inside the construct for now + auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_taskloop, + [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); }); +} + +void CodeGenFunction::EmitOMPTaskLoopSimdDirective( + const OMPTaskLoopSimdDirective &S) { + // emit the code inside the construct for now + auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); + CGM.getOpenMPRuntime().emitInlinedDirective( + *this, OMPD_taskloop_simd, + [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); }); +} + diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp index e3df5a4..4fb76710 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp @@ -108,7 +108,6 @@ llvm::GlobalVariable *CodeGenVTables::GetAddrOfVTT(const CXXRecordDecl *RD) { llvm::raw_svector_ostream Out(OutName); cast<ItaniumMangleContext>(CGM.getCXXABI().getMangleContext()) .mangleCXXVTT(RD, Out); - Out.flush(); StringRef Name = OutName.str(); // This will also defer the definition of the VTT. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp index 1b7f1d7..c8f3add 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp @@ -44,7 +44,6 @@ llvm::Constant *CodeGenModule::GetAddrOfThunk(GlobalDecl GD, Thunk.This, Out); else getCXXABI().getMangleContext().mangleThunk(MD, Thunk, Out); - Out.flush(); llvm::Type *Ty = getTypes().GetFunctionTypeForVTable(GD); return GetOrCreateLLVMFunction(Name, Ty, GD, /*ForVTable=*/true, @@ -103,8 +102,11 @@ static RValue PerformReturnAdjustment(CodeGenFunction &CGF, CGF.EmitBlock(AdjustNotNull); } - ReturnValue = CGF.CGM.getCXXABI().performReturnAdjustment(CGF, ReturnValue, - Thunk.Return); + auto ClassDecl = ResultType->getPointeeType()->getAsCXXRecordDecl(); + auto ClassAlign = CGF.CGM.getClassPointerAlignment(ClassDecl); + ReturnValue = CGF.CGM.getCXXABI().performReturnAdjustment(CGF, + Address(ReturnValue, ClassAlign), + Thunk.Return); if (NullCheckValue) { CGF.Builder.CreateBr(AdjustEnd); @@ -172,27 +174,29 @@ CodeGenFunction::GenerateVarArgsThunk(llvm::Function *Fn, // Find the first store of "this", which will be to the alloca associated // with "this". - llvm::Value *ThisPtr = &*AI; - llvm::BasicBlock *EntryBB = Fn->begin(); - llvm::Instruction *ThisStore = + Address ThisPtr(&*AI, CGM.getClassPointerAlignment(MD->getParent())); + llvm::BasicBlock *EntryBB = &Fn->front(); + llvm::BasicBlock::iterator ThisStore = std::find_if(EntryBB->begin(), EntryBB->end(), [&](llvm::Instruction &I) { - return isa<llvm::StoreInst>(I) && I.getOperand(0) == ThisPtr; - }); - assert(ThisStore && "Store of this should be in entry block?"); + return isa<llvm::StoreInst>(I) && + I.getOperand(0) == ThisPtr.getPointer(); + }); + assert(ThisStore != EntryBB->end() && + "Store of this should be in entry block?"); // Adjust "this", if necessary. - Builder.SetInsertPoint(ThisStore); + Builder.SetInsertPoint(&*ThisStore); llvm::Value *AdjustedThisPtr = CGM.getCXXABI().performThisAdjustment(*this, ThisPtr, Thunk.This); ThisStore->setOperand(0, AdjustedThisPtr); if (!Thunk.Return.isEmpty()) { // Fix up the returned value, if necessary. - for (llvm::Function::iterator I = Fn->begin(), E = Fn->end(); I != E; I++) { - llvm::Instruction *T = I->getTerminator(); + for (llvm::BasicBlock &BB : *Fn) { + llvm::Instruction *T = BB.getTerminator(); if (isa<llvm::ReturnInst>(T)) { RValue RV = RValue::get(T->getOperand(0)); T->eraseFromParent(); - Builder.SetInsertPoint(&*I); + Builder.SetInsertPoint(&BB); RV = PerformReturnAdjustment(*this, ResultType, RV, Thunk); Builder.CreateRet(RV.getScalarVal()); break; @@ -236,6 +240,17 @@ void CodeGenFunction::StartThunk(llvm::Function *Fn, GlobalDecl GD, // Since we didn't pass a GlobalDecl to StartFunction, do this ourselves. CGM.getCXXABI().EmitInstanceFunctionProlog(*this); CXXThisValue = CXXABIThisValue; + CurCodeDecl = MD; + CurFuncDecl = MD; +} + +void CodeGenFunction::FinishThunk() { + // Clear these to restore the invariants expected by + // StartFunction/FinishFunction. + CurCodeDecl = nullptr; + CurFuncDecl = nullptr; + + FinishFunction(); } void CodeGenFunction::EmitCallAndReturnForThunk(llvm::Value *Callee, @@ -245,9 +260,10 @@ void CodeGenFunction::EmitCallAndReturnForThunk(llvm::Value *Callee, const CXXMethodDecl *MD = cast<CXXMethodDecl>(CurGD.getDecl()); // Adjust the 'this' pointer if necessary - llvm::Value *AdjustedThisPtr = Thunk ? CGM.getCXXABI().performThisAdjustment( - *this, LoadCXXThis(), Thunk->This) - : LoadCXXThis(); + llvm::Value *AdjustedThisPtr = + Thunk ? CGM.getCXXABI().performThisAdjustment( + *this, LoadCXXThisAddress(), Thunk->This) + : LoadCXXThis(); if (CurFnInfo->usesInAlloca()) { // We don't handle return adjusting thunks, because they require us to call @@ -312,6 +328,8 @@ void CodeGenFunction::EmitCallAndReturnForThunk(llvm::Value *Callee, // Consider return adjustment if we have ThunkInfo. if (Thunk && !Thunk->Return.isEmpty()) RV = PerformReturnAdjustment(*this, ResultType, RV, *Thunk); + else if (llvm::CallInst* Call = dyn_cast<llvm::CallInst>(CallOrInvoke)) + Call->setTailCallKind(llvm::CallInst::TCK_Tail); // Emit return. if (!ResultType->isVoidType() && Slot.isNull()) @@ -320,7 +338,7 @@ void CodeGenFunction::EmitCallAndReturnForThunk(llvm::Value *Callee, // Disable the final ARC autorelease. AutoreleaseResult = false; - FinishFunction(); + FinishThunk(); } void CodeGenFunction::EmitMustTailThunk(const CXXMethodDecl *MD, @@ -345,9 +363,8 @@ void CodeGenFunction::EmitMustTailThunk(const CXXMethodDecl *MD, Args[ThisArgNo] = AdjustedThisPtr; } else { assert(ThisAI.isInAlloca() && "this is passed directly or inalloca"); - llvm::Value *ThisAddr = GetAddrOfLocalVar(CXXABIThisDecl); - llvm::Type *ThisType = - cast<llvm::PointerType>(ThisAddr->getType())->getElementType(); + Address ThisAddr = GetAddrOfLocalVar(CXXABIThisDecl); + llvm::Type *ThisType = ThisAddr.getElementType(); if (ThisType != AdjustedThisPtr->getType()) AdjustedThisPtr = Builder.CreateBitCast(AdjustedThisPtr, ThisType); Builder.CreateStore(AdjustedThisPtr, ThisAddr); @@ -502,8 +519,8 @@ void CodeGenVTables::EmitThunks(GlobalDecl GD) if (!ThunkInfoVector) return; - for (unsigned I = 0, E = ThunkInfoVector->size(); I != E; ++I) - emitThunk(GD, (*ThunkInfoVector)[I], /*ForVTable=*/false); + for (const ThunkInfo& Thunk : *ThunkInfoVector) + emitThunk(GD, Thunk, /*ForVTable=*/false); } llvm::Constant *CodeGenVTables::CreateVTableInitializer( @@ -565,6 +582,24 @@ llvm::Constant *CodeGenVTables::CreateVTableInitializer( break; } + if (CGM.getLangOpts().CUDA) { + // Emit NULL for methods we can't codegen on this + // side. Otherwise we'd end up with vtable with unresolved + // references. + const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl()); + // OK on device side: functions w/ __device__ attribute + // OK on host side: anything except __device__-only functions. + bool CanEmitMethod = CGM.getLangOpts().CUDAIsDevice + ? MD->hasAttr<CUDADeviceAttr>() + : (MD->hasAttr<CUDAHostAttr>() || + !MD->hasAttr<CUDADeviceAttr>()); + if (!CanEmitMethod) { + Init = llvm::ConstantExpr::getNullValue(Int8PtrTy); + break; + } + // Method is acceptable, continue processing as usual. + } + if (cast<CXXMethodDecl>(GD.getDecl())->isPure()) { // We have a pure virtual member function. if (!PureVirtualFn) { @@ -642,7 +677,6 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD, cast<ItaniumMangleContext>(CGM.getCXXABI().getMangleContext()) .mangleCXXCtorVTable(RD, Base.getBaseOffset().getQuantity(), Base.getBase(), Out); - Out.flush(); StringRef Name = OutName.str(); llvm::ArrayType *ArrayType = @@ -679,6 +713,12 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD, return VTable; } +static bool shouldEmitAvailableExternallyVTable(const CodeGenModule &CGM, + const CXXRecordDecl *RD) { + return CGM.getCodeGenOpts().OptimizationLevel > 0 && + CGM.getCXXABI().canSpeculativelyEmitVTable(RD); +} + /// Compute the required linkage of the v-table for the given class. /// /// Note that we only call this at the end of the translation unit. @@ -700,7 +740,12 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { switch (keyFunction->getTemplateSpecializationKind()) { case TSK_Undeclared: case TSK_ExplicitSpecialization: - assert(def && "Should not have been asked to emit this"); + assert((def || CodeGenOpts.OptimizationLevel > 0) && + "Shouldn't query vtable linkage without key function or " + "optimizations"); + if (!def && CodeGenOpts.OptimizationLevel > 0) + return llvm::GlobalVariable::AvailableExternallyLinkage; + if (keyFunction->isInlined()) return !Context.getLangOpts().AppleKext ? llvm::GlobalVariable::LinkOnceODRLinkage : @@ -742,16 +787,18 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { } switch (RD->getTemplateSpecializationKind()) { - case TSK_Undeclared: - case TSK_ExplicitSpecialization: - case TSK_ImplicitInstantiation: - return DiscardableODRLinkage; - - case TSK_ExplicitInstantiationDeclaration: - return llvm::GlobalVariable::ExternalLinkage; - - case TSK_ExplicitInstantiationDefinition: - return NonDiscardableODRLinkage; + case TSK_Undeclared: + case TSK_ExplicitSpecialization: + case TSK_ImplicitInstantiation: + return DiscardableODRLinkage; + + case TSK_ExplicitInstantiationDeclaration: + return shouldEmitAvailableExternallyVTable(*this, RD) + ? llvm::GlobalVariable::AvailableExternallyLinkage + : llvm::GlobalVariable::ExternalLinkage; + + case TSK_ExplicitInstantiationDefinition: + return NonDiscardableODRLinkage; } llvm_unreachable("Invalid TemplateSpecializationKind!"); @@ -819,7 +866,12 @@ bool CodeGenVTables::isVTableExternal(const CXXRecordDecl *RD) { /// we define that v-table? static bool shouldEmitVTableAtEndOfTranslationUnit(CodeGenModule &CGM, const CXXRecordDecl *RD) { - return !CGM.getVTables().isVTableExternal(RD); + // If vtable is internal then it has to be done. + if (!CGM.getVTables().isVTableExternal(RD)) + return true; + + // If it's external then maybe we will need it as available_externally. + return shouldEmitAvailableExternallyVTable(CGM, RD); } /// Given that at some point we emitted a reference to one or more @@ -832,13 +884,9 @@ void CodeGenModule::EmitDeferredVTables() { size_t savedSize = DeferredVTables.size(); #endif - typedef std::vector<const CXXRecordDecl *>::const_iterator const_iterator; - for (const_iterator i = DeferredVTables.begin(), - e = DeferredVTables.end(); i != e; ++i) { - const CXXRecordDecl *RD = *i; + for (const CXXRecordDecl *RD : DeferredVTables) if (shouldEmitVTableAtEndOfTranslationUnit(*this, RD)) VTables.GenerateClassData(RD); - } assert(savedSize == DeferredVTables.size() && "deferred extra v-tables during v-table emission?"); @@ -846,8 +894,12 @@ void CodeGenModule::EmitDeferredVTables() { } bool CodeGenModule::IsCFIBlacklistedRecord(const CXXRecordDecl *RD) { - // FIXME: Make this user configurable. - return RD->isInStdNamespace(); + if (RD->hasAttr<UuidAttr>() && + getContext().getSanitizerBlacklist().isBlacklistedType("attr:uuid")) + return true; + + return getContext().getSanitizerBlacklist().isBlacklistedType( + RD->getQualifiedNameAsString()); } void CodeGenModule::EmitVTableBitSetEntries(llvm::GlobalVariable *VTable, @@ -861,41 +913,46 @@ void CodeGenModule::EmitVTableBitSetEntries(llvm::GlobalVariable *VTable, CharUnits PointerWidth = Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0)); - std::vector<llvm::MDTuple *> BitsetEntries; + typedef std::pair<const CXXRecordDecl *, unsigned> BSEntry; + std::vector<BSEntry> BitsetEntries; // Create a bit set entry for each address point. for (auto &&AP : VTLayout.getAddressPoints()) { if (IsCFIBlacklistedRecord(AP.first.getBase())) continue; - BitsetEntries.push_back(CreateVTableBitSetEntry( - VTable, PointerWidth * AP.second, AP.first.getBase())); + BitsetEntries.push_back(std::make_pair(AP.first.getBase(), AP.second)); } // Sort the bit set entries for determinism. - std::sort(BitsetEntries.begin(), BitsetEntries.end(), [](llvm::MDTuple *T1, - llvm::MDTuple *T2) { - if (T1 == T2) + std::sort(BitsetEntries.begin(), BitsetEntries.end(), + [this](const BSEntry &E1, const BSEntry &E2) { + if (&E1 == &E2) return false; - StringRef S1 = cast<llvm::MDString>(T1->getOperand(0))->getString(); - StringRef S2 = cast<llvm::MDString>(T2->getOperand(0))->getString(); + std::string S1; + llvm::raw_string_ostream O1(S1); + getCXXABI().getMangleContext().mangleTypeName( + QualType(E1.first->getTypeForDecl(), 0), O1); + O1.flush(); + + std::string S2; + llvm::raw_string_ostream O2(S2); + getCXXABI().getMangleContext().mangleTypeName( + QualType(E2.first->getTypeForDecl(), 0), O2); + O2.flush(); + if (S1 < S2) return true; if (S1 != S2) return false; - uint64_t Offset1 = cast<llvm::ConstantInt>( - cast<llvm::ConstantAsMetadata>(T1->getOperand(2)) - ->getValue())->getZExtValue(); - uint64_t Offset2 = cast<llvm::ConstantInt>( - cast<llvm::ConstantAsMetadata>(T2->getOperand(2)) - ->getValue())->getZExtValue(); - assert(Offset1 != Offset2); - return Offset1 < Offset2; + return E1.second < E2.second; }); llvm::NamedMDNode *BitsetsMD = getModule().getOrInsertNamedMetadata("llvm.bitsets"); for (auto BitsetEntry : BitsetEntries) - BitsetsMD->addOperand(BitsetEntry); + CreateVTableBitSetEntry(BitsetsMD, VTable, + PointerWidth * BitsetEntry.second, + BitsetEntry.first); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h index e0195a2..c27e54a 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.h @@ -85,10 +85,6 @@ public: uint64_t getSecondaryVirtualPointerIndex(const CXXRecordDecl *RD, BaseSubobject Base); - /// getAddressPoint - Get the address point of the given subobject in the - /// class decl. - uint64_t getAddressPoint(BaseSubobject Base, const CXXRecordDecl *RD); - /// GenerateConstructionVTable - Generate a construction vtable for the given /// base subobject. llvm::GlobalVariable * diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h b/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h index 9205591..3ccc4cd 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h @@ -16,10 +16,10 @@ #define LLVM_CLANG_LIB_CODEGEN_CGVALUE_H #include "clang/AST/ASTContext.h" -#include "clang/AST/CharUnits.h" #include "clang/AST/Type.h" #include "llvm/IR/Value.h" #include "llvm/IR/Type.h" +#include "Address.h" namespace llvm { class Constant; @@ -38,6 +38,10 @@ namespace CodeGen { class RValue { enum Flavor { Scalar, Complex, Aggregate }; + // The shift to make to an aggregate's alignment to make it look + // like a pointer. + enum { AggAlignShift = 4 }; + // Stores first value and flavor. llvm::PointerIntPair<llvm::Value *, 2, Flavor> V1; // Stores second value and volatility. @@ -63,11 +67,21 @@ public: } /// getAggregateAddr() - Return the Value* of the address of the aggregate. - llvm::Value *getAggregateAddr() const { + Address getAggregateAddress() const { + assert(isAggregate() && "Not an aggregate!"); + auto align = reinterpret_cast<uintptr_t>(V2.getPointer()) >> AggAlignShift; + return Address(V1.getPointer(), CharUnits::fromQuantity(align)); + } + llvm::Value *getAggregatePointer() const { assert(isAggregate() && "Not an aggregate!"); return V1.getPointer(); } + static RValue getIgnored() { + // FIXME: should we make this a more explicit state? + return get(nullptr); + } + static RValue get(llvm::Value *V) { RValue ER; ER.V1.setPointer(V); @@ -89,11 +103,14 @@ public: // FIXME: Aggregate rvalues need to retain information about whether they are // volatile or not. Remove default to find all places that probably get this // wrong. - static RValue getAggregate(llvm::Value *V, bool Volatile = false) { + static RValue getAggregate(Address addr, bool isVolatile = false) { RValue ER; - ER.V1.setPointer(V); + ER.V1.setPointer(addr.getPointer()); ER.V1.setInt(Aggregate); - ER.V2.setInt(Volatile); + + auto align = static_cast<uintptr_t>(addr.getAlignment().getQuantity()); + ER.V2.setPointer(reinterpret_cast<llvm::Value*>(align << AggAlignShift)); + ER.V2.setInt(isVolatile); return ER; } }; @@ -103,6 +120,32 @@ enum ARCPreciseLifetime_t { ARCImpreciseLifetime, ARCPreciseLifetime }; +/// The source of the alignment of an l-value; an expression of +/// confidence in the alignment actually matching the estimate. +enum class AlignmentSource { + /// The l-value was an access to a declared entity or something + /// equivalently strong, like the address of an array allocated by a + /// language runtime. + Decl, + + /// The l-value was considered opaque, so the alignment was + /// determined from a type, but that type was an explicitly-aligned + /// typedef. + AttributedType, + + /// The l-value was considered opaque, so the alignment was + /// determined from a type. + Type +}; + +/// Given that the base address has the given alignment source, what's +/// our confidence in the alignment of the field? +static inline AlignmentSource getFieldAlignmentSource(AlignmentSource Source) { + // For now, we don't distinguish fields of opaque pointers from + // top-level declarations, but maybe we should. + return AlignmentSource::Decl; +} + /// LValue - This represents an lvalue references. Because C/C++ allow /// bitfields, this is not a simple LLVM pointer, it may be a pointer plus a /// bitrange. @@ -157,6 +200,12 @@ class LValue { // to make the default bitfield pattern all-zeroes. bool ImpreciseLifetime : 1; + unsigned AlignSource : 2; + + // This flag shows if a nontemporal load/stores should be used when accessing + // this lvalue. + bool Nontemporal : 1; + Expr *BaseIvarExp; /// Used by struct-path-aware TBAA. @@ -169,17 +218,21 @@ class LValue { private: void Initialize(QualType Type, Qualifiers Quals, - CharUnits Alignment, + CharUnits Alignment, AlignmentSource AlignSource, llvm::MDNode *TBAAInfo = nullptr) { + assert((!Alignment.isZero() || Type->isIncompleteType()) && + "initializing l-value with zero alignment!"); this->Type = Type; this->Quals = Quals; this->Alignment = Alignment.getQuantity(); assert(this->Alignment == Alignment.getQuantity() && "Alignment exceeds allowed max!"); + this->AlignSource = unsigned(AlignSource); // Initialize Objective-C flags. this->Ivar = this->ObjIsArray = this->NonGC = this->GlobalObjCRef = false; this->ImpreciseLifetime = false; + this->Nontemporal = false; this->ThreadLocalRef = false; this->BaseIvarExp = nullptr; @@ -229,6 +282,8 @@ public: void setARCPreciseLifetime(ARCPreciseLifetime_t value) { ImpreciseLifetime = (value == ARCImpreciseLifetime); } + bool isNontemporal() const { return Nontemporal; } + void setNontemporal(bool Value) { Nontemporal = Value; } bool isObjCWeak() const { return Quals.getObjCGCAttr() == Qualifiers::Weak; @@ -261,29 +316,50 @@ public: CharUnits getAlignment() const { return CharUnits::fromQuantity(Alignment); } void setAlignment(CharUnits A) { Alignment = A.getQuantity(); } + AlignmentSource getAlignmentSource() const { + return AlignmentSource(AlignSource); + } + void setAlignmentSource(AlignmentSource Source) { + AlignSource = unsigned(Source); + } + // simple lvalue - llvm::Value *getAddress() const { assert(isSimple()); return V; } - void setAddress(llvm::Value *address) { + llvm::Value *getPointer() const { + assert(isSimple()); + return V; + } + Address getAddress() const { return Address(getPointer(), getAlignment()); } + void setAddress(Address address) { assert(isSimple()); - V = address; + V = address.getPointer(); + Alignment = address.getAlignment().getQuantity(); } // vector elt lvalue - llvm::Value *getVectorAddr() const { assert(isVectorElt()); return V; } + Address getVectorAddress() const { + return Address(getVectorPointer(), getAlignment()); + } + llvm::Value *getVectorPointer() const { assert(isVectorElt()); return V; } llvm::Value *getVectorIdx() const { assert(isVectorElt()); return VectorIdx; } // extended vector elements. - llvm::Value *getExtVectorAddr() const { assert(isExtVectorElt()); return V; } + Address getExtVectorAddress() const { + return Address(getExtVectorPointer(), getAlignment()); + } + llvm::Value *getExtVectorPointer() const { + assert(isExtVectorElt()); + return V; + } llvm::Constant *getExtVectorElts() const { assert(isExtVectorElt()); return VectorElts; } // bitfield lvalue - llvm::Value *getBitFieldAddr() const { - assert(isBitField()); - return V; + Address getBitFieldAddress() const { + return Address(getBitFieldPointer(), getAlignment()); } + llvm::Value *getBitFieldPointer() const { assert(isBitField()); return V; } const CGBitFieldInfo &getBitFieldInfo() const { assert(isBitField()); return *BitFieldInfo; @@ -292,37 +368,40 @@ public: // global register lvalue llvm::Value *getGlobalReg() const { assert(isGlobalReg()); return V; } - static LValue MakeAddr(llvm::Value *address, QualType type, - CharUnits alignment, ASTContext &Context, + static LValue MakeAddr(Address address, QualType type, + ASTContext &Context, + AlignmentSource alignSource, llvm::MDNode *TBAAInfo = nullptr) { Qualifiers qs = type.getQualifiers(); qs.setObjCGCAttr(Context.getObjCGCAttrKind(type)); LValue R; R.LVType = Simple; - assert(address->getType()->isPointerTy()); - R.V = address; - R.Initialize(type, qs, alignment, TBAAInfo); + assert(address.getPointer()->getType()->isPointerTy()); + R.V = address.getPointer(); + R.Initialize(type, qs, address.getAlignment(), alignSource, TBAAInfo); return R; } - static LValue MakeVectorElt(llvm::Value *Vec, llvm::Value *Idx, - QualType type, CharUnits Alignment) { + static LValue MakeVectorElt(Address vecAddress, llvm::Value *Idx, + QualType type, AlignmentSource alignSource) { LValue R; R.LVType = VectorElt; - R.V = Vec; + R.V = vecAddress.getPointer(); R.VectorIdx = Idx; - R.Initialize(type, type.getQualifiers(), Alignment); + R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), + alignSource); return R; } - static LValue MakeExtVectorElt(llvm::Value *Vec, llvm::Constant *Elts, - QualType type, CharUnits Alignment) { + static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, + QualType type, AlignmentSource alignSource) { LValue R; R.LVType = ExtVectorElt; - R.V = Vec; + R.V = vecAddress.getPointer(); R.VectorElts = Elts; - R.Initialize(type, type.getQualifiers(), Alignment); + R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), + alignSource); return R; } @@ -332,29 +411,28 @@ public: /// bit-field refers to. /// \param Info - The information describing how to perform the bit-field /// access. - static LValue MakeBitfield(llvm::Value *Addr, + static LValue MakeBitfield(Address Addr, const CGBitFieldInfo &Info, - QualType type, CharUnits Alignment) { + QualType type, + AlignmentSource alignSource) { LValue R; R.LVType = BitField; - R.V = Addr; + R.V = Addr.getPointer(); R.BitFieldInfo = &Info; - R.Initialize(type, type.getQualifiers(), Alignment); + R.Initialize(type, type.getQualifiers(), Addr.getAlignment(), alignSource); return R; } - static LValue MakeGlobalReg(llvm::Value *Reg, - QualType type, - CharUnits Alignment) { + static LValue MakeGlobalReg(Address Reg, QualType type) { LValue R; R.LVType = GlobalReg; - R.V = Reg; - R.Initialize(type, type.getQualifiers(), Alignment); + R.V = Reg.getPointer(); + R.Initialize(type, type.getQualifiers(), Reg.getAlignment(), + AlignmentSource::Decl); return R; } RValue asAggregateRValue() const { - // FIMXE: Alignment return RValue::getAggregate(getAddress(), isVolatileQualified()); } }; @@ -407,7 +485,7 @@ public: /// ignored - Returns an aggregate value slot indicating that the /// aggregate value is being ignored. static AggValueSlot ignored() { - return forAddr(nullptr, CharUnits(), Qualifiers(), IsNotDestructed, + return forAddr(Address::invalid(), Qualifiers(), IsNotDestructed, DoesNotNeedGCBarriers, IsNotAliased); } @@ -421,15 +499,20 @@ public: /// for calling destructors on this object /// \param needsGC - true if the slot is potentially located /// somewhere that ObjC GC calls should be emitted for - static AggValueSlot forAddr(llvm::Value *addr, CharUnits align, + static AggValueSlot forAddr(Address addr, Qualifiers quals, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, IsZeroed_t isZeroed = IsNotZeroed) { AggValueSlot AV; - AV.Addr = addr; - AV.Alignment = align.getQuantity(); + if (addr.isValid()) { + AV.Addr = addr.getPointer(); + AV.Alignment = addr.getAlignment().getQuantity(); + } else { + AV.Addr = nullptr; + AV.Alignment = 0; + } AV.Quals = quals; AV.DestructedFlag = isDestructed; AV.ObjCGCFlag = needsGC; @@ -443,7 +526,7 @@ public: NeedsGCBarriers_t needsGC, IsAliased_t isAliased, IsZeroed_t isZeroed = IsNotZeroed) { - return forAddr(LV.getAddress(), LV.getAlignment(), + return forAddr(LV.getAddress(), LV.getQuals(), isDestructed, needsGC, isAliased, isZeroed); } @@ -471,11 +554,15 @@ public: NeedsGCBarriers_t requiresGCollection() const { return NeedsGCBarriers_t(ObjCGCFlag); } - - llvm::Value *getAddr() const { + + llvm::Value *getPointer() const { return Addr; } + Address getAddress() const { + return Address(Addr, getAlignment()); + } + bool isIgnored() const { return Addr == nullptr; } @@ -488,9 +575,12 @@ public: return IsAliased_t(AliasedFlag); } - // FIXME: Alignment? RValue asRValue() const { - return RValue::getAggregate(getAddr(), isVolatile()); + if (isIgnored()) { + return RValue::getIgnored(); + } else { + return RValue::getAggregate(getAddress(), isVolatile()); + } } void setZeroed(bool V = true) { ZeroedFlag = V; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp index 755e8aa..643c996 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp @@ -26,23 +26,16 @@ using namespace clang; using namespace CodeGen; -CodeGenABITypes::CodeGenABITypes(ASTContext &C, - llvm::Module &M, - const llvm::DataLayout &TD, +CodeGenABITypes::CodeGenABITypes(ASTContext &C, llvm::Module &M, CoverageSourceInfo *CoverageInfo) - : CGO(new CodeGenOptions), - HSO(new HeaderSearchOptions), - PPO(new PreprocessorOptions), - CGM(new CodeGen::CodeGenModule(C, *HSO, *PPO, *CGO, - M, TD, C.getDiagnostics(), - CoverageInfo)) { -} + : CGO(new CodeGenOptions), HSO(new HeaderSearchOptions), + PPO(new PreprocessorOptions), + CGM(new CodeGen::CodeGenModule(C, *HSO, *PPO, *CGO, M, C.getDiagnostics(), + CoverageInfo)) {} -CodeGenABITypes::~CodeGenABITypes() -{ - delete CGO; - delete CGM; -} +// Explicitly out-of-line because ~CodeGenModule() is private but +// CodeGenABITypes.h is part of clang's API. +CodeGenABITypes::~CodeGenABITypes() = default; const CGFunctionInfo & CodeGenABITypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD, @@ -51,8 +44,9 @@ CodeGenABITypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD, } const CGFunctionInfo & -CodeGenABITypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> Ty) { - return CGM->getTypes().arrangeFreeFunctionType(Ty); +CodeGenABITypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> Ty, + const FunctionDecl *FD) { + return CGM->getTypes().arrangeFreeFunctionType(Ty, FD); } const CGFunctionInfo & @@ -62,15 +56,14 @@ CodeGenABITypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> Ty) { const CGFunctionInfo & CodeGenABITypes::arrangeCXXMethodType(const CXXRecordDecl *RD, - const FunctionProtoType *FTP) { - return CGM->getTypes().arrangeCXXMethodType(RD, FTP); + const FunctionProtoType *FTP, + const CXXMethodDecl *MD) { + return CGM->getTypes().arrangeCXXMethodType(RD, FTP, MD); } -const CGFunctionInfo & -CodeGenABITypes::arrangeFreeFunctionCall(CanQualType returnType, - ArrayRef<CanQualType> argTypes, - FunctionType::ExtInfo info, - RequiredArgs args) { +const CGFunctionInfo &CodeGenABITypes::arrangeFreeFunctionCall( + CanQualType returnType, ArrayRef<CanQualType> argTypes, + FunctionType::ExtInfo info, RequiredArgs args) { return CGM->getTypes().arrangeLLVMFunctionInfo( returnType, /*IsInstanceMethod=*/false, /*IsChainCall=*/false, argTypes, info, args); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp index 0e7b6d8..abef543 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp @@ -26,10 +26,12 @@ #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/FunctionInfo.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IRReader/IRReader.h" #include "llvm/Linker/Linker.h" +#include "llvm/Object/FunctionIndexObjectFile.h" #include "llvm/Pass.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SourceMgr.h" @@ -53,40 +55,47 @@ namespace clang { std::unique_ptr<CodeGenerator> Gen; - std::unique_ptr<llvm::Module> TheModule, LinkModule; + std::unique_ptr<llvm::Module> TheModule; + SmallVector<std::pair<unsigned, std::unique_ptr<llvm::Module>>, 4> + LinkModules; + + // This is here so that the diagnostic printer knows the module a diagnostic + // refers to. + llvm::Module *CurLinkModule = nullptr; public: - BackendConsumer(BackendAction Action, DiagnosticsEngine &Diags, - const HeaderSearchOptions &HeaderSearchOpts, - const PreprocessorOptions &PPOpts, - const CodeGenOptions &CodeGenOpts, - const TargetOptions &TargetOpts, - const LangOptions &LangOpts, bool TimePasses, - const std::string &InFile, llvm::Module *LinkModule, - raw_pwrite_stream *OS, LLVMContext &C, - CoverageSourceInfo *CoverageInfo = nullptr) + BackendConsumer( + BackendAction Action, DiagnosticsEngine &Diags, + const HeaderSearchOptions &HeaderSearchOpts, + const PreprocessorOptions &PPOpts, const CodeGenOptions &CodeGenOpts, + const TargetOptions &TargetOpts, const LangOptions &LangOpts, + bool TimePasses, const std::string &InFile, + const SmallVectorImpl<std::pair<unsigned, llvm::Module *>> &LinkModules, + raw_pwrite_stream *OS, LLVMContext &C, + CoverageSourceInfo *CoverageInfo = nullptr) : Diags(Diags), Action(Action), CodeGenOpts(CodeGenOpts), TargetOpts(TargetOpts), LangOpts(LangOpts), AsmOutStream(OS), Context(nullptr), LLVMIRGeneration("LLVM IR Generation Time"), Gen(CreateLLVMCodeGen(Diags, InFile, HeaderSearchOpts, PPOpts, - CodeGenOpts, C, CoverageInfo)), - LinkModule(LinkModule) { + CodeGenOpts, C, CoverageInfo)) { llvm::TimePassesIsEnabled = TimePasses; + for (auto &I : LinkModules) + this->LinkModules.push_back( + std::make_pair(I.first, std::unique_ptr<llvm::Module>(I.second))); } - std::unique_ptr<llvm::Module> takeModule() { return std::move(TheModule); } - llvm::Module *takeLinkModule() { return LinkModule.release(); } + void releaseLinkModules() { + for (auto &I : LinkModules) + I.second.release(); + } void HandleCXXStaticMemberVarInstantiation(VarDecl *VD) override { Gen->HandleCXXStaticMemberVarInstantiation(VD); } void Initialize(ASTContext &Ctx) override { - if (Context) { - assert(Context == &Ctx); - return; - } - + assert(!Context && "initialized multiple times"); + Context = &Ctx; if (llvm::TimePassesIsEnabled) @@ -158,14 +167,6 @@ namespace clang { assert(TheModule.get() == M && "Unexpected module change during IR generation"); - // Link LinkModule into this module if present, preserving its validity. - if (LinkModule) { - if (Linker::LinkModules( - M, LinkModule.get(), - [=](const DiagnosticInfo &DI) { linkerDiagnosticHandler(DI); })) - return; - } - // Install an inline asm handler so that diagnostics get printed through // our diagnostics hooks. LLVMContext &Ctx = TheModule->getContext(); @@ -179,8 +180,16 @@ namespace clang { void *OldDiagnosticContext = Ctx.getDiagnosticContext(); Ctx.setDiagnosticHandler(DiagnosticHandler, this); + // Link LinkModule into this module if present, preserving its validity. + for (auto &I : LinkModules) { + unsigned LinkFlags = I.first; + CurLinkModule = I.second.get(); + if (Linker::linkModules(*M, std::move(I.second), LinkFlags)) + return; + } + EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts, - C.getTargetInfo().getTargetDescription(), + C.getTargetInfo().getDataLayoutString(), TheModule.get(), Action, AsmOutStream); Ctx.setInlineAsmDiagnosticHandler(OldHandler, OldContext); @@ -226,8 +235,6 @@ namespace clang { ((BackendConsumer*)Context)->InlineAsmDiagHandler2(SM, Loc); } - void linkerDiagnosticHandler(const llvm::DiagnosticInfo &DI); - static void DiagnosticHandler(const llvm::DiagnosticInfo &DI, void *Context) { ((BackendConsumer *)Context)->DiagnosticHandlerImpl(DI); @@ -256,6 +263,10 @@ namespace clang { const llvm::DiagnosticInfoOptimizationRemarkMissed &D); void OptimizationRemarkHandler( const llvm::DiagnosticInfoOptimizationRemarkAnalysis &D); + void OptimizationRemarkHandler( + const llvm::DiagnosticInfoOptimizationRemarkAnalysisFPCommute &D); + void OptimizationRemarkHandler( + const llvm::DiagnosticInfoOptimizationRemarkAnalysisAliasing &D); void OptimizationFailureHandler( const llvm::DiagnosticInfoOptimizationFailure &D); }; @@ -333,8 +344,7 @@ void BackendConsumer::InlineAsmDiagHandler2(const llvm::SMDiagnostic &D, DiagnosticBuilder B = Diags.Report(Loc, diag::note_fe_inline_asm_here); // Convert the SMDiagnostic ranges into SourceRange and attach them // to the diagnostic. - for (unsigned i = 0, e = D.getRanges().size(); i != e; ++i) { - std::pair<unsigned, unsigned> Range = D.getRanges()[i]; + for (const std::pair<unsigned, unsigned> &Range : D.getRanges()) { unsigned Column = D.getColumnNo(); B << SourceRange(Loc.getLocWithOffset(Range.first - Column), Loc.getLocWithOffset(Range.second - Column)); @@ -492,33 +502,46 @@ void BackendConsumer::OptimizationRemarkHandler( void BackendConsumer::OptimizationRemarkHandler( const llvm::DiagnosticInfoOptimizationRemarkAnalysis &D) { - // Optimization analysis remarks are active only if the -Rpass-analysis - // flag has a regular expression that matches the name of the pass - // name in \p D. - if (CodeGenOpts.OptimizationRemarkAnalysisPattern && - CodeGenOpts.OptimizationRemarkAnalysisPattern->match(D.getPassName())) + // Optimization analysis remarks are active if the pass name is set to + // llvm::DiagnosticInfo::AlwasyPrint or if the -Rpass-analysis flag has a + // regular expression that matches the name of the pass name in \p D. + + if (D.getPassName() == llvm::DiagnosticInfo::AlwaysPrint || + (CodeGenOpts.OptimizationRemarkAnalysisPattern && + CodeGenOpts.OptimizationRemarkAnalysisPattern->match(D.getPassName()))) EmitOptimizationMessage( D, diag::remark_fe_backend_optimization_remark_analysis); } -void BackendConsumer::OptimizationFailureHandler( - const llvm::DiagnosticInfoOptimizationFailure &D) { - EmitOptimizationMessage(D, diag::warn_fe_backend_optimization_failure); +void BackendConsumer::OptimizationRemarkHandler( + const llvm::DiagnosticInfoOptimizationRemarkAnalysisFPCommute &D) { + // Optimization analysis remarks are active if the pass name is set to + // llvm::DiagnosticInfo::AlwasyPrint or if the -Rpass-analysis flag has a + // regular expression that matches the name of the pass name in \p D. + + if (D.getPassName() == llvm::DiagnosticInfo::AlwaysPrint || + (CodeGenOpts.OptimizationRemarkAnalysisPattern && + CodeGenOpts.OptimizationRemarkAnalysisPattern->match(D.getPassName()))) + EmitOptimizationMessage( + D, diag::remark_fe_backend_optimization_remark_analysis_fpcommute); } -void BackendConsumer::linkerDiagnosticHandler(const DiagnosticInfo &DI) { - if (DI.getSeverity() != DS_Error) - return; - - std::string MsgStorage; - { - raw_string_ostream Stream(MsgStorage); - DiagnosticPrinterRawOStream DP(Stream); - DI.print(DP); - } +void BackendConsumer::OptimizationRemarkHandler( + const llvm::DiagnosticInfoOptimizationRemarkAnalysisAliasing &D) { + // Optimization analysis remarks are active if the pass name is set to + // llvm::DiagnosticInfo::AlwasyPrint or if the -Rpass-analysis flag has a + // regular expression that matches the name of the pass name in \p D. + + if (D.getPassName() == llvm::DiagnosticInfo::AlwaysPrint || + (CodeGenOpts.OptimizationRemarkAnalysisPattern && + CodeGenOpts.OptimizationRemarkAnalysisPattern->match(D.getPassName()))) + EmitOptimizationMessage( + D, diag::remark_fe_backend_optimization_remark_analysis_aliasing); +} - Diags.Report(diag::err_fe_cannot_link_module) - << LinkModule->getModuleIdentifier() << MsgStorage; +void BackendConsumer::OptimizationFailureHandler( + const llvm::DiagnosticInfoOptimizationFailure &D) { + EmitOptimizationMessage(D, diag::warn_fe_backend_optimization_failure); } /// \brief This function is invoked when the backend needs @@ -538,6 +561,13 @@ void BackendConsumer::DiagnosticHandlerImpl(const DiagnosticInfo &DI) { return; ComputeDiagID(Severity, backend_frame_larger_than, DiagID); break; + case DK_Linker: + assert(CurLinkModule); + // FIXME: stop eating the warnings and notes. + if (Severity != DS_Error) + return; + DiagID = diag::err_fe_cannot_link_module; + break; case llvm::DK_OptimizationRemark: // Optimization remarks are always handled completely by this // handler. There is no generic way of emitting them. @@ -554,6 +584,18 @@ void BackendConsumer::DiagnosticHandlerImpl(const DiagnosticInfo &DI) { OptimizationRemarkHandler( cast<DiagnosticInfoOptimizationRemarkAnalysis>(DI)); return; + case llvm::DK_OptimizationRemarkAnalysisFPCommute: + // Optimization remarks are always handled completely by this + // handler. There is no generic way of emitting them. + OptimizationRemarkHandler( + cast<DiagnosticInfoOptimizationRemarkAnalysisFPCommute>(DI)); + return; + case llvm::DK_OptimizationRemarkAnalysisAliasing: + // Optimization remarks are always handled completely by this + // handler. There is no generic way of emitting them. + OptimizationRemarkHandler( + cast<DiagnosticInfoOptimizationRemarkAnalysisAliasing>(DI)); + return; case llvm::DK_OptimizationFailure: // Optimization failures are always handled completely by this // handler. @@ -571,6 +613,12 @@ void BackendConsumer::DiagnosticHandlerImpl(const DiagnosticInfo &DI) { DI.print(DP); } + if (DiagID == diag::err_fe_cannot_link_module) { + Diags.Report(diag::err_fe_cannot_link_module) + << CurLinkModule->getModuleIdentifier() << MsgStorage; + return; + } + // Report the backend message using the usual diagnostic mechanism. FullSourceLoc Loc; Diags.Report(Loc, DiagID).AddString(MsgStorage); @@ -578,9 +626,8 @@ void BackendConsumer::DiagnosticHandlerImpl(const DiagnosticInfo &DI) { #undef ComputeDiagID CodeGenAction::CodeGenAction(unsigned _Act, LLVMContext *_VMContext) - : Act(_Act), LinkModule(nullptr), - VMContext(_VMContext ? _VMContext : new LLVMContext), - OwnsVMContext(!_VMContext) {} + : Act(_Act), VMContext(_VMContext ? _VMContext : new LLVMContext), + OwnsVMContext(!_VMContext) {} CodeGenAction::~CodeGenAction() { TheModule.reset(); @@ -595,9 +642,9 @@ void CodeGenAction::EndSourceFileAction() { if (!getCompilerInstance().hasASTConsumer()) return; - // If we were given a link module, release consumer's ownership of it. - if (LinkModule) - BEConsumer->takeLinkModule(); + // Take back ownership of link modules we passed to consumer. + if (!LinkModules.empty()) + BEConsumer->releaseLinkModules(); // Steal the module from the consumer. TheModule = BEConsumer->takeModule(); @@ -639,28 +686,29 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { if (BA != Backend_EmitNothing && !OS) return nullptr; - llvm::Module *LinkModuleToUse = LinkModule; - - // If we were not given a link module, and the user requested that one be - // loaded from bitcode, do so now. - const std::string &LinkBCFile = CI.getCodeGenOpts().LinkBitcodeFile; - if (!LinkModuleToUse && !LinkBCFile.empty()) { - auto BCBuf = CI.getFileManager().getBufferForFile(LinkBCFile); - if (!BCBuf) { - CI.getDiagnostics().Report(diag::err_cannot_open_file) - << LinkBCFile << BCBuf.getError().message(); - return nullptr; - } + // Load bitcode modules to link with, if we need to. + if (LinkModules.empty()) + for (auto &I : CI.getCodeGenOpts().LinkBitcodeFiles) { + const std::string &LinkBCFile = I.second; + + auto BCBuf = CI.getFileManager().getBufferForFile(LinkBCFile); + if (!BCBuf) { + CI.getDiagnostics().Report(diag::err_cannot_open_file) + << LinkBCFile << BCBuf.getError().message(); + LinkModules.clear(); + return nullptr; + } - ErrorOr<std::unique_ptr<llvm::Module>> ModuleOrErr = - getLazyBitcodeModule(std::move(*BCBuf), *VMContext); - if (std::error_code EC = ModuleOrErr.getError()) { - CI.getDiagnostics().Report(diag::err_cannot_open_file) - << LinkBCFile << EC.message(); - return nullptr; + ErrorOr<std::unique_ptr<llvm::Module>> ModuleOrErr = + getLazyBitcodeModule(std::move(*BCBuf), *VMContext); + if (std::error_code EC = ModuleOrErr.getError()) { + CI.getDiagnostics().Report(diag::err_cannot_open_file) << LinkBCFile + << EC.message(); + LinkModules.clear(); + return nullptr; + } + addLinkModule(ModuleOrErr.get().release(), I.first); } - LinkModuleToUse = ModuleOrErr.get().release(); - } CoverageSourceInfo *CoverageInfo = nullptr; // Add the preprocessor callback only when the coverage mapping is generated. @@ -669,11 +717,12 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { CI.getPreprocessor().addPPCallbacks( std::unique_ptr<PPCallbacks>(CoverageInfo)); } + std::unique_ptr<BackendConsumer> Result(new BackendConsumer( BA, CI.getDiagnostics(), CI.getHeaderSearchOpts(), CI.getPreprocessorOpts(), CI.getCodeGenOpts(), CI.getTargetOpts(), - CI.getLangOpts(), CI.getFrontendOpts().ShowTimers, InFile, - LinkModuleToUse, OS, *VMContext, CoverageInfo)); + CI.getLangOpts(), CI.getFrontendOpts().ShowTimers, InFile, LinkModules, + OS, *VMContext, CoverageInfo)); BEConsumer = Result.get(); return std::move(Result); } @@ -732,11 +781,43 @@ void CodeGenAction::ExecuteAction() { TheModule->setTargetTriple(TargetOpts.Triple); } + auto DiagHandler = [&](const DiagnosticInfo &DI) { + TheModule->getContext().diagnose(DI); + }; + + // If we are performing ThinLTO importing compilation (indicated by + // a non-empty index file option), then we need promote to global scope + // and rename any local values that are potentially exported to other + // modules. Do this early so that the rest of the compilation sees the + // promoted symbols. + std::unique_ptr<FunctionInfoIndex> Index; + if (!CI.getCodeGenOpts().ThinLTOIndexFile.empty()) { + ErrorOr<std::unique_ptr<FunctionInfoIndex>> IndexOrErr = + llvm::getFunctionIndexForFile(CI.getCodeGenOpts().ThinLTOIndexFile, + DiagHandler); + if (std::error_code EC = IndexOrErr.getError()) { + std::string Error = EC.message(); + errs() << "Error loading index file '" + << CI.getCodeGenOpts().ThinLTOIndexFile << "': " << Error + << "\n"; + return; + } + Index = std::move(IndexOrErr.get()); + assert(Index); + // Currently this requires creating a new Module object. + std::unique_ptr<llvm::Module> RenamedModule = + renameModuleForThinLTO(std::move(TheModule), Index.get()); + if (!RenamedModule) + return; + + TheModule = std::move(RenamedModule); + } + LLVMContext &Ctx = TheModule->getContext(); Ctx.setInlineAsmDiagnosticHandler(BitcodeInlineAsmDiagHandler); EmitBackendOutput(CI.getDiagnostics(), CI.getCodeGenOpts(), TargetOpts, - CI.getLangOpts(), CI.getTarget().getTargetDescription(), - TheModule.get(), BA, OS); + CI.getLangOpts(), CI.getTarget().getDataLayoutString(), + TheModule.get(), BA, OS, std::move(Index)); return; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp index ec3c75c..048a043 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "CodeGenFunction.h" +#include "CGBlocks.h" #include "CGCleanup.h" #include "CGCUDARuntime.h" #include "CGCXXABI.h" @@ -24,9 +25,11 @@ #include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/StmtCXX.h" +#include "clang/Basic/Builtins.h" #include "clang/Basic/TargetInfo.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/CodeGenOptions.h" +#include "clang/Sema/SemaDiagnostic.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/MDBuilder.h" @@ -36,12 +39,14 @@ using namespace CodeGen; CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()), - Builder(cgm.getModule().getContext(), llvm::ConstantFolder(), + Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(), CGBuilderInserterTy(this)), - CurFn(nullptr), CapturedStmtInfo(nullptr), + CurFn(nullptr), ReturnValue(Address::invalid()), + CapturedStmtInfo(nullptr), SanOpts(CGM.getLangOpts().Sanitize), IsSanitizerScope(false), CurFuncIsThunk(false), AutoreleaseResult(false), SawAsmBlock(false), - IsOutlinedSEHHelper(false), BlockInfo(nullptr), BlockPointer(nullptr), + IsOutlinedSEHHelper(false), + BlockInfo(nullptr), BlockPointer(nullptr), LambdaThisCaptureField(nullptr), NormalCleanupDest(nullptr), NextCleanupDestIndex(1), FirstBlockInfo(nullptr), EHResumeBlock(nullptr), ExceptionSlot(nullptr), EHSelectorSlot(nullptr), @@ -51,7 +56,7 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) CaseRangeBlock(nullptr), UnreachableBlock(nullptr), NumReturnExprs(0), NumSimpleReturnExprs(0), CXXABIThisDecl(nullptr), CXXABIThisValue(nullptr), CXXThisValue(nullptr), - CXXDefaultInitExprThis(nullptr), CXXStructorImplicitParamDecl(nullptr), + CXXStructorImplicitParamDecl(nullptr), CXXStructorImplicitParamValue(nullptr), OutermostConditional(nullptr), CurLexicalScope(nullptr), TerminateLandingPad(nullptr), TerminateHandler(nullptr), TrapBB(nullptr) { @@ -91,18 +96,69 @@ CodeGenFunction::~CodeGenFunction() { } } -LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { +CharUnits CodeGenFunction::getNaturalPointeeTypeAlignment(QualType T, + AlignmentSource *Source) { + return getNaturalTypeAlignment(T->getPointeeType(), Source, + /*forPointee*/ true); +} + +CharUnits CodeGenFunction::getNaturalTypeAlignment(QualType T, + AlignmentSource *Source, + bool forPointeeType) { + // Honor alignment typedef attributes even on incomplete types. + // We also honor them straight for C++ class types, even as pointees; + // there's an expressivity gap here. + if (auto TT = T->getAs<TypedefType>()) { + if (auto Align = TT->getDecl()->getMaxAlignment()) { + if (Source) *Source = AlignmentSource::AttributedType; + return getContext().toCharUnitsFromBits(Align); + } + } + + if (Source) *Source = AlignmentSource::Type; + CharUnits Alignment; - if (CGM.getCXXABI().isTypeInfoCalculable(T)) { - Alignment = getContext().getTypeAlignInChars(T); - unsigned MaxAlign = getContext().getLangOpts().MaxTypeAlign; - if (MaxAlign && Alignment.getQuantity() > MaxAlign && - !getContext().isAlignmentRequired(T)) - Alignment = CharUnits::fromQuantity(MaxAlign); + if (T->isIncompleteType()) { + Alignment = CharUnits::One(); // Shouldn't be used, but pessimistic is best. + } else { + // For C++ class pointees, we don't know whether we're pointing at a + // base or a complete object, so we generally need to use the + // non-virtual alignment. + const CXXRecordDecl *RD; + if (forPointeeType && (RD = T->getAsCXXRecordDecl())) { + Alignment = CGM.getClassPointerAlignment(RD); + } else { + Alignment = getContext().getTypeAlignInChars(T); + } + + // Cap to the global maximum type alignment unless the alignment + // was somehow explicit on the type. + if (unsigned MaxAlign = getLangOpts().MaxTypeAlign) { + if (Alignment.getQuantity() > MaxAlign && + !getContext().isAlignmentRequired(T)) + Alignment = CharUnits::fromQuantity(MaxAlign); + } } - return LValue::MakeAddr(V, T, Alignment, getContext(), CGM.getTBAAInfo(T)); + return Alignment; } +LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { + AlignmentSource AlignSource; + CharUnits Alignment = getNaturalTypeAlignment(T, &AlignSource); + return LValue::MakeAddr(Address(V, Alignment), T, getContext(), AlignSource, + CGM.getTBAAInfo(T)); +} + +/// Given a value of type T* that may not be to a complete object, +/// construct an l-value with the natural pointee alignment of T. +LValue +CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) { + AlignmentSource AlignSource; + CharUnits Align = getNaturalTypeAlignment(T, &AlignSource, /*pointee*/ true); + return MakeAddrLValue(Address(V, Align), T, AlignSource); +} + + llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) { return CGM.getTypes().ConvertTypeForMem(T); } @@ -295,7 +351,7 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { EscapeArgs[Pair.second] = Pair.first; llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration( &CGM.getModule(), llvm::Intrinsic::localescape); - CGBuilderTy(AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs); + CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs); } // Remove the AllocaInsertPt instruction, which is just a convenience for us. @@ -660,6 +716,14 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, } } + // If we're in C++ mode and the function name is "main", it is guaranteed + // to be norecurse by the standard (3.6.1.3 "The function main shall not be + // used within a program"). + if (getLangOpts().CPlusPlus) + if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) + if (FD->isMain()) + Fn->addFnAttr(llvm::Attribute::NoRecurse); + llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn); // Create a marker to make it easy to insert allocas into the entryblock @@ -696,7 +760,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, if (RetTy->isVoidType()) { // Void type; nothing to return. - ReturnValue = nullptr; + ReturnValue = Address::invalid(); // Count the implicit return. if (!endsWithReturn(D)) @@ -708,15 +772,16 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, auto AI = CurFn->arg_begin(); if (CurFnInfo->getReturnInfo().isSRetAfterThis()) ++AI; - ReturnValue = AI; + ReturnValue = Address(&*AI, CurFnInfo->getReturnInfo().getIndirectAlign()); } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca && !hasScalarEvaluationKind(CurFnInfo->getReturnType())) { // Load the sret pointer from the argument struct and return into that. unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex(); llvm::Function::arg_iterator EI = CurFn->arg_end(); --EI; - llvm::Value *Addr = Builder.CreateStructGEP(nullptr, EI, Idx); - ReturnValue = Builder.CreateLoad(Addr, "agg.result"); + llvm::Value *Addr = Builder.CreateStructGEP(nullptr, &*EI, Idx); + Addr = Builder.CreateAlignedLoad(Addr, getPointerAlign(), "agg.result"); + ReturnValue = Address(Addr, getNaturalTypeAlignment(RetTy)); } else { ReturnValue = CreateIRTemp(RetTy, "retval"); @@ -826,15 +891,11 @@ static void TryMarkNoThrow(llvm::Function *F) { // can't do this on functions that can be overwritten. if (F->mayBeOverridden()) return; - for (llvm::Function::iterator FI = F->begin(), FE = F->end(); FI != FE; ++FI) - for (llvm::BasicBlock::iterator - BI = FI->begin(), BE = FI->end(); BI != BE; ++BI) - if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(&*BI)) { - if (!Call->doesNotThrow()) - return; - } else if (isa<llvm::ResumeInst>(&*BI)) { + for (llvm::BasicBlock &BB : *F) + for (llvm::Instruction &I : BB) + if (I.mayThrow()) return; - } + F->setDoesNotThrow(); } @@ -859,7 +920,18 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, CGM.getCXXABI().buildThisParam(*this, Args); } - Args.append(FD->param_begin(), FD->param_end()); + for (auto *Param : FD->params()) { + Args.push_back(Param); + if (!Param->hasAttr<PassObjectSizeAttr>()) + continue; + + IdentifierInfo *NoID = nullptr; + auto *Implicit = ImplicitParamDecl::Create( + getContext(), Param->getDeclContext(), Param->getLocation(), NoID, + getContext().getSizeType()); + SizeArguments[Param] = Implicit; + Args.push_back(Implicit); + } if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD))) CGM.getCXXABI().addImplicitStructorParams(*this, ResTy, Args); @@ -885,8 +957,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin()); // Generate the body of the function. - PGO.checkGlobalDecl(GD); - PGO.assignRegionCounters(GD.getDecl(), CurFn); + PGO.assignRegionCounters(GD, CurFn); if (isa<CXXDestructorDecl>(FD)) EmitDestructorBody(Args); else if (isa<CXXConstructorDecl>(FD)) @@ -1207,6 +1278,22 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, return; } + // If the branch has a condition wrapped by __builtin_unpredictable, + // create metadata that specifies that the branch is unpredictable. + // Don't bother if not optimizing because that metadata would not be used. + llvm::MDNode *Unpredictable = nullptr; + if (CGM.getCodeGenOpts().OptimizationLevel != 0) { + if (const CallExpr *Call = dyn_cast<CallExpr>(Cond)) { + const Decl *TargetDecl = Call->getCalleeDecl(); + if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) { + if (FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) { + llvm::MDBuilder MDHelper(getLLVMContext()); + Unpredictable = MDHelper.createUnpredictable(); + } + } + } + } + // Create branch weights based on the number of times we get here and the // number of times the condition should be true. uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount); @@ -1219,7 +1306,7 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, ApplyDebugLocation DL(*this, Cond); CondV = EvaluateExprAsBool(Cond); } - Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights); + Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable); } /// ErrorUnsupported - Print out an error that codegen doesn't support the @@ -1236,20 +1323,18 @@ void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) { /// base element of the array /// \param sizeInChars - the total size of the VLA, in chars static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, - llvm::Value *dest, llvm::Value *src, + Address dest, Address src, llvm::Value *sizeInChars) { - std::pair<CharUnits,CharUnits> baseSizeAndAlign - = CGF.getContext().getTypeInfoInChars(baseType); - CGBuilderTy &Builder = CGF.Builder; + CharUnits baseSize = CGF.getContext().getTypeSizeInChars(baseType); llvm::Value *baseSizeInChars - = llvm::ConstantInt::get(CGF.IntPtrTy, baseSizeAndAlign.first.getQuantity()); + = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity()); - llvm::Type *i8p = Builder.getInt8PtrTy(); - - llvm::Value *begin = Builder.CreateBitCast(dest, i8p, "vla.begin"); - llvm::Value *end = Builder.CreateInBoundsGEP(dest, sizeInChars, "vla.end"); + Address begin = + Builder.CreateElementBitCast(dest, CGF.Int8Ty, "vla.begin"); + llvm::Value *end = + Builder.CreateInBoundsGEP(begin.getPointer(), sizeInChars, "vla.end"); llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock(); llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop"); @@ -1259,17 +1344,19 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, // count must be nonzero. CGF.EmitBlock(loopBB); - llvm::PHINode *cur = Builder.CreatePHI(i8p, 2, "vla.cur"); - cur->addIncoming(begin, originBB); + llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur"); + cur->addIncoming(begin.getPointer(), originBB); + + CharUnits curAlign = + dest.getAlignment().alignmentOfArrayElement(baseSize); // memcpy the individual element bit-pattern. - Builder.CreateMemCpy(cur, src, baseSizeInChars, - baseSizeAndAlign.second.getQuantity(), + Builder.CreateMemCpy(Address(cur, curAlign), src, baseSizeInChars, /*volatile*/ false); // Go to the next element. - llvm::Value *next = Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), - cur, 1, "vla.next"); + llvm::Value *next = + Builder.CreateInBoundsGEP(CGF.Int8Ty, cur, baseSizeInChars, "vla.next"); // Leave if that's the end of the VLA. llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone"); @@ -1280,7 +1367,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, } void -CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) { +CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) { // Ignore empty classes in C++. if (getLangOpts().CPlusPlus) { if (const RecordType *RT = Ty->getAs<RecordType>()) { @@ -1290,23 +1377,17 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) { } // Cast the dest ptr to the appropriate i8 pointer type. - unsigned DestAS = - cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace(); - llvm::Type *BP = Builder.getInt8PtrTy(DestAS); - if (DestPtr->getType() != BP) - DestPtr = Builder.CreateBitCast(DestPtr, BP); + if (DestPtr.getElementType() != Int8Ty) + DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty); // Get size and alignment info for this aggregate. - std::pair<CharUnits, CharUnits> TypeInfo = - getContext().getTypeInfoInChars(Ty); - CharUnits Size = TypeInfo.first; - CharUnits Align = TypeInfo.second; + CharUnits size = getContext().getTypeSizeInChars(Ty); llvm::Value *SizeVal; const VariableArrayType *vla; // Don't bother emitting a zero-byte memset. - if (Size.isZero()) { + if (size.isZero()) { // But note that getTypeInfo returns 0 for a VLA. if (const VariableArrayType *vlaType = dyn_cast_or_null<VariableArrayType>( @@ -1324,7 +1405,7 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) { return; } } else { - SizeVal = CGM.getSize(Size); + SizeVal = CGM.getSize(size); vla = nullptr; } @@ -1343,21 +1424,22 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) { /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, NullConstant, Twine()); - llvm::Value *SrcPtr = - Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()); + CharUnits NullAlign = DestPtr.getAlignment(); + NullVariable->setAlignment(NullAlign.getQuantity()); + Address SrcPtr(Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()), + NullAlign); if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal); // Get and call the appropriate llvm.memcpy overload. - Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, Align.getQuantity(), false); + Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false); return; } // Otherwise, just memset the whole thing to zero. This is legal // because in LLVM, all default initializers (other than the ones we just // handled above) are guaranteed to have a bit pattern of all zeros. - Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, - Align.getQuantity(), false); + Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false); } llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) { @@ -1376,7 +1458,7 @@ llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() { // If we already made the indirect branch for indirect goto, return its block. if (IndirectBranch) return IndirectBranch->getParent(); - CGBuilderTy TmpBuilder(createBasicBlock("indirectgoto")); + CGBuilderTy TmpBuilder(*this, createBasicBlock("indirectgoto")); // Create the PHI node that indirect gotos will add entries to. llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, 0, @@ -1391,7 +1473,7 @@ llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() { /// element type and a properly-typed first element pointer. llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, QualType &baseType, - llvm::Value *&addr) { + Address &addr) { const ArrayType *arrayType = origArrayType; // If it's a VLA, we have to load the stored size. Note that @@ -1430,8 +1512,7 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, QualType eltType; llvm::ArrayType *llvmArrayType = - dyn_cast<llvm::ArrayType>( - cast<llvm::PointerType>(addr->getType())->getElementType()); + dyn_cast<llvm::ArrayType>(addr.getElementType()); while (llvmArrayType) { assert(isa<ConstantArrayType>(arrayType)); assert(cast<ConstantArrayType>(arrayType)->getSize().getZExtValue() @@ -1459,12 +1540,13 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, arrayType = getContext().getAsArrayType(eltType); } - unsigned AddressSpace = addr->getType()->getPointerAddressSpace(); - llvm::Type *BaseType = ConvertType(eltType)->getPointerTo(AddressSpace); - addr = Builder.CreateBitCast(addr, BaseType, "array.begin"); + llvm::Type *baseType = ConvertType(eltType); + addr = Builder.CreateElementBitCast(addr, baseType, "array.begin"); } else { // Create the actual GEP. - addr = Builder.CreateInBoundsGEP(addr, gepIndices, "array.begin"); + addr = Address(Builder.CreateInBoundsGEP(addr.getPointer(), + gepIndices, "array.begin"), + addr.getAlignment()); } baseType = eltType; @@ -1649,9 +1731,13 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { } while (type->isVariablyModifiedType()); } -llvm::Value* CodeGenFunction::EmitVAListRef(const Expr* E) { +Address CodeGenFunction::EmitVAListRef(const Expr* E) { if (getContext().getBuiltinVaListType()->isArrayType()) - return EmitScalarExpr(E); + return EmitPointerWithAlignment(E); + return EmitLValue(E).getAddress(); +} + +Address CodeGenFunction::EmitMSVAListRef(const Expr *E) { return EmitLValue(E).getAddress(); } @@ -1713,9 +1799,10 @@ void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) { I->getAnnotation(), D->getLocation()); } -llvm::Value *CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, - llvm::Value *V) { +Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, + Address Addr) { assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute"); + llvm::Value *V = Addr.getPointer(); llvm::Type *VTy = V->getType(); llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation, CGM.Int8PtrTy); @@ -1730,7 +1817,7 @@ llvm::Value *CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, V = Builder.CreateBitCast(V, VTy); } - return V; + return Address(V, Addr.getAlignment()); } CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { } @@ -1773,3 +1860,80 @@ template void CGBuilderInserter<PreserveNames>::InsertHelper( llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB, llvm::BasicBlock::iterator InsertPt) const; #undef PreserveNames + +static bool hasRequiredFeatures(const SmallVectorImpl<StringRef> &ReqFeatures, + CodeGenModule &CGM, const FunctionDecl *FD, + std::string &FirstMissing) { + // If there aren't any required features listed then go ahead and return. + if (ReqFeatures.empty()) + return false; + + // Now build up the set of caller features and verify that all the required + // features are there. + llvm::StringMap<bool> CallerFeatureMap; + CGM.getFunctionFeatureMap(CallerFeatureMap, FD); + + // If we have at least one of the features in the feature list return + // true, otherwise return false. + return std::all_of( + ReqFeatures.begin(), ReqFeatures.end(), [&](StringRef Feature) { + SmallVector<StringRef, 1> OrFeatures; + Feature.split(OrFeatures, "|"); + return std::any_of(OrFeatures.begin(), OrFeatures.end(), + [&](StringRef Feature) { + if (!CallerFeatureMap.lookup(Feature)) { + FirstMissing = Feature.str(); + return false; + } + return true; + }); + }); +} + +// Emits an error if we don't have a valid set of target features for the +// called function. +void CodeGenFunction::checkTargetFeatures(const CallExpr *E, + const FunctionDecl *TargetDecl) { + // Early exit if this is an indirect call. + if (!TargetDecl) + return; + + // Get the current enclosing function if it exists. If it doesn't + // we can't check the target features anyhow. + const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl); + if (!FD) + return; + + // Grab the required features for the call. For a builtin this is listed in + // the td file with the default cpu, for an always_inline function this is any + // listed cpu and any listed features. + unsigned BuiltinID = TargetDecl->getBuiltinID(); + std::string MissingFeature; + if (BuiltinID) { + SmallVector<StringRef, 1> ReqFeatures; + const char *FeatureList = + CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID); + // Return if the builtin doesn't have any required features. + if (!FeatureList || StringRef(FeatureList) == "") + return; + StringRef(FeatureList).split(ReqFeatures, ","); + if (!hasRequiredFeatures(ReqFeatures, CGM, FD, MissingFeature)) + CGM.getDiags().Report(E->getLocStart(), diag::err_builtin_needs_feature) + << TargetDecl->getDeclName() + << CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID); + + } else if (TargetDecl->hasAttr<TargetAttr>()) { + // Get the required features for the callee. + SmallVector<StringRef, 1> ReqFeatures; + llvm::StringMap<bool> CalleeFeatureMap; + CGM.getFunctionFeatureMap(CalleeFeatureMap, TargetDecl); + for (const auto &F : CalleeFeatureMap) { + // Only positive features are "required". + if (F.getValue()) + ReqFeatures.push_back(F.getKey()); + } + if (!hasRequiredFeatures(ReqFeatures, CGM, FD, MissingFeature)) + CGM.getDiags().Report(E->getLocStart(), diag::err_function_needs_feature) + << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature; + } +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h index f2bc402..b4a9186 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h @@ -24,6 +24,7 @@ #include "clang/AST/CharUnits.h" #include "clang/AST/ExprCXX.h" #include "clang/AST/ExprObjC.h" +#include "clang/AST/ExprOpenMP.h" #include "clang/AST/Type.h" #include "clang/Basic/ABI.h" #include "clang/Basic/CapturedStmt.h" @@ -80,6 +81,8 @@ class CGFunctionInfo; class CGRecordLayout; class CGBlockInfo; class CGCXXABI; +class BlockByrefHelpers; +class BlockByrefInfo; class BlockFlags; class BlockFieldFlags; @@ -159,9 +162,9 @@ public: /// ReturnBlock - Unified return block. JumpDest ReturnBlock; - /// ReturnValue - The temporary alloca to hold the return value. This is null - /// iff the function has no return value. - llvm::Value *ReturnValue; + /// ReturnValue - The temporary alloca to hold the return + /// value. This is invalid iff the function has no return value. + Address ReturnValue; /// AllocaInsertPoint - This is an instruction in the entry block before which /// we prefer to insert allocas. @@ -247,10 +250,6 @@ public: ~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; } }; - /// BoundsChecking - Emit run-time bounds checks. Higher values mean - /// potentially higher performance penalties. - unsigned char BoundsChecking; - /// \brief Sanitizers enabled for this function. SanitizerSet SanOpts; @@ -294,6 +293,8 @@ public: llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack; llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack; + llvm::Instruction *CurrentFuncletPad = nullptr; + /// Header for data within LifetimeExtendedCleanupStack. struct LifetimeExtendedCleanupHeader { /// The size of the following cleanup object. @@ -327,7 +328,7 @@ public: /// A stack of exception code slots. Entering an __except block pushes a slot /// on the stack and leaving pops one. The __exception_code() intrinsic loads /// a value from the top of the stack. - SmallVector<llvm::Value *, 1> SEHCodeSlotStack; + SmallVector<Address, 1> SEHCodeSlotStack; /// Value returned by __exception_info intrinsic. llvm::Value *SEHInfo = nullptr; @@ -373,6 +374,11 @@ public: /// Returns true inside SEH __try blocks. bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); } + /// Returns true while emitting a cleanuppad. + bool isCleanupPadScope() const { + return CurrentFuncletPad && isa<llvm::CleanupPadInst>(CurrentFuncletPad); + } + /// pushFullExprCleanup - Push a cleanup to be run at the end of the /// current full-expression. Safe against the possibility that /// we're currently inside a conditionally-evaluated expression. @@ -419,13 +425,12 @@ public: /// complete-object destructor of an object of the given type at the /// given address. Does nothing if T is not a C++ class type with a /// non-trivial destructor. - void PushDestructorCleanup(QualType T, llvm::Value *Addr); + void PushDestructorCleanup(QualType T, Address Addr); /// PushDestructorCleanup - Push a cleanup to call the /// complete-object variant of the given destructor on the object at /// the given address. - void PushDestructorCleanup(const CXXDestructorDecl *Dtor, - llvm::Value *Addr); + void PushDestructorCleanup(const CXXDestructorDecl *Dtor, Address Addr); /// PopCleanupBlock - Will pop the cleanup entry on the stack and /// process all branch fixups. @@ -555,13 +560,14 @@ public: void rescopeLabels(); }; + typedef llvm::DenseMap<const Decl *, Address> DeclMapTy; + /// \brief The scope used to remap some variables as private in the OpenMP /// loop body (or other captured region emitted without outlining), and to /// restore old vars back on exit. class OMPPrivateScope : public RunCleanupsScope { - typedef llvm::DenseMap<const VarDecl *, llvm::Value *> VarDeclMapTy; - VarDeclMapTy SavedLocals; - VarDeclMapTy SavedPrivates; + DeclMapTy SavedLocals; + DeclMapTy SavedPrivates; private: OMPPrivateScope(const OMPPrivateScope &) = delete; @@ -578,13 +584,30 @@ public: /// been privatized already. bool addPrivate(const VarDecl *LocalVD, - const std::function<llvm::Value *()> &PrivateGen) { + llvm::function_ref<Address()> PrivateGen) { assert(PerformCleanup && "adding private to dead scope"); - if (SavedLocals.count(LocalVD) > 0) return false; - SavedLocals[LocalVD] = CGF.LocalDeclMap.lookup(LocalVD); - CGF.LocalDeclMap.erase(LocalVD); - SavedPrivates[LocalVD] = PrivateGen(); - CGF.LocalDeclMap[LocalVD] = SavedLocals[LocalVD]; + + // Only save it once. + if (SavedLocals.count(LocalVD)) return false; + + // Copy the existing local entry to SavedLocals. + auto it = CGF.LocalDeclMap.find(LocalVD); + if (it != CGF.LocalDeclMap.end()) { + SavedLocals.insert({LocalVD, it->second}); + } else { + SavedLocals.insert({LocalVD, Address::invalid()}); + } + + // Generate the private entry. + Address Addr = PrivateGen(); + QualType VarTy = LocalVD->getType(); + if (VarTy->isReferenceType()) { + Address Temp = CGF.CreateMemTemp(VarTy); + CGF.Builder.CreateStore(Addr.getPointer(), Temp); + Addr = Temp; + } + SavedPrivates.insert({LocalVD, Addr}); + return true; } @@ -597,19 +620,14 @@ public: /// private copies. /// \return true if at least one variable was privatized, false otherwise. bool Privatize() { - for (auto VDPair : SavedPrivates) { - CGF.LocalDeclMap[VDPair.first] = VDPair.second; - } + copyInto(SavedPrivates, CGF.LocalDeclMap); SavedPrivates.clear(); return !SavedLocals.empty(); } void ForceCleanup() { RunCleanupsScope::ForceCleanup(); - // Remap vars back to the original values. - for (auto I : SavedLocals) { - CGF.LocalDeclMap[I.first] = I.second; - } + copyInto(SavedLocals, CGF.LocalDeclMap); SavedLocals.clear(); } @@ -618,6 +636,25 @@ public: if (PerformCleanup) ForceCleanup(); } + + private: + /// Copy all the entries in the source map over the corresponding + /// entries in the destination, which must exist. + static void copyInto(const DeclMapTy &src, DeclMapTy &dest) { + for (auto &pair : src) { + if (!pair.second.isValid()) { + dest.erase(pair.first); + continue; + } + + auto it = dest.find(pair.first); + if (it != dest.end()) { + it->second = pair.second; + } else { + dest.insert(pair); + } + } + } }; /// \brief Takes the old cleanup stack size and emits the cleanup blocks @@ -665,6 +702,7 @@ public: llvm::BasicBlock *getEHResumeBlock(bool isCleanup); llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope); + llvm::BasicBlock *getMSVCDispatchBlock(EHScopeStack::stable_iterator scope); /// An object to manage conditionally-evaluated expressions. class ConditionalEvaluation { @@ -697,10 +735,11 @@ public: /// one branch or the other of a conditional expression. bool isInConditionalBranch() const { return OutermostConditional != nullptr; } - void setBeforeOutermostConditional(llvm::Value *value, llvm::Value *addr) { + void setBeforeOutermostConditional(llvm::Value *value, Address addr) { assert(isInConditionalBranch()); llvm::BasicBlock *block = OutermostConditional->getStartingBlock(); - new llvm::StoreInst(value, addr, &block->back()); + auto store = new llvm::StoreInst(value, addr.getPointer(), &block->back()); + store->setAlignment(addr.getAlignment().getQuantity()); } /// An RAII object to record that we're evaluating a statement @@ -859,15 +898,6 @@ public: } }; - /// getByrefValueFieldNumber - Given a declaration, returns the LLVM field - /// number that holds the value. - std::pair<llvm::Type *, unsigned> - getByRefValueLLVMField(const ValueDecl *VD) const; - - /// BuildBlockByrefAddress - Computes address location of the - /// variable which is declared as __block. - llvm::Value *BuildBlockByrefAddress(llvm::Value *BaseAddr, - const VarDecl *V); private: CGDebugInfo *DebugInfo; bool DisableDebugInfo; @@ -884,9 +914,14 @@ private: /// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C /// decls. - typedef llvm::DenseMap<const Decl*, llvm::Value*> DeclMapTy; DeclMapTy LocalDeclMap; + /// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this + /// will contain a mapping from said ParmVarDecl to its implicit "object_size" + /// parameter. + llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2> + SizeArguments; + /// Track escaped local variables with auto storage. Used during SEH /// outlining to produce a call to llvm.localescape. llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals; @@ -984,7 +1019,7 @@ public: /// if we need to evaluate a CXXDefaultInitExpr within the evaluation. class FieldConstructionScope { public: - FieldConstructionScope(CodeGenFunction &CGF, llvm::Value *This) + FieldConstructionScope(CodeGenFunction &CGF, Address This) : CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) { CGF.CXXDefaultInitExprThis = This; } @@ -994,7 +1029,7 @@ public: private: CodeGenFunction &CGF; - llvm::Value *OldCXXDefaultInitExprThis; + Address OldCXXDefaultInitExprThis; }; /// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this' @@ -1002,16 +1037,20 @@ public: class CXXDefaultInitExprScope { public: CXXDefaultInitExprScope(CodeGenFunction &CGF) - : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue) { - CGF.CXXThisValue = CGF.CXXDefaultInitExprThis; + : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue), + OldCXXThisAlignment(CGF.CXXThisAlignment) { + CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getPointer(); + CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment(); } ~CXXDefaultInitExprScope() { CGF.CXXThisValue = OldCXXThisValue; + CGF.CXXThisAlignment = OldCXXThisAlignment; } public: CodeGenFunction &CGF; llvm::Value *OldCXXThisValue; + CharUnits OldCXXThisAlignment; }; private: @@ -1020,10 +1059,12 @@ private: ImplicitParamDecl *CXXABIThisDecl; llvm::Value *CXXABIThisValue; llvm::Value *CXXThisValue; + CharUnits CXXABIThisAlignment; + CharUnits CXXThisAlignment; /// The value of 'this' to use when evaluating CXXDefaultInitExprs within /// this expression. - llvm::Value *CXXDefaultInitExprThis; + Address CXXDefaultInitExprThis = Address::invalid(); /// CXXStructorImplicitParamDecl - When generating code for a constructor or /// destructor, this will hold the implicit argument (e.g. VTT). @@ -1042,10 +1083,9 @@ private: /// handling code. SourceLocation CurEHLocation; - /// ByrefValueInfoMap - For each __block variable, contains a pair of the LLVM - /// type as well as the field number that contains the actual data. - llvm::DenseMap<const ValueDecl *, std::pair<llvm::Type *, - unsigned> > ByRefValueInfo; + /// BlockByrefInfos - For each __block variable, contains + /// information about the layout of the variable. + llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos; llvm::BasicBlock *TerminateLandingPad; llvm::BasicBlock *TerminateHandler; @@ -1086,15 +1126,15 @@ public: /// Returns a pointer to the function's exception object and selector slot, /// which is assigned in every landing pad. - llvm::Value *getExceptionSlot(); - llvm::Value *getEHSelectorSlot(); + Address getExceptionSlot(); + Address getEHSelectorSlot(); /// Returns the contents of the function's exception object and selector /// slots. llvm::Value *getExceptionFromSlot(); llvm::Value *getSelectorFromSlot(); - llvm::Value *getNormalCleanupDestSlot(); + Address getNormalCleanupDestSlot(); llvm::BasicBlock *getUnreachableBlock() { if (!UnreachableBlock) { @@ -1121,38 +1161,41 @@ public: // Cleanups //===--------------------------------------------------------------------===// - typedef void Destroyer(CodeGenFunction &CGF, llvm::Value *addr, QualType ty); + typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty); void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin, - llvm::Value *arrayEndPointer, + Address arrayEndPointer, QualType elementType, + CharUnits elementAlignment, Destroyer *destroyer); void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin, llvm::Value *arrayEnd, QualType elementType, + CharUnits elementAlignment, Destroyer *destroyer); void pushDestroy(QualType::DestructionKind dtorKind, - llvm::Value *addr, QualType type); + Address addr, QualType type); void pushEHDestroy(QualType::DestructionKind dtorKind, - llvm::Value *addr, QualType type); - void pushDestroy(CleanupKind kind, llvm::Value *addr, QualType type, + Address addr, QualType type); + void pushDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray); - void pushLifetimeExtendedDestroy(CleanupKind kind, llvm::Value *addr, + void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray); void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete, llvm::Value *CompletePtr, QualType ElementType); - void pushStackRestore(CleanupKind kind, llvm::Value *SPMem); - void emitDestroy(llvm::Value *addr, QualType type, Destroyer *destroyer, + void pushStackRestore(CleanupKind kind, Address SPMem); + void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray); - llvm::Function *generateDestroyHelper(llvm::Constant *addr, QualType type, + llvm::Function *generateDestroyHelper(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray, const VarDecl *VD); void emitArrayDestroy(llvm::Value *begin, llvm::Value *end, - QualType type, Destroyer *destroyer, + QualType elementType, CharUnits elementAlign, + Destroyer *destroyer, bool checkZeroLength, bool useEHCleanup); Destroyer *getDestroyer(QualType::DestructionKind destructionKind); @@ -1203,8 +1246,6 @@ public: void generateObjCSetterBody(const ObjCImplementationDecl *classImpl, const ObjCPropertyImplDecl *propImpl, llvm::Constant *AtomicHelperFn); - bool IndirectObjCSetterArg(const CGFunctionInfo &FI); - bool IvarTypeWithAggrGCObjects(QualType Ty); //===--------------------------------------------------------------------===// // Block Bits @@ -1213,10 +1254,6 @@ public: llvm::Value *EmitBlockLiteral(const BlockExpr *); llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info); static void destroyBlockInfos(CGBlockInfo *info); - llvm::Constant *BuildDescriptorBlockDecl(const BlockExpr *, - const CGBlockInfo &Info, - llvm::StructType *, - llvm::Constant *BlockVarLayout); llvm::Function *GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo &Info, @@ -1238,15 +1275,22 @@ public: void emitByrefStructureInit(const AutoVarEmission &emission); void enterByrefCleanup(const AutoVarEmission &emission); - llvm::Value *LoadBlockStruct() { - assert(BlockPointer && "no block pointer set!"); - return BlockPointer; - } + void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum, + llvm::Value *ptr); + + Address LoadBlockStruct(); + Address GetAddrOfBlockDecl(const VarDecl *var, bool ByRef); - void AllocateBlockCXXThisPointer(const CXXThisExpr *E); - void AllocateBlockDecl(const DeclRefExpr *E); - llvm::Value *GetAddrOfBlockDecl(const VarDecl *var, bool ByRef); - llvm::Type *BuildByRefType(const VarDecl *var); + /// BuildBlockByrefAddress - Computes the location of the + /// data in a variable which is declared as __block. + Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V, + bool followForward = true); + Address emitBlockByrefAddress(Address baseAddr, + const BlockByrefInfo &info, + bool followForward, + const llvm::Twine &name); + + const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var); void GenerateCode(GlobalDecl GD, llvm::Function *Fn, const CGFunctionInfo &FnInfo); @@ -1290,6 +1334,8 @@ public: void EmitCallAndReturnForThunk(llvm::Value *Callee, const ThunkInfo *Thunk); + void FinishThunk(); + /// Emit a musttail call for a thunk with a potentially adjusted this pointer. void EmitMustTailThunk(const CXXMethodDecl *MD, llvm::Value *AdjustedThisPtr, llvm::Value *Callee); @@ -1308,27 +1354,34 @@ public: void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init, ArrayRef<VarDecl *> ArrayIndexes); - /// InitializeVTablePointer - Initialize the vtable pointer of the given - /// subobject. - /// - void InitializeVTablePointer(BaseSubobject Base, - const CXXRecordDecl *NearestVBase, - CharUnits OffsetFromNearestVBase, - const CXXRecordDecl *VTableClass); + /// Struct with all informations about dynamic [sub]class needed to set vptr. + struct VPtr { + BaseSubobject Base; + const CXXRecordDecl *NearestVBase; + CharUnits OffsetFromNearestVBase; + const CXXRecordDecl *VTableClass; + }; + + /// Initialize the vtable pointer of the given subobject. + void InitializeVTablePointer(const VPtr &vptr); + + typedef llvm::SmallVector<VPtr, 4> VPtrsVector; typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy; - void InitializeVTablePointers(BaseSubobject Base, - const CXXRecordDecl *NearestVBase, - CharUnits OffsetFromNearestVBase, - bool BaseIsNonVirtualPrimaryBase, - const CXXRecordDecl *VTableClass, - VisitedVirtualBasesSetTy& VBases); + VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass); + + void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase, + CharUnits OffsetFromNearestVBase, + bool BaseIsNonVirtualPrimaryBase, + const CXXRecordDecl *VTableClass, + VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs); void InitializeVTablePointers(const CXXRecordDecl *ClassDecl); /// GetVTablePtr - Return the Value of the vtable pointer member pointed /// to by This. - llvm::Value *GetVTablePtr(llvm::Value *This, llvm::Type *Ty); + llvm::Value *GetVTablePtr(Address This, llvm::Type *VTableTy, + const CXXRecordDecl *VTableClass); enum CFITypeCheckKind { CFITCK_VCall, @@ -1495,49 +1548,85 @@ public: // Helpers //===--------------------------------------------------------------------===// - LValue MakeAddrLValue(llvm::Value *V, QualType T, - CharUnits Alignment = CharUnits()) { - return LValue::MakeAddr(V, T, Alignment, getContext(), + LValue MakeAddrLValue(Address Addr, QualType T, + AlignmentSource AlignSource = AlignmentSource::Type) { + return LValue::MakeAddr(Addr, T, getContext(), AlignSource, CGM.getTBAAInfo(T)); } + LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, + AlignmentSource AlignSource = AlignmentSource::Type) { + return LValue::MakeAddr(Address(V, Alignment), T, getContext(), + AlignSource, CGM.getTBAAInfo(T)); + } + + LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T); LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T); + CharUnits getNaturalTypeAlignment(QualType T, + AlignmentSource *Source = nullptr, + bool forPointeeType = false); + CharUnits getNaturalPointeeTypeAlignment(QualType T, + AlignmentSource *Source = nullptr); + + Address EmitLoadOfReference(Address Ref, const ReferenceType *RefTy, + AlignmentSource *Source = nullptr); + LValue EmitLoadOfReferenceLValue(Address Ref, const ReferenceType *RefTy); /// CreateTempAlloca - This creates a alloca and inserts it into the entry /// block. The caller is responsible for setting an appropriate alignment on /// the alloca. llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp"); + Address CreateTempAlloca(llvm::Type *Ty, CharUnits align, + const Twine &Name = "tmp"); - /// InitTempAlloca - Provide an initial value for the given alloca. - void InitTempAlloca(llvm::AllocaInst *Alloca, llvm::Value *Value); + /// CreateDefaultAlignedTempAlloca - This creates an alloca with the + /// default ABI alignment of the given LLVM type. + /// + /// IMPORTANT NOTE: This is *not* generally the right alignment for + /// any given AST type that happens to have been lowered to the + /// given IR type. This should only ever be used for function-local, + /// IR-driven manipulations like saving and restoring a value. Do + /// not hand this address off to arbitrary IRGen routines, and especially + /// do not pass it as an argument to a function that might expect a + /// properly ABI-aligned value. + Address CreateDefaultAlignTempAlloca(llvm::Type *Ty, + const Twine &Name = "tmp"); + + /// InitTempAlloca - Provide an initial value for the given alloca which + /// will be observable at all locations in the function. + /// + /// The address should be something that was returned from one of + /// the CreateTempAlloca or CreateMemTemp routines, and the + /// initializer must be valid in the entry block (i.e. it must + /// either be a constant or an argument value). + void InitTempAlloca(Address Alloca, llvm::Value *Value); /// CreateIRTemp - Create a temporary IR object of the given type, with /// appropriate alignment. This routine should only be used when an temporary /// value needs to be stored into an alloca (for example, to avoid explicit /// PHI construction), but the type is the IR type, not the type appropriate /// for storing in memory. - llvm::AllocaInst *CreateIRTemp(QualType T, const Twine &Name = "tmp"); + /// + /// That is, this is exactly equivalent to CreateMemTemp, but calling + /// ConvertType instead of ConvertTypeForMem. + Address CreateIRTemp(QualType T, const Twine &Name = "tmp"); /// CreateMemTemp - Create a temporary memory object of the given type, with /// appropriate alignment. - llvm::AllocaInst *CreateMemTemp(QualType T, const Twine &Name = "tmp"); + Address CreateMemTemp(QualType T, const Twine &Name = "tmp"); + Address CreateMemTemp(QualType T, CharUnits Align, const Twine &Name = "tmp"); /// CreateAggTemp - Create a temporary memory object for the given /// aggregate type. AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp") { - CharUnits Alignment = getContext().getTypeAlignInChars(T); - return AggValueSlot::forAddr(CreateMemTemp(T, Name), Alignment, + return AggValueSlot::forAddr(CreateMemTemp(T, Name), T.getQualifiers(), AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased); } - /// CreateInAllocaTmp - Create a temporary memory object for the given - /// aggregate type. - AggValueSlot CreateInAllocaTmp(QualType T, const Twine &Name = "inalloca"); - /// Emit a cast to void* in the appropriate address space. llvm::Value *EmitCastToVoidPtr(llvm::Value *value); @@ -1560,7 +1649,12 @@ public: // EmitVAListRef - Emit a "reference" to a va_list; this is either the address // or the value of the expression, depending on how va_list is defined. - llvm::Value *EmitVAListRef(const Expr *E); + Address EmitVAListRef(const Expr *E); + + /// Emit a "reference" to a __builtin_ms_va_list; this is + /// always the value of the expression, because a __builtin_ms_va_list is a + /// pointer to a char. + Address EmitMSVAListRef(const Expr *E); /// EmitAnyExprToTemp - Similary to EmitAnyExpr(), however, the result will /// always be accessible even if no aggregate location is provided. @@ -1568,10 +1662,10 @@ public: /// EmitAnyExprToMem - Emits the code necessary to evaluate an /// arbitrary expression into the given memory location. - void EmitAnyExprToMem(const Expr *E, llvm::Value *Location, + void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer); - void EmitAnyExprToExn(const Expr *E, llvm::Value *Addr); + void EmitAnyExprToExn(const Expr *E, Address Addr); /// EmitExprAsInit - Emits the code necessary to initialize a /// location in memory with the given initializer. @@ -1591,19 +1685,15 @@ public: /// /// The difference to EmitAggregateCopy is that tail padding is not copied. /// This is required for correctness when assigning non-POD structures in C++. - void EmitAggregateAssign(llvm::Value *DestPtr, llvm::Value *SrcPtr, + void EmitAggregateAssign(Address DestPtr, Address SrcPtr, QualType EltTy) { bool IsVolatile = hasVolatileMember(EltTy); - EmitAggregateCopy(DestPtr, SrcPtr, EltTy, IsVolatile, CharUnits::Zero(), - true); + EmitAggregateCopy(DestPtr, SrcPtr, EltTy, IsVolatile, true); } - void EmitAggregateCopyCtor(llvm::Value *DestPtr, llvm::Value *SrcPtr, - QualType DestTy, QualType SrcTy) { - CharUnits DestTypeAlign = getContext().getTypeAlignInChars(DestTy); - CharUnits SrcTypeAlign = getContext().getTypeAlignInChars(SrcTy); + void EmitAggregateCopyCtor(Address DestPtr, Address SrcPtr, + QualType DestTy, QualType SrcTy) { EmitAggregateCopy(DestPtr, SrcPtr, SrcTy, /*IsVolatile=*/false, - std::min(DestTypeAlign, SrcTypeAlign), /*IsAssignment=*/false); } @@ -1613,20 +1703,16 @@ public: /// volatile. /// \param isAssignment - If false, allow padding to be copied. This often /// yields more efficient. - void EmitAggregateCopy(llvm::Value *DestPtr, llvm::Value *SrcPtr, + void EmitAggregateCopy(Address DestPtr, Address SrcPtr, QualType EltTy, bool isVolatile=false, - CharUnits Alignment = CharUnits::Zero(), bool isAssignment = false); - /// StartBlock - Start new block named N. If insert block is a dummy block - /// then reuse it. - void StartBlock(const char *N); - /// GetAddrOfLocalVar - Return the address of a local variable. - llvm::Value *GetAddrOfLocalVar(const VarDecl *VD) { - llvm::Value *Res = LocalDeclMap[VD]; - assert(Res && "Invalid argument to GetAddrOfLocalVar(), no decl!"); - return Res; + Address GetAddrOfLocalVar(const VarDecl *VD) { + auto it = LocalDeclMap.find(VD); + assert(it != LocalDeclMap.end() && + "Invalid argument to GetAddrOfLocalVar(), no decl!"); + return it->second; } /// getOpaqueLValueMapping - Given an opaque value expression (which @@ -1661,19 +1747,31 @@ public: /// EmitNullInitialization - Generate code to set a value of the given type to /// null, If the type contains data member pointers, they will be initialized /// to -1 in accordance with the Itanium C++ ABI. - void EmitNullInitialization(llvm::Value *DestPtr, QualType Ty); - - // EmitVAArg - Generate code to get an argument from the passed in pointer - // and update it accordingly. The return value is a pointer to the argument. + void EmitNullInitialization(Address DestPtr, QualType Ty); + + /// Emits a call to an LLVM variable-argument intrinsic, either + /// \c llvm.va_start or \c llvm.va_end. + /// \param ArgValue A reference to the \c va_list as emitted by either + /// \c EmitVAListRef or \c EmitMSVAListRef. + /// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise, + /// calls \c llvm.va_end. + llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart); + + /// Generate code to get an argument from the passed in pointer + /// and update it accordingly. + /// \param VE The \c VAArgExpr for which to generate code. + /// \param VAListAddr Receives a reference to the \c va_list as emitted by + /// either \c EmitVAListRef or \c EmitMSVAListRef. + /// \returns A pointer to the argument. // FIXME: We should be able to get rid of this method and use the va_arg // instruction in LLVM instead once it works well enough. - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty); + Address EmitVAArg(VAArgExpr *VE, Address &VAListAddr); /// emitArrayLength - Compute the length of an array, even if it's a /// VLA, and drill down to the base element type. llvm::Value *emitArrayLength(const ArrayType *arrayType, QualType &baseType, - llvm::Value *&addr); + Address &addr); /// EmitVLASize - Capture all the sizes for the VLA expressions in /// the given variably-modified type and store them in the VLASizeMap. @@ -1694,6 +1792,7 @@ public: assert(CXXThisValue && "no 'this' value for this function"); return CXXThisValue; } + Address LoadCXXThisAddress(); /// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have /// virtual bases. @@ -1704,35 +1803,29 @@ public: return CXXStructorImplicitParamValue; } - /// LoadCXXStructorImplicitParam - Load the implicit parameter - /// for a constructor/destructor. - llvm::Value *LoadCXXStructorImplicitParam() { - assert(CXXStructorImplicitParamValue && - "no implicit argument value for this function"); - return CXXStructorImplicitParamValue; - } - /// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a /// complete class to the given direct base. - llvm::Value * - GetAddressOfDirectBaseInCompleteClass(llvm::Value *Value, + Address + GetAddressOfDirectBaseInCompleteClass(Address Value, const CXXRecordDecl *Derived, const CXXRecordDecl *Base, bool BaseIsVirtual); + static bool ShouldNullCheckClassCastValue(const CastExpr *Cast); + /// GetAddressOfBaseClass - This function will add the necessary delta to the /// load of 'this' and returns address of the base class. - llvm::Value *GetAddressOfBaseClass(llvm::Value *Value, - const CXXRecordDecl *Derived, - CastExpr::path_const_iterator PathBegin, - CastExpr::path_const_iterator PathEnd, - bool NullCheckValue, SourceLocation Loc); - - llvm::Value *GetAddressOfDerivedClass(llvm::Value *Value, - const CXXRecordDecl *Derived, - CastExpr::path_const_iterator PathBegin, - CastExpr::path_const_iterator PathEnd, - bool NullCheckValue); + Address GetAddressOfBaseClass(Address Value, + const CXXRecordDecl *Derived, + CastExpr::path_const_iterator PathBegin, + CastExpr::path_const_iterator PathEnd, + bool NullCheckValue, SourceLocation Loc); + + Address GetAddressOfDerivedClass(Address Value, + const CXXRecordDecl *Derived, + CastExpr::path_const_iterator PathBegin, + CastExpr::path_const_iterator PathEnd, + bool NullCheckValue); /// GetVTTParameter - Return the VTT parameter that should be passed to a /// base constructor/destructor with virtual bases. @@ -1751,23 +1844,31 @@ public: // they are substantially the same. void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor, const FunctionArgList &Args); + void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type, bool ForVirtualBase, bool Delegating, - llvm::Value *This, const CXXConstructExpr *E); + Address This, const CXXConstructExpr *E); + + /// Emit assumption load for all bases. Requires to be be called only on + /// most-derived class and not under construction of the object. + void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This); + + /// Emit assumption that vptr load == global vtable. + void EmitVTableAssumptionLoad(const VPtr &vptr, Address This); void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, - llvm::Value *This, llvm::Value *Src, - const CXXConstructExpr *E); + Address This, Address Src, + const CXXConstructExpr *E); void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D, const ConstantArrayType *ArrayTy, - llvm::Value *ArrayPtr, + Address ArrayPtr, const CXXConstructExpr *E, bool ZeroInitialization = false); void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D, llvm::Value *NumElements, - llvm::Value *ArrayPtr, + Address ArrayPtr, const CXXConstructExpr *E, bool ZeroInitialization = false); @@ -1775,15 +1876,15 @@ public: void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type, bool ForVirtualBase, bool Delegating, - llvm::Value *This); + Address This); void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType, - llvm::Type *ElementTy, llvm::Value *NewPtr, + llvm::Type *ElementTy, Address NewPtr, llvm::Value *NumElements, llvm::Value *AllocSizeWithoutCookie); void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType, - llvm::Value *Ptr); + Address Ptr); llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr); void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr); @@ -1797,9 +1898,9 @@ public: RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, const Expr *Arg, bool IsDelete); - llvm::Value* EmitCXXTypeidExpr(const CXXTypeidExpr *E); - llvm::Value *EmitDynamicCast(llvm::Value *V, const CXXDynamicCastExpr *DCE); - llvm::Value* EmitCXXUuidofExpr(const CXXUuidofExpr *E); + llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E); + llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE); + Address EmitCXXUuidofExpr(const CXXUuidofExpr *E); /// \brief Situations in which we might emit a check for the suitability of a /// pointer or glvalue. @@ -1896,12 +1997,9 @@ public: const VarDecl *Variable; - /// The alignment of the variable. - CharUnits Alignment; - - /// The address of the alloca. Null if the variable was emitted + /// The address of the alloca. Invalid if the variable was emitted /// as a global constant. - llvm::Value *Address; + Address Addr; llvm::Value *NRVOFlag; @@ -1916,14 +2014,14 @@ public: llvm::Value *SizeForLifetimeMarkers; struct Invalid {}; - AutoVarEmission(Invalid) : Variable(nullptr) {} + AutoVarEmission(Invalid) : Variable(nullptr), Addr(Address::invalid()) {} AutoVarEmission(const VarDecl &variable) - : Variable(&variable), Address(nullptr), NRVOFlag(nullptr), + : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr), IsByRef(false), IsConstantAggregate(false), SizeForLifetimeMarkers(nullptr) {} - bool wasEmittedAsGlobal() const { return Address == nullptr; } + bool wasEmittedAsGlobal() const { return !Addr.isValid(); } public: static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); } @@ -1938,19 +2036,17 @@ public: /// Returns the raw, allocated address, which is not necessarily /// the address of the object itself. - llvm::Value *getAllocatedAddress() const { - return Address; + Address getAllocatedAddress() const { + return Addr; } /// Returns the address of the object within this declaration. /// Note that this does not chase the forwarding pointer for /// __block decls. - llvm::Value *getObjectAddress(CodeGenFunction &CGF) const { - if (!IsByRef) return Address; + Address getObjectAddress(CodeGenFunction &CGF) const { + if (!IsByRef) return Addr; - auto F = CGF.getByRefValueLLVMField(Variable); - return CGF.Builder.CreateStructGEP(F.first, Address, F.second, - Variable->getNameAsString()); + return CGF.emitBlockByrefAddress(Addr, Variable, /*forward*/ false); } }; AutoVarEmission EmitAutoVarAlloca(const VarDecl &var); @@ -1962,9 +2058,35 @@ public: void EmitStaticVarDecl(const VarDecl &D, llvm::GlobalValue::LinkageTypes Linkage); + class ParamValue { + llvm::Value *Value; + unsigned Alignment; + ParamValue(llvm::Value *V, unsigned A) : Value(V), Alignment(A) {} + public: + static ParamValue forDirect(llvm::Value *value) { + return ParamValue(value, 0); + } + static ParamValue forIndirect(Address addr) { + assert(!addr.getAlignment().isZero()); + return ParamValue(addr.getPointer(), addr.getAlignment().getQuantity()); + } + + bool isIndirect() const { return Alignment != 0; } + llvm::Value *getAnyValue() const { return Value; } + + llvm::Value *getDirectValue() const { + assert(!isIndirect()); + return Value; + } + + Address getIndirectAddress() const { + assert(isIndirect()); + return Address(Value, CharUnits::fromQuantity(Alignment)); + } + }; + /// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl. - void EmitParmDecl(const VarDecl &D, llvm::Value *Arg, bool ArgIsPointer, - unsigned ArgNo); + void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo); /// protectFromPeepholes - Protect a value that we're intending to /// store to the side, but which will probably be used later, from @@ -2001,11 +2123,11 @@ public: /// \return True if the statement was handled. bool EmitSimpleStmt(const Stmt *S); - llvm::Value *EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false, - AggValueSlot AVS = AggValueSlot::ignored()); - llvm::Value *EmitCompoundStmtWithoutScope(const CompoundStmt &S, - bool GetLast = false, - AggValueSlot AVS = + Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false, + AggValueSlot AVS = AggValueSlot::ignored()); + Address EmitCompoundStmtWithoutScope(const CompoundStmt &S, + bool GetLast = false, + AggValueSlot AVS = AggValueSlot::ignored()); /// EmitLabel - Emit the block for the given label. It is legal to call this @@ -2018,8 +2140,6 @@ public: void EmitIndirectGotoStmt(const IndirectGotoStmt &S); void EmitIfStmt(const IfStmt &S); - void EmitCondBrHints(llvm::LLVMContext &Context, llvm::BranchInst *CondBr, - ArrayRef<const Attr *> Attrs); void EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> Attrs = None); void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = None); @@ -2077,19 +2197,20 @@ public: /// either be an alloca or a call to llvm.localrecover if there are nested /// outlined functions. ParentFP is the frame pointer of the outermost parent /// frame. - llvm::Value *recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, - llvm::Value *ParentVar, - llvm::Value *ParentFP); + Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, + Address ParentVar, + llvm::Value *ParentFP); void EmitCXXForRangeStmt(const CXXForRangeStmt &S, ArrayRef<const Attr *> Attrs = None); LValue InitCapturedStruct(const CapturedStmt &S); llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K); - void GenerateCapturedStmtFunctionProlog(const CapturedStmt &S); - llvm::Function *GenerateCapturedStmtFunctionEpilog(const CapturedStmt &S); llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S); - llvm::Value *GenerateCapturedStmtArgument(const CapturedStmt &S); + Address GenerateCapturedStmtArgument(const CapturedStmt &S); + llvm::Function *GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S); + void GenerateOpenMPCapturedVars(const CapturedStmt &S, + SmallVectorImpl<llvm::Value *> &CapturedVars); /// \brief Perform element by element copying of arrays with type \a /// OriginalType from \a SrcAddr to \a DestAddr using copying procedure /// generated by \a CopyGen. @@ -2100,8 +2221,8 @@ public: /// \param CopyGen Copying procedure that copies value of single array element /// to another single array element. void EmitOMPAggregateAssign( - llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType, - const llvm::function_ref<void(llvm::Value *, llvm::Value *)> &CopyGen); + Address DestAddr, Address SrcAddr, QualType OriginalType, + const llvm::function_ref<void(Address, Address)> &CopyGen); /// \brief Emit proper copying of data from one variable to another. /// /// \param OriginalType Original type of the copied variables. @@ -2113,8 +2234,8 @@ public: /// the base array element). /// \param Copy Actual copygin expression for copying data from \a SrcVD to \a /// DestVD. - void EmitOMPCopy(CodeGenFunction &CGF, QualType OriginalType, - llvm::Value *DestAddr, llvm::Value *SrcAddr, + void EmitOMPCopy(QualType OriginalType, + Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy); /// \brief Emit atomic update code for constructs: \a X = \a X \a BO \a E or @@ -2213,10 +2334,14 @@ public: void EmitOMPOrderedDirective(const OMPOrderedDirective &S); void EmitOMPAtomicDirective(const OMPAtomicDirective &S); void EmitOMPTargetDirective(const OMPTargetDirective &S); + void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S); void EmitOMPTeamsDirective(const OMPTeamsDirective &S); void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S); void EmitOMPCancelDirective(const OMPCancelDirective &S); + void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S); + void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S); + void EmitOMPDistributeDirective(const OMPDistributeDirective &S); /// \brief Emit inner loop of the worksharing/simd construct. /// @@ -2249,8 +2374,8 @@ private: void EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind, const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered, - llvm::Value *LB, llvm::Value *UB, llvm::Value *ST, - llvm::Value *IL, llvm::Value *Chunk); + Address LB, Address UB, Address ST, + Address IL, llvm::Value *Chunk); /// \brief Emit code for sections directive. OpenMPDirectiveKind EmitSections(const OMPExecutableDirective &S); @@ -2297,7 +2422,7 @@ public: /// that the address will be used to access the object. LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK); - RValue convertTempToRValue(llvm::Value *addr, QualType type, + RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc); void EmitAtomicInit(Expr *E, LValue lvalue); @@ -2338,12 +2463,14 @@ public: /// EmitLoadOfScalar - Load a scalar value from an address, taking /// care to appropriately convert from the memory representation to /// the LLVM value representation. - llvm::Value *EmitLoadOfScalar(llvm::Value *Addr, bool Volatile, - unsigned Alignment, QualType Ty, + llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, + AlignmentSource AlignSource = + AlignmentSource::Type, llvm::MDNode *TBAAInfo = nullptr, QualType TBAABaseTy = QualType(), - uint64_t TBAAOffset = 0); + uint64_t TBAAOffset = 0, + bool isNontemporal = false); /// EmitLoadOfScalar - Load a scalar value from an address, taking /// care to appropriately convert from the memory representation to @@ -2354,11 +2481,12 @@ public: /// EmitStoreOfScalar - Store a scalar value to an address, taking /// care to appropriately convert from the memory representation to /// the LLVM value representation. - void EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr, - bool Volatile, unsigned Alignment, QualType Ty, + void EmitStoreOfScalar(llvm::Value *Value, Address Addr, + bool Volatile, QualType Ty, + AlignmentSource AlignSource = AlignmentSource::Type, llvm::MDNode *TBAAInfo = nullptr, bool isInit = false, QualType TBAABaseTy = QualType(), - uint64_t TBAAOffset = 0); + uint64_t TBAAOffset = 0, bool isNontemporal = false); /// EmitStoreOfScalar - Store a scalar value to an address, taking /// care to appropriately convert from the memory representation to @@ -2405,13 +2533,14 @@ public: // Note: only available for agg return types LValue EmitVAArgExprLValue(const VAArgExpr *E); LValue EmitDeclRefLValue(const DeclRefExpr *E); - LValue EmitReadRegister(const VarDecl *VD); LValue EmitStringLiteralLValue(const StringLiteral *E); LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E); LValue EmitPredefinedLValue(const PredefinedExpr *E); LValue EmitUnaryOpLValue(const UnaryOperator *E); LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E, bool Accessed = false); + LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, + bool IsLowerBound = true); LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E); LValue EmitMemberExpr(const MemberExpr *E); LValue EmitObjCIsaExpr(const ObjCIsaExpr *E); @@ -2422,10 +2551,13 @@ public: LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E); LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e); - llvm::Value *EmitExtVectorElementLValue(LValue V); + Address EmitExtVectorElementLValue(LValue V); RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc); + Address EmitArrayToPointerDecay(const Expr *Array, + AlignmentSource *AlignSource = nullptr); + class ConstantEmission { llvm::PointerIntPair<llvm::Constant*, 1, bool> ValueAndIsReference; ConstantEmission(llvm::Constant *C, bool isReference) @@ -2497,23 +2629,20 @@ public: /// EmitCall - Generate a call of the given function, expecting the given /// result type, and using the given argument list which specifies both the /// LLVM arguments and the types they were derived from. - /// - /// \param TargetDecl - If given, the decl of the function in a direct call; - /// used to set attributes on the call (noreturn, etc.). - RValue EmitCall(const CGFunctionInfo &FnInfo, - llvm::Value *Callee, - ReturnValueSlot ReturnValue, - const CallArgList &Args, - const Decl *TargetDecl = nullptr, + RValue EmitCall(const CGFunctionInfo &FnInfo, llvm::Value *Callee, + ReturnValueSlot ReturnValue, const CallArgList &Args, + CGCalleeInfo CalleeInfo = CGCalleeInfo(), llvm::Instruction **callOrInvoke = nullptr); RValue EmitCall(QualType FnType, llvm::Value *Callee, const CallExpr *E, ReturnValueSlot ReturnValue, - const Decl *TargetDecl = nullptr, + CGCalleeInfo CalleeInfo = CGCalleeInfo(), llvm::Value *Chain = nullptr); RValue EmitCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue = ReturnValueSlot()); + void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl); + llvm::CallInst *EmitRuntimeCall(llvm::Value *callee, const Twine &name = ""); llvm::CallInst *EmitRuntimeCall(llvm::Value *callee, @@ -2528,8 +2657,6 @@ public: llvm::CallSite EmitCallOrInvoke(llvm::Value *Callee, ArrayRef<llvm::Value *> Args, const Twine &Name = ""); - llvm::CallSite EmitCallOrInvoke(llvm::Value *Callee, - const Twine &Name = ""); llvm::CallSite EmitRuntimeCallOrInvoke(llvm::Value *callee, ArrayRef<llvm::Value*> args, const Twine &name = ""); @@ -2565,6 +2692,10 @@ public: NestedNameSpecifier *Qualifier, bool IsArrow, const Expr *Base); // Compute the object pointer. + Address EmitCXXMemberDataPointerAddress(const Expr *E, Address base, + llvm::Value *memberPtr, + const MemberPointerType *memberPtrType, + AlignmentSource *AlignSource = nullptr); RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E, ReturnValueSlot ReturnValue); @@ -2599,7 +2730,7 @@ public: unsigned Modifier, const CallExpr *E, SmallVectorImpl<llvm::Value *> &Ops, - llvm::Value *Align = nullptr); + Address PtrOp0, Address PtrOp1); llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E); @@ -2612,8 +2743,6 @@ public: bool negateForRightShift); llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name); - // Helper functions for EmitAArch64BuiltinExpr. - llvm::Value *vectorWrapScalar8(llvm::Value *Op); llvm::Value *vectorWrapScalar16(llvm::Value *Op); llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E); @@ -2623,6 +2752,8 @@ public: llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E); + llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, + const CallExpr *E); llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E); llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E); @@ -2643,24 +2774,23 @@ public: } // ARC primitives. - void EmitARCInitWeak(llvm::Value *value, llvm::Value *addr); - void EmitARCDestroyWeak(llvm::Value *addr); - llvm::Value *EmitARCLoadWeak(llvm::Value *addr); - llvm::Value *EmitARCLoadWeakRetained(llvm::Value *addr); - llvm::Value *EmitARCStoreWeak(llvm::Value *value, llvm::Value *addr, - bool ignored); - void EmitARCCopyWeak(llvm::Value *dst, llvm::Value *src); - void EmitARCMoveWeak(llvm::Value *dst, llvm::Value *src); + void EmitARCInitWeak(Address addr, llvm::Value *value); + void EmitARCDestroyWeak(Address addr); + llvm::Value *EmitARCLoadWeak(Address addr); + llvm::Value *EmitARCLoadWeakRetained(Address addr); + llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored); + void EmitARCCopyWeak(Address dst, Address src); + void EmitARCMoveWeak(Address dst, Address src); llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value); llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value); llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value, bool resultIgnored); - llvm::Value *EmitARCStoreStrongCall(llvm::Value *addr, llvm::Value *value, + llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value, bool resultIgnored); llvm::Value *EmitARCRetain(QualType type, llvm::Value *value); llvm::Value *EmitARCRetainNonBlock(llvm::Value *value); llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory); - void EmitARCDestroyStrong(llvm::Value *addr, ARCPreciseLifetime_t precise); + void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise); void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise); llvm::Value *EmitARCAutorelease(llvm::Value *value); llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value); @@ -2673,8 +2803,6 @@ public: EmitARCStoreStrong(const BinaryOperator *e, bool ignored); llvm::Value *EmitObjCThrowOperand(const Expr *expr); - - llvm::Value *EmitObjCProduceObject(QualType T, llvm::Value *Ptr); llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr); llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr); @@ -2707,17 +2835,16 @@ public: /// scalar type, returning the result. llvm::Value *EmitScalarExpr(const Expr *E , bool IgnoreResultAssign = false); - /// EmitScalarConversion - Emit a conversion from the specified type to the - /// specified destination type, both of which are LLVM scalar types. + /// Emit a conversion from the specified type to the specified destination + /// type, both of which are LLVM scalar types. llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy, - QualType DstTy); + QualType DstTy, SourceLocation Loc); - /// EmitComplexToScalarConversion - Emit a conversion from the specified - /// complex type to the specified destination type, where the destination type - /// is an LLVM scalar type. + /// Emit a conversion from the specified complex type to the specified + /// destination type, where the destination type is an LLVM scalar type. llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, - QualType DstTy); - + QualType DstTy, + SourceLocation Loc); /// EmitAggExpr - Emit the computation of the specified expression /// of aggregate type. The result is computed into the given slot, @@ -2728,11 +2855,6 @@ public: /// aggregate type into a temporary LValue. LValue EmitAggExprToLValue(const Expr *E); - /// EmitGCMemmoveCollectable - Emit special API for structs with object - /// pointers. - void EmitGCMemmoveCollectable(llvm::Value *DestPtr, llvm::Value *SrcPtr, - QualType Ty); - /// EmitExtendGCLifetime - Given a pointer to an Objective-C object, /// make sure it survives garbage collection until this point. void EmitExtendGCLifetime(llvm::Value *object); @@ -2753,6 +2875,9 @@ public: /// EmitLoadOfComplex - Load a complex number from the specified l-value. ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc); + Address emitAddrOfRealComponent(Address complex, QualType complexType); + Address emitAddrOfImagComponent(Address complex, QualType complexType); + /// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the /// global variable that has already been created for it. If the initializer /// has a different type than GV does, this may free GV and return a different @@ -2787,7 +2912,7 @@ public: /// variables. void GenerateCXXGlobalInitFunc(llvm::Function *Fn, ArrayRef<llvm::Function *> CXXThreadLocals, - llvm::GlobalVariable *Guard = nullptr); + Address Guard = Address::invalid()); /// GenerateCXXGlobalDtorsFunc - Generates code for destroying global /// variables. @@ -2802,8 +2927,7 @@ public: void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest); - void EmitSynthesizedCXXCopyCtor(llvm::Value *Dest, llvm::Value *Src, - const Expr *Exp); + void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp); void enterFullExpression(const ExprWithCleanups *E) { if (E->getNumObjects() == 0) return; @@ -2815,7 +2939,7 @@ public: void EmitLambdaExpr(const LambdaExpr *E, AggValueSlot Dest); - RValue EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest = nullptr); + RValue EmitAtomicExpr(AtomicExpr *E); //===--------------------------------------------------------------------===// // Annotations Emission @@ -2832,7 +2956,7 @@ public: /// Emit field annotations for the given field & value. Returns the /// annotation result. - llvm::Value *EmitFieldAnnotations(const FieldDecl *D, llvm::Value *V); + Address EmitFieldAnnotations(const FieldDecl *D, Address V); //===--------------------------------------------------------------------===// // Internal Helpers @@ -2885,6 +3009,11 @@ public: StringRef CheckName, ArrayRef<llvm::Constant *> StaticArgs, ArrayRef<llvm::Value *> DynamicArgs); + /// \brief Emit a slow path cross-DSO CFI check which calls __cfi_slowpath + /// if Cond if false. + void EmitCfiSlowPathCheck(llvm::Value *Cond, llvm::ConstantInt *TypeId, + llvm::Value *Ptr); + /// \brief Create a basic block that will call the trap intrinsic, and emit a /// conditional branch to it, for the -ftrapv checks. void EmitTrapCheck(llvm::Value *Checked); @@ -2920,6 +3049,12 @@ private: llvm::SmallVector<std::pair<llvm::Instruction *, llvm::Value *>, 4> DeferredReplacements; + /// Set the address of a local variable. + void setAddrOfLocalVar(const VarDecl *VD, Address Addr) { + assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!"); + LocalDeclMap.insert({VD, Addr}); + } + /// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty /// from function arguments into \arg Dst. See ABIArgInfo::Expand. /// @@ -2942,6 +3077,18 @@ private: std::string &ConstraintStr, SourceLocation Loc); + /// \brief Attempts to statically evaluate the object size of E. If that + /// fails, emits code to figure the size of E out for us. This is + /// pass_object_size aware. + llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type, + llvm::IntegerType *ResType); + + /// \brief Emits the size of E, as required by __builtin_object_size. This + /// function is aware of pass_object_size parameters, and will act accordingly + /// if E is a parameter with the pass_object_size attribute. + llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type, + llvm::IntegerType *ResType); + public: #ifndef NDEBUG // Determine whether the given argument is an Objective-C method @@ -2966,12 +3113,11 @@ public: /// EmitCallArgs - Emit call arguments for a function. template <typename T> void EmitCallArgs(CallArgList &Args, const T *CallArgTypeInfo, - CallExpr::const_arg_iterator ArgBeg, - CallExpr::const_arg_iterator ArgEnd, + llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange, const FunctionDecl *CalleeDecl = nullptr, unsigned ParamsToSkip = 0) { SmallVector<QualType, 16> ArgTypes; - CallExpr::const_arg_iterator Arg = ArgBeg; + CallExpr::const_arg_iterator Arg = ArgRange.begin(); assert((ParamsToSkip == 0 || CallArgTypeInfo) && "Can't skip parameters if type info is not provided"); @@ -2984,7 +3130,7 @@ public: for (auto I = CallArgTypeInfo->param_type_begin() + ParamsToSkip, E = CallArgTypeInfo->param_type_end(); I != E; ++I, ++Arg) { - assert(Arg != ArgEnd && "Running over edge of argument list!"); + assert(Arg != ArgRange.end() && "Running over edge of argument list!"); assert((isGenericMethod || ((*I)->isVariablyModifiedType() || (*I).getNonReferenceType()->isObjCRetainableType() || @@ -2992,7 +3138,7 @@ public: .getCanonicalType((*I).getNonReferenceType()) .getTypePtr() == getContext() - .getCanonicalType(Arg->getType()) + .getCanonicalType((*Arg)->getType()) .getTypePtr())) && "type mismatch in call argument!"); ArgTypes.push_back(*I); @@ -3001,23 +3147,46 @@ public: // Either we've emitted all the call args, or we have a call to variadic // function. - assert( - (Arg == ArgEnd || !CallArgTypeInfo || CallArgTypeInfo->isVariadic()) && - "Extra arguments in non-variadic function!"); + assert((Arg == ArgRange.end() || !CallArgTypeInfo || + CallArgTypeInfo->isVariadic()) && + "Extra arguments in non-variadic function!"); // If we still have any arguments, emit them using the type of the argument. - for (; Arg != ArgEnd; ++Arg) - ArgTypes.push_back(getVarArgType(*Arg)); + for (auto *A : llvm::make_range(Arg, ArgRange.end())) + ArgTypes.push_back(getVarArgType(A)); - EmitCallArgs(Args, ArgTypes, ArgBeg, ArgEnd, CalleeDecl, ParamsToSkip); + EmitCallArgs(Args, ArgTypes, ArgRange, CalleeDecl, ParamsToSkip); } void EmitCallArgs(CallArgList &Args, ArrayRef<QualType> ArgTypes, - CallExpr::const_arg_iterator ArgBeg, - CallExpr::const_arg_iterator ArgEnd, + llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange, const FunctionDecl *CalleeDecl = nullptr, unsigned ParamsToSkip = 0); + /// EmitPointerWithAlignment - Given an expression with a pointer + /// type, emit the value and compute our best estimate of the + /// alignment of the pointee. + /// + /// Note that this function will conservatively fall back on the type + /// when it doesn't + /// + /// \param Source - If non-null, this will be initialized with + /// information about the source of the alignment. Note that this + /// function will conservatively fall back on the type when it + /// doesn't recognize the expression, which means that sometimes + /// + /// a worst-case One + /// reasonable way to use this information is when there's a + /// language guarantee that the pointer must be aligned to some + /// stricter value, and we're simply trying to ensure that + /// sufficiently obvious uses of under-aligned objects don't get + /// miscompiled; for example, a placement new into the address of + /// a local variable. In such a case, it's quite reasonable to + /// just ignore the returned alignment when it isn't from an + /// explicit source. + Address EmitPointerWithAlignment(const Expr *Addr, + AlignmentSource *Source = nullptr); + private: QualType getVarArgType(const Expr *Arg); @@ -3027,16 +3196,11 @@ private: void EmitDeclMetadata(); - CodeGenModule::ByrefHelpers * - buildByrefHelpers(llvm::StructType &byrefType, - const AutoVarEmission &emission); + BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType, + const AutoVarEmission &emission); void AddObjCARCExceptionMetadata(llvm::Instruction *Inst); - /// GetPointeeAlignment - Given an expression with a pointer type, emit the - /// value and compute our best estimate of the alignment of the pointee. - std::pair<llvm::Value*, unsigned> EmitPointerWithAlignment(const Expr *Addr); - llvm::Value *GetValueForARMHint(unsigned BuiltinID); }; @@ -3059,17 +3223,23 @@ struct DominatingLLVMValue { static saved_type save(CodeGenFunction &CGF, llvm::Value *value) { if (!needsSaving(value)) return saved_type(value, false); - // Otherwise we need an alloca. - llvm::Value *alloca = - CGF.CreateTempAlloca(value->getType(), "cond-cleanup.save"); + // Otherwise, we need an alloca. + auto align = CharUnits::fromQuantity( + CGF.CGM.getDataLayout().getPrefTypeAlignment(value->getType())); + Address alloca = + CGF.CreateTempAlloca(value->getType(), align, "cond-cleanup.save"); CGF.Builder.CreateStore(value, alloca); - return saved_type(alloca, true); + return saved_type(alloca.getPointer(), true); } static llvm::Value *restore(CodeGenFunction &CGF, saved_type value) { + // If the value says it wasn't saved, trust that it's still dominating. if (!value.getInt()) return value.getPointer(); - return CGF.Builder.CreateLoad(value.getPointer()); + + // Otherwise, it should be an alloca instruction, as set up in save(). + auto alloca = cast<llvm::AllocaInst>(value.getPointer()); + return CGF.Builder.CreateAlignedLoad(alloca, alloca->getAlignment()); } }; @@ -3082,6 +3252,28 @@ template <class T> struct DominatingPointer<T,true> : DominatingLLVMValue { } }; +/// A specialization of DominatingValue for Address. +template <> struct DominatingValue<Address> { + typedef Address type; + + struct saved_type { + DominatingLLVMValue::saved_type SavedValue; + CharUnits Alignment; + }; + + static bool needsSaving(type value) { + return DominatingLLVMValue::needsSaving(value.getPointer()); + } + static saved_type save(CodeGenFunction &CGF, type value) { + return { DominatingLLVMValue::save(CGF, value.getPointer()), + value.getAlignment() }; + } + static type restore(CodeGenFunction &CGF, saved_type value) { + return Address(DominatingLLVMValue::restore(CGF, value.SavedValue), + value.Alignment); + } +}; + /// A specialization of DominatingValue for RValue. template <> struct DominatingValue<RValue> { typedef RValue type; @@ -3090,15 +3282,17 @@ template <> struct DominatingValue<RValue> { AggregateAddress, ComplexAddress }; llvm::Value *Value; - Kind K; - saved_type(llvm::Value *v, Kind k) : Value(v), K(k) {} + unsigned K : 3; + unsigned Align : 29; + saved_type(llvm::Value *v, Kind k, unsigned a = 0) + : Value(v), K(k), Align(a) {} public: static bool needsSaving(RValue value); static saved_type save(CodeGenFunction &CGF, RValue value); RValue restore(CodeGenFunction &CGF); - // implementations in CGExprCXX.cpp + // implementations in CGCleanup.cpp }; static bool needsSaving(type value) { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp index c9c48c7..173b0dc 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "CodeGenModule.h" +#include "CGBlocks.h" #include "CGCUDARuntime.h" #include "CGCXXABI.h" #include "CGCall.h" @@ -52,6 +53,7 @@ #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/ConvertUTF.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MD5.h" using namespace clang; using namespace CodeGen; @@ -64,8 +66,10 @@ static CGCXXABI *createCXXABI(CodeGenModule &CGM) { case TargetCXXABI::GenericARM: case TargetCXXABI::iOS: case TargetCXXABI::iOS64: + case TargetCXXABI::WatchOS: case TargetCXXABI::GenericMIPS: case TargetCXXABI::GenericItanium: + case TargetCXXABI::WebAssembly: return CreateItaniumCXXABI(CGM); case TargetCXXABI::Microsoft: return CreateMicrosoftCXXABI(CGM); @@ -77,17 +81,16 @@ static CGCXXABI *createCXXABI(CodeGenModule &CGM) { CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, const PreprocessorOptions &PPO, const CodeGenOptions &CGO, llvm::Module &M, - const llvm::DataLayout &TD, DiagnosticsEngine &diags, CoverageSourceInfo *CoverageInfo) : Context(C), LangOpts(C.getLangOpts()), HeaderSearchOpts(HSO), PreprocessorOpts(PPO), CodeGenOpts(CGO), TheModule(M), Diags(diags), - TheDataLayout(TD), Target(C.getTargetInfo()), ABI(createCXXABI(*this)), + Target(C.getTargetInfo()), ABI(createCXXABI(*this)), VMContext(M.getContext()), TBAA(nullptr), TheTargetCodeGenInfo(nullptr), Types(*this), VTables(*this), ObjCRuntime(nullptr), OpenCLRuntime(nullptr), OpenMPRuntime(nullptr), CUDARuntime(nullptr), - DebugInfo(nullptr), ARCData(nullptr), - NoObjCARCExceptionsMetadata(nullptr), RRData(nullptr), PGOReader(nullptr), + DebugInfo(nullptr), ObjCData(nullptr), + NoObjCARCExceptionsMetadata(nullptr), PGOReader(nullptr), CFConstantStringClassRef(nullptr), ConstantStringClassRef(nullptr), NSConstantStringType(nullptr), NSConcreteGlobalBlock(nullptr), NSConcreteStackBlock(nullptr), BlockObjectAssign(nullptr), @@ -106,7 +109,9 @@ CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, DoubleTy = llvm::Type::getDoubleTy(LLVMContext); PointerWidthInBits = C.getTargetInfo().getPointerWidth(0); PointerAlignInBytes = - C.toCharUnitsFromBits(C.getTargetInfo().getPointerAlign(0)).getQuantity(); + C.toCharUnitsFromBits(C.getTargetInfo().getPointerAlign(0)).getQuantity(); + IntAlignInBytes = + C.toCharUnitsFromBits(C.getTargetInfo().getIntAlign()).getQuantity(); IntTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getIntWidth()); IntPtrTy = llvm::IntegerType::get(LLVMContext, PointerWidthInBits); Int8PtrTy = Int8Ty->getPointerTo(0); @@ -139,9 +144,8 @@ CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, Block.GlobalUniqueCount = 0; - if (C.getLangOpts().ObjCAutoRefCount) - ARCData = new ARCEntrypoints(); - RRData = new RREntrypoints(); + if (C.getLangOpts().ObjC1) + ObjCData = new ObjCEntrypoints(); if (!CodeGenOpts.InstrProfileInput.empty()) { auto ReaderOrErr = @@ -169,8 +173,7 @@ CodeGenModule::~CodeGenModule() { delete TheTargetCodeGenInfo; delete TBAA; delete DebugInfo; - delete ARCData; - delete RRData; + delete ObjCData; } void CodeGenModule::createObjCRuntime() { @@ -186,6 +189,7 @@ void CodeGenModule::createObjCRuntime() { case ObjCRuntime::FragileMacOSX: case ObjCRuntime::MacOSX: case ObjCRuntime::iOS: + case ObjCRuntime::WatchOS: ObjCRuntime = CreateMacObjCRuntime(*this); return; } @@ -232,12 +236,27 @@ void CodeGenModule::applyReplacements() { OldF->replaceAllUsesWith(Replacement); if (NewF) { NewF->removeFromParent(); - OldF->getParent()->getFunctionList().insertAfter(OldF, NewF); + OldF->getParent()->getFunctionList().insertAfter(OldF->getIterator(), + NewF); } OldF->eraseFromParent(); } } +void CodeGenModule::addGlobalValReplacement(llvm::GlobalValue *GV, llvm::Constant *C) { + GlobalValReplacements.push_back(std::make_pair(GV, C)); +} + +void CodeGenModule::applyGlobalValReplacements() { + for (auto &I : GlobalValReplacements) { + llvm::GlobalValue *GV = I.first; + llvm::Constant *C = I.second; + + GV->replaceAllUsesWith(C); + GV->eraseFromParent(); + } +} + // This is only used in aliases that we created and we know they have a // linear structure. static const llvm::GlobalObject *getAliasedGlobal(const llvm::GlobalAlias &GA) { @@ -340,6 +359,7 @@ void InstrProfStats::reportDiagnostics(DiagnosticsEngine &Diags, void CodeGenModule::Release() { EmitDeferred(); + applyGlobalValReplacements(); applyReplacements(); checkAliases(); EmitCXXGlobalInitFunc(); @@ -355,8 +375,11 @@ void CodeGenModule::Release() { if (llvm::Function *CudaDtorFunction = CUDARuntime->makeModuleDtorFunction()) AddGlobalDtor(CudaDtorFunction); } - if (PGOReader && PGOStats.hasDiagnostics()) - PGOStats.reportDiagnostics(getDiags(), getCodeGenOpts().MainFileName); + if (PGOReader) { + getModule().setMaximumFunctionCount(PGOReader->getMaximumFunctionCount()); + if (PGOStats.hasDiagnostics()) + PGOStats.reportDiagnostics(getDiags(), getCodeGenOpts().MainFileName); + } EmitCtorList(GlobalCtors, "llvm.global_ctors"); EmitCtorList(GlobalDtors, "llvm.global_dtors"); EmitGlobalAnnotations(); @@ -370,11 +393,32 @@ void CodeGenModule::Release() { (Context.getLangOpts().Modules || !LinkerOptionsMetadata.empty())) { EmitModuleLinkOptions(); } - if (CodeGenOpts.DwarfVersion) + if (CodeGenOpts.DwarfVersion) { // We actually want the latest version when there are conflicts. // We can change from Warning to Latest if such mode is supported. getModule().addModuleFlag(llvm::Module::Warning, "Dwarf Version", CodeGenOpts.DwarfVersion); + } + if (CodeGenOpts.EmitCodeView) { + // Indicate that we want CodeView in the metadata. + getModule().addModuleFlag(llvm::Module::Warning, "CodeView", 1); + } + if (CodeGenOpts.OptimizationLevel > 0 && CodeGenOpts.StrictVTablePointers) { + // We don't support LTO with 2 with different StrictVTablePointers + // FIXME: we could support it by stripping all the information introduced + // by StrictVTablePointers. + + getModule().addModuleFlag(llvm::Module::Error, "StrictVTablePointers",1); + + llvm::Metadata *Ops[2] = { + llvm::MDString::get(VMContext, "StrictVTablePointers"), + llvm::ConstantAsMetadata::get(llvm::ConstantInt::get( + llvm::Type::getInt32Ty(VMContext), 1))}; + + getModule().addModuleFlag(llvm::Module::Require, + "StrictVTablePointersRequirement", + llvm::MDNode::get(VMContext, Ops)); + } if (DebugInfo) // We support a single version in the linked module. The LLVM // parser will drop debug info with a different version number @@ -399,6 +443,11 @@ void CodeGenModule::Release() { getModule().addModuleFlag(llvm::Module::Error, "min_enum_size", EnumWidth); } + if (CodeGenOpts.SanitizeCfiCrossDso) { + // Indicate that we want cross-DSO control flow integrity checks. + getModule().addModuleFlag(llvm::Module::Override, "Cross-DSO CFI", 1); + } + if (uint32_t PLevel = Context.getLangOpts().PICLevel) { llvm::PICLevel::Level PL = llvm::PICLevel::Default; switch (PLevel) { @@ -450,12 +499,6 @@ llvm::MDNode *CodeGenModule::getTBAAStructInfo(QualType QTy) { return TBAA->getTBAAStructInfo(QTy); } -llvm::MDNode *CodeGenModule::getTBAAStructTypeInfo(QualType QTy) { - if (!TBAA) - return nullptr; - return TBAA->getTBAAStructTypeInfo(QTy); -} - llvm::MDNode *CodeGenModule::getTBAAStructTagInfo(QualType BaseTy, llvm::MDNode *AccessN, uint64_t O) { @@ -468,9 +511,9 @@ llvm::MDNode *CodeGenModule::getTBAAStructTagInfo(QualType BaseTy, /// and struct-path aware TBAA, the tag has the same format: /// base type, access type and offset. /// When ConvertTypeToTag is true, we create a tag based on the scalar type. -void CodeGenModule::DecorateInstruction(llvm::Instruction *Inst, - llvm::MDNode *TBAAInfo, - bool ConvertTypeToTag) { +void CodeGenModule::DecorateInstructionWithTBAA(llvm::Instruction *Inst, + llvm::MDNode *TBAAInfo, + bool ConvertTypeToTag) { if (ConvertTypeToTag && TBAA) Inst->setMetadata(llvm::LLVMContext::MD_tbaa, TBAA->getTBAAScalarTagInfo(TBAAInfo)); @@ -478,6 +521,16 @@ void CodeGenModule::DecorateInstruction(llvm::Instruction *Inst, Inst->setMetadata(llvm::LLVMContext::MD_tbaa, TBAAInfo); } +void CodeGenModule::DecorateInstructionWithInvariantGroup( + llvm::Instruction *I, const CXXRecordDecl *RD) { + llvm::Metadata *MD = CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); + auto *MetaDataNode = dyn_cast<llvm::MDNode>(MD); + // Check if we have to wrap MDString in MDNode. + if (!MetaDataNode) + MetaDataNode = llvm::MDNode::get(getLLVMContext(), MD); + I->setMetadata(llvm::LLVMContext::MD_invariant_group, MetaDataNode); +} + void CodeGenModule::Error(SourceLocation loc, StringRef message) { unsigned diagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error, "%0"); getDiags().Report(Context.getFullLoc(loc), diagID) << message; @@ -692,6 +745,21 @@ void CodeGenModule::setFunctionDLLStorageClass(GlobalDecl GD, llvm::Function *F) F->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass); } +llvm::ConstantInt * +CodeGenModule::CreateCfiIdForTypeMetadata(llvm::Metadata *MD) { + llvm::MDString *MDS = dyn_cast<llvm::MDString>(MD); + if (!MDS) return nullptr; + + llvm::MD5 md5; + llvm::MD5::MD5Result result; + md5.update(MDS->getString()); + md5.final(result); + uint64_t id = 0; + for (int i = 0; i < 8; ++i) + id |= static_cast<uint64_t>(result[i]) << (i * 8); + return llvm::ConstantInt::get(Int64Ty, id); +} + void CodeGenModule::setFunctionDefinitionAttributes(const FunctionDecl *D, llvm::Function *F) { setNonAliasAttributes(D, F); @@ -737,6 +805,21 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, if (!hasUnwindExceptions(LangOpts)) B.addAttribute(llvm::Attribute::NoUnwind); + if (LangOpts.getStackProtector() == LangOptions::SSPOn) + B.addAttribute(llvm::Attribute::StackProtect); + else if (LangOpts.getStackProtector() == LangOptions::SSPStrong) + B.addAttribute(llvm::Attribute::StackProtectStrong); + else if (LangOpts.getStackProtector() == LangOptions::SSPReq) + B.addAttribute(llvm::Attribute::StackProtectReq); + + if (!D) { + F->addAttributes(llvm::AttributeSet::FunctionIndex, + llvm::AttributeSet::get( + F->getContext(), + llvm::AttributeSet::FunctionIndex, B)); + return; + } + if (D->hasAttr<NakedAttr>()) { // Naked implies noinline: we should not be inlining such functions. B.addAttribute(llvm::Attribute::Naked); @@ -761,13 +844,6 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, if (D->hasAttr<MinSizeAttr>()) B.addAttribute(llvm::Attribute::MinSize); - if (LangOpts.getStackProtector() == LangOptions::SSPOn) - B.addAttribute(llvm::Attribute::StackProtect); - else if (LangOpts.getStackProtector() == LangOptions::SSPStrong) - B.addAttribute(llvm::Attribute::StackProtectStrong); - else if (LangOpts.getStackProtector() == LangOptions::SSPReq) - B.addAttribute(llvm::Attribute::StackProtectReq); - F->addAttributes(llvm::AttributeSet::FunctionIndex, llvm::AttributeSet::get( F->getContext(), llvm::AttributeSet::FunctionIndex, B)); @@ -778,10 +854,8 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, F->addFnAttr(llvm::Attribute::NoInline); // OptimizeNone wins over OptimizeForSize, MinSize, AlwaysInline. - assert(!F->hasFnAttribute(llvm::Attribute::OptimizeForSize) && - "OptimizeNone and OptimizeForSize on same function!"); - assert(!F->hasFnAttribute(llvm::Attribute::MinSize) && - "OptimizeNone and MinSize on same function!"); + F->removeFnAttr(llvm::Attribute::OptimizeForSize); + F->removeFnAttr(llvm::Attribute::MinSize); assert(!F->hasFnAttribute(llvm::Attribute::AlwaysInline) && "OptimizeNone and AlwaysInline on same function!"); @@ -800,19 +874,24 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, if (alignment) F->setAlignment(alignment); - // C++ ABI requires 2-byte alignment for member functions. - if (F->getAlignment() < 2 && isa<CXXMethodDecl>(D)) - F->setAlignment(2); + // Some C++ ABIs require 2-byte alignment for member functions, in order to + // reserve a bit for differentiating between virtual and non-virtual member + // functions. If the current target's C++ ABI requires this and this is a + // member function, set its alignment accordingly. + if (getTarget().getCXXABI().areMemberFunctionsAligned()) { + if (F->getAlignment() < 2 && isa<CXXMethodDecl>(D)) + F->setAlignment(2); + } } void CodeGenModule::SetCommonAttributes(const Decl *D, llvm::GlobalValue *GV) { - if (const auto *ND = dyn_cast<NamedDecl>(D)) + if (const auto *ND = dyn_cast_or_null<NamedDecl>(D)) setGlobalVisibility(GV, ND); else GV->setVisibility(llvm::GlobalValue::DefaultVisibility); - if (D->hasAttr<UsedAttr>()) + if (D && D->hasAttr<UsedAttr>()) addUsedGlobal(GV); } @@ -830,8 +909,9 @@ void CodeGenModule::setNonAliasAttributes(const Decl *D, llvm::GlobalObject *GO) { SetCommonAttributes(D, GO); - if (const SectionAttr *SA = D->getAttr<SectionAttr>()) - GO->setSection(SA->getName()); + if (D) + if (const SectionAttr *SA = D->getAttr<SectionAttr>()) + GO->setSection(SA->getName()); getTargetCodeGenInfo().setTargetAttributes(D, GO, *this); } @@ -872,6 +952,49 @@ static void setLinkageAndVisibilityForGV(llvm::GlobalValue *GV, } } +void CodeGenModule::CreateFunctionBitSetEntry(const FunctionDecl *FD, + llvm::Function *F) { + // Only if we are checking indirect calls. + if (!LangOpts.Sanitize.has(SanitizerKind::CFIICall)) + return; + + // Non-static class methods are handled via vtable pointer checks elsewhere. + if (isa<CXXMethodDecl>(FD) && !cast<CXXMethodDecl>(FD)->isStatic()) + return; + + // Additionally, if building with cross-DSO support... + if (CodeGenOpts.SanitizeCfiCrossDso) { + // Don't emit entries for function declarations. In cross-DSO mode these are + // handled with better precision at run time. + if (!FD->hasBody()) + return; + // Skip available_externally functions. They won't be codegen'ed in the + // current module anyway. + if (getContext().GetGVALinkageForFunction(FD) == GVA_AvailableExternally) + return; + } + + llvm::NamedMDNode *BitsetsMD = + getModule().getOrInsertNamedMetadata("llvm.bitsets"); + + llvm::Metadata *MD = CreateMetadataIdentifierForType(FD->getType()); + llvm::Metadata *BitsetOps[] = { + MD, llvm::ConstantAsMetadata::get(F), + llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(Int64Ty, 0))}; + BitsetsMD->addOperand(llvm::MDTuple::get(getLLVMContext(), BitsetOps)); + + // Emit a hash-based bit set entry for cross-DSO calls. + if (CodeGenOpts.SanitizeCfiCrossDso) { + if (auto TypeId = CreateCfiIdForTypeMetadata(MD)) { + llvm::Metadata *BitsetOps2[] = { + llvm::ConstantAsMetadata::get(TypeId), + llvm::ConstantAsMetadata::get(F), + llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(Int64Ty, 0))}; + BitsetsMD->addOperand(llvm::MDTuple::get(getLLVMContext(), BitsetOps2)); + } + } +} + void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, bool IsIncompleteFunction, bool IsThunk) { @@ -913,6 +1036,8 @@ void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, if (FD->isReplaceableGlobalAllocationFunction()) F->addAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::NoBuiltin); + + CreateFunctionBitSetEntry(FD, F); } void CodeGenModule::addUsedGlobal(llvm::GlobalValue *GV) { @@ -1104,9 +1229,16 @@ void CodeGenModule::EmitDeferred() { llvm::GlobalValue *GV = G.GV; G.GV = nullptr; - assert(!GV || GV == GetGlobalValue(getMangledName(D))); - if (!GV) - GV = GetGlobalValue(getMangledName(D)); + // We should call GetAddrOfGlobal with IsForDefinition set to true in order + // to get GlobalValue with exactly the type we need, not something that + // might had been created for another decl with the same mangled name but + // different type. + // FIXME: Support for variables is not implemented yet. + if (isa<FunctionDecl>(D.getDecl())) + GV = cast<llvm::GlobalValue>(GetAddrOfGlobal(D, /*IsForDefinition=*/true)); + else + if (!GV) + GV = GetGlobalValue(getMangledName(D)); // Check to see if we've already emitted this. This is necessary // for a couple of reasons: first, decls can end up in the @@ -1208,7 +1340,7 @@ bool CodeGenModule::isInSanitizerBlacklist(llvm::Function *Fn, if (SanitizerBL.isBlacklistedFunction(Fn->getName())) return true; // Blacklist by location. - if (!Loc.isInvalid()) + if (Loc.isValid()) return SanitizerBL.isBlacklistedLocation(Loc); // If location is unknown, this may be a compiler-generated function. Assume // it's located in the main file. @@ -1271,7 +1403,7 @@ bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) { return true; } -llvm::Constant *CodeGenModule::GetAddrOfUuidDescriptor( +ConstantAddress CodeGenModule::GetAddrOfUuidDescriptor( const CXXUuidofExpr* E) { // Sema has verified that IIDSource has a __declspec(uuid()), and that its // well-formed. @@ -1279,9 +1411,12 @@ llvm::Constant *CodeGenModule::GetAddrOfUuidDescriptor( std::string Name = "_GUID_" + Uuid.lower(); std::replace(Name.begin(), Name.end(), '-', '_'); + // Contains a 32-bit field. + CharUnits Alignment = CharUnits::fromQuantity(4); + // Look for an existing global. if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name)) - return GV; + return ConstantAddress(GV, Alignment); llvm::Constant *Init = EmitUuidofInitializer(Uuid); assert(Init && "failed to initialize as constant"); @@ -1291,20 +1426,22 @@ llvm::Constant *CodeGenModule::GetAddrOfUuidDescriptor( /*isConstant=*/true, llvm::GlobalValue::LinkOnceODRLinkage, Init, Name); if (supportsCOMDAT()) GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); - return GV; + return ConstantAddress(GV, Alignment); } -llvm::Constant *CodeGenModule::GetWeakRefReference(const ValueDecl *VD) { +ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) { const AliasAttr *AA = VD->getAttr<AliasAttr>(); assert(AA && "No alias?"); + CharUnits Alignment = getContext().getDeclAlign(VD); llvm::Type *DeclTy = getTypes().ConvertTypeForMem(VD->getType()); // See if there is already something with the target's name in the module. llvm::GlobalValue *Entry = GetGlobalValue(AA->getAliasee()); if (Entry) { unsigned AS = getContext().getTargetAddressSpace(VD->getType()); - return llvm::ConstantExpr::getBitCast(Entry, DeclTy->getPointerTo(AS)); + auto Ptr = llvm::ConstantExpr::getBitCast(Entry, DeclTy->getPointerTo(AS)); + return ConstantAddress(Ptr, Alignment); } llvm::Constant *Aliasee; @@ -1321,7 +1458,7 @@ llvm::Constant *CodeGenModule::GetWeakRefReference(const ValueDecl *VD) { F->setLinkage(llvm::Function::ExternalWeakLinkage); WeakRefReferences.insert(F); - return Aliasee; + return ConstantAddress(Aliasee, Alignment); } void CodeGenModule::EmitGlobal(GlobalDecl GD) { @@ -1435,7 +1572,7 @@ namespace { unsigned BuiltinID = FD->getBuiltinID(); if (!BuiltinID || !BI.isLibFunction(BuiltinID)) return true; - StringRef BuiltinName = BI.GetName(BuiltinID); + StringRef BuiltinName = BI.getName(BuiltinID); if (BuiltinName.startswith("__builtin_") && Name == BuiltinName.slice(strlen("__builtin_"), StringRef::npos)) { Result = true; @@ -1444,6 +1581,35 @@ namespace { return true; } }; + + struct DLLImportFunctionVisitor + : public RecursiveASTVisitor<DLLImportFunctionVisitor> { + bool SafeToInline = true; + + bool VisitVarDecl(VarDecl *VD) { + // A thread-local variable cannot be imported. + SafeToInline = !VD->getTLSKind(); + return SafeToInline; + } + + // Make sure we're not referencing non-imported vars or functions. + bool VisitDeclRefExpr(DeclRefExpr *E) { + ValueDecl *VD = E->getDecl(); + if (isa<FunctionDecl>(VD)) + SafeToInline = VD->hasAttr<DLLImportAttr>(); + else if (VarDecl *V = dyn_cast<VarDecl>(VD)) + SafeToInline = !V->hasGlobalStorage() || V->hasAttr<DLLImportAttr>(); + return SafeToInline; + } + bool VisitCXXDeleteExpr(CXXDeleteExpr *E) { + SafeToInline = E->getOperatorDelete()->hasAttr<DLLImportAttr>(); + return SafeToInline; + } + bool VisitCXXNewExpr(CXXNewExpr *E) { + SafeToInline = E->getOperatorNew()->hasAttr<DLLImportAttr>(); + return SafeToInline; + } + }; } // isTriviallyRecursive - Check if this function calls another @@ -1474,6 +1640,15 @@ CodeGenModule::shouldEmitFunction(GlobalDecl GD) { const auto *F = cast<FunctionDecl>(GD.getDecl()); if (CodeGenOpts.OptimizationLevel == 0 && !F->hasAttr<AlwaysInlineAttr>()) return false; + + if (F->hasAttr<DLLImportAttr>()) { + // Check whether it would be safe to inline this dllimport function. + DLLImportFunctionVisitor Visitor; + Visitor.TraverseFunctionDecl(const_cast<FunctionDecl*>(F)); + if (!Visitor.SafeToInline) + return false; + } + // PR9614. Avoid cases where the source code is lying to us. An available // externally function should have an equivalent function somewhere else, // but a function that calls itself is clearly not equivalent to the real @@ -1537,6 +1712,9 @@ void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) { llvm_unreachable("Invalid argument to EmitGlobalDefinition()"); } +static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old, + llvm::Function *NewFn); + /// GetOrCreateLLVMFunction - If the specified mangled name is not in the /// module, create and return an llvm Function with the specified type. If there /// is something in the module with the specified name, return it potentially @@ -1549,7 +1727,8 @@ CodeGenModule::GetOrCreateLLVMFunction(StringRef MangledName, llvm::Type *Ty, GlobalDecl GD, bool ForVTable, bool DontDefer, bool IsThunk, - llvm::AttributeSet ExtraAttrs) { + llvm::AttributeSet ExtraAttrs, + bool IsForDefinition) { const Decl *D = GD.getDecl(); // Lookup the entry, lazily creating it if necessary. @@ -1565,11 +1744,33 @@ CodeGenModule::GetOrCreateLLVMFunction(StringRef MangledName, if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>()) Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass); - if (Entry->getType()->getElementType() == Ty) + // If there are two attempts to define the same mangled name, issue an + // error. + if (IsForDefinition && !Entry->isDeclaration()) { + GlobalDecl OtherGD; + // Check that GD is not yet in ExplicitDefinitions is required to make + // sure that we issue an error only once. + if (lookupRepresentativeDecl(MangledName, OtherGD) && + (GD.getCanonicalDecl().getDecl() != + OtherGD.getCanonicalDecl().getDecl()) && + DiagnosedConflictingDefinitions.insert(GD).second) { + getDiags().Report(D->getLocation(), + diag::err_duplicate_mangled_name); + getDiags().Report(OtherGD.getDecl()->getLocation(), + diag::note_previous_definition); + } + } + + if ((isa<llvm::Function>(Entry) || isa<llvm::GlobalAlias>(Entry)) && + (Entry->getType()->getElementType() == Ty)) { return Entry; + } // Make sure the result is of the correct type. - return llvm::ConstantExpr::getBitCast(Entry, Ty->getPointerTo()); + // (If function is requested for a definition, we always need to create a new + // function, not just return a bitcast.) + if (!IsForDefinition) + return llvm::ConstantExpr::getBitCast(Entry, Ty->getPointerTo()); } // This function doesn't have a complete type (for example, the return @@ -1584,10 +1785,36 @@ CodeGenModule::GetOrCreateLLVMFunction(StringRef MangledName, FTy = llvm::FunctionType::get(VoidTy, false); IsIncompleteFunction = true; } - - llvm::Function *F = llvm::Function::Create(FTy, - llvm::Function::ExternalLinkage, - MangledName, &getModule()); + + llvm::Function *F = + llvm::Function::Create(FTy, llvm::Function::ExternalLinkage, + Entry ? StringRef() : MangledName, &getModule()); + + // If we already created a function with the same mangled name (but different + // type) before, take its name and add it to the list of functions to be + // replaced with F at the end of CodeGen. + // + // This happens if there is a prototype for a function (e.g. "int f()") and + // then a definition of a different type (e.g. "int f(int x)"). + if (Entry) { + F->takeName(Entry); + + // This might be an implementation of a function without a prototype, in + // which case, try to do special replacement of calls which match the new + // prototype. The really key thing here is that we also potentially drop + // arguments from the call site so as to make a direct call, which makes the + // inliner happier and suppresses a number of optimizer warnings (!) about + // dropping arguments. + if (!Entry->use_empty()) { + ReplaceUsesOfNonProtoTypeWithRealFunction(Entry, F); + Entry->removeDeadConstantUsers(); + } + + llvm::Constant *BC = llvm::ConstantExpr::getBitCast( + F, Entry->getType()->getElementType()->getPointerTo()); + addGlobalValReplacement(Entry, BC); + } + assert(F->getName() == MangledName && "name was uniqued!"); if (D) SetFunctionAttributes(GD, F, IsIncompleteFunction, IsThunk); @@ -1660,13 +1887,19 @@ CodeGenModule::GetOrCreateLLVMFunction(StringRef MangledName, llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty, bool ForVTable, - bool DontDefer) { + bool DontDefer, + bool IsForDefinition) { // If there was no specific requested type, just convert it now. - if (!Ty) - Ty = getTypes().ConvertType(cast<ValueDecl>(GD.getDecl())->getType()); - + if (!Ty) { + const auto *FD = cast<FunctionDecl>(GD.getDecl()); + auto CanonTy = Context.getCanonicalType(FD->getType()); + Ty = getTypes().ConvertFunctionType(CanonTy, FD); + } + StringRef MangledName = getMangledName(GD); - return GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer); + return GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer, + /*IsThunk=*/false, llvm::AttributeSet(), + IsForDefinition); } /// CreateRuntimeFunction - Create a new runtime function with the specified @@ -1781,7 +2014,7 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, if (D->getTLSKind()) { if (D->getTLSKind() == VarDecl::TLS_Dynamic) - CXXThreadLocals.push_back(std::make_pair(D, GV)); + CXXThreadLocals.push_back(D); setTLSMode(GV, *D); } @@ -1805,6 +2038,33 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, return GV; } +llvm::Constant * +CodeGenModule::GetAddrOfGlobal(GlobalDecl GD, + bool IsForDefinition) { + if (isa<CXXConstructorDecl>(GD.getDecl())) + return getAddrOfCXXStructor(cast<CXXConstructorDecl>(GD.getDecl()), + getFromCtorType(GD.getCtorType()), + /*FnInfo=*/nullptr, /*FnType=*/nullptr, + /*DontDefer=*/false, IsForDefinition); + else if (isa<CXXDestructorDecl>(GD.getDecl())) + return getAddrOfCXXStructor(cast<CXXDestructorDecl>(GD.getDecl()), + getFromDtorType(GD.getDtorType()), + /*FnInfo=*/nullptr, /*FnType=*/nullptr, + /*DontDefer=*/false, IsForDefinition); + else if (isa<CXXMethodDecl>(GD.getDecl())) { + auto FInfo = &getTypes().arrangeCXXMethodDeclaration( + cast<CXXMethodDecl>(GD.getDecl())); + auto Ty = getTypes().GetFunctionType(*FInfo); + return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false, + IsForDefinition); + } else if (isa<FunctionDecl>(GD.getDecl())) { + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); + llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); + return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false, + IsForDefinition); + } else + return GetAddrOfGlobalVar(cast<VarDecl>(GD.getDecl())); +} llvm::GlobalVariable * CodeGenModule::CreateOrReplaceCXXRuntimeVariable(StringRef Name, @@ -1893,8 +2153,8 @@ void CodeGenModule::EmitTentativeDefinition(const VarDecl *D) { } CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const { - return Context.toCharUnitsFromBits( - TheDataLayout.getTypeStoreSizeInBits(Ty)); + return Context.toCharUnitsFromBits( + getDataLayout().getTypeStoreSizeInBits(Ty)); } unsigned CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D, @@ -1986,7 +2246,18 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) { const VarDecl *InitDecl; const Expr *InitExpr = D->getAnyInitializer(InitDecl); - if (!InitExpr) { + // CUDA E.2.4.1 "__shared__ variables cannot have an initialization as part + // of their declaration." + if (getLangOpts().CPlusPlus && getLangOpts().CUDAIsDevice + && D->hasAttr<CUDASharedAttr>()) { + if (InitExpr) { + const auto *C = dyn_cast<CXXConstructExpr>(InitExpr); + if (C == nullptr || !C->getConstructor()->hasTrivialBody()) + Error(D->getLocation(), + "__shared__ variable cannot have an initialization."); + } + Init = llvm::UndefValue::get(getTypes().ConvertType(ASTTy)); + } else if (!InitExpr) { // This is a tentative definition; tentative definitions are // implicitly initialized with { 0 }. // @@ -2072,6 +2343,17 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) { if (D->hasAttr<AnnotateAttr>()) AddGlobalAnnotations(D, GV); + // CUDA B.2.1 "The __device__ qualifier declares a variable that resides on + // the device. [...]" + // CUDA B.2.2 "The __constant__ qualifier, optionally used together with + // __device__, declares a variable that: [...] + // Is accessible from all the threads within the grid and from the host + // through the runtime library (cudaGetSymbolAddress() / cudaGetSymbolSize() + // / cudaMemcpyToSymbol() / cudaMemcpyFromSymbol())." + if (GV && LangOpts.CUDA && LangOpts.CUDAIsDevice && + (D->hasAttr<CUDAConstantAttr>() || D->hasAttr<CUDADeviceAttr>())) { + GV->setExternallyInitialized(true); + } GV->setInitializer(Init); // If it is safe to mark the global 'constant', do so now. @@ -2091,12 +2373,17 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) { llvm::GlobalValue::LinkageTypes Linkage = getLLVMLinkageVarDefinition(D, GV->isConstant()); - // On Darwin, the backing variable for a C++11 thread_local variable always - // has internal linkage; all accesses should just be calls to the + // On Darwin, if the normal linkage of a C++ thread_local variable is + // LinkOnce or Weak, we keep the normal linkage to prevent multiple + // copies within a linkage unit; otherwise, the backing variable has + // internal linkage and all accesses should just be calls to the // Itanium-specified entry point, which has the normal linkage of the - // variable. + // variable. This is to preserve the ability to change the implementation + // behind the scenes. if (!D->isStaticLocal() && D->getTLSKind() == VarDecl::TLS_Dynamic && - Context.getTargetInfo().getTriple().isMacOSX()) + Context.getTargetInfo().getTriple().isOSDarwin() && + !llvm::GlobalVariable::isLinkOnceLinkage(Linkage) && + !llvm::GlobalVariable::isWeakLinkage(Linkage)) Linkage = llvm::GlobalValue::InternalLinkage; GV->setLinkage(Linkage); @@ -2115,7 +2402,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) { if (D->getTLSKind() && !GV->isThreadLocal()) { if (D->getTLSKind() == VarDecl::TLS_Dynamic) - CXXThreadLocals.push_back(std::make_pair(D, GV)); + CXXThreadLocals.push_back(D); setTLSMode(GV, *D); } @@ -2166,7 +2453,7 @@ static bool isVarDeclStrongDefinition(const ASTContext &Context, // Declarations with a required alignment do not have common linakge in MSVC // mode. - if (Context.getLangOpts().MSVCCompat) { + if (Context.getTargetInfo().getCXXABI().isMicrosoft()) { if (D->hasAttr<AlignedAttr>()) return true; QualType VarType = D->getType(); @@ -2263,6 +2550,7 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, llvm::Type *newRetTy = newFn->getReturnType(); SmallVector<llvm::Value*, 4> newArgs; + SmallVector<llvm::OperandBundleDef, 1> newBundles; for (llvm::Value::use_iterator ui = old->use_begin(), ue = old->use_end(); ui != ue; ) { @@ -2330,16 +2618,19 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, // over the required information. newArgs.append(callSite.arg_begin(), callSite.arg_begin() + argNo); + // Copy over any operand bundles. + callSite.getOperandBundlesAsDefs(newBundles); + llvm::CallSite newCall; if (callSite.isCall()) { - newCall = llvm::CallInst::Create(newFn, newArgs, "", + newCall = llvm::CallInst::Create(newFn, newArgs, newBundles, "", callSite.getInstruction()); } else { auto *oldInvoke = cast<llvm::InvokeInst>(callSite.getInstruction()); newCall = llvm::InvokeInst::Create(newFn, oldInvoke->getNormalDest(), oldInvoke->getUnwindDest(), - newArgs, "", + newArgs, newBundles, "", callSite.getInstruction()); } newArgs.clear(); // for the next iteration @@ -2357,6 +2648,7 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, // Copy debug location attached to CI. if (callSite->getDebugLoc()) newCall->setDebugLoc(callSite->getDebugLoc()); + callSite->eraseFromParent(); } } @@ -2397,66 +2689,14 @@ void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD, llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); // Get or create the prototype for the function. - if (!GV) { - llvm::Constant *C = - GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer*/ true); - - // Strip off a bitcast if we got one back. - if (auto *CE = dyn_cast<llvm::ConstantExpr>(C)) { - assert(CE->getOpcode() == llvm::Instruction::BitCast); - GV = cast<llvm::GlobalValue>(CE->getOperand(0)); - } else { - GV = cast<llvm::GlobalValue>(C); - } - } + if (!GV || (GV->getType()->getElementType() != Ty)) + GV = cast<llvm::GlobalValue>(GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, + /*DontDefer=*/true, + /*IsForDefinition=*/true)); - if (!GV->isDeclaration()) { - getDiags().Report(D->getLocation(), diag::err_duplicate_mangled_name); - GlobalDecl OldGD = Manglings.lookup(GV->getName()); - if (auto *Prev = OldGD.getDecl()) - getDiags().Report(Prev->getLocation(), diag::note_previous_definition); + // Already emitted. + if (!GV->isDeclaration()) return; - } - - if (GV->getType()->getElementType() != Ty) { - // If the types mismatch then we have to rewrite the definition. - assert(GV->isDeclaration() && "Shouldn't replace non-declaration"); - - // F is the Function* for the one with the wrong type, we must make a new - // Function* and update everything that used F (a declaration) with the new - // Function* (which will be a definition). - // - // This happens if there is a prototype for a function - // (e.g. "int f()") and then a definition of a different type - // (e.g. "int f(int x)"). Move the old function aside so that it - // doesn't interfere with GetAddrOfFunction. - GV->setName(StringRef()); - auto *NewFn = cast<llvm::Function>(GetAddrOfFunction(GD, Ty)); - - // This might be an implementation of a function without a - // prototype, in which case, try to do special replacement of - // calls which match the new prototype. The really key thing here - // is that we also potentially drop arguments from the call site - // so as to make a direct call, which makes the inliner happier - // and suppresses a number of optimizer warnings (!) about - // dropping arguments. - if (!GV->use_empty()) { - ReplaceUsesOfNonProtoTypeWithRealFunction(GV, NewFn); - GV->removeDeadConstantUsers(); - } - - // Replace uses of F with the Function we will endow with a body. - if (!GV->use_empty()) { - llvm::Constant *NewPtrForOldDecl = - llvm::ConstantExpr::getBitCast(NewFn, GV->getType()); - GV->replaceAllUsesWith(NewPtrForOldDecl); - } - - // Ok, delete the old function now, which is dead. - GV->eraseFromParent(); - - GV = NewFn; - } // We need to set linkage and visibility on the function before // generating code for it because various parts of IR generation @@ -2521,8 +2761,7 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) { // Create the new alias itself, but don't set a name yet. auto *GA = llvm::GlobalAlias::create( - cast<llvm::PointerType>(Aliasee->getType()), - llvm::Function::ExternalLinkage, "", Aliasee, &getModule()); + DeclTy, 0, llvm::Function::ExternalLinkage, "", Aliasee, &getModule()); if (Entry) { if (GA->getAliasee() == Entry) { @@ -2612,7 +2851,7 @@ GetConstantStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map, return *Map.insert(std::make_pair(String, nullptr)).first; } -llvm::Constant * +ConstantAddress CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { unsigned StringLength = 0; bool isUTF16 = false; @@ -2622,7 +2861,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { StringLength); if (auto *C = Entry.second) - return C; + return ConstantAddress(C, CharUnits::fromQuantity(C->getAlignment())); llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty); llvm::Constant *Zeros[] = { Zero, Zero }; @@ -2658,7 +2897,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { // String pointer. llvm::Constant *C = nullptr; if (isUTF16) { - ArrayRef<uint16_t> Arr = llvm::makeArrayRef<uint16_t>( + auto Arr = llvm::makeArrayRef( reinterpret_cast<uint16_t *>(const_cast<char *>(Entry.first().data())), Entry.first().size() / 2); C = llvm::ConstantDataArray::get(VMContext, Arr); @@ -2699,25 +2938,28 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { Ty = getTypes().ConvertType(getContext().LongTy); Fields[3] = llvm::ConstantInt::get(Ty, StringLength); + CharUnits Alignment = getPointerAlign(); + // The struct. C = llvm::ConstantStruct::get(STy, Fields); GV = new llvm::GlobalVariable(getModule(), C->getType(), true, llvm::GlobalVariable::PrivateLinkage, C, "_unnamed_cfstring_"); GV->setSection("__DATA,__cfstring"); + GV->setAlignment(Alignment.getQuantity()); Entry.second = GV; - return GV; + return ConstantAddress(GV, Alignment); } -llvm::GlobalVariable * +ConstantAddress CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) { unsigned StringLength = 0; llvm::StringMapEntry<llvm::GlobalVariable *> &Entry = GetConstantStringEntry(CFConstantStringMap, Literal, StringLength); if (auto *C = Entry.second) - return C; + return ConstantAddress(C, CharUnits::fromQuantity(C->getAlignment())); llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty); llvm::Constant *Zeros[] = { Zero, Zero }; @@ -2810,10 +3052,12 @@ CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) { Fields[2] = llvm::ConstantInt::get(Ty, StringLength); // The struct. + CharUnits Alignment = getPointerAlign(); C = llvm::ConstantStruct::get(NSConstantStringType, Fields); GV = new llvm::GlobalVariable(getModule(), C->getType(), true, llvm::GlobalVariable::PrivateLinkage, C, "_unnamed_nsstring_"); + GV->setAlignment(Alignment.getQuantity()); const char *NSStringSection = "__OBJC,__cstring_object,regular,no_dead_strip"; const char *NSStringNonFragileABISection = "__DATA,__objc_stringobj,regular,no_dead_strip"; @@ -2823,7 +3067,7 @@ CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) { : NSStringSection); Entry.second = GV; - return GV; + return ConstantAddress(GV, Alignment); } QualType CodeGenModule::getObjCFastEnumerationStateType() { @@ -2902,7 +3146,7 @@ CodeGenModule::GetConstantArrayFromStringLiteral(const StringLiteral *E) { static llvm::GlobalVariable * GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT, CodeGenModule &CGM, StringRef GlobalName, - unsigned Alignment) { + CharUnits Alignment) { // OpenCL v1.2 s6.5.3: a string literal is in the constant address space. unsigned AddrSpace = 0; if (CGM.getLangOpts().OpenCL) @@ -2913,7 +3157,7 @@ GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT, auto *GV = new llvm::GlobalVariable( M, C->getType(), !CGM.getLangOpts().WritableStrings, LT, C, GlobalName, nullptr, llvm::GlobalVariable::NotThreadLocal, AddrSpace); - GV->setAlignment(Alignment); + GV->setAlignment(Alignment.getQuantity()); GV->setUnnamedAddr(true); if (GV->isWeakForLinker()) { assert(CGM.supportsCOMDAT() && "Only COFF uses weak string literals"); @@ -2925,20 +3169,19 @@ GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT, /// GetAddrOfConstantStringFromLiteral - Return a pointer to a /// constant array for the given string literal. -llvm::GlobalVariable * +ConstantAddress CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S, StringRef Name) { - auto Alignment = - getContext().getAlignOfGlobalVarInChars(S->getType()).getQuantity(); + CharUnits Alignment = getContext().getAlignOfGlobalVarInChars(S->getType()); llvm::Constant *C = GetConstantArrayFromStringLiteral(S); llvm::GlobalVariable **Entry = nullptr; if (!LangOpts.WritableStrings) { Entry = &ConstantStringMap[C]; if (auto GV = *Entry) { - if (Alignment > GV->getAlignment()) - GV->setAlignment(Alignment); - return GV; + if (Alignment.getQuantity() > GV->getAlignment()) + GV->setAlignment(Alignment.getQuantity()); + return ConstantAddress(GV, Alignment); } } @@ -2954,7 +3197,6 @@ CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S, getCXXABI().getMangleContext().shouldMangleStringLiteral(S)) { llvm::raw_svector_ostream Out(MangledNameBuffer); getCXXABI().getMangleContext().mangleStringLiteral(S, Out); - Out.flush(); LT = llvm::GlobalValue::LinkOnceODRLinkage; GlobalVariableName = MangledNameBuffer; @@ -2969,12 +3211,12 @@ CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S, SanitizerMD->reportGlobalToASan(GV, S->getStrTokenLoc(0), "<string literal>", QualType()); - return GV; + return ConstantAddress(GV, Alignment); } /// GetAddrOfConstantStringFromObjCEncode - Return a pointer to a constant /// array for the given ObjCEncodeExpr node. -llvm::GlobalVariable * +ConstantAddress CodeGenModule::GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *E) { std::string Str; getContext().getObjCEncodingForType(E->getEncodedType(), Str); @@ -2985,14 +3227,11 @@ CodeGenModule::GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *E) { /// GetAddrOfConstantCString - Returns a pointer to a character array containing /// the literal and a terminating '\0' character. /// The result has pointer to array type. -llvm::GlobalVariable *CodeGenModule::GetAddrOfConstantCString( - const std::string &Str, const char *GlobalName, unsigned Alignment) { +ConstantAddress CodeGenModule::GetAddrOfConstantCString( + const std::string &Str, const char *GlobalName) { StringRef StrWithNull(Str.c_str(), Str.size() + 1); - if (Alignment == 0) { - Alignment = getContext() - .getAlignOfGlobalVarInChars(getContext().CharTy) - .getQuantity(); - } + CharUnits Alignment = + getContext().getAlignOfGlobalVarInChars(getContext().CharTy); llvm::Constant *C = llvm::ConstantDataArray::getString(getLLVMContext(), StrWithNull, false); @@ -3002,9 +3241,9 @@ llvm::GlobalVariable *CodeGenModule::GetAddrOfConstantCString( if (!LangOpts.WritableStrings) { Entry = &ConstantStringMap[C]; if (auto GV = *Entry) { - if (Alignment > GV->getAlignment()) - GV->setAlignment(Alignment); - return GV; + if (Alignment.getQuantity() > GV->getAlignment()) + GV->setAlignment(Alignment.getQuantity()); + return ConstantAddress(GV, Alignment); } } @@ -3016,10 +3255,10 @@ llvm::GlobalVariable *CodeGenModule::GetAddrOfConstantCString( GlobalName, Alignment); if (Entry) *Entry = GV; - return GV; + return ConstantAddress(GV, Alignment); } -llvm::Constant *CodeGenModule::GetAddrOfGlobalTemporary( +ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( const MaterializeTemporaryExpr *E, const Expr *Init) { assert((E->getStorageDuration() == SD_Static || E->getStorageDuration() == SD_Thread) && "not a global temporary"); @@ -3031,9 +3270,10 @@ llvm::Constant *CodeGenModule::GetAddrOfGlobalTemporary( if (Init == E->GetTemporaryExpr()) MaterializedType = E->getType(); - llvm::Constant *&Slot = MaterializedGlobalTemporaryMap[E]; - if (Slot) - return Slot; + CharUnits Align = getContext().getTypeAlignInChars(MaterializedType); + + if (llvm::Constant *Slot = MaterializedGlobalTemporaryMap[E]) + return ConstantAddress(Slot, Align); // FIXME: If an externally-visible declaration extends multiple temporaries, // we need to give each temporary the same name in every translation unit (and @@ -3042,7 +3282,6 @@ llvm::Constant *CodeGenModule::GetAddrOfGlobalTemporary( llvm::raw_svector_ostream Out(Name); getCXXABI().getMangleContext().mangleReferenceTemporary( VD, E->getManglingNumber(), Out); - Out.flush(); APValue *Value = nullptr; if (E->getStorageDuration() == SD_Static) { @@ -3098,14 +3337,13 @@ llvm::Constant *CodeGenModule::GetAddrOfGlobalTemporary( /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, AddrSpace); setGlobalVisibility(GV, VD); - GV->setAlignment( - getContext().getTypeAlignInChars(MaterializedType).getQuantity()); + GV->setAlignment(Align.getQuantity()); if (supportsCOMDAT() && GV->isWeakForLinker()) GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); if (VD->getTLSKind()) setTLSMode(GV, *VD); - Slot = GV; - return GV; + MaterializedGlobalTemporaryMap[E] = GV; + return ConstantAddress(GV, Align); } /// EmitObjCPropertyImplementations - Emit information for synthesized @@ -3367,11 +3605,8 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { auto *Import = cast<ImportDecl>(D); // Ignore import declarations that come from imported modules. - if (clang::Module *Owner = Import->getImportedOwningModule()) { - if (getLangOpts().CurrentModule.empty() || - Owner->getTopLevelModule()->Name == getLangOpts().CurrentModule) - break; - } + if (Import->getImportedOwningModule()) + break; if (CGDebugInfo *DI = getModuleDebugInfo()) DI->EmitImportDecl(*Import); @@ -3412,7 +3647,7 @@ void CodeGenModule::AddDeferredUnusedCoverageMapping(Decl *D) { case Decl::ObjCMethod: case Decl::CXXConstructor: case Decl::CXXDestructor: { - if (!cast<FunctionDecl>(D)->hasBody()) + if (!cast<FunctionDecl>(D)->doesThisDeclarationHaveABody()) return; auto I = DeferredEmptyCoverageMappingDecls.find(D); if (I == DeferredEmptyCoverageMappingDecls.end()) @@ -3541,10 +3776,12 @@ bool CodeGenModule::lookupRepresentativeDecl(StringRef MangledName, void CodeGenModule::EmitDeclMetadata() { llvm::NamedMDNode *GlobalMetadata = nullptr; - // StaticLocalDeclMap for (auto &I : MangledDeclNames) { llvm::GlobalValue *Addr = getModule().getNamedValue(I.second); - EmitGlobalDeclMetadata(*this, GlobalMetadata, I.first, Addr); + // Some mangled names don't necessarily have an associated GlobalValue + // in this module, e.g. if we mangled it for DebugInfo. + if (Addr) + EmitGlobalDeclMetadata(*this, GlobalMetadata, I.first, Addr); } } @@ -3562,7 +3799,7 @@ void CodeGenFunction::EmitDeclMetadata() { for (auto &I : LocalDeclMap) { const Decl *D = I.first; - llvm::Value *Addr = I.second; + llvm::Value *Addr = I.second.getPointer(); if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Addr)) { llvm::Value *DAddr = GetPointerConstant(getLLVMContext(), D); Alloca->setMetadata( @@ -3643,12 +3880,6 @@ llvm::Constant *CodeGenModule::EmitUuidofInitializer(StringRef Uuid) { return llvm::ConstantStruct::getAnon(Fields); } -llvm::Constant * -CodeGenModule::getAddrOfCXXCatchHandlerType(QualType Ty, - QualType CatchHandlerType) { - return getCXXABI().getAddrOfCXXCatchHandlerType(Ty, CatchHandlerType); -} - llvm::Constant *CodeGenModule::GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH) { // Return a bogus pointer if RTTI is disabled, unless it's for EH. @@ -3671,22 +3902,82 @@ void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) { VD->getAnyInitializer() && !VD->getAnyInitializer()->isConstantInitializer(getContext(), /*ForRef=*/false); + + Address Addr(GetAddrOfGlobalVar(VD), getContext().getDeclAlign(VD)); if (auto InitFunction = getOpenMPRuntime().emitThreadPrivateVarDefinition( - VD, GetAddrOfGlobalVar(VD), RefExpr->getLocStart(), PerformInit)) + VD, Addr, RefExpr->getLocStart(), PerformInit)) CXXGlobalInits.push_back(InitFunction); } } -llvm::MDTuple *CodeGenModule::CreateVTableBitSetEntry( - llvm::GlobalVariable *VTable, CharUnits Offset, const CXXRecordDecl *RD) { - std::string OutName; - llvm::raw_string_ostream Out(OutName); - getCXXABI().getMangleContext().mangleCXXVTableBitSet(RD, Out); +llvm::Metadata *CodeGenModule::CreateMetadataIdentifierForType(QualType T) { + llvm::Metadata *&InternalId = MetadataIdMap[T.getCanonicalType()]; + if (InternalId) + return InternalId; + + if (isExternallyVisible(T->getLinkage())) { + std::string OutName; + llvm::raw_string_ostream Out(OutName); + getCXXABI().getMangleContext().mangleTypeName(T, Out); + + InternalId = llvm::MDString::get(getLLVMContext(), Out.str()); + } else { + InternalId = llvm::MDNode::getDistinct(getLLVMContext(), + llvm::ArrayRef<llvm::Metadata *>()); + } + + return InternalId; +} +void CodeGenModule::CreateVTableBitSetEntry(llvm::NamedMDNode *BitsetsMD, + llvm::GlobalVariable *VTable, + CharUnits Offset, + const CXXRecordDecl *RD) { + llvm::Metadata *MD = + CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); llvm::Metadata *BitsetOps[] = { - llvm::MDString::get(getLLVMContext(), Out.str()), - llvm::ConstantAsMetadata::get(VTable), + MD, llvm::ConstantAsMetadata::get(VTable), llvm::ConstantAsMetadata::get( llvm::ConstantInt::get(Int64Ty, Offset.getQuantity()))}; - return llvm::MDTuple::get(getLLVMContext(), BitsetOps); + BitsetsMD->addOperand(llvm::MDTuple::get(getLLVMContext(), BitsetOps)); + + if (CodeGenOpts.SanitizeCfiCrossDso) { + if (auto TypeId = CreateCfiIdForTypeMetadata(MD)) { + llvm::Metadata *BitsetOps2[] = { + llvm::ConstantAsMetadata::get(TypeId), + llvm::ConstantAsMetadata::get(VTable), + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(Int64Ty, Offset.getQuantity()))}; + BitsetsMD->addOperand(llvm::MDTuple::get(getLLVMContext(), BitsetOps2)); + } + } +} + +// Fills in the supplied string map with the set of target features for the +// passed in function. +void CodeGenModule::getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap, + const FunctionDecl *FD) { + StringRef TargetCPU = Target.getTargetOpts().CPU; + if (const auto *TD = FD->getAttr<TargetAttr>()) { + // If we have a TargetAttr build up the feature map based on that. + TargetAttr::ParsedTargetAttr ParsedAttr = TD->parse(); + + // Make a copy of the features as passed on the command line into the + // beginning of the additional features from the function to override. + ParsedAttr.first.insert(ParsedAttr.first.begin(), + Target.getTargetOpts().FeaturesAsWritten.begin(), + Target.getTargetOpts().FeaturesAsWritten.end()); + + if (ParsedAttr.second != "") + TargetCPU = ParsedAttr.second; + + // Now populate the feature map, first with the TargetCPU which is either + // the default or a new one from the target attribute string. Then we'll use + // the passed in features (FeaturesAsWritten) along with the new ones from + // the attribute. + Target.initFeatureMap(FeatureMap, getDiags(), TargetCPU, ParsedAttr.first); + } else { + Target.initFeatureMap(FeatureMap, getDiags(), TargetCPU, + Target.getTargetOpts().Features); + } } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h index dd167a2..3311383 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h @@ -15,6 +15,7 @@ #define LLVM_CLANG_LIB_CODEGEN_CODEGENMODULE_H #include "CGVTables.h" +#include "CodeGenTypeCache.h" #include "CodeGenTypes.h" #include "SanitizerMetadata.h" #include "clang/AST/Attr.h" @@ -30,7 +31,6 @@ #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringMap.h" -#include "llvm/IR/CallingConv.h" #include "llvm/IR/Module.h" #include "llvm/IR/ValueHandle.h" @@ -108,65 +108,14 @@ struct OrderGlobalInits { } }; -struct CodeGenTypeCache { - /// void - llvm::Type *VoidTy; +struct ObjCEntrypoints { + ObjCEntrypoints() { memset(this, 0, sizeof(*this)); } - /// i8, i16, i32, and i64 - llvm::IntegerType *Int8Ty, *Int16Ty, *Int32Ty, *Int64Ty; - /// float, double - llvm::Type *FloatTy, *DoubleTy; - - /// int - llvm::IntegerType *IntTy; - - /// intptr_t, size_t, and ptrdiff_t, which we assume are the same size. - union { - llvm::IntegerType *IntPtrTy; - llvm::IntegerType *SizeTy; - llvm::IntegerType *PtrDiffTy; - }; - - /// void* in address space 0 - union { - llvm::PointerType *VoidPtrTy; - llvm::PointerType *Int8PtrTy; - }; - - /// void** in address space 0 - union { - llvm::PointerType *VoidPtrPtrTy; - llvm::PointerType *Int8PtrPtrTy; - }; - - /// The width of a pointer into the generic address space. - unsigned char PointerWidthInBits; - - /// The size and alignment of a pointer into the generic address - /// space. - union { - unsigned char PointerAlignInBytes; - unsigned char PointerSizeInBytes; - unsigned char SizeSizeInBytes; // sizeof(size_t) - }; - - llvm::CallingConv::ID RuntimeCC; - llvm::CallingConv::ID getRuntimeCC() const { return RuntimeCC; } - llvm::CallingConv::ID BuiltinCC; - llvm::CallingConv::ID getBuiltinCC() const { return BuiltinCC; } -}; - -struct RREntrypoints { - RREntrypoints() { memset(this, 0, sizeof(*this)); } - /// void objc_autoreleasePoolPop(void*); + /// void objc_autoreleasePoolPop(void*); llvm::Constant *objc_autoreleasePoolPop; /// void *objc_autoreleasePoolPush(void); llvm::Constant *objc_autoreleasePoolPush; -}; - -struct ARCEntrypoints { - ARCEntrypoints() { memset(this, 0, sizeof(*this)); } /// id objc_autorelease(id); llvm::Constant *objc_autorelease; @@ -257,6 +206,36 @@ public: void reportDiagnostics(DiagnosticsEngine &Diags, StringRef MainFile); }; +/// A pair of helper functions for a __block variable. +class BlockByrefHelpers : public llvm::FoldingSetNode { + // MSVC requires this type to be complete in order to process this + // header. +public: + llvm::Constant *CopyHelper; + llvm::Constant *DisposeHelper; + + /// The alignment of the field. This is important because + /// different offsets to the field within the byref struct need to + /// have different helper functions. + CharUnits Alignment; + + BlockByrefHelpers(CharUnits alignment) : Alignment(alignment) {} + BlockByrefHelpers(const BlockByrefHelpers &) = default; + virtual ~BlockByrefHelpers(); + + void Profile(llvm::FoldingSetNodeID &id) const { + id.AddInteger(Alignment.getQuantity()); + profileImpl(id); + } + virtual void profileImpl(llvm::FoldingSetNodeID &id) const = 0; + + virtual bool needsCopy() const { return true; } + virtual void emitCopy(CodeGenFunction &CGF, Address dest, Address src) = 0; + + virtual bool needsDispose() const { return true; } + virtual void emitDispose(CodeGenFunction &CGF, Address field) = 0; +}; + /// This class organizes the cross-function state that is used while generating /// LLVM code. class CodeGenModule : public CodeGenTypeCache { @@ -285,7 +264,6 @@ private: const CodeGenOptions &CodeGenOpts; llvm::Module &TheModule; DiagnosticsEngine &Diags; - const llvm::DataLayout &TheDataLayout; const TargetInfo &Target; std::unique_ptr<CGCXXABI> ABI; llvm::LLVMContext &VMContext; @@ -307,9 +285,8 @@ private: CGOpenMPRuntime* OpenMPRuntime; CGCUDARuntime* CUDARuntime; CGDebugInfo* DebugInfo; - ARCEntrypoints *ARCData; + ObjCEntrypoints *ObjCData; llvm::MDNode *NoObjCARCExceptionsMetadata; - RREntrypoints *RRData; std::unique_ptr<llvm::IndexedInstrProfReader> PGOReader; InstrProfStats PGOStats; @@ -343,6 +320,17 @@ private: typedef llvm::StringMap<llvm::TrackingVH<llvm::Constant> > ReplacementsTy; ReplacementsTy Replacements; + /// List of global values to be replaced with something else. Used when we + /// want to replace a GlobalValue but can't identify it by its mangled name + /// anymore (because the name is already taken). + llvm::SmallVector<std::pair<llvm::GlobalValue *, llvm::Constant *>, 8> + GlobalValReplacements; + + /// Set of global decls for which we already diagnosed mangled name conflict. + /// Required to not issue a warning (on a mangling conflict) multiple times + /// for the same decl. + llvm::DenseSet<GlobalDecl> DiagnosedConflictingDefinitions; + /// A queue of (optional) vtables to consider emitting. std::vector<const CXXRecordDecl*> DeferredVTables; @@ -390,13 +378,12 @@ private: StaticExternCMap StaticExternCValues; /// \brief thread_local variables defined or used in this TU. - std::vector<std::pair<const VarDecl *, llvm::GlobalVariable *> > - CXXThreadLocals; + std::vector<const VarDecl *> CXXThreadLocals; /// \brief thread_local variables with initializers that need to run /// before any thread_local variable in this TU is odr-used. std::vector<llvm::Function *> CXXThreadLocalInits; - std::vector<llvm::GlobalVariable *> CXXThreadLocalInitVars; + std::vector<const VarDecl *> CXXThreadLocalInitVars; /// Global variables with initializers that need to run before main. std::vector<llvm::Function *> CXXGlobalInits; @@ -491,12 +478,16 @@ private: llvm::DenseMap<const Decl *, bool> DeferredEmptyCoverageMappingDecls; std::unique_ptr<CoverageMappingModuleGen> CoverageMapping; + + /// Mapping from canonical types to their metadata identifiers. We need to + /// maintain this mapping because identifiers may be formed from distinct + /// MDNodes. + llvm::DenseMap<QualType, llvm::Metadata *> MetadataIdMap; + public: - CodeGenModule(ASTContext &C, - const HeaderSearchOptions &headersearchopts, + CodeGenModule(ASTContext &C, const HeaderSearchOptions &headersearchopts, const PreprocessorOptions &ppopts, - const CodeGenOptions &CodeGenOpts, - llvm::Module &M, const llvm::DataLayout &TD, + const CodeGenOptions &CodeGenOpts, llvm::Module &M, DiagnosticsEngine &Diags, CoverageSourceInfo *CoverageInfo = nullptr); @@ -534,14 +525,9 @@ public: return *CUDARuntime; } - ARCEntrypoints &getARCEntrypoints() const { - assert(getLangOpts().ObjCAutoRefCount && ARCData != nullptr); - return *ARCData; - } - - RREntrypoints &getRREntrypoints() const { - assert(RRData != nullptr); - return *RRData; + ObjCEntrypoints &getObjCEntrypoints() const { + assert(ObjCData != nullptr); + return *ObjCData; } InstrProfStats &getPGOStats() { return PGOStats; } @@ -614,7 +600,9 @@ public: const CodeGenOptions &getCodeGenOpts() const { return CodeGenOpts; } llvm::Module &getModule() const { return TheModule; } DiagnosticsEngine &getDiags() const { return Diags; } - const llvm::DataLayout &getDataLayout() const { return TheDataLayout; } + const llvm::DataLayout &getDataLayout() const { + return TheModule.getDataLayout(); + } const TargetInfo &getTarget() const { return Target; } const llvm::Triple &getTriple() const; bool supportsCOMDAT() const; @@ -645,8 +633,6 @@ public: llvm::MDNode *getTBAAInfo(QualType QTy); llvm::MDNode *getTBAAInfoForVTablePtr(); llvm::MDNode *getTBAAStructInfo(QualType QTy); - /// Return the MDNode in the type DAG for the given struct type. - llvm::MDNode *getTBAAStructTypeInfo(QualType QTy); /// Return the path-aware tag for given base type, access node and offset. llvm::MDNode *getTBAAStructTagInfo(QualType BaseTy, llvm::MDNode *AccessN, uint64_t O); @@ -660,9 +646,13 @@ public: /// is the same as the type. For struct-path aware TBAA, the tag /// is different from the type: base type, access type and offset. /// When ConvertTypeToTag is true, we create a tag based on the scalar type. - void DecorateInstruction(llvm::Instruction *Inst, - llvm::MDNode *TBAAInfo, - bool ConvertTypeToTag = true); + void DecorateInstructionWithTBAA(llvm::Instruction *Inst, + llvm::MDNode *TBAAInfo, + bool ConvertTypeToTag = true); + + /// Adds !invariant.barrier !tag to instruction + void DecorateInstructionWithInvariantGroup(llvm::Instruction *I, + const CXXRecordDecl *RD); /// Emit the given number of characters as a value of type size_t. llvm::ConstantInt *getSize(CharUnits numChars); @@ -683,18 +673,7 @@ public: llvm_unreachable("unknown visibility!"); } - llvm::Constant *GetAddrOfGlobal(GlobalDecl GD) { - if (isa<CXXConstructorDecl>(GD.getDecl())) - return getAddrOfCXXStructor(cast<CXXConstructorDecl>(GD.getDecl()), - getFromCtorType(GD.getCtorType())); - else if (isa<CXXDestructorDecl>(GD.getDecl())) - return getAddrOfCXXStructor(cast<CXXDestructorDecl>(GD.getDecl()), - getFromDtorType(GD.getDtorType())); - else if (isa<FunctionDecl>(GD.getDecl())) - return GetAddrOfFunction(GD); - else - return GetAddrOfGlobalVar(cast<VarDecl>(GD.getDecl())); - } + llvm::Constant *GetAddrOfGlobal(GlobalDecl GD, bool IsForDefinition = false); /// Will return a global variable of the given type. If a variable with a /// different type already exists then a new variable with the right type @@ -706,6 +685,7 @@ public: llvm::Function * CreateGlobalInitOrDestructFunction(llvm::FunctionType *ty, const Twine &name, + const CGFunctionInfo &FI, SourceLocation Loc = SourceLocation(), bool TLS = false); @@ -724,24 +704,37 @@ public: /// Return the address of the given function. If Ty is non-null, then this /// function will use the specified type if it has to create it. - llvm::Constant *GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty = 0, + llvm::Constant *GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty = nullptr, bool ForVTable = false, - bool DontDefer = false); + bool DontDefer = false, + bool IsForDefinition = false); /// Get the address of the RTTI descriptor for the given type. llvm::Constant *GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH = false); - llvm::Constant *getAddrOfCXXCatchHandlerType(QualType Ty, - QualType CatchHandlerType); - /// Get the address of a uuid descriptor . - llvm::Constant *GetAddrOfUuidDescriptor(const CXXUuidofExpr* E); + ConstantAddress GetAddrOfUuidDescriptor(const CXXUuidofExpr* E); /// Get the address of the thunk for the given global decl. llvm::Constant *GetAddrOfThunk(GlobalDecl GD, const ThunkInfo &Thunk); /// Get a reference to the target of VD. - llvm::Constant *GetWeakRefReference(const ValueDecl *VD); + ConstantAddress GetWeakRefReference(const ValueDecl *VD); + + /// Returns the assumed alignment of an opaque pointer to the given class. + CharUnits getClassPointerAlignment(const CXXRecordDecl *CD); + + /// Returns the assumed alignment of a virtual base of a class. + CharUnits getVBaseAlignment(CharUnits DerivedAlign, + const CXXRecordDecl *Derived, + const CXXRecordDecl *VBase); + + /// Given a class pointer with an actual known alignment, and the + /// expected alignment of an object at a dynamic offset w.r.t that + /// pointer, return the alignment to assume at the offset. + CharUnits getDynamicOffsetAlignment(CharUnits ActualAlign, + const CXXRecordDecl *Class, + CharUnits ExpectedTargetAlign); CharUnits computeNonVirtualBaseClassOffset(const CXXRecordDecl *DerivedClass, @@ -755,35 +748,7 @@ public: CastExpr::path_const_iterator PathBegin, CastExpr::path_const_iterator PathEnd); - /// A pair of helper functions for a __block variable. - class ByrefHelpers : public llvm::FoldingSetNode { - public: - llvm::Constant *CopyHelper; - llvm::Constant *DisposeHelper; - - /// The alignment of the field. This is important because - /// different offsets to the field within the byref struct need to - /// have different helper functions. - CharUnits Alignment; - - ByrefHelpers(CharUnits alignment) : Alignment(alignment) {} - virtual ~ByrefHelpers(); - - void Profile(llvm::FoldingSetNodeID &id) const { - id.AddInteger(Alignment.getQuantity()); - profileImpl(id); - } - virtual void profileImpl(llvm::FoldingSetNodeID &id) const = 0; - - virtual bool needsCopy() const { return true; } - virtual void emitCopy(CodeGenFunction &CGF, - llvm::Value *dest, llvm::Value *src) = 0; - - virtual bool needsDispose() const { return true; } - virtual void emitDispose(CodeGenFunction &CGF, llvm::Value *field) = 0; - }; - - llvm::FoldingSet<ByrefHelpers> ByrefHelpersCache; + llvm::FoldingSet<BlockByrefHelpers> ByrefHelpersCache; /// Fetches the global unique block count. int getUniqueBlockCount() { return ++Block.GlobalUniqueCount; } @@ -798,23 +763,23 @@ public: llvm::Constant *GetAddrOfGlobalBlock(const BlockExpr *BE, const char *); /// Return a pointer to a constant CFString object for the given string. - llvm::Constant *GetAddrOfConstantCFString(const StringLiteral *Literal); + ConstantAddress GetAddrOfConstantCFString(const StringLiteral *Literal); /// Return a pointer to a constant NSString object for the given string. Or a /// user defined String object as defined via /// -fconstant-string-class=class_name option. - llvm::GlobalVariable *GetAddrOfConstantString(const StringLiteral *Literal); + ConstantAddress GetAddrOfConstantString(const StringLiteral *Literal); /// Return a constant array for the given string. llvm::Constant *GetConstantArrayFromStringLiteral(const StringLiteral *E); /// Return a pointer to a constant array for the given string literal. - llvm::GlobalVariable * + ConstantAddress GetAddrOfConstantStringFromLiteral(const StringLiteral *S, StringRef Name = ".str"); /// Return a pointer to a constant array for the given ObjCEncodeExpr node. - llvm::GlobalVariable * + ConstantAddress GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *); /// Returns a pointer to a character array containing the literal and a @@ -822,18 +787,17 @@ public: /// /// \param GlobalName If provided, the name to use for the global (if one is /// created). - llvm::GlobalVariable * + ConstantAddress GetAddrOfConstantCString(const std::string &Str, - const char *GlobalName = nullptr, - unsigned Alignment = 0); + const char *GlobalName = nullptr); /// Returns a pointer to a constant global variable for the given file-scope /// compound literal expression. - llvm::Constant *GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr*E); + ConstantAddress GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr*E); /// \brief Returns a pointer to a global variable representing a temporary /// with static or thread storage duration. - llvm::Constant *GetAddrOfGlobalTemporary(const MaterializeTemporaryExpr *E, + ConstantAddress GetAddrOfGlobalTemporary(const MaterializeTemporaryExpr *E, const Expr *Inner); /// \brief Retrieve the record type that describes the state of an @@ -847,11 +811,11 @@ public: StructorType Type); /// Return the address of the constructor/destructor of the given type. - llvm::GlobalValue * + llvm::Constant * getAddrOfCXXStructor(const CXXMethodDecl *MD, StructorType Type, const CGFunctionInfo *FnInfo = nullptr, llvm::FunctionType *FnType = nullptr, - bool DontDefer = false); + bool DontDefer = false, bool IsForDefinition = false); /// Given a builtin id for a function like "__builtin_fabsf", return a /// Function* for "fabsf". @@ -948,6 +912,11 @@ public: QualType DestType, CodeGenFunction *CGF = nullptr); + /// \brief Emit type info if type of an expression is a variably modified + /// type. Also emit proper debug info for cast types. + void EmitExplicitCastExprType(const ExplicitCastExpr *E, + CodeGenFunction *CGF = nullptr); + /// Return the result of value-initializing the given type, i.e. a null /// expression of the given type. This is usually, but not always, an LLVM /// null constant. @@ -998,16 +967,19 @@ public: /// function type. /// /// \param Info - The function type information. - /// \param TargetDecl - The decl these attributes are being constructed - /// for. If supplied the attributes applied to this decl may contribute to the - /// function attributes and calling convention. + /// \param CalleeInfo - The callee information these attributes are being + /// constructed for. If valid, the attributes applied to this decl may + /// contribute to the function attributes and calling convention. /// \param PAL [out] - On return, the attribute list to use. /// \param CallingConv [out] - On return, the LLVM calling convention to use. void ConstructAttributeList(const CGFunctionInfo &Info, - const Decl *TargetDecl, - AttributeListType &PAL, - unsigned &CallingConv, - bool AttrOnCallSite); + CGCalleeInfo CalleeInfo, AttributeListType &PAL, + unsigned &CallingConv, bool AttrOnCallSite); + + // Fills in the supplied string map with the set of target features for the + // passed in function. + void getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap, + const FunctionDecl *FD); StringRef getMangledName(GlobalDecl GD); StringRef getBlockMangledName(GlobalDecl GD, const BlockDecl *BD); @@ -1016,9 +988,6 @@ public: void EmitVTable(CXXRecordDecl *Class); - /// Emit the RTTI descriptors for the builtin types. - void EmitFundamentalRTTIDescriptors(); - /// \brief Appends Opts to the "Linker Options" metadata value. void AppendLinkerOptions(StringRef Opts); @@ -1122,6 +1091,8 @@ public: void addReplacement(StringRef Name, llvm::Constant *C); + void addGlobalValReplacement(llvm::GlobalValue *GV, llvm::Constant *C); + /// \brief Emit a code for threadprivate directive. /// \param D Threadprivate declaration. void EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D); @@ -1135,10 +1106,21 @@ public: void EmitVTableBitSetEntries(llvm::GlobalVariable *VTable, const VTableLayout &VTLayout); - /// Create a bitset entry for the given vtable. - llvm::MDTuple *CreateVTableBitSetEntry(llvm::GlobalVariable *VTable, - CharUnits Offset, - const CXXRecordDecl *RD); + /// Generate a cross-DSO type identifier for type. + llvm::ConstantInt *CreateCfiIdForTypeMetadata(llvm::Metadata *MD); + + /// Create a metadata identifier for the given type. This may either be an + /// MDString (for external identifiers) or a distinct unnamed MDNode (for + /// internal identifiers). + llvm::Metadata *CreateMetadataIdentifierForType(QualType T); + + /// Create a bitset entry for the given function and add it to BitsetsMD. + void CreateFunctionBitSetEntry(const FunctionDecl *FD, llvm::Function *F); + + /// Create a bitset entry for the given vtable and add it to BitsetsMD. + void CreateVTableBitSetEntry(llvm::NamedMDNode *BitsetsMD, + llvm::GlobalVariable *VTable, CharUnits Offset, + const CXXRecordDecl *RD); /// \breif Get the declaration of std::terminate for the platform. llvm::Constant *getTerminateFn(); @@ -1148,7 +1130,8 @@ private: GetOrCreateLLVMFunction(StringRef MangledName, llvm::Type *Ty, GlobalDecl D, bool ForVTable, bool DontDefer = false, bool IsThunk = false, - llvm::AttributeSet ExtraAttrs = llvm::AttributeSet()); + llvm::AttributeSet ExtraAttrs = llvm::AttributeSet(), + bool IsForDefinition = false); llvm::Constant *GetOrCreateLLVMGlobal(StringRef MangledName, llvm::PointerType *PTy, @@ -1194,7 +1177,7 @@ private: // FIXME: Hardcoding priority here is gross. void AddGlobalCtor(llvm::Function *Ctor, int Priority = 65535, - llvm::Constant *AssociatedData = 0); + llvm::Constant *AssociatedData = nullptr); void AddGlobalDtor(llvm::Function *Dtor, int Priority = 65535); /// Generates a global array of functions and priorities using the given list @@ -1202,15 +1185,15 @@ private: /// as a LLVM constructor or destructor array. void EmitCtorList(const CtorList &Fns, const char *GlobalName); - /// Emit the RTTI descriptors for the given type. - void EmitFundamentalRTTIDescriptor(QualType Type); - /// Emit any needed decls for which code generation was deferred. void EmitDeferred(); /// Call replaceAllUsesWith on all pairs in Replacements. void applyReplacements(); + /// Call replaceAllUsesWith on all pairs in GlobalValReplacements. + void applyGlobalValReplacements(); + void checkAliases(); /// Emit any vtables which we deferred and still have a use for. @@ -1258,4 +1241,4 @@ private: } // end namespace CodeGen } // end namespace clang -#endif +#endif // LLVM_CLANG_LIB_CODEGEN_CODEGENMODULE_H diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp index 8dffefc..3877433 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp @@ -28,58 +28,20 @@ using namespace CodeGen; void CodeGenPGO::setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage) { - StringRef RawFuncName = Name; - - // Function names may be prefixed with a binary '1' to indicate - // that the backend should not modify the symbols due to any platform - // naming convention. Do not include that '1' in the PGO profile name. - if (RawFuncName[0] == '\1') - RawFuncName = RawFuncName.substr(1); - - FuncName = RawFuncName; - if (llvm::GlobalValue::isLocalLinkage(Linkage)) { - // For local symbols, prepend the main file name to distinguish them. - // Do not include the full path in the file name since there's no guarantee - // that it will stay the same, e.g., if the files are checked out from - // version control in different locations. - if (CGM.getCodeGenOpts().MainFileName.empty()) - FuncName = FuncName.insert(0, "<unknown>:"); - else - FuncName = FuncName.insert(0, CGM.getCodeGenOpts().MainFileName + ":"); - } + llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader(); + FuncName = llvm::getPGOFuncName( + Name, Linkage, CGM.getCodeGenOpts().MainFileName, + PGOReader ? PGOReader->getVersion() : llvm::IndexedInstrProf::Version); // If we're generating a profile, create a variable for the name. if (CGM.getCodeGenOpts().ProfileInstrGenerate) - createFuncNameVar(Linkage); + FuncNameVar = llvm::createPGOFuncNameVar(CGM.getModule(), Linkage, FuncName); } void CodeGenPGO::setFuncName(llvm::Function *Fn) { setFuncName(Fn->getName(), Fn->getLinkage()); } -void CodeGenPGO::createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage) { - // We generally want to match the function's linkage, but available_externally - // and extern_weak both have the wrong semantics, and anything that doesn't - // need to link across compilation units doesn't need to be visible at all. - if (Linkage == llvm::GlobalValue::ExternalWeakLinkage) - Linkage = llvm::GlobalValue::LinkOnceAnyLinkage; - else if (Linkage == llvm::GlobalValue::AvailableExternallyLinkage) - Linkage = llvm::GlobalValue::LinkOnceODRLinkage; - else if (Linkage == llvm::GlobalValue::InternalLinkage || - Linkage == llvm::GlobalValue::ExternalLinkage) - Linkage = llvm::GlobalValue::PrivateLinkage; - - auto *Value = - llvm::ConstantDataArray::getString(CGM.getLLVMContext(), FuncName, false); - FuncNameVar = - new llvm::GlobalVariable(CGM.getModule(), Value->getType(), true, Linkage, - Value, "__llvm_profile_name_" + FuncName); - - // Hide the symbol so that we correctly get a copy for each executable. - if (!llvm::GlobalValue::isLocalLinkage(FuncNameVar->getLinkage())) - FuncNameVar->setVisibility(llvm::GlobalValue::HiddenVisibility); -} - namespace { /// \brief Stable hasher for PGO region counters. /// @@ -604,7 +566,7 @@ struct ComputeRegionCounts : public ConstStmtVisitor<ComputeRegionCounts> { RecordNextStmtCount = true; } }; -} +} // end anonymous namespace void PGOHash::combine(HashType Type) { // Check that we never combine 0 and only have six bits. @@ -643,27 +605,24 @@ uint64_t PGOHash::finalize() { return endian::read<uint64_t, little, unaligned>(Result); } -void CodeGenPGO::checkGlobalDecl(GlobalDecl GD) { - // Make sure we only emit coverage mapping for one constructor/destructor. - // Clang emits several functions for the constructor and the destructor of - // a class. Every function is instrumented, but we only want to provide - // coverage for one of them. Because of that we only emit the coverage mapping - // for the base constructor/destructor. - if ((isa<CXXConstructorDecl>(GD.getDecl()) && - GD.getCtorType() != Ctor_Base) || - (isa<CXXDestructorDecl>(GD.getDecl()) && - GD.getDtorType() != Dtor_Base)) { - SkipCoverageMapping = true; - } -} - -void CodeGenPGO::assignRegionCounters(const Decl *D, llvm::Function *Fn) { +void CodeGenPGO::assignRegionCounters(GlobalDecl GD, llvm::Function *Fn) { + const Decl *D = GD.getDecl(); bool InstrumentRegions = CGM.getCodeGenOpts().ProfileInstrGenerate; llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader(); if (!InstrumentRegions && !PGOReader) return; if (D->isImplicit()) return; + // Constructors and destructors may be represented by several functions in IR. + // If so, instrument only base variant, others are implemented by delegation + // to the base one, it would be counted twice otherwise. + if (CGM.getTarget().getCXXABI().hasConstructorVariants() && + ((isa<CXXConstructorDecl>(GD.getDecl()) && + GD.getCtorType() != Ctor_Base) || + (isa<CXXDestructorDecl>(GD.getDecl()) && + GD.getDtorType() != Dtor_Base))) { + return; + } CGM.ClearUnusedCoverageMapping(D); setFuncName(Fn); @@ -763,7 +722,7 @@ CodeGenPGO::applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, return; uint64_t MaxFunctionCount = PGOReader->getMaximumFunctionCount(); - uint64_t FunctionCount = getRegionCount(0); + uint64_t FunctionCount = getRegionCount(nullptr); if (FunctionCount >= (uint64_t)(0.3 * (double)MaxFunctionCount)) // Turn on InlineHint attribute for hot functions. // FIXME: 30% is from preliminary tuning on SPEC, it may not be optimal. @@ -779,7 +738,7 @@ CodeGenPGO::applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S) { if (!CGM.getCodeGenOpts().ProfileInstrGenerate || !RegionCounterMap) return; - if (!Builder.GetInsertPoint()) + if (!Builder.GetInsertBlock()) return; unsigned Counter = (*RegionCounterMap)[S]; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h index de6f369..6bf29ec 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h @@ -78,13 +78,11 @@ public: setCurrentRegionCount(*Count); } - /// Check if we need to emit coverage mapping for a given declaration - void checkGlobalDecl(GlobalDecl GD); /// Assign counters to regions and configure them for PGO of a given /// function. Does nothing if instrumentation is not enabled and either /// generates global variables or associates PGO data with each of the /// counters depending on whether we are generating or using instrumentation. - void assignRegionCounters(const Decl *D, llvm::Function *Fn); + void assignRegionCounters(GlobalDecl GD, llvm::Function *Fn); /// Emit a coverage mapping range with a counter zero /// for an unused declaration. void emitEmptyCounterMapping(const Decl *D, StringRef FuncName, @@ -92,14 +90,12 @@ public: private: void setFuncName(llvm::Function *Fn); void setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage); - void createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage); void mapRegionCounters(const Decl *D); void computeRegionCounts(const Decl *D); void applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, llvm::Function *Fn); void loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader, bool IsInMainFile); - void emitCounterVariables(); void emitCounterRegionMapping(const Decl *D); public: diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.cpp index 53ba02a..c3c925c 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTBAA.cpp @@ -155,7 +155,6 @@ CodeGenTBAA::getTBAAInfo(QualType QTy) { SmallString<256> OutName; llvm::raw_svector_ostream Out(OutName); MContext.mangleTypeName(QualType(ETy, 0), Out); - Out.flush(); return MetadataCache[Ty] = createTBAAScalarType(OutName, getChar()); } @@ -271,7 +270,6 @@ CodeGenTBAA::getTBAAStructTypeInfo(QualType QTy) { // Don't use the mangler for C code. llvm::raw_svector_ostream Out(OutName); MContext.mangleTypeName(QualType(Ty, 0), Out); - Out.flush(); } else { OutName = RD->getName(); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypeCache.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypeCache.h new file mode 100644 index 0000000..c32b66d --- /dev/null +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypeCache.h @@ -0,0 +1,108 @@ +//===--- CodeGenTypeCache.h - Commonly used LLVM types and info -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This structure provides a set of common types useful during IR emission. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENTYPECACHE_H +#define LLVM_CLANG_LIB_CODEGEN_CODEGENTYPECACHE_H + +#include "clang/AST/CharUnits.h" +#include "llvm/IR/CallingConv.h" + +namespace llvm { + class Type; + class IntegerType; + class PointerType; +} + +namespace clang { +namespace CodeGen { + +/// This structure provides a set of types that are commonly used +/// during IR emission. It's initialized once in CodeGenModule's +/// constructor and then copied around into new CodeGenFunctions. +struct CodeGenTypeCache { + /// void + llvm::Type *VoidTy; + + /// i8, i16, i32, and i64 + llvm::IntegerType *Int8Ty, *Int16Ty, *Int32Ty, *Int64Ty; + /// float, double + llvm::Type *FloatTy, *DoubleTy; + + /// int + llvm::IntegerType *IntTy; + + /// intptr_t, size_t, and ptrdiff_t, which we assume are the same size. + union { + llvm::IntegerType *IntPtrTy; + llvm::IntegerType *SizeTy; + llvm::IntegerType *PtrDiffTy; + }; + + /// void* in address space 0 + union { + llvm::PointerType *VoidPtrTy; + llvm::PointerType *Int8PtrTy; + }; + + /// void** in address space 0 + union { + llvm::PointerType *VoidPtrPtrTy; + llvm::PointerType *Int8PtrPtrTy; + }; + + /// The size and alignment of the builtin C type 'int'. This comes + /// up enough in various ABI lowering tasks to be worth pre-computing. + union { + unsigned char IntSizeInBytes; + unsigned char IntAlignInBytes; + }; + CharUnits getIntSize() const { + return CharUnits::fromQuantity(IntSizeInBytes); + } + CharUnits getIntAlign() const { + return CharUnits::fromQuantity(IntAlignInBytes); + } + + /// The width of a pointer into the generic address space. + unsigned char PointerWidthInBits; + + /// The size and alignment of a pointer into the generic address space. + union { + unsigned char PointerAlignInBytes; + unsigned char PointerSizeInBytes; + unsigned char SizeSizeInBytes; // sizeof(size_t) + unsigned char SizeAlignInBytes; + }; + CharUnits getSizeSize() const { + return CharUnits::fromQuantity(SizeSizeInBytes); + } + CharUnits getSizeAlign() const { + return CharUnits::fromQuantity(SizeAlignInBytes); + } + CharUnits getPointerSize() const { + return CharUnits::fromQuantity(PointerSizeInBytes); + } + CharUnits getPointerAlign() const { + return CharUnits::fromQuantity(PointerAlignInBytes); + } + + llvm::CallingConv::ID RuntimeCC; + llvm::CallingConv::ID getRuntimeCC() const { return RuntimeCC; } + llvm::CallingConv::ID BuiltinCC; + llvm::CallingConv::ID getBuiltinCC() const { return BuiltinCC; } +}; + +} // end namespace CodeGen +} // end namespace clang + +#endif diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp index a4a8654..fcda053 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp @@ -31,7 +31,6 @@ using namespace CodeGen; CodeGenTypes::CodeGenTypes(CodeGenModule &cgm) : CGM(cgm), Context(cgm.getContext()), TheModule(cgm.getModule()), - TheDataLayout(cgm.getDataLayout()), Target(cgm.getTarget()), TheCXXABI(cgm.getCXXABI()), TheABIInfo(cgm.getTargetCodeGenInfo().getABIInfo()) { SkippedLayout = false; @@ -295,6 +294,76 @@ static llvm::Type *getTypeForFormat(llvm::LLVMContext &VMContext, llvm_unreachable("Unknown float format!"); } +llvm::Type *CodeGenTypes::ConvertFunctionType(QualType QFT, + const FunctionDecl *FD) { + assert(QFT.isCanonical()); + const Type *Ty = QFT.getTypePtr(); + const FunctionType *FT = cast<FunctionType>(QFT.getTypePtr()); + // First, check whether we can build the full function type. If the + // function type depends on an incomplete type (e.g. a struct or enum), we + // cannot lower the function type. + if (!isFuncTypeConvertible(FT)) { + // This function's type depends on an incomplete tag type. + + // Force conversion of all the relevant record types, to make sure + // we re-convert the FunctionType when appropriate. + if (const RecordType *RT = FT->getReturnType()->getAs<RecordType>()) + ConvertRecordDeclType(RT->getDecl()); + if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FT)) + for (unsigned i = 0, e = FPT->getNumParams(); i != e; i++) + if (const RecordType *RT = FPT->getParamType(i)->getAs<RecordType>()) + ConvertRecordDeclType(RT->getDecl()); + + SkippedLayout = true; + + // Return a placeholder type. + return llvm::StructType::get(getLLVMContext()); + } + + // While we're converting the parameter types for a function, we don't want + // to recursively convert any pointed-to structs. Converting directly-used + // structs is ok though. + if (!RecordsBeingLaidOut.insert(Ty).second) { + SkippedLayout = true; + return llvm::StructType::get(getLLVMContext()); + } + + // The function type can be built; call the appropriate routines to + // build it. + const CGFunctionInfo *FI; + if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FT)) { + FI = &arrangeFreeFunctionType( + CanQual<FunctionProtoType>::CreateUnsafe(QualType(FPT, 0)), FD); + } else { + const FunctionNoProtoType *FNPT = cast<FunctionNoProtoType>(FT); + FI = &arrangeFreeFunctionType( + CanQual<FunctionNoProtoType>::CreateUnsafe(QualType(FNPT, 0))); + } + + llvm::Type *ResultType = nullptr; + // If there is something higher level prodding our CGFunctionInfo, then + // don't recurse into it again. + if (FunctionsBeingProcessed.count(FI)) { + + ResultType = llvm::StructType::get(getLLVMContext()); + SkippedLayout = true; + } else { + + // Otherwise, we're good to go, go ahead and convert it. + ResultType = GetFunctionType(*FI); + } + + RecordsBeingLaidOut.erase(Ty); + + if (SkippedLayout) + TypeCache.clear(); + + if (RecordsBeingLaidOut.empty()) + while (!DeferredRecords.empty()) + ConvertRecordDeclType(DeferredRecords.pop_back_val()); + return ResultType; +} + /// ConvertType - Convert the specified type to its LLVM form. llvm::Type *CodeGenTypes::ConvertType(QualType T) { T = Context.getCanonicalType(T); @@ -389,9 +458,19 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) { case BuiltinType::OCLImage1dBuffer: case BuiltinType::OCLImage2d: case BuiltinType::OCLImage2dArray: + case BuiltinType::OCLImage2dDepth: + case BuiltinType::OCLImage2dArrayDepth: + case BuiltinType::OCLImage2dMSAA: + case BuiltinType::OCLImage2dArrayMSAA: + case BuiltinType::OCLImage2dMSAADepth: + case BuiltinType::OCLImage2dArrayMSAADepth: case BuiltinType::OCLImage3d: case BuiltinType::OCLSampler: case BuiltinType::OCLEvent: + case BuiltinType::OCLClkEvent: + case BuiltinType::OCLQueue: + case BuiltinType::OCLNDRange: + case BuiltinType::OCLReserveID: ResultType = CGM.getOpenCLRuntime().convertOpenCLSpecificType(Ty); break; @@ -476,75 +555,9 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) { break; } case Type::FunctionNoProto: - case Type::FunctionProto: { - const FunctionType *FT = cast<FunctionType>(Ty); - // First, check whether we can build the full function type. If the - // function type depends on an incomplete type (e.g. a struct or enum), we - // cannot lower the function type. - if (!isFuncTypeConvertible(FT)) { - // This function's type depends on an incomplete tag type. - - // Force conversion of all the relevant record types, to make sure - // we re-convert the FunctionType when appropriate. - if (const RecordType *RT = FT->getReturnType()->getAs<RecordType>()) - ConvertRecordDeclType(RT->getDecl()); - if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FT)) - for (unsigned i = 0, e = FPT->getNumParams(); i != e; i++) - if (const RecordType *RT = FPT->getParamType(i)->getAs<RecordType>()) - ConvertRecordDeclType(RT->getDecl()); - - // Return a placeholder type. - ResultType = llvm::StructType::get(getLLVMContext()); - - SkippedLayout = true; - break; - } - - // While we're converting the parameter types for a function, we don't want - // to recursively convert any pointed-to structs. Converting directly-used - // structs is ok though. - if (!RecordsBeingLaidOut.insert(Ty).second) { - ResultType = llvm::StructType::get(getLLVMContext()); - - SkippedLayout = true; - break; - } - - // The function type can be built; call the appropriate routines to - // build it. - const CGFunctionInfo *FI; - if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FT)) { - FI = &arrangeFreeFunctionType( - CanQual<FunctionProtoType>::CreateUnsafe(QualType(FPT, 0))); - } else { - const FunctionNoProtoType *FNPT = cast<FunctionNoProtoType>(FT); - FI = &arrangeFreeFunctionType( - CanQual<FunctionNoProtoType>::CreateUnsafe(QualType(FNPT, 0))); - } - - // If there is something higher level prodding our CGFunctionInfo, then - // don't recurse into it again. - if (FunctionsBeingProcessed.count(FI)) { - - ResultType = llvm::StructType::get(getLLVMContext()); - SkippedLayout = true; - } else { - - // Otherwise, we're good to go, go ahead and convert it. - ResultType = GetFunctionType(*FI); - } - - RecordsBeingLaidOut.erase(Ty); - - if (SkippedLayout) - TypeCache.clear(); - - if (RecordsBeingLaidOut.empty()) - while (!DeferredRecords.empty()) - ConvertRecordDeclType(DeferredRecords.pop_back_val()); + case Type::FunctionProto: + ResultType = ConvertFunctionType(T); break; - } - case Type::ObjCObject: ResultType = ConvertType(cast<ObjCObjectType>(Ty)->getBaseType()); break; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h index 1580e21..a96f23c4 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h @@ -122,7 +122,6 @@ class CodeGenTypes { // Some of this stuff should probably be left on the CGM. ASTContext &Context; llvm::Module &TheModule; - const llvm::DataLayout &TheDataLayout; const TargetInfo &Target; CGCXXABI &TheCXXABI; @@ -159,7 +158,6 @@ class CodeGenTypes { SmallVector<const RecordDecl *, 8> DeferredRecords; -private: /// This map keeps cache of llvm::Types and maps clang::Type to /// corresponding llvm::Type. llvm::DenseMap<const Type *, llvm::Type *> TypeCache; @@ -168,7 +166,9 @@ public: CodeGenTypes(CodeGenModule &cgm); ~CodeGenTypes(); - const llvm::DataLayout &getDataLayout() const { return TheDataLayout; } + const llvm::DataLayout &getDataLayout() const { + return TheModule.getDataLayout(); + } ASTContext &getContext() const { return Context; } const ABIInfo &getABIInfo() const { return TheABIInfo; } const TargetInfo &getTarget() const { return Target; } @@ -178,6 +178,14 @@ public: /// ConvertType - Convert type T into a llvm::Type. llvm::Type *ConvertType(QualType T); + /// \brief Converts the GlobalDecl into an llvm::Type. This should be used + /// when we know the target of the function we want to convert. This is + /// because some functions (explicitly, those with pass_object_size + /// parameters) may not have the same signature as their type portrays, and + /// can only be called directly. + llvm::Type *ConvertFunctionType(QualType FT, + const FunctionDecl *FD = nullptr); + /// ConvertTypeForMem - Convert type T into a llvm::Type. This differs from /// ConvertType in that it is used to convert to the memory representation for /// a type. For example, the scalar representation for _Bool is i1, but the @@ -264,11 +272,12 @@ public: const CGFunctionInfo &arrangeMSMemberPointerThunk(const CXXMethodDecl *MD); const CGFunctionInfo &arrangeMSCtorClosure(const CXXConstructorDecl *CD, CXXCtorType CT); - - const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionProtoType> Ty); + const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionProtoType> Ty, + const FunctionDecl *FD); const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionNoProtoType> Ty); const CGFunctionInfo &arrangeCXXMethodType(const CXXRecordDecl *RD, - const FunctionProtoType *FTP); + const FunctionProtoType *FTP, + const CXXMethodDecl *MD); /// "Arrange" the LLVM information for a call or type with the given /// signature. This is largely an internal method; other clients diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp index eca9159..eb6edea 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp @@ -47,17 +47,6 @@ public: Optional<SourceLocation> LocEnd) : Count(Count), LocStart(LocStart), LocEnd(LocEnd) {} - SourceMappingRegion(SourceMappingRegion &&Region) - : Count(std::move(Region.Count)), LocStart(std::move(Region.LocStart)), - LocEnd(std::move(Region.LocEnd)) {} - - SourceMappingRegion &operator=(SourceMappingRegion &&RHS) { - Count = std::move(RHS.Count); - LocStart = std::move(RHS.LocStart); - LocEnd = std::move(RHS.LocEnd); - return *this; - } - const Counter &getCounter() const { return Count; } void setCounter(Counter C) { Count = C; } @@ -66,7 +55,7 @@ public: void setStartLoc(SourceLocation Loc) { LocStart = Loc; } - const SourceLocation &getStartLoc() const { + SourceLocation getStartLoc() const { assert(LocStart && "Region has no start location"); return *LocStart; } @@ -75,7 +64,7 @@ public: void setEndLoc(SourceLocation Loc) { LocEnd = Loc; } - const SourceLocation &getEndLoc() const { + SourceLocation getEndLoc() const { assert(LocEnd && "Region has no end location"); return *LocEnd; } @@ -174,7 +163,7 @@ public: unsigned Depth = 0; for (SourceLocation Parent = getIncludeOrExpansionLoc(Loc); - !Parent.isInvalid(); Parent = getIncludeOrExpansionLoc(Parent)) + Parent.isValid(); Parent = getIncludeOrExpansionLoc(Parent)) ++Depth; FileLocs.push_back(std::make_pair(Loc, Depth)); } @@ -255,7 +244,7 @@ public: assert(Region.hasEndLoc() && "incomplete region"); SourceLocation LocStart = Region.getStartLoc(); - assert(!SM.getFileID(LocStart).isInvalid() && "region in invalid file"); + assert(SM.getFileID(LocStart).isValid() && "region in invalid file"); auto CovFileID = getCoverageFileID(LocStart); // Ignore regions that don't have a file, such as builtin macros. @@ -413,8 +402,8 @@ struct CounterCoverageMappingBuilder SourceRegions.emplace_back(Region.getCounter(), NestedLoc, EndLoc); EndLoc = getPreciseTokenLocEnd(getIncludeOrExpansionLoc(EndLoc)); - assert(!EndLoc.isInvalid() && - "File exit was not handled before popRegions"); + if (EndLoc.isInvalid()) + llvm::report_fatal_error("File exit not handled before popRegions"); } Region.setEndLoc(EndLoc); @@ -426,7 +415,7 @@ struct CounterCoverageMappingBuilder MostRecentLocation = getIncludeOrExpansionLoc(EndLoc); assert(SM.isWrittenInSameFile(Region.getStartLoc(), EndLoc)); - SourceRegions.push_back(std::move(Region)); + SourceRegions.push_back(Region); } RegionStack.pop_back(); } @@ -496,12 +485,12 @@ struct CounterCoverageMappingBuilder llvm::SmallSet<SourceLocation, 8> StartLocs; Optional<Counter> ParentCounter; - for (auto I = RegionStack.rbegin(), E = RegionStack.rend(); I != E; ++I) { - if (!I->hasStartLoc()) + for (SourceMappingRegion &I : llvm::reverse(RegionStack)) { + if (!I.hasStartLoc()) continue; - SourceLocation Loc = I->getStartLoc(); + SourceLocation Loc = I.getStartLoc(); if (!isNestedIn(Loc, ParentFile)) { - ParentCounter = I->getCounter(); + ParentCounter = I.getCounter(); break; } @@ -510,11 +499,11 @@ struct CounterCoverageMappingBuilder // correct count. We avoid creating redundant regions by stopping once // we've seen this region. if (StartLocs.insert(Loc).second) - SourceRegions.emplace_back(I->getCounter(), Loc, + SourceRegions.emplace_back(I.getCounter(), Loc, getEndOfFileOrMacro(Loc)); Loc = getIncludeOrExpansionLoc(Loc); } - I->setStartLoc(getPreciseTokenLocEnd(Loc)); + I.setStartLoc(getPreciseTokenLocEnd(Loc)); } if (ParentCounter) { @@ -580,7 +569,7 @@ struct CounterCoverageMappingBuilder } void VisitStmt(const Stmt *S) { - if (!S->getLocStart().isInvalid()) + if (S->getLocStart().isValid()) extendRegion(S); for (const Stmt *Child : S->children()) if (Child) @@ -796,7 +785,7 @@ struct CounterCoverageMappingBuilder else pushRegion(Count, getStart(S)); - if (const CaseStmt *CS = dyn_cast<CaseStmt>(S)) { + if (const auto *CS = dyn_cast<CaseStmt>(S)) { Visit(CS->getLHS()); if (const Expr *RHS = CS->getRHS()) Visit(RHS); @@ -842,7 +831,6 @@ struct CounterCoverageMappingBuilder } void VisitCXXCatchStmt(const CXXCatchStmt *S) { - extendRegion(S); propagateCounts(getRegionCounter(S), S->getHandlerBlock()); } @@ -891,7 +879,7 @@ static bool isMachO(const CodeGenModule &CGM) { } static StringRef getCoverageSection(const CodeGenModule &CGM) { - return isMachO(CGM) ? "__DATA,__llvm_covmap" : "__llvm_covmap"; + return llvm::getInstrProfCoverageSectionName(isMachO(CGM)); } static void dump(llvm::raw_ostream &OS, StringRef FunctionName, @@ -922,24 +910,23 @@ static void dump(llvm::raw_ostream &OS, StringRef FunctionName, } void CoverageMappingModuleGen::addFunctionMappingRecord( - llvm::GlobalVariable *FunctionName, StringRef FunctionNameValue, - uint64_t FunctionHash, const std::string &CoverageMapping) { + llvm::GlobalVariable *NamePtr, StringRef NameValue, + uint64_t FuncHash, const std::string &CoverageMapping) { llvm::LLVMContext &Ctx = CGM.getLLVMContext(); - auto *Int32Ty = llvm::Type::getInt32Ty(Ctx); - auto *Int64Ty = llvm::Type::getInt64Ty(Ctx); - auto *Int8PtrTy = llvm::Type::getInt8PtrTy(Ctx); if (!FunctionRecordTy) { - llvm::Type *FunctionRecordTypes[] = {Int8PtrTy, Int32Ty, Int32Ty, Int64Ty}; + #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) LLVMType, + llvm::Type *FunctionRecordTypes[] = { + #include "llvm/ProfileData/InstrProfData.inc" + }; FunctionRecordTy = llvm::StructType::get(Ctx, makeArrayRef(FunctionRecordTypes), /*isPacked=*/true); } + #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Init, llvm::Constant *FunctionRecordVals[] = { - llvm::ConstantExpr::getBitCast(FunctionName, Int8PtrTy), - llvm::ConstantInt::get(Int32Ty, FunctionNameValue.size()), - llvm::ConstantInt::get(Int32Ty, CoverageMapping.size()), - llvm::ConstantInt::get(Int64Ty, FunctionHash)}; + #include "llvm/ProfileData/InstrProfData.inc" + }; FunctionRecords.push_back(llvm::ConstantStruct::get( FunctionRecordTy, makeArrayRef(FunctionRecordVals))); CoverageMappings += CoverageMapping; @@ -961,7 +948,7 @@ void CoverageMappingModuleGen::addFunctionMappingRecord( Expressions, Regions); if (Reader.read()) return; - dump(llvm::outs(), FunctionNameValue, Expressions, Regions); + dump(llvm::outs(), NameValue, Expressions, Regions); } } @@ -1023,7 +1010,7 @@ void CoverageMappingModuleGen::emit() { auto CovData = new llvm::GlobalVariable(CGM.getModule(), CovDataTy, true, llvm::GlobalValue::InternalLinkage, CovDataVal, - "__llvm_coverage_mapping"); + llvm::getCoverageMappingVarName()); CovData->setSection(getCoverageSection(CGM)); CovData->setAlignment(8); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h b/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h index a795188..85cd154 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h @@ -96,6 +96,9 @@ enum CleanupKind : unsigned { /// and catch blocks. class EHScopeStack { public: + /* Should switch to alignof(uint64_t) instead of 8, when EHCleanupScope can */ + enum { ScopeStackAlignment = 8 }; + /// A saved depth on the scope stack. This is necessary because /// pushing scopes onto the stack invalidates iterators. class stable_iterator { @@ -141,7 +144,15 @@ public: class Cleanup { // Anchor the construction vtable. virtual void anchor(); + + protected: + ~Cleanup() = default; + public: + Cleanup(const Cleanup &) = default; + Cleanup(Cleanup &&) {} + Cleanup() = default; + /// Generation flags. class Flags { enum { @@ -168,10 +179,6 @@ public: void setIsEHCleanupKind() { flags |= F_IsEHCleanupKind; } }; - // Provide a virtual destructor to suppress a very common warning - // that unfortunately cannot be suppressed without this. Cleanups - // should not rely on this destructor ever being called. - virtual ~Cleanup() {} /// Emit the cleanup. For normal cleanups, this is run in the /// same EH context as when the cleanup was pushed, i.e. the @@ -184,7 +191,8 @@ public: /// ConditionalCleanup stores the saved form of its parameters, /// then restores them and performs the cleanup. - template <class T, class... As> class ConditionalCleanup : public Cleanup { + template <class T, class... As> + class ConditionalCleanup final : public Cleanup { typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple; SavedTuple Saved; @@ -248,6 +256,7 @@ private: SmallVector<BranchFixup, 8> BranchFixups; char *allocate(size_t Size); + void deallocate(size_t Size); void *pushCleanup(CleanupKind K, size_t DataSize); @@ -259,6 +268,8 @@ public: /// Push a lazily-created cleanup on the stack. template <class T, class... As> void pushCleanup(CleanupKind Kind, As... A) { + static_assert(llvm::AlignOf<T>::Alignment <= ScopeStackAlignment, + "Cleanup's alignment is too large."); void *Buffer = pushCleanup(Kind, sizeof(T)); Cleanup *Obj = new (Buffer) T(A...); (void) Obj; @@ -267,6 +278,8 @@ public: /// Push a lazily-created cleanup on the stack. Tuple version. template <class T, class... As> void pushCleanupTuple(CleanupKind Kind, std::tuple<As...> A) { + static_assert(llvm::AlignOf<T>::Alignment <= ScopeStackAlignment, + "Cleanup's alignment is too large."); void *Buffer = pushCleanup(Kind, sizeof(T)); Cleanup *Obj = new (Buffer) T(std::move(A)); (void) Obj; @@ -287,6 +300,8 @@ public: /// stack is modified. template <class T, class... As> T *pushCleanupWithExtra(CleanupKind Kind, size_t N, As... A) { + static_assert(llvm::AlignOf<T>::Alignment <= ScopeStackAlignment, + "Cleanup's alignment is too large."); void *Buffer = pushCleanup(Kind, sizeof(T) + T::getExtraSize(N)); return new (Buffer) T(N, A...); } @@ -346,7 +361,6 @@ public: return InnermostEHScope; } - stable_iterator getInnermostActiveEHScope() const; /// An unstable reference to a scope-stack depth. Invalidated by /// pushes but not pops. @@ -377,9 +391,6 @@ public: /// to the EH stack. iterator find(stable_iterator save) const; - /// Removes the cleanup pointed to by the given stable_iterator. - void removeCleanup(stable_iterator save); - /// Add a branch fixup to the current cleanup scope. BranchFixup &addBranchFixup() { assert(hasNormalCleanups() && "adding fixup in scope without cleanups"); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp index 2be9ceb..0c4008f 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -69,6 +69,45 @@ public: return RAA_Default; } + bool isThisCompleteObject(GlobalDecl GD) const override { + // The Itanium ABI has separate complete-object vs. base-object + // variants of both constructors and destructors. + if (isa<CXXDestructorDecl>(GD.getDecl())) { + switch (GD.getDtorType()) { + case Dtor_Complete: + case Dtor_Deleting: + return true; + + case Dtor_Base: + return false; + + case Dtor_Comdat: + llvm_unreachable("emitting dtor comdat as function?"); + } + llvm_unreachable("bad dtor kind"); + } + if (isa<CXXConstructorDecl>(GD.getDecl())) { + switch (GD.getCtorType()) { + case Ctor_Complete: + return true; + + case Ctor_Base: + return false; + + case Ctor_CopyingClosure: + case Ctor_DefaultClosure: + llvm_unreachable("closure ctors in Itanium ABI?"); + + case Ctor_Comdat: + llvm_unreachable("emitting ctor comdat as function?"); + } + llvm_unreachable("bad dtor kind"); + } + + // No other kinds. + return false; + } + bool isZeroInitializable(const MemberPointerType *MPT) override; llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT) override; @@ -76,13 +115,14 @@ public: llvm::Value * EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF, const Expr *E, - llvm::Value *&This, + Address This, + llvm::Value *&ThisPtrForCall, llvm::Value *MemFnPtr, const MemberPointerType *MPT) override; llvm::Value * EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E, - llvm::Value *Base, + Address Base, llvm::Value *MemPtr, const MemberPointerType *MPT) override; @@ -111,9 +151,22 @@ public: const MemberPointerType *MPT) override; void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE, - llvm::Value *Ptr, QualType ElementType, + Address Ptr, QualType ElementType, const CXXDestructorDecl *Dtor) override; + /// Itanium says that an _Unwind_Exception has to be "double-word" + /// aligned (and thus the end of it is also so-aligned), meaning 16 + /// bytes. Of course, that was written for the actual Itanium, + /// which is a 64-bit platform. Classically, the ABI doesn't really + /// specify the alignment on other platforms, but in practice + /// libUnwind declares the struct with __attribute__((aligned)), so + /// we assume that alignment here. (It's generally 16 bytes, but + /// some targets overwrite it.) + CharUnits getAlignmentOfExnObject() { + auto align = CGM.getContext().getTargetDefaultAlignForAttributeAligned(); + return CGM.getContext().toCharUnitsFromBits(align); + } + void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override; void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override; @@ -126,34 +179,34 @@ public: void EmitFundamentalRTTIDescriptor(QualType Type); void EmitFundamentalRTTIDescriptors(); llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override; - llvm::Constant * + CatchTypeInfo getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) override { - return getAddrOfRTTIDescriptor(Ty); + return CatchTypeInfo{getAddrOfRTTIDescriptor(Ty), 0}; } bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override; void EmitBadTypeidCall(CodeGenFunction &CGF) override; llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy, - llvm::Value *ThisPtr, + Address ThisPtr, llvm::Type *StdTypeInfoPtrTy) override; bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr, QualType SrcRecordTy) override; - llvm::Value *EmitDynamicCastCall(CodeGenFunction &CGF, llvm::Value *Value, + llvm::Value *EmitDynamicCastCall(CodeGenFunction &CGF, Address Value, QualType SrcRecordTy, QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) override; - llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, llvm::Value *Value, + llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, Address Value, QualType SrcRecordTy, QualType DestTy) override; bool EmitBadCastCall(CodeGenFunction &CGF) override; llvm::Value * - GetVirtualBaseClassOffset(CodeGenFunction &CGF, llvm::Value *This, + GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This, const CXXRecordDecl *ClassDecl, const CXXRecordDecl *BaseClassDecl) override; @@ -185,15 +238,29 @@ public: void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type, bool ForVirtualBase, - bool Delegating, llvm::Value *This) override; + bool Delegating, Address This) override; void emitVTableDefinitions(CodeGenVTables &CGVT, const CXXRecordDecl *RD) override; + bool isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF, + CodeGenFunction::VPtr Vptr) override; + + bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) override { + return true; + } + + llvm::Constant * + getVTableAddressPoint(BaseSubobject Base, + const CXXRecordDecl *VTableClass) override; + llvm::Value *getVTableAddressPointInStructor( CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, - BaseSubobject Base, const CXXRecordDecl *NearestVBase, - bool &NeedsVirtualOffset) override; + BaseSubobject Base, const CXXRecordDecl *NearestVBase) override; + + llvm::Value *getVTableAddressPointInStructorWithVTT( + CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, + BaseSubobject Base, const CXXRecordDecl *NearestVBase); llvm::Constant * getVTableAddressPointForConstExpr(BaseSubobject Base, @@ -203,18 +270,19 @@ public: CharUnits VPtrOffset) override; llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, - llvm::Value *This, - llvm::Type *Ty, + Address This, llvm::Type *Ty, SourceLocation Loc) override; llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType, - llvm::Value *This, + Address This, const CXXMemberCallExpr *CE) override; void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override; + bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const override; + void setThunkLinkage(llvm::Function *Thunk, bool ForVTable, GlobalDecl GD, bool ReturnAdjustment) override { // Allow inlining of thunks by emitting them with available_externally @@ -223,10 +291,10 @@ public: Thunk->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage); } - llvm::Value *performThisAdjustment(CodeGenFunction &CGF, llvm::Value *This, + llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This, const ThisAdjustment &TA) override; - llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret, + llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret, const ReturnAdjustment &RA) override; size_t getSrcArgforCopyCtor(const CXXConstructorDecl *, @@ -240,13 +308,13 @@ public: { return "__cxa_deleted_virtual"; } CharUnits getArrayCookieSizeImpl(QualType elementType) override; - llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF, - llvm::Value *NewPtr, - llvm::Value *NumElements, - const CXXNewExpr *expr, - QualType ElementType) override; + Address InitializeArrayCookie(CodeGenFunction &CGF, + Address NewPtr, + llvm::Value *NumElements, + const CXXNewExpr *expr, + QualType ElementType) override; llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, - llvm::Value *allocPtr, + Address allocPtr, CharUnits cookieSize) override; void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, @@ -259,10 +327,9 @@ public: llvm::Value *Val); void EmitThreadLocalInitFuncs( CodeGenModule &CGM, - ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>> - CXXThreadLocals, + ArrayRef<const VarDecl *> CXXThreadLocals, ArrayRef<llvm::Function *> CXXThreadLocalInits, - ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) override; + ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override; bool usesThreadWrapperFunction() const override { return true; } LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD, @@ -302,6 +369,41 @@ public: friend class ItaniumRTTIBuilder; void emitCXXStructor(const CXXMethodDecl *MD, StructorType Type) override; + + private: + bool hasAnyUsedVirtualInlineFunction(const CXXRecordDecl *RD) const { + const auto &VtableLayout = + CGM.getItaniumVTableContext().getVTableLayout(RD); + + for (const auto &VtableComponent : VtableLayout.vtable_components()) { + if (!VtableComponent.isUsedFunctionPointerKind()) + continue; + + const CXXMethodDecl *Method = VtableComponent.getFunctionDecl(); + if (Method->getCanonicalDecl()->isInlined()) + return true; + } + return false; + } + + bool isVTableHidden(const CXXRecordDecl *RD) const { + const auto &VtableLayout = + CGM.getItaniumVTableContext().getVTableLayout(RD); + + for (const auto &VtableComponent : VtableLayout.vtable_components()) { + if (VtableComponent.isRTTIKind()) { + const CXXRecordDecl *RTTIDecl = VtableComponent.getRTTIDecl(); + if (RTTIDecl->getVisibility() == Visibility::HiddenVisibility) + return true; + } else if (VtableComponent.isUsedFunctionPointerKind()) { + const CXXMethodDecl *Method = VtableComponent.getFunctionDecl(); + if (Method->getVisibility() == Visibility::HiddenVisibility && + !Method->isDefined()) + return true; + } + } + return false; + } }; class ARMCXXABI : public ItaniumCXXABI { @@ -320,12 +422,12 @@ public: QualType ResTy) override; CharUnits getArrayCookieSizeImpl(QualType elementType) override; - llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF, - llvm::Value *NewPtr, - llvm::Value *NumElements, - const CXXNewExpr *expr, - QualType ElementType) override; - llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, llvm::Value *allocPtr, + Address InitializeArrayCookie(CodeGenFunction &CGF, + Address NewPtr, + llvm::Value *NumElements, + const CXXNewExpr *expr, + QualType ElementType) override; + llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, Address allocPtr, CharUnits cookieSize) override; }; @@ -336,6 +438,20 @@ public: // ARM64 libraries are prepared for non-unique RTTI. bool shouldRTTIBeUnique() const override { return false; } }; + +class WebAssemblyCXXABI final : public ItaniumCXXABI { +public: + explicit WebAssemblyCXXABI(CodeGen::CodeGenModule &CGM) + : ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true, + /*UseARMGuardVarABI=*/true) {} + +private: + bool HasThisReturn(GlobalDecl GD) const override { + return isa<CXXConstructorDecl>(GD.getDecl()) || + (isa<CXXDestructorDecl>(GD.getDecl()) && + GD.getDtorType() != Dtor_Deleting); + } +}; } CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) { @@ -344,6 +460,7 @@ CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) { // between the ARM and iOS ABIs. case TargetCXXABI::GenericARM: case TargetCXXABI::iOS: + case TargetCXXABI::WatchOS: return new ARMCXXABI(CGM); case TargetCXXABI::iOS64: @@ -359,6 +476,9 @@ CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) { case TargetCXXABI::GenericMIPS: return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true); + case TargetCXXABI::WebAssembly: + return new WebAssemblyCXXABI(CGM); + case TargetCXXABI::GenericItanium: if (CGM.getContext().getTargetInfo().getTriple().getArch() == llvm::Triple::le32) { @@ -404,7 +524,8 @@ ItaniumCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) { /// If the member is non-virtual, memptr.ptr is the address of /// the function to call. llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( - CodeGenFunction &CGF, const Expr *E, llvm::Value *&This, + CodeGenFunction &CGF, const Expr *E, Address ThisAddr, + llvm::Value *&ThisPtrForCall, llvm::Value *MemFnPtr, const MemberPointerType *MPT) { CGBuilderTy &Builder = CGF.Builder; @@ -413,9 +534,8 @@ llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( const CXXRecordDecl *RD = cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl()); - llvm::FunctionType *FTy = - CGM.getTypes().GetFunctionType( - CGM.getTypes().arrangeCXXMethodType(RD, FPT)); + llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType( + CGM.getTypes().arrangeCXXMethodType(RD, FPT, /*FD=*/nullptr)); llvm::Constant *ptrdiff_1 = llvm::ConstantInt::get(CGM.PtrDiffTy, 1); @@ -433,9 +553,11 @@ llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( // Apply the adjustment and cast back to the original struct type // for consistency. + llvm::Value *This = ThisAddr.getPointer(); llvm::Value *Ptr = Builder.CreateBitCast(This, Builder.getInt8PtrTy()); Ptr = Builder.CreateInBoundsGEP(Ptr, Adj); This = Builder.CreateBitCast(Ptr, This->getType(), "this.adjusted"); + ThisPtrForCall = This; // Load the function pointer. llvm::Value *FnAsInt = Builder.CreateExtractValue(MemFnPtr, 0, "memptr.ptr"); @@ -457,7 +579,11 @@ llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( // Cast the adjusted this to a pointer to vtable pointer and load. llvm::Type *VTableTy = Builder.getInt8PtrTy(); - llvm::Value *VTable = CGF.GetVTablePtr(This, VTableTy); + CharUnits VTablePtrAlign = + CGF.CGM.getDynamicOffsetAlignment(ThisAddr.getAlignment(), RD, + CGF.getPointerAlign()); + llvm::Value *VTable = + CGF.GetVTablePtr(Address(This, VTablePtrAlign), VTableTy, RD); // Apply the offset. llvm::Value *VTableOffset = FnAsInt; @@ -467,7 +593,9 @@ llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( // Load the virtual function to call. VTable = Builder.CreateBitCast(VTable, FTy->getPointerTo()->getPointerTo()); - llvm::Value *VirtualFn = Builder.CreateLoad(VTable, "memptr.virtualfn"); + llvm::Value *VirtualFn = + Builder.CreateAlignedLoad(VTable, CGF.getPointerAlign(), + "memptr.virtualfn"); CGF.EmitBranch(FnEnd); // In the non-virtual path, the function pointer is actually a @@ -487,24 +615,23 @@ llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( /// Compute an l-value by applying the given pointer-to-member to a /// base object. llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress( - CodeGenFunction &CGF, const Expr *E, llvm::Value *Base, llvm::Value *MemPtr, + CodeGenFunction &CGF, const Expr *E, Address Base, llvm::Value *MemPtr, const MemberPointerType *MPT) { assert(MemPtr->getType() == CGM.PtrDiffTy); CGBuilderTy &Builder = CGF.Builder; - unsigned AS = Base->getType()->getPointerAddressSpace(); - // Cast to char*. - Base = Builder.CreateBitCast(Base, Builder.getInt8Ty()->getPointerTo(AS)); + Base = Builder.CreateElementBitCast(Base, CGF.Int8Ty); // Apply the offset, which we assume is non-null. - llvm::Value *Addr = Builder.CreateInBoundsGEP(Base, MemPtr, "memptr.offset"); + llvm::Value *Addr = + Builder.CreateInBoundsGEP(Base.getPointer(), MemPtr, "memptr.offset"); // Cast the address to the appropriate pointer type, adopting the // address space of the base pointer. - llvm::Type *PType - = CGF.ConvertTypeForMem(MPT->getPointeeType())->getPointerTo(AS); + llvm::Type *PType = CGF.ConvertTypeForMem(MPT->getPointeeType()) + ->getPointerTo(Base.getAddressSpace()); return Builder.CreateBitCast(Addr, PType); } @@ -858,7 +985,8 @@ bool ItaniumCXXABI::classifyReturnType(CGFunctionInfo &FI) const { // FIXME: Use canCopyArgument() when it is fixed to handle lazily declared // special members. if (RD->hasNonTrivialDestructor() || RD->hasNonTrivialCopyConstructor()) { - FI.getReturnInfo() = ABIArgInfo::getIndirect(0, /*ByVal=*/false); + auto Align = CGM.getContext().getTypeAlignInChars(FI.getReturnType()); + FI.getReturnInfo() = ABIArgInfo::getIndirect(Align, /*ByVal=*/false); return true; } return false; @@ -874,7 +1002,7 @@ bool ItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) { /// at entry -2 in the vtable. void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE, - llvm::Value *Ptr, + Address Ptr, QualType ElementType, const CXXDestructorDecl *Dtor) { bool UseGlobalDelete = DE->isGlobalDelete(); @@ -883,16 +1011,20 @@ void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF, // to pass to the deallocation function. // Grab the vtable pointer as an intptr_t*. - llvm::Value *VTable = CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo()); + auto *ClassDecl = + cast<CXXRecordDecl>(ElementType->getAs<RecordType>()->getDecl()); + llvm::Value *VTable = + CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo(), ClassDecl); // Track back to entry -2 and pull out the offset there. llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64( VTable, -2, "complete-offset.ptr"); - llvm::LoadInst *Offset = CGF.Builder.CreateLoad(OffsetPtr); - Offset->setAlignment(CGF.PointerAlignInBytes); + llvm::Value *Offset = + CGF.Builder.CreateAlignedLoad(OffsetPtr, CGF.getPointerAlign()); // Apply the offset. - llvm::Value *CompletePtr = CGF.Builder.CreateBitCast(Ptr, CGF.Int8PtrTy); + llvm::Value *CompletePtr = + CGF.Builder.CreateBitCast(Ptr.getPointer(), CGF.Int8PtrTy); CompletePtr = CGF.Builder.CreateInBoundsGEP(CompletePtr, Offset); // If we're supposed to call the global delete, make sure we do so @@ -954,7 +1086,8 @@ void ItaniumCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { llvm::CallInst *ExceptionPtr = CGF.EmitNounwindRuntimeCall( AllocExceptionFn, llvm::ConstantInt::get(SizeTy, TypeSize), "exception"); - CGF.EmitAnyExprToExn(E->getSubExpr(), ExceptionPtr); + CharUnits ExnAlign = getAlignmentOfExnObject(); + CGF.EmitAnyExprToExn(E->getSubExpr(), Address(ExceptionPtr, ExnAlign)); // Now throw the exception. llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType, @@ -1023,25 +1156,25 @@ static CharUnits computeOffsetHint(ASTContext &Context, CharUnits Offset; // Now walk all possible inheritance paths. - for (CXXBasePaths::paths_iterator I = Paths.begin(), E = Paths.end(); I != E; - ++I) { - if (I->Access != AS_public) // Ignore non-public inheritance. + for (const CXXBasePath &Path : Paths) { + if (Path.Access != AS_public) // Ignore non-public inheritance. continue; ++NumPublicPaths; - for (CXXBasePath::iterator J = I->begin(), JE = I->end(); J != JE; ++J) { + for (const CXXBasePathElement &PathElement : Path) { // If the path contains a virtual base class we can't give any hint. // -1: no hint. - if (J->Base->isVirtual()) + if (PathElement.Base->isVirtual()) return CharUnits::fromQuantity(-1ULL); if (NumPublicPaths > 1) // Won't use offsets, skip computation. continue; // Accumulate the base class offsets. - const ASTRecordLayout &L = Context.getASTRecordLayout(J->Class); - Offset += L.getBaseClassOffset(J->Base->getType()->getAsCXXRecordDecl()); + const ASTRecordLayout &L = Context.getASTRecordLayout(PathElement.Class); + Offset += L.getBaseClassOffset( + PathElement.Base->getType()->getAsCXXRecordDecl()); } } @@ -1078,14 +1211,16 @@ void ItaniumCXXABI::EmitBadTypeidCall(CodeGenFunction &CGF) { llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy, - llvm::Value *ThisPtr, + Address ThisPtr, llvm::Type *StdTypeInfoPtrTy) { + auto *ClassDecl = + cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl()); llvm::Value *Value = - CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo()); + CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo(), ClassDecl); // Load the type info. Value = CGF.Builder.CreateConstInBoundsGEP1_64(Value, -1ULL); - return CGF.Builder.CreateLoad(Value); + return CGF.Builder.CreateAlignedLoad(Value, CGF.getPointerAlign()); } bool ItaniumCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr, @@ -1094,7 +1229,7 @@ bool ItaniumCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr, } llvm::Value *ItaniumCXXABI::EmitDynamicCastCall( - CodeGenFunction &CGF, llvm::Value *Value, QualType SrcRecordTy, + CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy, QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) { llvm::Type *PtrDiffLTy = CGF.ConvertType(CGF.getContext().getPointerDiffType()); @@ -1113,6 +1248,7 @@ llvm::Value *ItaniumCXXABI::EmitDynamicCastCall( computeOffsetHint(CGF.getContext(), SrcDecl, DestDecl).getQuantity()); // Emit the call to __dynamic_cast. + llvm::Value *Value = ThisAddr.getPointer(); Value = CGF.EmitCastToVoidPtr(Value); llvm::Value *args[] = {Value, SrcRTTI, DestRTTI, OffsetHint}; @@ -1136,22 +1272,28 @@ llvm::Value *ItaniumCXXABI::EmitDynamicCastCall( } llvm::Value *ItaniumCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF, - llvm::Value *Value, + Address ThisAddr, QualType SrcRecordTy, QualType DestTy) { llvm::Type *PtrDiffLTy = CGF.ConvertType(CGF.getContext().getPointerDiffType()); llvm::Type *DestLTy = CGF.ConvertType(DestTy); + auto *ClassDecl = + cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl()); // Get the vtable pointer. - llvm::Value *VTable = CGF.GetVTablePtr(Value, PtrDiffLTy->getPointerTo()); + llvm::Value *VTable = CGF.GetVTablePtr(ThisAddr, PtrDiffLTy->getPointerTo(), + ClassDecl); // Get the offset-to-top from the vtable. llvm::Value *OffsetToTop = CGF.Builder.CreateConstInBoundsGEP1_64(VTable, -2ULL); - OffsetToTop = CGF.Builder.CreateLoad(OffsetToTop, "offset.to.top"); + OffsetToTop = + CGF.Builder.CreateAlignedLoad(OffsetToTop, CGF.getPointerAlign(), + "offset.to.top"); // Finally, add the offset to the pointer. + llvm::Value *Value = ThisAddr.getPointer(); Value = CGF.EmitCastToVoidPtr(Value); Value = CGF.Builder.CreateInBoundsGEP(Value, OffsetToTop); @@ -1167,10 +1309,10 @@ bool ItaniumCXXABI::EmitBadCastCall(CodeGenFunction &CGF) { llvm::Value * ItaniumCXXABI::GetVirtualBaseClassOffset(CodeGenFunction &CGF, - llvm::Value *This, + Address This, const CXXRecordDecl *ClassDecl, const CXXRecordDecl *BaseClassDecl) { - llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy); + llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy, ClassDecl); CharUnits VBaseOffsetOffset = CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(ClassDecl, BaseClassDecl); @@ -1182,7 +1324,8 @@ ItaniumCXXABI::GetVirtualBaseClassOffset(CodeGenFunction &CGF, CGM.PtrDiffTy->getPointerTo()); llvm::Value *VBaseOffset = - CGF.Builder.CreateLoad(VBaseOffsetPtr, "vbase.offset"); + CGF.Builder.CreateAlignedLoad(VBaseOffsetPtr, CGF.getPointerAlign(), + "vbase.offset"); return VBaseOffset; } @@ -1293,7 +1436,7 @@ unsigned ItaniumCXXABI::addImplicitConstructorArgs( void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type, bool ForVirtualBase, - bool Delegating, llvm::Value *This) { + bool Delegating, Address This) { GlobalDecl GD(DD, Type); llvm::Value *VTT = CGF.GetVTTParameter(GD, ForVirtualBase, Delegating); QualType VTTTy = getContext().getPointerType(getContext().VoidPtrTy); @@ -1305,8 +1448,8 @@ void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF, if (!Callee) Callee = CGM.getAddrOfCXXStructor(DD, getFromDtorType(Type)); - CGF.EmitCXXMemberOrOperatorCall(DD, Callee, ReturnValueSlot(), This, VTT, - VTTTy, nullptr); + CGF.EmitCXXMemberOrOperatorCall(DD, Callee, ReturnValueSlot(), + This.getPointer(), VTT, VTTTy, nullptr); } void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, @@ -1356,41 +1499,29 @@ void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, CGM.EmitVTableBitSetEntries(VTable, VTLayout); } +bool ItaniumCXXABI::isVirtualOffsetNeededForVTableField( + CodeGenFunction &CGF, CodeGenFunction::VPtr Vptr) { + if (Vptr.NearestVBase == nullptr) + return false; + return NeedsVTTParameter(CGF.CurGD); +} + llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor( CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base, - const CXXRecordDecl *NearestVBase, bool &NeedsVirtualOffset) { - bool NeedsVTTParam = CGM.getCXXABI().NeedsVTTParameter(CGF.CurGD); - NeedsVirtualOffset = (NeedsVTTParam && NearestVBase); - - llvm::Value *VTableAddressPoint; - if (NeedsVTTParam && (Base.getBase()->getNumVBases() || NearestVBase)) { - // Get the secondary vpointer index. - uint64_t VirtualPointerIndex = - CGM.getVTables().getSecondaryVirtualPointerIndex(VTableClass, Base); - - /// Load the VTT. - llvm::Value *VTT = CGF.LoadCXXVTT(); - if (VirtualPointerIndex) - VTT = CGF.Builder.CreateConstInBoundsGEP1_64(VTT, VirtualPointerIndex); - - // And load the address point from the VTT. - VTableAddressPoint = CGF.Builder.CreateLoad(VTT); - } else { - llvm::Constant *VTable = - CGM.getCXXABI().getAddrOfVTable(VTableClass, CharUnits()); - uint64_t AddressPoint = CGM.getItaniumVTableContext() - .getVTableLayout(VTableClass) - .getAddressPoint(Base); - VTableAddressPoint = - CGF.Builder.CreateConstInBoundsGEP2_64(VTable, 0, AddressPoint); - } + const CXXRecordDecl *NearestVBase) { - return VTableAddressPoint; + if ((Base.getBase()->getNumVBases() || NearestVBase != nullptr) && + NeedsVTTParameter(CGF.CurGD)) { + return getVTableAddressPointInStructorWithVTT(CGF, VTableClass, Base, + NearestVBase); + } + return getVTableAddressPoint(Base, VTableClass); } -llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr( - BaseSubobject Base, const CXXRecordDecl *VTableClass) { - auto *VTable = getAddrOfVTable(VTableClass, CharUnits()); +llvm::Constant * +ItaniumCXXABI::getVTableAddressPoint(BaseSubobject Base, + const CXXRecordDecl *VTableClass) { + llvm::GlobalValue *VTable = getAddrOfVTable(VTableClass, CharUnits()); // Find the appropriate vtable within the vtable group. uint64_t AddressPoint = CGM.getItaniumVTableContext() @@ -1405,6 +1536,30 @@ llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr( VTable, Indices); } +llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT( + CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base, + const CXXRecordDecl *NearestVBase) { + assert((Base.getBase()->getNumVBases() || NearestVBase != nullptr) && + NeedsVTTParameter(CGF.CurGD) && "This class doesn't have VTT"); + + // Get the secondary vpointer index. + uint64_t VirtualPointerIndex = + CGM.getVTables().getSecondaryVirtualPointerIndex(VTableClass, Base); + + /// Load the VTT. + llvm::Value *VTT = CGF.LoadCXXVTT(); + if (VirtualPointerIndex) + VTT = CGF.Builder.CreateConstInBoundsGEP1_64(VTT, VirtualPointerIndex); + + // And load the address point from the VTT. + return CGF.Builder.CreateAlignedLoad(VTT, CGF.getPointerAlign()); +} + +llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr( + BaseSubobject Base, const CXXRecordDecl *VTableClass) { + return getVTableAddressPoint(Base, VTableClass); +} + llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) { assert(VPtrOffset.isZero() && "Itanium ABI only supports zero vptr offsets"); @@ -1416,11 +1571,9 @@ llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, // Queue up this v-table for possible deferred emission. CGM.addDeferredVTable(RD); - SmallString<256> OutName; - llvm::raw_svector_ostream Out(OutName); + SmallString<256> Name; + llvm::raw_svector_ostream Out(Name); getMangleContext().mangleCXXVTable(RD, Out); - Out.flush(); - StringRef Name = OutName.str(); ItaniumVTableContext &VTContext = CGM.getItaniumVTableContext(); llvm::ArrayType *ArrayType = llvm::ArrayType::get( @@ -1440,26 +1593,27 @@ llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, llvm::Value *ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, - llvm::Value *This, + Address This, llvm::Type *Ty, SourceLocation Loc) { GD = GD.getCanonicalDecl(); Ty = Ty->getPointerTo()->getPointerTo(); - llvm::Value *VTable = CGF.GetVTablePtr(This, Ty); + auto *MethodDecl = cast<CXXMethodDecl>(GD.getDecl()); + llvm::Value *VTable = CGF.GetVTablePtr(This, Ty, MethodDecl->getParent()); if (CGF.SanOpts.has(SanitizerKind::CFIVCall)) - CGF.EmitVTablePtrCheckForCall(cast<CXXMethodDecl>(GD.getDecl()), VTable, + CGF.EmitVTablePtrCheckForCall(MethodDecl, VTable, CodeGenFunction::CFITCK_VCall, Loc); uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn"); - return CGF.Builder.CreateLoad(VFuncPtr); + return CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); } llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall( CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType, - llvm::Value *This, const CXXMemberCallExpr *CE) { + Address This, const CXXMemberCallExpr *CE) { assert(CE == nullptr || CE->arg_begin() == CE->arg_end()); assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete); @@ -1470,8 +1624,9 @@ llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall( getVirtualFunctionPointer(CGF, GlobalDecl(Dtor, DtorType), This, Ty, CE ? CE->getLocStart() : SourceLocation()); - CGF.EmitCXXMemberOrOperatorCall(Dtor, Callee, ReturnValueSlot(), This, - /*ImplicitParam=*/nullptr, QualType(), CE); + CGF.EmitCXXMemberOrOperatorCall(Dtor, Callee, ReturnValueSlot(), + This.getPointer(), /*ImplicitParam=*/nullptr, + QualType(), CE); return nullptr; } @@ -1481,30 +1636,41 @@ void ItaniumCXXABI::emitVirtualInheritanceTables(const CXXRecordDecl *RD) { VTables.EmitVTTDefinition(VTT, CGM.getVTableLinkage(RD), RD); } +bool ItaniumCXXABI::canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const { + // We don't emit available_externally vtables if we are in -fapple-kext mode + // because kext mode does not permit devirtualization. + if (CGM.getLangOpts().AppleKext) + return false; + + // If we don't have any inline virtual functions, and if vtable is not hidden, + // then we are safe to emit available_externally copy of vtable. + // FIXME we can still emit a copy of the vtable if we + // can emit definition of the inline functions. + return !hasAnyUsedVirtualInlineFunction(RD) && !isVTableHidden(RD); +} static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, - llvm::Value *Ptr, + Address InitialPtr, int64_t NonVirtualAdjustment, int64_t VirtualAdjustment, bool IsReturnAdjustment) { if (!NonVirtualAdjustment && !VirtualAdjustment) - return Ptr; + return InitialPtr.getPointer(); - llvm::Type *Int8PtrTy = CGF.Int8PtrTy; - llvm::Value *V = CGF.Builder.CreateBitCast(Ptr, Int8PtrTy); + Address V = CGF.Builder.CreateElementBitCast(InitialPtr, CGF.Int8Ty); + // In a base-to-derived cast, the non-virtual adjustment is applied first. if (NonVirtualAdjustment && !IsReturnAdjustment) { - // Perform the non-virtual adjustment for a base-to-derived cast. - V = CGF.Builder.CreateConstInBoundsGEP1_64(V, NonVirtualAdjustment); + V = CGF.Builder.CreateConstInBoundsByteGEP(V, + CharUnits::fromQuantity(NonVirtualAdjustment)); } + // Perform the virtual adjustment if we have one. + llvm::Value *ResultPtr; if (VirtualAdjustment) { llvm::Type *PtrDiffTy = CGF.ConvertType(CGF.getContext().getPointerDiffType()); - // Perform the virtual adjustment. - llvm::Value *VTablePtrPtr = - CGF.Builder.CreateBitCast(V, Int8PtrTy->getPointerTo()); - + Address VTablePtrPtr = CGF.Builder.CreateElementBitCast(V, CGF.Int8PtrTy); llvm::Value *VTablePtr = CGF.Builder.CreateLoad(VTablePtrPtr); llvm::Value *OffsetPtr = @@ -1513,23 +1679,28 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, OffsetPtr = CGF.Builder.CreateBitCast(OffsetPtr, PtrDiffTy->getPointerTo()); // Load the adjustment offset from the vtable. - llvm::Value *Offset = CGF.Builder.CreateLoad(OffsetPtr); + llvm::Value *Offset = + CGF.Builder.CreateAlignedLoad(OffsetPtr, CGF.getPointerAlign()); // Adjust our pointer. - V = CGF.Builder.CreateInBoundsGEP(V, Offset); + ResultPtr = CGF.Builder.CreateInBoundsGEP(V.getPointer(), Offset); + } else { + ResultPtr = V.getPointer(); } + // In a derived-to-base conversion, the non-virtual adjustment is + // applied second. if (NonVirtualAdjustment && IsReturnAdjustment) { - // Perform the non-virtual adjustment for a derived-to-base cast. - V = CGF.Builder.CreateConstInBoundsGEP1_64(V, NonVirtualAdjustment); + ResultPtr = CGF.Builder.CreateConstInBoundsGEP1_64(ResultPtr, + NonVirtualAdjustment); } // Cast back to the original type. - return CGF.Builder.CreateBitCast(V, Ptr->getType()); + return CGF.Builder.CreateBitCast(ResultPtr, InitialPtr.getType()); } llvm::Value *ItaniumCXXABI::performThisAdjustment(CodeGenFunction &CGF, - llvm::Value *This, + Address This, const ThisAdjustment &TA) { return performTypeAdjustment(CGF, This, TA.NonVirtual, TA.Virtual.Itanium.VCallOffsetOffset, @@ -1537,7 +1708,7 @@ llvm::Value *ItaniumCXXABI::performThisAdjustment(CodeGenFunction &CGF, } llvm::Value * -ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret, +ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret, const ReturnAdjustment &RA) { return performTypeAdjustment(CGF, Ret, RA.NonVirtual, RA.Virtual.Itanium.VBaseOffsetOffset, @@ -1550,8 +1721,7 @@ void ARMCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF, return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV, ResultType); // Destructor thunks in the ARM ABI have indeterminate results. - llvm::Type *T = - cast<llvm::PointerType>(CGF.ReturnValue->getType())->getElementType(); + llvm::Type *T = CGF.ReturnValue.getElementType(); RValue Undef = RValue::get(llvm::UndefValue::get(T)); return ItaniumCXXABI::EmitReturnFromThunk(CGF, Undef, ResultType); } @@ -1565,18 +1735,17 @@ CharUnits ItaniumCXXABI::getArrayCookieSizeImpl(QualType elementType) { CGM.getContext().getTypeAlignInChars(elementType)); } -llvm::Value *ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, - llvm::Value *NewPtr, - llvm::Value *NumElements, - const CXXNewExpr *expr, - QualType ElementType) { +Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, + Address NewPtr, + llvm::Value *NumElements, + const CXXNewExpr *expr, + QualType ElementType) { assert(requiresArrayCookie(expr)); - unsigned AS = NewPtr->getType()->getPointerAddressSpace(); + unsigned AS = NewPtr.getAddressSpace(); ASTContext &Ctx = getContext(); - QualType SizeTy = Ctx.getSizeType(); - CharUnits SizeSize = Ctx.getTypeSizeInChars(SizeTy); + CharUnits SizeSize = CGF.getSizeSize(); // The size of the cookie. CharUnits CookieSize = @@ -1584,49 +1753,45 @@ llvm::Value *ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, assert(CookieSize == getArrayCookieSizeImpl(ElementType)); // Compute an offset to the cookie. - llvm::Value *CookiePtr = NewPtr; + Address CookiePtr = NewPtr; CharUnits CookieOffset = CookieSize - SizeSize; if (!CookieOffset.isZero()) - CookiePtr = CGF.Builder.CreateConstInBoundsGEP1_64(CookiePtr, - CookieOffset.getQuantity()); + CookiePtr = CGF.Builder.CreateConstInBoundsByteGEP(CookiePtr, CookieOffset); // Write the number of elements into the appropriate slot. - llvm::Type *NumElementsTy = CGF.ConvertType(SizeTy)->getPointerTo(AS); - llvm::Value *NumElementsPtr = - CGF.Builder.CreateBitCast(CookiePtr, NumElementsTy); + Address NumElementsPtr = + CGF.Builder.CreateElementBitCast(CookiePtr, CGF.SizeTy); llvm::Instruction *SI = CGF.Builder.CreateStore(NumElements, NumElementsPtr); + + // Handle the array cookie specially in ASan. if (CGM.getLangOpts().Sanitize.has(SanitizerKind::Address) && AS == 0 && expr->getOperatorNew()->isReplaceableGlobalAllocationFunction()) { // The store to the CookiePtr does not need to be instrumented. CGM.getSanitizerMetadata()->disableSanitizerForInstruction(SI); llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.VoidTy, NumElementsTy, false); + llvm::FunctionType::get(CGM.VoidTy, NumElementsPtr.getType(), false); llvm::Constant *F = CGM.CreateRuntimeFunction(FTy, "__asan_poison_cxx_array_cookie"); - CGF.Builder.CreateCall(F, NumElementsPtr); + CGF.Builder.CreateCall(F, NumElementsPtr.getPointer()); } // Finally, compute a pointer to the actual data buffer by skipping // over the cookie completely. - return CGF.Builder.CreateConstInBoundsGEP1_64(NewPtr, - CookieSize.getQuantity()); + return CGF.Builder.CreateConstInBoundsByteGEP(NewPtr, CookieSize); } llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF, - llvm::Value *allocPtr, + Address allocPtr, CharUnits cookieSize) { // The element size is right-justified in the cookie. - llvm::Value *numElementsPtr = allocPtr; - CharUnits numElementsOffset = - cookieSize - CharUnits::fromQuantity(CGF.SizeSizeInBytes); + Address numElementsPtr = allocPtr; + CharUnits numElementsOffset = cookieSize - CGF.getSizeSize(); if (!numElementsOffset.isZero()) numElementsPtr = - CGF.Builder.CreateConstInBoundsGEP1_64(numElementsPtr, - numElementsOffset.getQuantity()); + CGF.Builder.CreateConstInBoundsByteGEP(numElementsPtr, numElementsOffset); - unsigned AS = allocPtr->getType()->getPointerAddressSpace(); - numElementsPtr = - CGF.Builder.CreateBitCast(numElementsPtr, CGF.SizeTy->getPointerTo(AS)); + unsigned AS = allocPtr.getAddressSpace(); + numElementsPtr = CGF.Builder.CreateElementBitCast(numElementsPtr, CGF.SizeTy); if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address) || AS != 0) return CGF.Builder.CreateLoad(numElementsPtr); // In asan mode emit a function call instead of a regular load and let the @@ -1638,7 +1803,7 @@ llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF, llvm::FunctionType::get(CGF.SizeTy, CGF.SizeTy->getPointerTo(0), false); llvm::Constant *F = CGM.CreateRuntimeFunction(FTy, "__asan_load_cxx_array_cookie"); - return CGF.Builder.CreateCall(F, numElementsPtr); + return CGF.Builder.CreateCall(F, numElementsPtr.getPointer()); } CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) { @@ -1654,47 +1819,41 @@ CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) { CGM.getContext().getTypeAlignInChars(elementType)); } -llvm::Value *ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, - llvm::Value *newPtr, - llvm::Value *numElements, - const CXXNewExpr *expr, - QualType elementType) { +Address ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, + Address newPtr, + llvm::Value *numElements, + const CXXNewExpr *expr, + QualType elementType) { assert(requiresArrayCookie(expr)); - // NewPtr is a char*, but we generalize to arbitrary addrspaces. - unsigned AS = newPtr->getType()->getPointerAddressSpace(); - // The cookie is always at the start of the buffer. - llvm::Value *cookie = newPtr; + Address cookie = newPtr; // The first element is the element size. - cookie = CGF.Builder.CreateBitCast(cookie, CGF.SizeTy->getPointerTo(AS)); + cookie = CGF.Builder.CreateElementBitCast(cookie, CGF.SizeTy); llvm::Value *elementSize = llvm::ConstantInt::get(CGF.SizeTy, getContext().getTypeSizeInChars(elementType).getQuantity()); CGF.Builder.CreateStore(elementSize, cookie); // The second element is the element count. - cookie = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.SizeTy, cookie, 1); + cookie = CGF.Builder.CreateConstInBoundsGEP(cookie, 1, CGF.getSizeSize()); CGF.Builder.CreateStore(numElements, cookie); // Finally, compute a pointer to the actual data buffer by skipping // over the cookie completely. CharUnits cookieSize = ARMCXXABI::getArrayCookieSizeImpl(elementType); - return CGF.Builder.CreateConstInBoundsGEP1_64(newPtr, - cookieSize.getQuantity()); + return CGF.Builder.CreateConstInBoundsByteGEP(newPtr, cookieSize); } llvm::Value *ARMCXXABI::readArrayCookieImpl(CodeGenFunction &CGF, - llvm::Value *allocPtr, + Address allocPtr, CharUnits cookieSize) { // The number of elements is at offset sizeof(size_t) relative to // the allocated pointer. - llvm::Value *numElementsPtr - = CGF.Builder.CreateConstInBoundsGEP1_64(allocPtr, CGF.SizeSizeInBytes); + Address numElementsPtr + = CGF.Builder.CreateConstInBoundsByteGEP(allocPtr, CGF.getSizeSize()); - unsigned AS = allocPtr->getType()->getPointerAddressSpace(); - numElementsPtr = - CGF.Builder.CreateBitCast(numElementsPtr, CGF.SizeTy->getPointerTo(AS)); + numElementsPtr = CGF.Builder.CreateElementBitCast(numElementsPtr, CGF.SizeTy); return CGF.Builder.CreateLoad(numElementsPtr); } @@ -1735,7 +1894,7 @@ static llvm::Constant *getGuardAbortFn(CodeGenModule &CGM, } namespace { - struct CallGuardAbort : EHScopeStack::Cleanup { + struct CallGuardAbort final : EHScopeStack::Cleanup { llvm::GlobalVariable *Guard; CallGuardAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {} @@ -1764,12 +1923,21 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, bool useInt8GuardVariable = !threadsafe && var->hasInternalLinkage(); llvm::IntegerType *guardTy; + CharUnits guardAlignment; if (useInt8GuardVariable) { guardTy = CGF.Int8Ty; + guardAlignment = CharUnits::One(); } else { // Guard variables are 64 bits in the generic ABI and size width on ARM // (i.e. 32-bit on AArch32, 64-bit on AArch64). - guardTy = (UseARMGuardVarABI ? CGF.SizeTy : CGF.Int64Ty); + if (UseARMGuardVarABI) { + guardTy = CGF.SizeTy; + guardAlignment = CGF.getSizeAlign(); + } else { + guardTy = CGF.Int64Ty; + guardAlignment = CharUnits::fromQuantity( + CGM.getDataLayout().getABITypeAlignment(guardTy)); + } } llvm::PointerType *guardPtrTy = guardTy->getPointerTo(); @@ -1782,7 +1950,6 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, { llvm::raw_svector_ostream out(guardName); getMangleContext().mangleStaticGuardVariable(&D, out); - out.flush(); } // Create the guard variable with a zero-initializer. @@ -1794,11 +1961,14 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, guard->setVisibility(var->getVisibility()); // If the variable is thread-local, so is its guard variable. guard->setThreadLocalMode(var->getThreadLocalMode()); + guard->setAlignment(guardAlignment.getQuantity()); - // The ABI says: It is suggested that it be emitted in the same COMDAT group - // as the associated data object + // The ABI says: "It is suggested that it be emitted in the same COMDAT + // group as the associated data object." In practice, this doesn't work for + // non-ELF object formats, so only do it for ELF. llvm::Comdat *C = var->getComdat(); - if (!D.isLocalVarDecl() && C) { + if (!D.isLocalVarDecl() && C && + CGM.getTarget().getTriple().isOSBinFormatELF()) { guard->setComdat(C); CGF.CurFn->setComdat(C); } else if (CGM.supportsCOMDAT() && guard->isWeakForLinker()) { @@ -1808,6 +1978,8 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, CGM.setStaticLocalDeclGuardAddress(&D, guard); } + Address guardAddr = Address(guard, guardAlignment); + // Test whether the variable has completed initialization. // // Itanium C++ ABI 3.3.2: @@ -1827,8 +1999,7 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, // Load the first byte of the guard variable. llvm::LoadInst *LI = - Builder.CreateLoad(Builder.CreateBitCast(guard, CGM.Int8PtrTy)); - LI->setAlignment(1); + Builder.CreateLoad(Builder.CreateElementBitCast(guardAddr, CGM.Int8Ty)); // Itanium ABI: // An implementation supporting thread-safety on multiprocessor @@ -1898,9 +2069,10 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, CGF.PopCleanupBlock(); // Call __cxa_guard_release. This cannot throw. - CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy), guard); + CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy), + guardAddr.getPointer()); } else { - Builder.CreateStore(llvm::ConstantInt::get(guardTy, 1), guard); + Builder.CreateStore(llvm::ConstantInt::get(guardTy, 1), guardAddr); } CGF.EmitBlock(EndBlock); @@ -1914,7 +2086,7 @@ static void emitGlobalDtorWithCXAAtExit(CodeGenFunction &CGF, const char *Name = "__cxa_atexit"; if (TLS) { const llvm::Triple &T = CGF.getTarget().getTriple(); - Name = T.isMacOSX() ? "_tlv_atexit" : "__cxa_thread_atexit"; + Name = T.isOSDarwin() ? "_tlv_atexit" : "__cxa_thread_atexit"; } // We're assuming that the destructor function is something we can @@ -1970,10 +2142,10 @@ void ItaniumCXXABI::registerGlobalDtor(CodeGenFunction &CGF, static bool isThreadWrapperReplaceable(const VarDecl *VD, CodeGen::CodeGenModule &CGM) { assert(!VD->isStaticLocal() && "static local VarDecls don't need wrappers!"); - // OS X prefers to have references to thread local variables to go through + // Darwin prefers to have references to thread local variables to go through // the thread wrapper instead of directly referencing the backing variable. return VD->getTLSKind() == VarDecl::TLS_Dynamic && - CGM.getTarget().getTriple().isMacOSX(); + CGM.getTarget().getTriple().isOSDarwin(); } /// Get the appropriate linkage for the wrapper function. This is essentially @@ -1989,12 +2161,10 @@ getThreadLocalWrapperLinkage(const VarDecl *VD, CodeGen::CodeGenModule &CGM) { return VarLinkage; // If the thread wrapper is replaceable, give it appropriate linkage. - if (isThreadWrapperReplaceable(VD, CGM)) { - if (llvm::GlobalVariable::isLinkOnceLinkage(VarLinkage) || - llvm::GlobalVariable::isWeakODRLinkage(VarLinkage)) - return llvm::GlobalVariable::WeakAnyLinkage; - return VarLinkage; - } + if (isThreadWrapperReplaceable(VD, CGM)) + if (!llvm::GlobalVariable::isLinkOnceLinkage(VarLinkage) && + !llvm::GlobalVariable::isWeakODRLinkage(VarLinkage)) + return VarLinkage; return llvm::GlobalValue::WeakODRLinkage; } @@ -2006,7 +2176,6 @@ ItaniumCXXABI::getOrCreateThreadLocalWrapper(const VarDecl *VD, { llvm::raw_svector_ostream Out(WrapperName); getMangleContext().mangleItaniumThreadLocalWrapper(VD, Out); - Out.flush(); } if (llvm::Value *V = CGM.getModule().getNamedValue(WrapperName)) @@ -2021,22 +2190,29 @@ ItaniumCXXABI::getOrCreateThreadLocalWrapper(const VarDecl *VD, llvm::Function::Create(FnTy, getThreadLocalWrapperLinkage(VD, CGM), WrapperName.str(), &CGM.getModule()); // Always resolve references to the wrapper at link time. - if (!Wrapper->hasLocalLinkage() && !isThreadWrapperReplaceable(VD, CGM)) + if (!Wrapper->hasLocalLinkage() && !(isThreadWrapperReplaceable(VD, CGM) && + !llvm::GlobalVariable::isLinkOnceLinkage(Wrapper->getLinkage()) && + !llvm::GlobalVariable::isWeakODRLinkage(Wrapper->getLinkage()))) Wrapper->setVisibility(llvm::GlobalValue::HiddenVisibility); + + if (isThreadWrapperReplaceable(VD, CGM)) { + Wrapper->setCallingConv(llvm::CallingConv::CXX_FAST_TLS); + Wrapper->addFnAttr(llvm::Attribute::NoUnwind); + } return Wrapper; } void ItaniumCXXABI::EmitThreadLocalInitFuncs( - CodeGenModule &CGM, - ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>> - CXXThreadLocals, ArrayRef<llvm::Function *> CXXThreadLocalInits, - ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) { + CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals, + ArrayRef<llvm::Function *> CXXThreadLocalInits, + ArrayRef<const VarDecl *> CXXThreadLocalInitVars) { llvm::Function *InitFunc = nullptr; if (!CXXThreadLocalInits.empty()) { // Generate a guarded initialization function. llvm::FunctionType *FTy = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false); - InitFunc = CGM.CreateGlobalInitOrDestructFunction(FTy, "__tls_init", + const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction(); + InitFunc = CGM.CreateGlobalInitOrDestructFunction(FTy, "__tls_init", FI, SourceLocation(), /*TLS=*/true); llvm::GlobalVariable *Guard = new llvm::GlobalVariable( @@ -2044,12 +2220,17 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( llvm::GlobalVariable::InternalLinkage, llvm::ConstantInt::get(CGM.Int8Ty, 0), "__tls_guard"); Guard->setThreadLocal(true); + + CharUnits GuardAlign = CharUnits::One(); + Guard->setAlignment(GuardAlign.getQuantity()); + CodeGenFunction(CGM) - .GenerateCXXGlobalInitFunc(InitFunc, CXXThreadLocalInits, Guard); + .GenerateCXXGlobalInitFunc(InitFunc, CXXThreadLocalInits, + Address(Guard, GuardAlign)); } - for (unsigned I = 0, N = CXXThreadLocals.size(); I != N; ++I) { - const VarDecl *VD = CXXThreadLocals[I].first; - llvm::GlobalVariable *Var = CXXThreadLocals[I].second; + for (const VarDecl *VD : CXXThreadLocals) { + llvm::GlobalVariable *Var = + cast<llvm::GlobalVariable>(CGM.GetGlobalValue(CGM.getMangledName(VD))); // Some targets require that all access to thread local variables go through // the thread wrapper. This means that we cannot attempt to create a thread @@ -2062,7 +2243,6 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( { llvm::raw_svector_ostream Out(InitFnName); getMangleContext().mangleItaniumThreadLocalInit(VD, Out); - Out.flush(); } // If we have a definition for the variable, emit the initialization @@ -2092,7 +2272,7 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Var); llvm::LLVMContext &Context = CGM.getModule().getContext(); llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, "", Wrapper); - CGBuilderTy Builder(Entry); + CGBuilderTy Builder(CGM, Entry); if (InitIsInitFunc) { if (Init) Builder.CreateCall(Init); @@ -2114,9 +2294,8 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( // the referenced object. llvm::Value *Val = Var; if (VD->getType()->isReferenceType()) { - llvm::LoadInst *LI = Builder.CreateLoad(Val); - LI->setAlignment(CGM.getContext().getDeclAlign(VD).getQuantity()); - Val = LI; + CharUnits Align = CGM.getContext().getDeclAlign(VD); + Val = Builder.CreateAlignedLoad(Val, Align); } if (Val->getType() != Wrapper->getReturnType()) Val = Builder.CreatePointerBitCastOrAddrSpaceCast( @@ -2128,18 +2307,19 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD, QualType LValType) { - QualType T = VD->getType(); - llvm::Type *Ty = CGF.getTypes().ConvertTypeForMem(T); - llvm::Value *Val = CGF.CGM.GetAddrOfGlobalVar(VD, Ty); + llvm::Value *Val = CGF.CGM.GetAddrOfGlobalVar(VD); llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Val); - Val = CGF.Builder.CreateCall(Wrapper); + llvm::CallInst *CallVal = CGF.Builder.CreateCall(Wrapper); + if (isThreadWrapperReplaceable(VD, CGF.CGM)) + CallVal->setCallingConv(llvm::CallingConv::CXX_FAST_TLS); LValue LV; if (VD->getType()->isReferenceType()) - LV = CGF.MakeNaturalAlignAddrLValue(Val, LValType); + LV = CGF.MakeNaturalAlignAddrLValue(CallVal, LValType); else - LV = CGF.MakeAddrLValue(Val, LValType, CGF.getContext().getDeclAlign(VD)); + LV = CGF.MakeAddrLValue(CallVal, LValType, + CGF.getContext().getDeclAlign(VD)); // FIXME: need setObjCGCLValueClass? return LV; } @@ -2255,11 +2435,9 @@ public: llvm::GlobalVariable *ItaniumRTTIBuilder::GetAddrOfTypeName( QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage) { - SmallString<256> OutName; - llvm::raw_svector_ostream Out(OutName); + SmallString<256> Name; + llvm::raw_svector_ostream Out(Name); CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(Ty, Out); - Out.flush(); - StringRef Name = OutName.str(); // We know that the mangled name of the type starts at index 4 of the // mangled name of the typename, so we can just index into it in order to @@ -2278,11 +2456,9 @@ llvm::GlobalVariable *ItaniumRTTIBuilder::GetAddrOfTypeName( llvm::Constant * ItaniumRTTIBuilder::GetAddrOfExternalRTTIDescriptor(QualType Ty) { // Mangle the RTTI name. - SmallString<256> OutName; - llvm::raw_svector_ostream Out(OutName); + SmallString<256> Name; + llvm::raw_svector_ostream Out(Name); CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, Out); - Out.flush(); - StringRef Name = OutName.str(); // Look for an existing global. llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name); @@ -2346,9 +2522,19 @@ static bool TypeInfoIsInStandardLibrary(const BuiltinType *Ty) { case BuiltinType::OCLImage1dBuffer: case BuiltinType::OCLImage2d: case BuiltinType::OCLImage2dArray: + case BuiltinType::OCLImage2dDepth: + case BuiltinType::OCLImage2dArrayDepth: + case BuiltinType::OCLImage2dMSAA: + case BuiltinType::OCLImage2dArrayMSAA: + case BuiltinType::OCLImage2dMSAADepth: + case BuiltinType::OCLImage2dArrayMSAADepth: case BuiltinType::OCLImage3d: case BuiltinType::OCLSampler: case BuiltinType::OCLEvent: + case BuiltinType::OCLClkEvent: + case BuiltinType::OCLQueue: + case BuiltinType::OCLNDRange: + case BuiltinType::OCLReserveID: return true; case BuiltinType::Dependent: @@ -2678,11 +2864,9 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) { Ty = CGM.getContext().getCanonicalType(Ty); // Check if we've already emitted an RTTI descriptor for this type. - SmallString<256> OutName; - llvm::raw_svector_ostream Out(OutName); + SmallString<256> Name; + llvm::raw_svector_ostream Out(Name); CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, Out); - Out.flush(); - StringRef Name = OutName.str(); llvm::GlobalVariable *OldGV = CGM.getModule().getNamedGlobal(Name); if (OldGV && !OldGV->isDeclaration()) { @@ -2818,9 +3002,6 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) { new llvm::GlobalVariable(M, Init->getType(), /*Constant=*/true, Linkage, Init, Name); - if (CGM.supportsCOMDAT() && GV->isWeakForLinker()) - GV->setComdat(M.getOrInsertComdat(GV->getName())); - // If there's already an old global variable, replace it with the new one. if (OldGV) { GV->takeName(OldGV); @@ -2830,6 +3011,9 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) { OldGV->eraseFromParent(); } + if (CGM.supportsCOMDAT() && GV->isWeakForLinker()) + GV->setComdat(M.getOrInsertComdat(GV->getName())); + // The Itanium ABI specifies that type_info objects must be globally // unique, with one exception: if the type is an incomplete class // type or a (possibly indirect) pointer to one. That exception @@ -3232,15 +3416,13 @@ static void emitConstructorDestructorAlias(CodeGenModule &CGM, return; auto *Aliasee = cast<llvm::GlobalValue>(CGM.GetAddrOfGlobal(TargetDecl)); - llvm::PointerType *AliasType = Aliasee->getType(); // Create the alias with no name. - auto *Alias = llvm::GlobalAlias::create(AliasType, Linkage, "", Aliasee, - &CGM.getModule()); + auto *Alias = llvm::GlobalAlias::create(Linkage, "", Aliasee); // Switch any previous uses to the alias. if (Entry) { - assert(Entry->getType() == AliasType && + assert(Entry->getType() == Aliasee->getType() && "declaration exists with different type"); Alias->takeName(Entry); Entry->replaceAllUsesWith(Alias); @@ -3278,7 +3460,7 @@ void ItaniumCXXABI::emitCXXStructor(const CXXMethodDecl *MD, if (CGType == StructorCodegen::RAUW) { StringRef MangledName = CGM.getMangledName(CompleteDecl); - auto *Aliasee = cast<llvm::GlobalValue>(CGM.GetAddrOfGlobal(BaseDecl)); + auto *Aliasee = CGM.GetAddrOfGlobal(BaseDecl); CGM.addReplacement(MangledName, Aliasee); return; } @@ -3345,7 +3527,7 @@ namespace { /// of the caught type, so we have to assume the actual thrown /// exception type might have a throwing destructor, even if the /// caught type's destructor is trivial or nothrow. - struct CallEndCatch : EHScopeStack::Cleanup { + struct CallEndCatch final : EHScopeStack::Cleanup { CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {} bool MightThrow; @@ -3379,7 +3561,7 @@ static llvm::Value *CallBeginCatch(CodeGenFunction &CGF, /// parameter during catch initialization. static void InitCatchParam(CodeGenFunction &CGF, const VarDecl &CatchParam, - llvm::Value *ParamAddr, + Address ParamAddr, SourceLocation Loc) { // Load the exception from where the landing pad saved it. llvm::Value *Exn = CGF.getExceptionFromSlot(); @@ -3433,12 +3615,13 @@ static void InitCatchParam(CodeGenFunction &CGF, cast<llvm::PointerType>(LLVMCatchTy)->getElementType(); // Create the temporary and write the adjusted pointer into it. - llvm::Value *ExnPtrTmp = CGF.CreateTempAlloca(PtrTy, "exn.byref.tmp"); + Address ExnPtrTmp = + CGF.CreateTempAlloca(PtrTy, CGF.getPointerAlign(), "exn.byref.tmp"); llvm::Value *Casted = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy); CGF.Builder.CreateStore(Casted, ExnPtrTmp); // Bind the reference to the temporary. - AdjustedExn = ExnPtrTmp; + AdjustedExn = ExnPtrTmp.getPointer(); } } @@ -3483,8 +3666,7 @@ static void InitCatchParam(CodeGenFunction &CGF, llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy); LValue srcLV = CGF.MakeNaturalAlignAddrLValue(Cast, CatchType); - LValue destLV = CGF.MakeAddrLValue(ParamAddr, CatchType, - CGF.getContext().getDeclAlign(&CatchParam)); + LValue destLV = CGF.MakeAddrLValue(ParamAddr, CatchType); switch (TEK) { case TEK_Complex: CGF.EmitStoreOfComplex(CGF.EmitLoadOfComplex(srcLV, Loc), destLV, @@ -3502,6 +3684,8 @@ static void InitCatchParam(CodeGenFunction &CGF, } assert(isa<RecordType>(CatchType) && "unexpected catch type!"); + auto catchRD = CatchType->getAsCXXRecordDecl(); + CharUnits caughtExnAlignment = CGF.CGM.getClassPointerAlignment(catchRD); llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok @@ -3510,7 +3694,8 @@ static void InitCatchParam(CodeGenFunction &CGF, const Expr *copyExpr = CatchParam.getInit(); if (!copyExpr) { llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, true); - llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy); + Address adjustedExn(CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy), + caughtExnAlignment); CGF.EmitAggregateCopy(ParamAddr, adjustedExn, CatchType); return; } @@ -3521,7 +3706,8 @@ static void InitCatchParam(CodeGenFunction &CGF, CGF.EmitNounwindRuntimeCall(getGetExceptionPtrFn(CGF.CGM), Exn); // Cast that to the appropriate type. - llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy); + Address adjustedExn(CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy), + caughtExnAlignment); // The copy expression is defined in terms of an OpaqueValueExpr. // Find it and map it to the adjusted expression. @@ -3533,9 +3719,8 @@ static void InitCatchParam(CodeGenFunction &CGF, CGF.EHStack.pushTerminate(); // Perform the copy construction. - CharUnits Alignment = CGF.getContext().getDeclAlign(&CatchParam); CGF.EmitAggExpr(copyExpr, - AggValueSlot::forAddr(ParamAddr, Alignment, Qualifiers(), + AggValueSlot::forAddr(ParamAddr, Qualifiers(), AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased)); @@ -3619,7 +3804,7 @@ static llvm::Constant *getClangCallTerminateFn(CodeGenModule &CGM) { // Set up the function. llvm::BasicBlock *entry = llvm::BasicBlock::Create(CGM.getLLVMContext(), "", fn); - CGBuilderTy builder(entry); + CGBuilderTy builder(CGM, entry); // Pull the exception pointer out of the parameter list. llvm::Value *exn = &*fn->arg_begin(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp index 3433990..93210d5 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -15,6 +15,7 @@ //===----------------------------------------------------------------------===// #include "CGCXXABI.h" +#include "CGCleanup.h" #include "CGVTables.h" #include "CodeGenModule.h" #include "CodeGenTypes.h" @@ -45,7 +46,7 @@ public: : CGCXXABI(CGM), BaseClassDescriptorType(nullptr), ClassHierarchyDescriptorType(nullptr), CompleteObjectLocatorType(nullptr), CatchableTypeType(nullptr), - ThrowInfoType(nullptr), CatchHandlerTypeType(nullptr) {} + ThrowInfoType(nullptr) {} bool HasThisReturn(GlobalDecl GD) const override; bool hasMostDerivedReturn(GlobalDecl GD) const override; @@ -56,6 +57,27 @@ public: bool isSRetParameterAfterThis() const override { return true; } + bool isThisCompleteObject(GlobalDecl GD) const override { + // The Microsoft ABI doesn't use separate complete-object vs. + // base-object variants of constructors, but it does of destructors. + if (isa<CXXDestructorDecl>(GD.getDecl())) { + switch (GD.getDtorType()) { + case Dtor_Complete: + case Dtor_Deleting: + return true; + + case Dtor_Base: + return false; + + case Dtor_Comdat: llvm_unreachable("emitting dtor comdat as function?"); + } + llvm_unreachable("bad dtor kind"); + } + + // No other kinds. + return false; + } + size_t getSrcArgforCopyCtor(const CXXConstructorDecl *CD, FunctionArgList &Args) const override { assert(Args.size() >= 2 && @@ -68,11 +90,30 @@ public: return 1; } + std::vector<CharUnits> getVBPtrOffsets(const CXXRecordDecl *RD) override { + std::vector<CharUnits> VBPtrOffsets; + const ASTContext &Context = getContext(); + const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD); + + const VBTableGlobals &VBGlobals = enumerateVBTables(RD); + for (const VPtrInfo *VBT : *VBGlobals.VBTables) { + const ASTRecordLayout &SubobjectLayout = + Context.getASTRecordLayout(VBT->BaseWithVPtr); + CharUnits Offs = VBT->NonVirtualOffset; + Offs += SubobjectLayout.getVBPtrOffset(); + if (VBT->getVBaseWithVPtr()) + Offs += Layout.getVBaseClassOffset(VBT->getVBaseWithVPtr()); + VBPtrOffsets.push_back(Offs); + } + llvm::array_pod_sort(VBPtrOffsets.begin(), VBPtrOffsets.end()); + return VBPtrOffsets; + } + StringRef GetPureVirtualCallName() override { return "_purecall"; } StringRef GetDeletedVirtualCallName() override { return "_purecall"; } void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE, - llvm::Value *Ptr, QualType ElementType, + Address Ptr, QualType ElementType, const CXXDestructorDecl *Dtor) override; void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override; @@ -84,31 +125,39 @@ public: const VPtrInfo *Info); llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override; - llvm::Constant * + CatchTypeInfo getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) override; + /// MSVC needs an extra flag to indicate a catchall. + CatchTypeInfo getCatchAllTypeInfo() override { + return CatchTypeInfo{nullptr, 0x40}; + } + bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override; void EmitBadTypeidCall(CodeGenFunction &CGF) override; llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy, - llvm::Value *ThisPtr, + Address ThisPtr, llvm::Type *StdTypeInfoPtrTy) override; bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr, QualType SrcRecordTy) override; - llvm::Value *EmitDynamicCastCall(CodeGenFunction &CGF, llvm::Value *Value, + llvm::Value *EmitDynamicCastCall(CodeGenFunction &CGF, Address Value, QualType SrcRecordTy, QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) override; - llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, llvm::Value *Value, + llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, Address Value, QualType SrcRecordTy, QualType DestTy) override; bool EmitBadCastCall(CodeGenFunction &CGF) override; + bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const override { + return false; + } llvm::Value * - GetVirtualBaseClassOffset(CodeGenFunction &CGF, llvm::Value *This, + GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This, const CXXRecordDecl *ClassDecl, const CXXRecordDecl *BaseClassDecl) override; @@ -182,9 +231,9 @@ public: return MD->getParent(); } - llvm::Value * + Address adjustThisArgumentForVirtualFunctionCall(CodeGenFunction &CGF, GlobalDecl GD, - llvm::Value *This, + Address This, bool VirtualCall) override; void addImplicitStructorParams(CodeGenFunction &CGF, QualType &ResTy, @@ -203,7 +252,7 @@ public: void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type, bool ForVirtualBase, - bool Delegating, llvm::Value *This) override; + bool Delegating, Address This) override; void emitVTableBitSetEntries(VPtrInfo *Info, const CXXRecordDecl *RD, llvm::GlobalVariable *VTable); @@ -211,10 +260,22 @@ public: void emitVTableDefinitions(CodeGenVTables &CGVT, const CXXRecordDecl *RD) override; + bool isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF, + CodeGenFunction::VPtr Vptr) override; + + /// Don't initialize vptrs if dynamic class + /// is marked with with the 'novtable' attribute. + bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) override { + return !VTableClass->hasAttr<MSNoVTableAttr>(); + } + + llvm::Constant * + getVTableAddressPoint(BaseSubobject Base, + const CXXRecordDecl *VTableClass) override; + llvm::Value *getVTableAddressPointInStructor( CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, - BaseSubobject Base, const CXXRecordDecl *NearestVBase, - bool &NeedsVirtualOffset) override; + BaseSubobject Base, const CXXRecordDecl *NearestVBase) override; llvm::Constant * getVTableAddressPointForConstExpr(BaseSubobject Base, @@ -224,13 +285,13 @@ public: CharUnits VPtrOffset) override; llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, - llvm::Value *This, llvm::Type *Ty, + Address This, llvm::Type *Ty, SourceLocation Loc) override; llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType, - llvm::Value *This, + Address This, const CXXMemberCallExpr *CE) override; void adjustCallArgsForDestructorThunk(CodeGenFunction &CGF, GlobalDecl GD, @@ -253,7 +314,6 @@ public: SmallString<256> OutName; llvm::raw_svector_ostream Out(OutName); getMangleContext().mangleCXXVirtualDisplacementMap(SrcRD, DstRD, Out); - Out.flush(); StringRef MangledName = OutName.str(); if (auto *VDispMap = CGM.getModule().getNamedGlobal(MangledName)) @@ -310,18 +370,16 @@ public: Thunk->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage); } - llvm::Value *performThisAdjustment(CodeGenFunction &CGF, llvm::Value *This, + llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This, const ThisAdjustment &TA) override; - llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret, + llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret, const ReturnAdjustment &RA) override; void EmitThreadLocalInitFuncs( - CodeGenModule &CGM, - ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>> - CXXThreadLocals, + CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals, ArrayRef<llvm::Function *> CXXThreadLocalInits, - ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) override; + ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override; bool usesThreadWrapperFunction() const override { return false; } LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD, @@ -360,13 +418,13 @@ public: QualType elementType) override; bool requiresArrayCookie(const CXXNewExpr *expr) override; CharUnits getArrayCookieSizeImpl(QualType type) override; - llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF, - llvm::Value *NewPtr, - llvm::Value *NumElements, - const CXXNewExpr *expr, - QualType ElementType) override; + Address InitializeArrayCookie(CodeGenFunction &CGF, + Address NewPtr, + llvm::Value *NumElements, + const CXXNewExpr *expr, + QualType ElementType) override; llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, - llvm::Value *allocPtr, + Address allocPtr, CharUnits cookieSize) override; friend struct MSRTTIBuilder; @@ -493,14 +551,6 @@ private: return llvm::Constant::getAllOnesValue(CGM.IntTy); } - llvm::Constant *getConstantOrZeroInt(llvm::Constant *C) { - return C ? C : getZeroInt(); - } - - llvm::Value *getValueOrZeroInt(llvm::Value *C) { - return C ? C : getZeroInt(); - } - CharUnits getVirtualFunctionPrologueThisAdjustment(GlobalDecl GD); void @@ -511,13 +561,13 @@ private: /// the vbptr to the virtual base. Optionally returns the address of the /// vbptr itself. llvm::Value *GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF, - llvm::Value *Base, + Address Base, llvm::Value *VBPtrOffset, llvm::Value *VBTableOffset, llvm::Value **VBPtr = nullptr); llvm::Value *GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF, - llvm::Value *Base, + Address Base, int32_t VBPtrOffset, int32_t VBTableOffset, llvm::Value **VBPtr = nullptr) { @@ -527,14 +577,14 @@ private: return GetVBaseOffsetFromVBPtr(CGF, Base, VBPOffset, VBTOffset, VBPtr); } - std::pair<llvm::Value *, llvm::Value *> - performBaseAdjustment(CodeGenFunction &CGF, llvm::Value *Value, + std::pair<Address, llvm::Value *> + performBaseAdjustment(CodeGenFunction &CGF, Address Value, QualType SrcRecordTy); /// \brief Performs a full virtual base adjustment. Used to dereference /// pointers to members of virtual bases. llvm::Value *AdjustVirtualBase(CodeGenFunction &CGF, const Expr *E, - const CXXRecordDecl *RD, llvm::Value *Base, + const CXXRecordDecl *RD, Address Base, llvm::Value *VirtualBaseAdjustmentOffset, llvm::Value *VBPtrOffset /* optional */); @@ -570,17 +620,6 @@ public: return RD->hasAttr<MSInheritanceAttr>(); } - bool isTypeInfoCalculable(QualType Ty) const override { - if (!CGCXXABI::isTypeInfoCalculable(Ty)) - return false; - if (const auto *MPT = Ty->getAs<MemberPointerType>()) { - const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl(); - if (!RD->hasAttr<MSInheritanceAttr>()) - return false; - } - return true; - } - llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT) override; llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT, @@ -600,7 +639,7 @@ public: llvm::Value * EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E, - llvm::Value *Base, llvm::Value *MemPtr, + Address Base, llvm::Value *MemPtr, const MemberPointerType *MPT) override; llvm::Value *EmitNonNullMemberPointerConversion( @@ -623,23 +662,12 @@ public: llvm::Value * EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF, const Expr *E, - llvm::Value *&This, llvm::Value *MemPtr, + Address This, llvm::Value *&ThisPtrForCall, + llvm::Value *MemPtr, const MemberPointerType *MPT) override; void emitCXXStructor(const CXXMethodDecl *MD, StructorType Type) override; - llvm::StructType *getCatchHandlerTypeType() { - if (!CatchHandlerTypeType) { - llvm::Type *FieldTypes[] = { - CGM.IntTy, // Flags - CGM.Int8PtrTy, // TypeDescriptor - }; - CatchHandlerTypeType = llvm::StructType::create( - CGM.getLLVMContext(), FieldTypes, "eh.CatchHandlerType"); - } - return CatchHandlerTypeType; - } - llvm::StructType *getCatchableTypeType() { if (CatchableTypeType) return CatchableTypeType; @@ -755,7 +783,6 @@ private: llvm::StructType *CatchableTypeType; llvm::DenseMap<uint32_t, llvm::StructType *> CatchableTypeArrayTypeMap; llvm::StructType *ThrowInfoType; - llvm::StructType *CatchHandlerTypeType; }; } @@ -823,7 +850,7 @@ MicrosoftCXXABI::getRecordArgABI(const CXXRecordDecl *RD) const { void MicrosoftCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE, - llvm::Value *Ptr, + Address Ptr, QualType ElementType, const CXXDestructorDecl *Dtor) { // FIXME: Provide a source location here even though there's no @@ -848,11 +875,15 @@ void MicrosoftCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) { } namespace { -struct CallEndCatchMSVC : EHScopeStack::Cleanup { - CallEndCatchMSVC() {} +struct CatchRetScope final : EHScopeStack::Cleanup { + llvm::CatchPadInst *CPI; + + CatchRetScope(llvm::CatchPadInst *CPI) : CPI(CPI) {} + void Emit(CodeGenFunction &CGF, Flags flags) override { - CGF.EmitNounwindRuntimeCall( - CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_endcatch)); + llvm::BasicBlock *BB = CGF.createBasicBlock("catchret.dest"); + CGF.Builder.CreateCatchRet(CPI, BB); + CGF.EmitBlock(BB); } }; } @@ -862,50 +893,59 @@ void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF, // In the MS ABI, the runtime handles the copy, and the catch handler is // responsible for destruction. VarDecl *CatchParam = S->getExceptionDecl(); - llvm::Value *Exn = CGF.getExceptionFromSlot(); - llvm::Function *BeginCatch = - CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_begincatch); + llvm::BasicBlock *CatchPadBB = CGF.Builder.GetInsertBlock(); + llvm::CatchPadInst *CPI = + cast<llvm::CatchPadInst>(CatchPadBB->getFirstNonPHI()); + CGF.CurrentFuncletPad = CPI; // If this is a catch-all or the catch parameter is unnamed, we don't need to // emit an alloca to the object. if (!CatchParam || !CatchParam->getDeclName()) { - llvm::Value *Args[2] = {Exn, llvm::Constant::getNullValue(CGF.Int8PtrTy)}; - CGF.EmitNounwindRuntimeCall(BeginCatch, Args); - CGF.EHStack.pushCleanup<CallEndCatchMSVC>(NormalCleanup); + CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI); return; } CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam); - llvm::Value *ParamAddr = - CGF.Builder.CreateBitCast(var.getObjectAddress(CGF), CGF.Int8PtrTy); - llvm::Value *Args[2] = {Exn, ParamAddr}; - CGF.EmitNounwindRuntimeCall(BeginCatch, Args); - CGF.EHStack.pushCleanup<CallEndCatchMSVC>(NormalCleanup); + CPI->setArgOperand(2, var.getObjectAddress(CGF).getPointer()); + CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI); CGF.EmitAutoVarCleanups(var); } -std::pair<llvm::Value *, llvm::Value *> -MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, llvm::Value *Value, +/// We need to perform a generic polymorphic operation (like a typeid +/// or a cast), which requires an object with a vfptr. Adjust the +/// address to point to an object with a vfptr. +std::pair<Address, llvm::Value *> +MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value, QualType SrcRecordTy) { Value = CGF.Builder.CreateBitCast(Value, CGF.Int8PtrTy); const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl(); const ASTContext &Context = getContext(); + // If the class itself has a vfptr, great. This check implicitly + // covers non-virtual base subobjects: a class with its own virtual + // functions would be a candidate to be a primary base. if (Context.getASTRecordLayout(SrcDecl).hasExtendableVFPtr()) return std::make_pair(Value, llvm::ConstantInt::get(CGF.Int32Ty, 0)); - // Perform a base adjustment. - const CXXBaseSpecifier *PolymorphicBase = std::find_if( - SrcDecl->vbases_begin(), SrcDecl->vbases_end(), - [&](const CXXBaseSpecifier &Base) { - const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); - return Context.getASTRecordLayout(BaseDecl).hasExtendableVFPtr(); - }); - llvm::Value *Offset = GetVirtualBaseClassOffset( - CGF, Value, SrcDecl, PolymorphicBase->getType()->getAsCXXRecordDecl()); - Value = CGF.Builder.CreateInBoundsGEP(Value, Offset); + // Okay, one of the vbases must have a vfptr, or else this isn't + // actually a polymorphic class. + const CXXRecordDecl *PolymorphicBase = nullptr; + for (auto &Base : SrcDecl->vbases()) { + const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); + if (Context.getASTRecordLayout(BaseDecl).hasExtendableVFPtr()) { + PolymorphicBase = BaseDecl; + break; + } + } + assert(PolymorphicBase && "polymorphic class has no apparent vfptr?"); + + llvm::Value *Offset = + GetVirtualBaseClassOffset(CGF, Value, SrcDecl, PolymorphicBase); + llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP(Value.getPointer(), Offset); Offset = CGF.Builder.CreateTrunc(Offset, CGF.Int32Ty); - return std::make_pair(Value, Offset); + CharUnits VBaseAlign = + CGF.CGM.getVBaseAlignment(Value.getAlignment(), SrcDecl, PolymorphicBase); + return std::make_pair(Address(Ptr, VBaseAlign), Offset); } bool MicrosoftCXXABI::shouldTypeidBeNullChecked(bool IsDeref, @@ -934,12 +974,12 @@ void MicrosoftCXXABI::EmitBadTypeidCall(CodeGenFunction &CGF) { llvm::Value *MicrosoftCXXABI::EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy, - llvm::Value *ThisPtr, + Address ThisPtr, llvm::Type *StdTypeInfoPtrTy) { llvm::Value *Offset; std::tie(ThisPtr, Offset) = performBaseAdjustment(CGF, ThisPtr, SrcRecordTy); - return CGF.Builder.CreateBitCast( - emitRTtypeidCall(CGF, ThisPtr).getInstruction(), StdTypeInfoPtrTy); + auto Typeid = emitRTtypeidCall(CGF, ThisPtr.getPointer()).getInstruction(); + return CGF.Builder.CreateBitCast(Typeid, StdTypeInfoPtrTy); } bool MicrosoftCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr, @@ -950,7 +990,7 @@ bool MicrosoftCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr, } llvm::Value *MicrosoftCXXABI::EmitDynamicCastCall( - CodeGenFunction &CGF, llvm::Value *Value, QualType SrcRecordTy, + CodeGenFunction &CGF, Address This, QualType SrcRecordTy, QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) { llvm::Type *DestLTy = CGF.ConvertType(DestTy); @@ -960,7 +1000,8 @@ llvm::Value *MicrosoftCXXABI::EmitDynamicCastCall( CGF.CGM.GetAddrOfRTTIDescriptor(DestRecordTy.getUnqualifiedType()); llvm::Value *Offset; - std::tie(Value, Offset) = performBaseAdjustment(CGF, Value, SrcRecordTy); + std::tie(This, Offset) = performBaseAdjustment(CGF, This, SrcRecordTy); + llvm::Value *ThisPtr = This.getPointer(); // PVOID __RTDynamicCast( // PVOID inptr, @@ -974,14 +1015,14 @@ llvm::Value *MicrosoftCXXABI::EmitDynamicCastCall( llvm::FunctionType::get(CGF.Int8PtrTy, ArgTypes, false), "__RTDynamicCast"); llvm::Value *Args[] = { - Value, Offset, SrcRTTI, DestRTTI, + ThisPtr, Offset, SrcRTTI, DestRTTI, llvm::ConstantInt::get(CGF.Int32Ty, DestTy->isReferenceType())}; - Value = CGF.EmitRuntimeCallOrInvoke(Function, Args).getInstruction(); - return CGF.Builder.CreateBitCast(Value, DestLTy); + ThisPtr = CGF.EmitRuntimeCallOrInvoke(Function, Args).getInstruction(); + return CGF.Builder.CreateBitCast(ThisPtr, DestLTy); } llvm::Value * -MicrosoftCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF, llvm::Value *Value, +MicrosoftCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF, Address Value, QualType SrcRecordTy, QualType DestTy) { llvm::Value *Offset; @@ -993,7 +1034,7 @@ MicrosoftCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF, llvm::Value *Value, llvm::Constant *Function = CGF.CGM.CreateRuntimeFunction( llvm::FunctionType::get(CGF.Int8PtrTy, ArgTypes, false), "__RTCastToVoid"); - llvm::Value *Args[] = {Value}; + llvm::Value *Args[] = {Value.getPointer()}; return CGF.EmitRuntimeCall(Function, Args); } @@ -1002,7 +1043,7 @@ bool MicrosoftCXXABI::EmitBadCastCall(CodeGenFunction &CGF) { } llvm::Value *MicrosoftCXXABI::GetVirtualBaseClassOffset( - CodeGenFunction &CGF, llvm::Value *This, const CXXRecordDecl *ClassDecl, + CodeGenFunction &CGF, Address This, const CXXRecordDecl *ClassDecl, const CXXRecordDecl *BaseClassDecl) { const ASTContext &Context = getContext(); int64_t VBPtrChars = @@ -1040,15 +1081,16 @@ bool MicrosoftCXXABI::classifyReturnType(CGFunctionInfo &FI) const { if (!RD) return false; + CharUnits Align = CGM.getContext().getTypeAlignInChars(FI.getReturnType()); if (FI.isInstanceMethod()) { // If it's an instance method, aggregates are always returned indirectly via // the second parameter. - FI.getReturnInfo() = ABIArgInfo::getIndirect(0, /*ByVal=*/false); + FI.getReturnInfo() = ABIArgInfo::getIndirect(Align, /*ByVal=*/false); FI.getReturnInfo().setSRetAfterThis(FI.isInstanceMethod()); return true; } else if (!RD->isPOD()) { // If it's a free function, non-POD types are returned indirectly. - FI.getReturnInfo() = ABIArgInfo::getIndirect(0, /*ByVal=*/false); + FI.getReturnInfo() = ABIArgInfo::getIndirect(Align, /*ByVal=*/false); return true; } @@ -1100,8 +1142,7 @@ void MicrosoftCXXABI::initializeHiddenVirtualInheritanceMembers( const VBOffsets &VBaseMap = Layout.getVBaseOffsetsMap(); CGBuilderTy &Builder = CGF.Builder; - unsigned AS = - cast<llvm::PointerType>(getThisValue(CGF)->getType())->getAddressSpace(); + unsigned AS = getThisAddress(CGF).getAddressSpace(); llvm::Value *Int8This = nullptr; // Initialize lazily. for (VBOffsets::const_iterator I = VBaseMap.begin(), E = VBaseMap.end(); @@ -1110,7 +1151,7 @@ void MicrosoftCXXABI::initializeHiddenVirtualInheritanceMembers( continue; llvm::Value *VBaseOffset = - GetVirtualBaseClassOffset(CGF, getThisValue(CGF), RD, I->first); + GetVirtualBaseClassOffset(CGF, getThisAddress(CGF), RD, I->first); // FIXME: it doesn't look right that we SExt in GetVirtualBaseClassOffset() // just to Trunc back immediately. VBaseOffset = Builder.CreateTruncOrBitCast(VBaseOffset, CGF.Int32Ty); @@ -1131,7 +1172,8 @@ void MicrosoftCXXABI::initializeHiddenVirtualInheritanceMembers( VtorDispPtr = Builder.CreateBitCast( VtorDispPtr, CGF.Int32Ty->getPointerTo(AS), "vtordisp.ptr"); - Builder.CreateStore(VtorDispValue, VtorDispPtr); + Builder.CreateAlignedStore(VtorDispValue, VtorDispPtr, + CharUnits::fromQuantity(4)); } } @@ -1162,8 +1204,8 @@ void MicrosoftCXXABI::EmitCXXConstructors(const CXXConstructorDecl *D) { void MicrosoftCXXABI::EmitVBPtrStores(CodeGenFunction &CGF, const CXXRecordDecl *RD) { - llvm::Value *ThisInt8Ptr = - CGF.Builder.CreateBitCast(getThisValue(CGF), CGM.Int8PtrTy, "this.int8"); + Address This = getThisAddress(CGF); + This = CGF.Builder.CreateElementBitCast(This, CGM.Int8Ty, "this.int8"); const ASTContext &Context = getContext(); const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD); @@ -1177,11 +1219,10 @@ void MicrosoftCXXABI::EmitVBPtrStores(CodeGenFunction &CGF, Offs += SubobjectLayout.getVBPtrOffset(); if (VBT->getVBaseWithVPtr()) Offs += Layout.getVBaseClassOffset(VBT->getVBaseWithVPtr()); - llvm::Value *VBPtr = - CGF.Builder.CreateConstInBoundsGEP1_64(ThisInt8Ptr, Offs.getQuantity()); + Address VBPtr = CGF.Builder.CreateConstInBoundsByteGEP(This, Offs); llvm::Value *GVPtr = CGF.Builder.CreateConstInBoundsGEP2_32(GV->getValueType(), GV, 0, 0); - VBPtr = CGF.Builder.CreateBitCast(VBPtr, GVPtr->getType()->getPointerTo(0), + VBPtr = CGF.Builder.CreateElementBitCast(VBPtr, GVPtr->getType(), "vbptr." + VBT->ReusingBase->getName()); CGF.Builder.CreateStore(GVPtr, VBPtr); } @@ -1255,8 +1296,9 @@ MicrosoftCXXABI::getVirtualFunctionPrologueThisAdjustment(GlobalDecl GD) { return Adjustment; } -llvm::Value *MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall( - CodeGenFunction &CGF, GlobalDecl GD, llvm::Value *This, bool VirtualCall) { +Address MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall( + CodeGenFunction &CGF, GlobalDecl GD, Address This, + bool VirtualCall) { if (!VirtualCall) { // If the call of a virtual function is not virtual, we just have to // compensate for the adjustment the virtual function does in its prologue. @@ -1264,11 +1306,9 @@ llvm::Value *MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall( if (Adjustment.isZero()) return This; - unsigned AS = cast<llvm::PointerType>(This->getType())->getAddressSpace(); - llvm::Type *charPtrTy = CGF.Int8Ty->getPointerTo(AS); - This = CGF.Builder.CreateBitCast(This, charPtrTy); + This = CGF.Builder.CreateElementBitCast(This, CGF.Int8Ty); assert(Adjustment.isPositive()); - return CGF.Builder.CreateConstGEP1_32(This, Adjustment.getQuantity()); + return CGF.Builder.CreateConstByteGEP(This, Adjustment); } GD = GD.getCanonicalDecl(); @@ -1288,8 +1328,6 @@ llvm::Value *MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall( MicrosoftVTableContext::MethodVFTableLocation ML = CGM.getMicrosoftVTableContext().getMethodVFTableLocation(LookupGD); - unsigned AS = cast<llvm::PointerType>(This->getType())->getAddressSpace(); - llvm::Type *charPtrTy = CGF.Int8Ty->getPointerTo(AS); CharUnits StaticOffset = ML.VFPtrOffset; // Base destructors expect 'this' to point to the beginning of the base @@ -1298,27 +1336,34 @@ llvm::Value *MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall( if (isa<CXXDestructorDecl>(MD) && GD.getDtorType() == Dtor_Base) StaticOffset = CharUnits::Zero(); + Address Result = This; if (ML.VBase) { - This = CGF.Builder.CreateBitCast(This, charPtrTy); + Result = CGF.Builder.CreateElementBitCast(Result, CGF.Int8Ty); + + const CXXRecordDecl *Derived = MD->getParent(); + const CXXRecordDecl *VBase = ML.VBase; llvm::Value *VBaseOffset = - GetVirtualBaseClassOffset(CGF, This, MD->getParent(), ML.VBase); - This = CGF.Builder.CreateInBoundsGEP(This, VBaseOffset); + GetVirtualBaseClassOffset(CGF, Result, Derived, VBase); + llvm::Value *VBasePtr = + CGF.Builder.CreateInBoundsGEP(Result.getPointer(), VBaseOffset); + CharUnits VBaseAlign = + CGF.CGM.getVBaseAlignment(Result.getAlignment(), Derived, VBase); + Result = Address(VBasePtr, VBaseAlign); } if (!StaticOffset.isZero()) { assert(StaticOffset.isPositive()); - This = CGF.Builder.CreateBitCast(This, charPtrTy); + Result = CGF.Builder.CreateElementBitCast(Result, CGF.Int8Ty); if (ML.VBase) { // Non-virtual adjustment might result in a pointer outside the allocated // object, e.g. if the final overrider class is laid out after the virtual // base that declares a method in the most derived class. // FIXME: Update the code that emits this adjustment in thunks prologues. - This = CGF.Builder.CreateConstGEP1_32(This, StaticOffset.getQuantity()); + Result = CGF.Builder.CreateConstByteGEP(Result, StaticOffset); } else { - This = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, This, - StaticOffset.getQuantity()); + Result = CGF.Builder.CreateConstInBoundsByteGEP(Result, StaticOffset); } } - return This; + return Result; } void MicrosoftCXXABI::addImplicitStructorParams(CodeGenFunction &CGF, @@ -1439,7 +1484,7 @@ unsigned MicrosoftCXXABI::addImplicitConstructorArgs( void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type, bool ForVirtualBase, - bool Delegating, llvm::Value *This) { + bool Delegating, Address This) { llvm::Value *Callee = CGM.getAddrOfCXXStructor(DD, getFromDtorType(Type)); if (DD->isVirtual()) { @@ -1449,7 +1494,7 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF, This, false); } - CGF.EmitCXXStructorCall(DD, Callee, ReturnValueSlot(), This, + CGF.EmitCXXStructorCall(DD, Callee, ReturnValueSlot(), This.getPointer(), /*ImplicitParam=*/nullptr, /*ImplicitParamTy=*/QualType(), nullptr, getFromDtorType(Type)); @@ -1478,15 +1523,14 @@ void MicrosoftCXXABI::emitVTableBitSetEntries(VPtrInfo *Info, if (Info->PathToBaseWithVPtr.empty()) { if (!CGM.IsCFIBlacklistedRecord(RD)) - BitsetsMD->addOperand( - CGM.CreateVTableBitSetEntry(VTable, AddressPoint, RD)); + CGM.CreateVTableBitSetEntry(BitsetsMD, VTable, AddressPoint, RD); return; } // Add a bitset entry for the least derived base belonging to this vftable. if (!CGM.IsCFIBlacklistedRecord(Info->PathToBaseWithVPtr.back())) - BitsetsMD->addOperand(CGM.CreateVTableBitSetEntry( - VTable, AddressPoint, Info->PathToBaseWithVPtr.back())); + CGM.CreateVTableBitSetEntry(BitsetsMD, VTable, AddressPoint, + Info->PathToBaseWithVPtr.back()); // Add a bitset entry for each derived class that is laid out at the same // offset as the least derived base. @@ -1505,14 +1549,12 @@ void MicrosoftCXXABI::emitVTableBitSetEntries(VPtrInfo *Info, if (!Offset.isZero()) return; if (!CGM.IsCFIBlacklistedRecord(DerivedRD)) - BitsetsMD->addOperand( - CGM.CreateVTableBitSetEntry(VTable, AddressPoint, DerivedRD)); + CGM.CreateVTableBitSetEntry(BitsetsMD, VTable, AddressPoint, DerivedRD); } // Finally do the same for the most derived class. if (Info->FullOffsetInMDC.isZero() && !CGM.IsCFIBlacklistedRecord(RD)) - BitsetsMD->addOperand( - CGM.CreateVTableBitSetEntry(VTable, AddressPoint, RD)); + CGM.CreateVTableBitSetEntry(BitsetsMD, VTable, AddressPoint, RD); } void MicrosoftCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, @@ -1542,14 +1584,15 @@ void MicrosoftCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, } } +bool MicrosoftCXXABI::isVirtualOffsetNeededForVTableField( + CodeGenFunction &CGF, CodeGenFunction::VPtr Vptr) { + return Vptr.NearestVBase != nullptr; +} + llvm::Value *MicrosoftCXXABI::getVTableAddressPointInStructor( CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base, - const CXXRecordDecl *NearestVBase, bool &NeedsVirtualOffset) { - NeedsVirtualOffset = (NearestVBase != nullptr); - - (void)getAddrOfVTable(VTableClass, Base.getBaseOffset()); - VFTableIdTy ID(VTableClass, Base.getBaseOffset()); - llvm::GlobalValue *VTableAddressPoint = VFTablesMap[ID]; + const CXXRecordDecl *NearestVBase) { + llvm::Constant *VTableAddressPoint = getVTableAddressPoint(Base, VTableClass); if (!VTableAddressPoint) { assert(Base.getBase()->getNumVBases() && !getContext().getASTRecordLayout(Base.getBase()).hasOwnVFPtr()); @@ -1564,11 +1607,17 @@ static void mangleVFTableName(MicrosoftMangleContext &MangleContext, MangleContext.mangleCXXVFTable(RD, VFPtr->MangledPath, Out); } -llvm::Constant *MicrosoftCXXABI::getVTableAddressPointForConstExpr( - BaseSubobject Base, const CXXRecordDecl *VTableClass) { +llvm::Constant * +MicrosoftCXXABI::getVTableAddressPoint(BaseSubobject Base, + const CXXRecordDecl *VTableClass) { (void)getAddrOfVTable(VTableClass, Base.getBaseOffset()); VFTableIdTy ID(VTableClass, Base.getBaseOffset()); - llvm::GlobalValue *VFTable = VFTablesMap[ID]; + return VFTablesMap[ID]; +} + +llvm::Constant *MicrosoftCXXABI::getVTableAddressPointForConstExpr( + BaseSubobject Base, const CXXRecordDecl *VTableClass) { + llvm::Constant *VFTable = getVTableAddressPoint(Base, VTableClass); assert(VFTable && "Couldn't find a vftable for the given base?"); return VFTable; } @@ -1578,6 +1627,7 @@ llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, // getAddrOfVTable may return 0 if asked to get an address of a vtable which // shouldn't be used in the given record type. We want to cache this result in // VFTablesMap, thus a simple zero check is not sufficient. + VFTableIdTy ID(RD, VPtrOffset); VTablesMapTy::iterator I; bool Inserted; @@ -1631,10 +1681,11 @@ llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, if (llvm::GlobalValue *VFTable = CGM.getModule().getNamedGlobal(VFTableName)) { VFTablesMap[ID] = VFTable; - return VTableAliasIsRequred - ? cast<llvm::GlobalVariable>( - cast<llvm::GlobalAlias>(VFTable)->getBaseObject()) - : cast<llvm::GlobalVariable>(VFTable); + VTable = VTableAliasIsRequred + ? cast<llvm::GlobalVariable>( + cast<llvm::GlobalAlias>(VFTable)->getBaseObject()) + : cast<llvm::GlobalVariable>(VFTable); + return VTable; } uint64_t NumVTableSlots = @@ -1678,9 +1729,10 @@ llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, if (C) C->setSelectionKind(llvm::Comdat::Largest); } - VFTable = llvm::GlobalAlias::create( - cast<llvm::PointerType>(VTableGEP->getType()), VFTableLinkage, - VFTableName.str(), VTableGEP, &CGM.getModule()); + VFTable = llvm::GlobalAlias::create(CGM.Int8PtrTy, + /*AddressSpace=*/0, VFTableLinkage, + VFTableName.str(), VTableGEP, + &CGM.getModule()); VFTable->setUnnamedAddr(true); } else { // We don't need a GlobalAlias to be a symbol for the VTable if we won't @@ -1746,16 +1798,18 @@ getClassAtVTableLocation(ASTContext &Ctx, GlobalDecl GD, llvm::Value *MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, - llvm::Value *This, + Address This, llvm::Type *Ty, SourceLocation Loc) { GD = GD.getCanonicalDecl(); CGBuilderTy &Builder = CGF.Builder; Ty = Ty->getPointerTo()->getPointerTo(); - llvm::Value *VPtr = + Address VPtr = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true); - llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty); + + auto *MethodDecl = cast<CXXMethodDecl>(GD.getDecl()); + llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty, MethodDecl->getParent()); MicrosoftVTableContext::MethodVFTableLocation ML = CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD); @@ -1765,12 +1819,12 @@ llvm::Value *MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, llvm::Value *VFuncPtr = Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn"); - return Builder.CreateLoad(VFuncPtr); + return Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); } llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType, - llvm::Value *This, const CXXMemberCallExpr *CE) { + Address This, const CXXMemberCallExpr *CE) { assert(CE == nullptr || CE->arg_begin() == CE->arg_end()); assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete); @@ -1789,7 +1843,8 @@ llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( DtorType == Dtor_Deleting); This = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true); - RValue RV = CGF.EmitCXXStructorCall(Dtor, Callee, ReturnValueSlot(), This, + RValue RV = CGF.EmitCXXStructorCall(Dtor, Callee, ReturnValueSlot(), + This.getPointer(), ImplicitParam, Context.IntTy, CE, StructorType::Deleting); return RV.getScalarVal(); @@ -1832,7 +1887,6 @@ llvm::Function *MicrosoftCXXABI::EmitVirtualMemPtrThunk( SmallString<256> ThunkName; llvm::raw_svector_ostream Out(ThunkName); getMangleContext().mangleVirtualMemPtrThunk(MD, Out); - Out.flush(); // If the thunk has been generated previously, just return it. if (llvm::GlobalValue *GV = CGM.getModule().getNamedValue(ThunkName)) @@ -1882,10 +1936,12 @@ llvm::Function *MicrosoftCXXABI::EmitVirtualMemPtrThunk( // Load the vfptr and then callee from the vftable. The callee should have // adjusted 'this' so that the vfptr is at offset zero. llvm::Value *VTable = CGF.GetVTablePtr( - getThisValue(CGF), ThunkTy->getPointerTo()->getPointerTo()); + getThisAddress(CGF), ThunkTy->getPointerTo()->getPointerTo(), MD->getParent()); + llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn"); - llvm::Value *Callee = CGF.Builder.CreateLoad(VFuncPtr); + llvm::Value *Callee = + CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); CGF.EmitMustTailThunk(MD, getThisValue(CGF), Callee); @@ -1908,7 +1964,6 @@ MicrosoftCXXABI::getAddrOfVBTable(const VPtrInfo &VBT, const CXXRecordDecl *RD, SmallString<256> OutName; llvm::raw_svector_ostream Out(OutName); getMangleContext().mangleCXXVBTable(RD, VBT.MangledPath, Out); - Out.flush(); StringRef Name = OutName.str(); llvm::ArrayType *VBTableType = @@ -1978,22 +2033,30 @@ void MicrosoftCXXABI::emitVBTableDefinition(const VPtrInfo &VBT, } llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, - llvm::Value *This, + Address This, const ThisAdjustment &TA) { if (TA.isEmpty()) - return This; + return This.getPointer(); - llvm::Value *V = CGF.Builder.CreateBitCast(This, CGF.Int8PtrTy); + This = CGF.Builder.CreateElementBitCast(This, CGF.Int8Ty); - if (!TA.Virtual.isEmpty()) { + llvm::Value *V; + if (TA.Virtual.isEmpty()) { + V = This.getPointer(); + } else { assert(TA.Virtual.Microsoft.VtordispOffset < 0); // Adjust the this argument based on the vtordisp value. - llvm::Value *VtorDispPtr = - CGF.Builder.CreateConstGEP1_32(V, TA.Virtual.Microsoft.VtordispOffset); - VtorDispPtr = - CGF.Builder.CreateBitCast(VtorDispPtr, CGF.Int32Ty->getPointerTo()); + Address VtorDispPtr = + CGF.Builder.CreateConstInBoundsByteGEP(This, + CharUnits::fromQuantity(TA.Virtual.Microsoft.VtordispOffset)); + VtorDispPtr = CGF.Builder.CreateElementBitCast(VtorDispPtr, CGF.Int32Ty); llvm::Value *VtorDisp = CGF.Builder.CreateLoad(VtorDispPtr, "vtordisp"); - V = CGF.Builder.CreateGEP(V, CGF.Builder.CreateNeg(VtorDisp)); + V = CGF.Builder.CreateGEP(This.getPointer(), + CGF.Builder.CreateNeg(VtorDisp)); + + // Unfortunately, having applied the vtordisp means that we no + // longer really have a known alignment for the vbptr step. + // We'll assume the vbptr is pointer-aligned. if (TA.Virtual.Microsoft.VBPtrOffset) { // If the final overrider is defined in a virtual base other than the one @@ -2003,7 +2066,8 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, assert(TA.Virtual.Microsoft.VBOffsetOffset >= 0); llvm::Value *VBPtr; llvm::Value *VBaseOffset = - GetVBaseOffsetFromVBPtr(CGF, V, -TA.Virtual.Microsoft.VBPtrOffset, + GetVBaseOffsetFromVBPtr(CGF, Address(V, CGF.getPointerAlign()), + -TA.Virtual.Microsoft.VBPtrOffset, TA.Virtual.Microsoft.VBOffsetOffset, &VBPtr); V = CGF.Builder.CreateInBoundsGEP(VBPtr, VBaseOffset); } @@ -2021,20 +2085,21 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, } llvm::Value * -MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret, +MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret, const ReturnAdjustment &RA) { if (RA.isEmpty()) - return Ret; + return Ret.getPointer(); - llvm::Value *V = CGF.Builder.CreateBitCast(Ret, CGF.Int8PtrTy); + auto OrigTy = Ret.getType(); + Ret = CGF.Builder.CreateElementBitCast(Ret, CGF.Int8Ty); + llvm::Value *V = Ret.getPointer(); if (RA.Virtual.Microsoft.VBIndex) { assert(RA.Virtual.Microsoft.VBIndex > 0); - const ASTContext &Context = getContext(); - int32_t IntSize = Context.getTypeSizeInChars(Context.IntTy).getQuantity(); + int32_t IntSize = CGF.getIntSize().getQuantity(); llvm::Value *VBPtr; llvm::Value *VBaseOffset = - GetVBaseOffsetFromVBPtr(CGF, V, RA.Virtual.Microsoft.VBPtrOffset, + GetVBaseOffsetFromVBPtr(CGF, Ret, RA.Virtual.Microsoft.VBPtrOffset, IntSize * RA.Virtual.Microsoft.VBIndex, &VBPtr); V = CGF.Builder.CreateInBoundsGEP(VBPtr, VBaseOffset); } @@ -2043,7 +2108,7 @@ MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret, V = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, V, RA.NonVirtual); // Cast back to the original type. - return CGF.Builder.CreateBitCast(V, Ret->getType()); + return CGF.Builder.CreateBitCast(V, OrigTy); } bool MicrosoftCXXABI::requiresArrayCookie(const CXXDeleteExpr *expr, @@ -2068,37 +2133,34 @@ CharUnits MicrosoftCXXABI::getArrayCookieSizeImpl(QualType type) { } llvm::Value *MicrosoftCXXABI::readArrayCookieImpl(CodeGenFunction &CGF, - llvm::Value *allocPtr, + Address allocPtr, CharUnits cookieSize) { - unsigned AS = allocPtr->getType()->getPointerAddressSpace(); - llvm::Value *numElementsPtr = - CGF.Builder.CreateBitCast(allocPtr, CGF.SizeTy->getPointerTo(AS)); + Address numElementsPtr = + CGF.Builder.CreateElementBitCast(allocPtr, CGF.SizeTy); return CGF.Builder.CreateLoad(numElementsPtr); } -llvm::Value* MicrosoftCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, - llvm::Value *newPtr, - llvm::Value *numElements, - const CXXNewExpr *expr, - QualType elementType) { +Address MicrosoftCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, + Address newPtr, + llvm::Value *numElements, + const CXXNewExpr *expr, + QualType elementType) { assert(requiresArrayCookie(expr)); // The size of the cookie. CharUnits cookieSize = getArrayCookieSizeImpl(elementType); // Compute an offset to the cookie. - llvm::Value *cookiePtr = newPtr; + Address cookiePtr = newPtr; // Write the number of elements into the appropriate slot. - unsigned AS = newPtr->getType()->getPointerAddressSpace(); - llvm::Value *numElementsPtr - = CGF.Builder.CreateBitCast(cookiePtr, CGF.SizeTy->getPointerTo(AS)); + Address numElementsPtr + = CGF.Builder.CreateElementBitCast(cookiePtr, CGF.SizeTy); CGF.Builder.CreateStore(numElements, numElementsPtr); // Finally, compute a pointer to the actual data buffer by skipping // over the cookie completely. - return CGF.Builder.CreateConstInBoundsGEP1_64(newPtr, - cookieSize.getQuantity()); + return CGF.Builder.CreateConstInBoundsByteGEP(newPtr, cookieSize); } static void emitGlobalDtorWithTLRegDtor(CodeGenFunction &CGF, const VarDecl &VD, @@ -2130,11 +2192,9 @@ void MicrosoftCXXABI::registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D, } void MicrosoftCXXABI::EmitThreadLocalInitFuncs( - CodeGenModule &CGM, - ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>> - CXXThreadLocals, + CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals, ArrayRef<llvm::Function *> CXXThreadLocalInits, - ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) { + ArrayRef<const VarDecl *> CXXThreadLocalInitVars) { // This will create a GV in the .CRT$XDU section. It will point to our // initialization function. The CRT will call all of these function // pointers at start-up time and, eventually, at thread-creation time. @@ -2152,7 +2212,8 @@ void MicrosoftCXXABI::EmitThreadLocalInitFuncs( std::vector<llvm::Function *> NonComdatInits; for (size_t I = 0, E = CXXThreadLocalInitVars.size(); I != E; ++I) { - llvm::GlobalVariable *GV = CXXThreadLocalInitVars[I]; + llvm::GlobalVariable *GV = cast<llvm::GlobalVariable>( + CGM.GetGlobalValue(CGM.getMangledName(CXXThreadLocalInitVars[I]))); llvm::Function *F = CXXThreadLocalInits[I]; // If the GV is already in a comdat group, then we have to join it. @@ -2166,8 +2227,8 @@ void MicrosoftCXXABI::EmitThreadLocalInitFuncs( llvm::FunctionType *FTy = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false); llvm::Function *InitFunc = CGM.CreateGlobalInitOrDestructFunction( - FTy, "__tls_init", SourceLocation(), - /*TLS=*/true); + FTy, "__tls_init", CGM.getTypes().arrangeNullaryFunction(), + SourceLocation(), /*TLS=*/true); CodeGenFunction(CGM).GenerateCXXGlobalInitFunc(InitFunc, NonComdatInits); AddToXDU(InitFunc); @@ -2181,17 +2242,18 @@ LValue MicrosoftCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, return LValue(); } -static llvm::GlobalVariable *getInitThreadEpochPtr(CodeGenModule &CGM) { +static ConstantAddress getInitThreadEpochPtr(CodeGenModule &CGM) { StringRef VarName("_Init_thread_epoch"); + CharUnits Align = CGM.getIntAlign(); if (auto *GV = CGM.getModule().getNamedGlobal(VarName)) - return GV; + return ConstantAddress(GV, Align); auto *GV = new llvm::GlobalVariable( CGM.getModule(), CGM.IntTy, /*Constant=*/false, llvm::GlobalVariable::ExternalLinkage, /*Initializer=*/nullptr, VarName, /*InsertBefore=*/nullptr, llvm::GlobalVariable::GeneralDynamicTLSModel); - GV->setAlignment(CGM.getTarget().getIntAlign() / 8); - return GV; + GV->setAlignment(Align.getQuantity()); + return ConstantAddress(GV, Align); } static llvm::Constant *getInitThreadHeaderFn(CodeGenModule &CGM) { @@ -2228,10 +2290,10 @@ static llvm::Constant *getInitThreadAbortFn(CodeGenModule &CGM) { } namespace { -struct ResetGuardBit : EHScopeStack::Cleanup { - llvm::GlobalVariable *Guard; +struct ResetGuardBit final : EHScopeStack::Cleanup { + Address Guard; unsigned GuardNum; - ResetGuardBit(llvm::GlobalVariable *Guard, unsigned GuardNum) + ResetGuardBit(Address Guard, unsigned GuardNum) : Guard(Guard), GuardNum(GuardNum) {} void Emit(CodeGenFunction &CGF, Flags flags) override { @@ -2245,9 +2307,9 @@ struct ResetGuardBit : EHScopeStack::Cleanup { } }; -struct CallInitThreadAbort : EHScopeStack::Cleanup { - llvm::GlobalVariable *Guard; - CallInitThreadAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {} +struct CallInitThreadAbort final : EHScopeStack::Cleanup { + llvm::Value *Guard; + CallInitThreadAbort(Address Guard) : Guard(Guard.getPointer()) {} void Emit(CodeGenFunction &CGF, Flags flags) override { // Calling _Init_thread_abort will reset the guard's state. @@ -2280,6 +2342,7 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, CGBuilderTy &Builder = CGF.Builder; llvm::IntegerType *GuardTy = CGF.Int32Ty; llvm::ConstantInt *Zero = llvm::ConstantInt::get(GuardTy, 0); + CharUnits GuardAlign = CharUnits::fromQuantity(4); // Get the guard variable for this function if we have one already. GuardInfo *GI = nullptr; @@ -2320,7 +2383,6 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, Out); else getMangleContext().mangleStaticGuardVariable(&D, Out); - Out.flush(); } // Create the guard variable with a zero-initializer. Just absorb linkage, @@ -2330,6 +2392,7 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, GV->getLinkage(), Zero, GuardName.str()); GuardVar->setVisibility(GV->getVisibility()); GuardVar->setDLLStorageClass(GV->getDLLStorageClass()); + GuardVar->setAlignment(GuardAlign.getQuantity()); if (GuardVar->isWeakForLinker()) GuardVar->setComdat( CGM.getModule().getOrInsertComdat(GuardVar->getName())); @@ -2339,6 +2402,8 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, GI->Guard = GuardVar; } + ConstantAddress GuardAddr(GuardVar, GuardAlign); + assert(GuardVar->getLinkage() == GV->getLinkage() && "static local from the same function had different linkage"); @@ -2351,7 +2416,7 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, // Test our bit from the guard variable. llvm::ConstantInt *Bit = llvm::ConstantInt::get(GuardTy, 1U << GuardNum); - llvm::LoadInst *LI = Builder.CreateLoad(GuardVar); + llvm::LoadInst *LI = Builder.CreateLoad(GuardAddr); llvm::Value *IsInitialized = Builder.CreateICmpNE(Builder.CreateAnd(LI, Bit), Zero); llvm::BasicBlock *InitBlock = CGF.createBasicBlock("init"); @@ -2361,8 +2426,8 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, // Set our bit in the guard variable and emit the initializer and add a global // destructor if appropriate. CGF.EmitBlock(InitBlock); - Builder.CreateStore(Builder.CreateOr(LI, Bit), GuardVar); - CGF.EHStack.pushCleanup<ResetGuardBit>(EHCleanup, GuardVar, GuardNum); + Builder.CreateStore(Builder.CreateOr(LI, Bit), GuardAddr); + CGF.EHStack.pushCleanup<ResetGuardBit>(EHCleanup, GuardAddr, GuardNum); CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); CGF.PopCleanupBlock(); Builder.CreateBr(EndBlock); @@ -2382,11 +2447,8 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, // The algorithm is almost identical to what can be found in the appendix // found in N2325. - unsigned IntAlign = CGM.getTarget().getIntAlign() / 8; - // This BasicBLock determines whether or not we have any work to do. - llvm::LoadInst *FirstGuardLoad = - Builder.CreateAlignedLoad(GuardVar, IntAlign); + llvm::LoadInst *FirstGuardLoad = Builder.CreateLoad(GuardAddr); FirstGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered); llvm::LoadInst *InitThreadEpoch = Builder.CreateLoad(getInitThreadEpochPtr(CGM)); @@ -2399,9 +2461,9 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, // This BasicBlock attempts to determine whether or not this thread is // responsible for doing the initialization. CGF.EmitBlock(AttemptInitBlock); - CGF.EmitNounwindRuntimeCall(getInitThreadHeaderFn(CGM), GuardVar); - llvm::LoadInst *SecondGuardLoad = - Builder.CreateAlignedLoad(GuardVar, IntAlign); + CGF.EmitNounwindRuntimeCall(getInitThreadHeaderFn(CGM), + GuardAddr.getPointer()); + llvm::LoadInst *SecondGuardLoad = Builder.CreateLoad(GuardAddr); SecondGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered); llvm::Value *ShouldDoInit = Builder.CreateICmpEQ(SecondGuardLoad, getAllOnesInt()); @@ -2410,10 +2472,11 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, // Ok, we ended up getting selected as the initializing thread. CGF.EmitBlock(InitBlock); - CGF.EHStack.pushCleanup<CallInitThreadAbort>(EHCleanup, GuardVar); + CGF.EHStack.pushCleanup<CallInitThreadAbort>(EHCleanup, GuardAddr); CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); CGF.PopCleanupBlock(); - CGF.EmitNounwindRuntimeCall(getInitThreadFooterFn(CGM), GuardVar); + CGF.EmitNounwindRuntimeCall(getInitThreadFooterFn(CGM), + GuardAddr.getPointer()); Builder.CreateBr(EndBlock); CGF.EmitBlock(EndBlock); @@ -2768,19 +2831,28 @@ bool MicrosoftCXXABI::MemberPointerConstantIsNull(const MemberPointerType *MPT, llvm::Value * MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF, - llvm::Value *This, + Address This, llvm::Value *VBPtrOffset, llvm::Value *VBTableOffset, llvm::Value **VBPtrOut) { CGBuilderTy &Builder = CGF.Builder; // Load the vbtable pointer from the vbptr in the instance. - This = Builder.CreateBitCast(This, CGM.Int8PtrTy); + This = Builder.CreateElementBitCast(This, CGM.Int8Ty); llvm::Value *VBPtr = - Builder.CreateInBoundsGEP(This, VBPtrOffset, "vbptr"); + Builder.CreateInBoundsGEP(This.getPointer(), VBPtrOffset, "vbptr"); if (VBPtrOut) *VBPtrOut = VBPtr; VBPtr = Builder.CreateBitCast(VBPtr, - CGM.Int32Ty->getPointerTo(0)->getPointerTo(0)); - llvm::Value *VBTable = Builder.CreateLoad(VBPtr, "vbtable"); + CGM.Int32Ty->getPointerTo(0)->getPointerTo(This.getAddressSpace())); + + CharUnits VBPtrAlign; + if (auto CI = dyn_cast<llvm::ConstantInt>(VBPtrOffset)) { + VBPtrAlign = This.getAlignment().alignmentAtOffset( + CharUnits::fromQuantity(CI->getSExtValue())); + } else { + VBPtrAlign = CGF.getPointerAlign(); + } + + llvm::Value *VBTable = Builder.CreateAlignedLoad(VBPtr, VBPtrAlign, "vbtable"); // Translate from byte offset to table index. It improves analyzability. llvm::Value *VBTableIndex = Builder.CreateAShr( @@ -2790,16 +2862,17 @@ MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF, // Load an i32 offset from the vb-table. llvm::Value *VBaseOffs = Builder.CreateInBoundsGEP(VBTable, VBTableIndex); VBaseOffs = Builder.CreateBitCast(VBaseOffs, CGM.Int32Ty->getPointerTo(0)); - return Builder.CreateLoad(VBaseOffs, "vbase_offs"); + return Builder.CreateAlignedLoad(VBaseOffs, CharUnits::fromQuantity(4), + "vbase_offs"); } // Returns an adjusted base cast to i8*, since we do more address arithmetic on // it. llvm::Value *MicrosoftCXXABI::AdjustVirtualBase( CodeGenFunction &CGF, const Expr *E, const CXXRecordDecl *RD, - llvm::Value *Base, llvm::Value *VBTableOffset, llvm::Value *VBPtrOffset) { + Address Base, llvm::Value *VBTableOffset, llvm::Value *VBPtrOffset) { CGBuilderTy &Builder = CGF.Builder; - Base = Builder.CreateBitCast(Base, CGM.Int8PtrTy); + Base = Builder.CreateElementBitCast(Base, CGM.Int8Ty); llvm::BasicBlock *OriginalBB = nullptr; llvm::BasicBlock *SkipAdjustBB = nullptr; llvm::BasicBlock *VBaseAdjustBB = nullptr; @@ -2844,7 +2917,7 @@ llvm::Value *MicrosoftCXXABI::AdjustVirtualBase( Builder.CreateBr(SkipAdjustBB); CGF.EmitBlock(SkipAdjustBB); llvm::PHINode *Phi = Builder.CreatePHI(CGM.Int8PtrTy, 2, "memptr.base"); - Phi->addIncoming(Base, OriginalBB); + Phi->addIncoming(Base.getPointer(), OriginalBB); Phi->addIncoming(AdjustedBase, VBaseAdjustBB); return Phi; } @@ -2852,10 +2925,10 @@ llvm::Value *MicrosoftCXXABI::AdjustVirtualBase( } llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress( - CodeGenFunction &CGF, const Expr *E, llvm::Value *Base, llvm::Value *MemPtr, + CodeGenFunction &CGF, const Expr *E, Address Base, llvm::Value *MemPtr, const MemberPointerType *MPT) { assert(MPT->isMemberDataPointer()); - unsigned AS = Base->getType()->getPointerAddressSpace(); + unsigned AS = Base.getAddressSpace(); llvm::Type *PType = CGF.ConvertTypeForMem(MPT->getPointeeType())->getPointerTo(AS); CGBuilderTy &Builder = CGF.Builder; @@ -2877,17 +2950,19 @@ llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress( VirtualBaseAdjustmentOffset = Builder.CreateExtractValue(MemPtr, I++); } + llvm::Value *Addr; if (VirtualBaseAdjustmentOffset) { - Base = AdjustVirtualBase(CGF, E, RD, Base, VirtualBaseAdjustmentOffset, + Addr = AdjustVirtualBase(CGF, E, RD, Base, VirtualBaseAdjustmentOffset, VBPtrOffset); + } else { + Addr = Base.getPointer(); } // Cast to char*. - Base = Builder.CreateBitCast(Base, Builder.getInt8Ty()->getPointerTo(AS)); + Addr = Builder.CreateBitCast(Addr, CGF.Int8Ty->getPointerTo(AS)); // Apply the offset, which we assume is non-null. - llvm::Value *Addr = - Builder.CreateInBoundsGEP(Base, FieldOffset, "memptr.offset"); + Addr = Builder.CreateInBoundsGEP(Addr, FieldOffset, "memptr.offset"); // Cast the address to the appropriate pointer type, adopting the address // space of the base pointer. @@ -3050,7 +3125,8 @@ llvm::Value *MicrosoftCXXABI::EmitNonNullMemberPointerConversion( } else { llvm::Value *Idxs[] = {getZeroInt(), VBIndex}; VirtualBaseAdjustmentOffset = - Builder.CreateLoad(Builder.CreateInBoundsGEP(VDispMap, Idxs)); + Builder.CreateAlignedLoad(Builder.CreateInBoundsGEP(VDispMap, Idxs), + CharUnits::fromQuantity(4)); } DstVBIndexEqZero = @@ -3131,7 +3207,7 @@ llvm::Constant *MicrosoftCXXABI::EmitMemberPointerConversion( if (CK == CK_ReinterpretMemberPointer) return Src; - CGBuilderTy Builder(CGM.getLLVMContext()); + CGBuilderTy Builder(CGM, CGM.getLLVMContext()); auto *Dst = cast<llvm::Constant>(EmitNonNullMemberPointerConversion( SrcTy, DstTy, CK, PathBegin, PathEnd, Src, Builder)); @@ -3139,15 +3215,15 @@ llvm::Constant *MicrosoftCXXABI::EmitMemberPointerConversion( } llvm::Value *MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer( - CodeGenFunction &CGF, const Expr *E, llvm::Value *&This, - llvm::Value *MemPtr, const MemberPointerType *MPT) { + CodeGenFunction &CGF, const Expr *E, Address This, + llvm::Value *&ThisPtrForCall, llvm::Value *MemPtr, + const MemberPointerType *MPT) { assert(MPT->isMemberFunctionPointer()); const FunctionProtoType *FPT = MPT->getPointeeType()->castAs<FunctionProtoType>(); const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl(); - llvm::FunctionType *FTy = - CGM.getTypes().GetFunctionType( - CGM.getTypes().arrangeCXXMethodType(RD, FPT)); + llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType( + CGM.getTypes().arrangeCXXMethodType(RD, FPT, /*FD=*/nullptr)); CGBuilderTy &Builder = CGF.Builder; MSInheritanceAttr::Spelling Inheritance = RD->getMSInheritanceModel(); @@ -3171,15 +3247,18 @@ llvm::Value *MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer( } if (VirtualBaseAdjustmentOffset) { - This = AdjustVirtualBase(CGF, E, RD, This, VirtualBaseAdjustmentOffset, - VBPtrOffset); + ThisPtrForCall = AdjustVirtualBase(CGF, E, RD, This, + VirtualBaseAdjustmentOffset, VBPtrOffset); + } else { + ThisPtrForCall = This.getPointer(); } if (NonVirtualBaseAdjustment) { // Apply the adjustment and cast back to the original struct type. - llvm::Value *Ptr = Builder.CreateBitCast(This, Builder.getInt8PtrTy()); + llvm::Value *Ptr = Builder.CreateBitCast(ThisPtrForCall, CGF.Int8PtrTy); Ptr = Builder.CreateInBoundsGEP(Ptr, NonVirtualBaseAdjustment); - This = Builder.CreateBitCast(Ptr, This->getType(), "this.adjusted"); + ThisPtrForCall = Builder.CreateBitCast(Ptr, ThisPtrForCall->getType(), + "this.adjusted"); } return Builder.CreateBitCast(FunctionPointer, FTy->getPointerTo()); @@ -3404,7 +3483,7 @@ llvm::GlobalVariable *MSRTTIBuilder::getClassHierarchyDescriptor() { auto Type = ABI.getClassHierarchyDescriptorType(); auto CHD = new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage, /*Initializer=*/nullptr, - StringRef(MangledName)); + MangledName); if (CHD->isWeakForLinker()) CHD->setComdat(CGM.getModule().getOrInsertComdat(CHD->getName())); @@ -3442,7 +3521,7 @@ MSRTTIBuilder::getBaseClassArray(SmallVectorImpl<MSRTTIClass> &Classes) { auto *BCA = new llvm::GlobalVariable(Module, ArrType, /*Constant=*/true, Linkage, - /*Initializer=*/nullptr, StringRef(MangledName)); + /*Initializer=*/nullptr, MangledName); if (BCA->isWeakForLinker()) BCA->setComdat(CGM.getModule().getOrInsertComdat(BCA->getName())); @@ -3484,7 +3563,7 @@ MSRTTIBuilder::getBaseClassDescriptor(const MSRTTIClass &Class) { auto Type = ABI.getBaseClassDescriptorType(); auto BCD = new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage, - /*Initializer=*/nullptr, StringRef(MangledName)); + /*Initializer=*/nullptr, MangledName); if (BCD->isWeakForLinker()) BCD->setComdat(CGM.getModule().getOrInsertComdat(BCD->getName())); @@ -3530,7 +3609,7 @@ MSRTTIBuilder::getCompleteObjectLocator(const VPtrInfo *Info) { // Forward-declare the complete object locator. llvm::StructType *Type = ABI.getCompleteObjectLocatorType(); auto COL = new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage, - /*Initializer=*/nullptr, StringRef(MangledName)); + /*Initializer=*/nullptr, MangledName); // Initialize the CompleteObjectLocator. llvm::Constant *Fields[] = { @@ -3582,7 +3661,7 @@ static QualType decomposeTypeForEH(ASTContext &Context, QualType T, return T; } -llvm::Constant * +CatchTypeInfo MicrosoftCXXABI::getAddrOfCXXCatchHandlerType(QualType Type, QualType CatchHandlerType) { // TypeDescriptors for exceptions never have qualified pointer types, @@ -3601,28 +3680,8 @@ MicrosoftCXXABI::getAddrOfCXXCatchHandlerType(QualType Type, if (IsReference) Flags |= 8; - SmallString<256> MangledName; - { - llvm::raw_svector_ostream Out(MangledName); - getMangleContext().mangleCXXCatchHandlerType(Type, Flags, Out); - } - - if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(MangledName)) - return llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy); - - llvm::Constant *Fields[] = { - llvm::ConstantInt::get(CGM.IntTy, Flags), // Flags - getAddrOfRTTIDescriptor(Type), // TypeDescriptor - }; - llvm::StructType *CatchHandlerTypeType = getCatchHandlerTypeType(); - auto *Var = new llvm::GlobalVariable( - CGM.getModule(), CatchHandlerTypeType, /*Constant=*/true, - llvm::GlobalValue::PrivateLinkage, - llvm::ConstantStruct::get(CatchHandlerTypeType, Fields), - StringRef(MangledName)); - Var->setUnnamedAddr(true); - Var->setSection("llvm.metadata"); - return Var; + return CatchTypeInfo{getAddrOfRTTIDescriptor(Type)->stripPointerCasts(), + Flags}; } /// \brief Gets a TypeDescriptor. Returns a llvm::Constant * rather than a @@ -3658,7 +3717,7 @@ llvm::Constant *MicrosoftCXXABI::getAddrOfRTTIDescriptor(QualType Type) { CGM.getModule(), TypeDescriptorType, /*Constant=*/false, getLinkageForRTTI(Type), llvm::ConstantStruct::get(TypeDescriptorType, Fields), - StringRef(MangledName)); + MangledName); if (Var->isWeakForLinker()) Var->setComdat(CGM.getModule().getOrInsertComdat(Var->getName())); return llvm::ConstantExpr::getBitCast(Var, CGM.Int8PtrTy); @@ -3725,7 +3784,6 @@ MicrosoftCXXABI::getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD, SmallString<256> ThunkName; llvm::raw_svector_ostream Out(ThunkName); getMangleContext().mangleCXXCtor(CD, CT, Out); - Out.flush(); // If the thunk has been generated previously, just return it. if (llvm::GlobalValue *GV = CGM.getModule().getNamedValue(ThunkName)) @@ -3803,9 +3861,7 @@ MicrosoftCXXABI::getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD, CodeGenFunction::RunCleanupsScope Cleanups(CGF); const auto *FPT = CD->getType()->castAs<FunctionProtoType>(); - ConstExprIterator ArgBegin(ArgVec.data()), - ArgEnd(ArgVec.data() + ArgVec.size()); - CGF.EmitCallArgs(Args, FPT, ArgBegin, ArgEnd, CD, IsCopy ? 1 : 0); + CGF.EmitCallArgs(Args, FPT, llvm::makeArrayRef(ArgVec), CD, IsCopy ? 1 : 0); // Insert any ABI-specific implicit constructor arguments. unsigned ExtraArgs = addImplicitConstructorArgs(CGF, CD, Ctor_Complete, @@ -3904,7 +3960,7 @@ llvm::Constant *MicrosoftCXXABI::getCatchableType(QualType T, llvm::StructType *CTType = getCatchableTypeType(); auto *GV = new llvm::GlobalVariable( CGM.getModule(), CTType, /*Constant=*/true, getLinkageForRTTI(T), - llvm::ConstantStruct::get(CTType, Fields), StringRef(MangledName)); + llvm::ConstantStruct::get(CTType, Fields), MangledName); GV->setUnnamedAddr(true); GV->setSection(".xdata"); if (GV->isWeakForLinker()) @@ -4022,7 +4078,7 @@ llvm::GlobalVariable *MicrosoftCXXABI::getCatchableTypeArray(QualType T) { } CTA = new llvm::GlobalVariable( CGM.getModule(), CTAType, /*Constant=*/true, getLinkageForRTTI(T), - llvm::ConstantStruct::get(CTAType, Fields), StringRef(MangledName)); + llvm::ConstantStruct::get(CTAType, Fields), MangledName); CTA->setUnnamedAddr(true); CTA->setSection(".xdata"); if (CTA->isWeakForLinker()) @@ -4102,7 +4158,7 @@ void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { QualType ThrowType = SubExpr->getType(); // The exception object lives on the stack and it's address is passed to the // runtime function. - llvm::AllocaInst *AI = CGF.CreateMemTemp(ThrowType); + Address AI = CGF.CreateMemTemp(ThrowType); CGF.EmitAnyExprToMem(SubExpr, AI, ThrowType.getQualifiers(), /*IsInit=*/true); @@ -4111,6 +4167,9 @@ void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { llvm::GlobalVariable *TI = getThrowInfo(ThrowType); // Call into the runtime to throw the exception. - llvm::Value *Args[] = {CGF.Builder.CreateBitCast(AI, CGM.Int8PtrTy), TI}; + llvm::Value *Args[] = { + CGF.Builder.CreateBitCast(AI.getPointer(), CGM.Int8PtrTy), + TI + }; CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(), Args); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp index def56a9..0be5c55 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp @@ -30,7 +30,6 @@ using namespace clang; namespace { class CodeGeneratorImpl : public CodeGenerator { DiagnosticsEngine &Diags; - std::unique_ptr<const llvm::DataLayout> TD; ASTContext *Ctx; const HeaderSearchOptions &HeaderSearchOpts; // Only used for debug info. const PreprocessorOptions &PreprocessorOpts; // Only used for debug info. @@ -99,14 +98,10 @@ namespace { Ctx = &Context; M->setTargetTriple(Ctx->getTargetInfo().getTriple().getTriple()); - M->setDataLayout(Ctx->getTargetInfo().getTargetDescription()); - TD.reset( - new llvm::DataLayout(Ctx->getTargetInfo().getTargetDescription())); - Builder.reset(new CodeGen::CodeGenModule(Context, - HeaderSearchOpts, - PreprocessorOpts, - CodeGenOpts, *M, *TD, - Diags, CoverageInfo)); + M->setDataLayout(Ctx->getTargetInfo().getDataLayoutString()); + Builder.reset(new CodeGen::CodeGenModule(Context, HeaderSearchOpts, + PreprocessorOpts, CodeGenOpts, + *M, Diags, CoverageInfo)); for (size_t i = 0, e = CodeGenOpts.DependentLibraries.size(); i < e; ++i) HandleDependentLibrary(CodeGenOpts.DependentLibraries[i]); @@ -180,7 +175,7 @@ namespace { // For MSVC compatibility, treat declarations of static data members with // inline initializers as definitions. - if (Ctx->getLangOpts().MSVCCompat) { + if (Ctx->getTargetInfo().getCXXABI().isMicrosoft()) { for (Decl *Member : D->decls()) { if (VarDecl *VD = dyn_cast<VarDecl>(Member)) { if (Ctx->isMSStaticDataMemberInlineDefinition(VD) && diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp index 9c9b123..b397eb3 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp @@ -18,6 +18,9 @@ #include "clang/Basic/TargetInfo.h" #include "clang/CodeGen/BackendUtil.h" #include "clang/Frontend/CodeGenOptions.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Lex/HeaderSearch.h" #include "clang/Serialization/ASTWriter.h" #include "llvm/ADT/StringRef.h" #include "llvm/Bitcode/BitstreamReader.h" @@ -30,6 +33,7 @@ #include "llvm/Object/ObjectFile.h" #include "llvm/Support/TargetRegistry.h" #include <memory> + using namespace clang; #define DEBUG_TYPE "pchcontainer" @@ -39,6 +43,7 @@ class PCHContainerGenerator : public ASTConsumer { DiagnosticsEngine &Diags; const std::string MainFileName; ASTContext *Ctx; + ModuleMap &MMap; const HeaderSearchOptions &HeaderSearchOpts; const PreprocessorOptions &PreprocessorOpts; CodeGenOptions CodeGenOpts; @@ -50,34 +55,139 @@ class PCHContainerGenerator : public ASTConsumer { raw_pwrite_stream *OS; std::shared_ptr<PCHBuffer> Buffer; + /// Visit every type and emit debug info for it. + struct DebugTypeVisitor : public RecursiveASTVisitor<DebugTypeVisitor> { + clang::CodeGen::CGDebugInfo &DI; + ASTContext &Ctx; + DebugTypeVisitor(clang::CodeGen::CGDebugInfo &DI, ASTContext &Ctx) + : DI(DI), Ctx(Ctx) {} + + /// Determine whether this type can be represented in DWARF. + static bool CanRepresent(const Type *Ty) { + return !Ty->isDependentType() && !Ty->isUndeducedType(); + } + + bool VisitImportDecl(ImportDecl *D) { + auto *Import = cast<ImportDecl>(D); + if (!Import->getImportedOwningModule()) + DI.EmitImportDecl(*Import); + return true; + } + + bool VisitTypeDecl(TypeDecl *D) { + QualType QualTy = Ctx.getTypeDeclType(D); + if (!QualTy.isNull() && CanRepresent(QualTy.getTypePtr())) + DI.getOrCreateStandaloneType(QualTy, D->getLocation()); + return true; + } + + bool VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) { + QualType QualTy(D->getTypeForDecl(), 0); + if (!QualTy.isNull() && CanRepresent(QualTy.getTypePtr())) + DI.getOrCreateStandaloneType(QualTy, D->getLocation()); + return true; + } + + bool VisitFunctionDecl(FunctionDecl *D) { + if (isa<CXXMethodDecl>(D)) + // This is not yet supported. Constructing the `this' argument + // mandates a CodeGenFunction. + return true; + + SmallVector<QualType, 16> ArgTypes; + for (auto i : D->params()) + ArgTypes.push_back(i->getType()); + QualType RetTy = D->getReturnType(); + QualType FnTy = Ctx.getFunctionType(RetTy, ArgTypes, + FunctionProtoType::ExtProtoInfo()); + if (CanRepresent(FnTy.getTypePtr())) + DI.EmitFunctionDecl(D, D->getLocation(), FnTy); + return true; + } + + bool VisitObjCMethodDecl(ObjCMethodDecl *D) { + if (!D->getClassInterface()) + return true; + + bool selfIsPseudoStrong, selfIsConsumed; + SmallVector<QualType, 16> ArgTypes; + ArgTypes.push_back(D->getSelfType(Ctx, D->getClassInterface(), + selfIsPseudoStrong, selfIsConsumed)); + ArgTypes.push_back(Ctx.getObjCSelType()); + for (auto i : D->params()) + ArgTypes.push_back(i->getType()); + QualType RetTy = D->getReturnType(); + QualType FnTy = Ctx.getFunctionType(RetTy, ArgTypes, + FunctionProtoType::ExtProtoInfo()); + if (CanRepresent(FnTy.getTypePtr())) + DI.EmitFunctionDecl(D, D->getLocation(), FnTy); + return true; + } + }; + public: - PCHContainerGenerator(DiagnosticsEngine &diags, - const HeaderSearchOptions &HSO, - const PreprocessorOptions &PPO, const TargetOptions &TO, - const LangOptions &LO, const std::string &MainFileName, + PCHContainerGenerator(CompilerInstance &CI, const std::string &MainFileName, const std::string &OutputFileName, raw_pwrite_stream *OS, std::shared_ptr<PCHBuffer> Buffer) - : Diags(diags), HeaderSearchOpts(HSO), PreprocessorOpts(PPO), - TargetOpts(TO), LangOpts(LO), OS(OS), Buffer(Buffer) { + : Diags(CI.getDiagnostics()), Ctx(nullptr), + MMap(CI.getPreprocessor().getHeaderSearchInfo().getModuleMap()), + HeaderSearchOpts(CI.getHeaderSearchOpts()), + PreprocessorOpts(CI.getPreprocessorOpts()), + TargetOpts(CI.getTargetOpts()), LangOpts(CI.getLangOpts()), OS(OS), + Buffer(Buffer) { // The debug info output isn't affected by CodeModel and // ThreadModel, but the backend expects them to be nonempty. CodeGenOpts.CodeModel = "default"; CodeGenOpts.ThreadModel = "single"; + CodeGenOpts.DebugTypeExtRefs = true; CodeGenOpts.setDebugInfo(CodeGenOptions::FullDebugInfo); - CodeGenOpts.SplitDwarfFile = OutputFileName; } - virtual ~PCHContainerGenerator() {} + ~PCHContainerGenerator() override = default; void Initialize(ASTContext &Context) override { + assert(!Ctx && "initialized multiple times"); + Ctx = &Context; VMContext.reset(new llvm::LLVMContext()); M.reset(new llvm::Module(MainFileName, *VMContext)); - M->setDataLayout(Ctx->getTargetInfo().getTargetDescription()); - Builder.reset(new CodeGen::CodeGenModule(*Ctx, HeaderSearchOpts, - PreprocessorOpts, CodeGenOpts, *M, - M->getDataLayout(), Diags)); + M->setDataLayout(Ctx->getTargetInfo().getDataLayoutString()); + Builder.reset(new CodeGen::CodeGenModule( + *Ctx, HeaderSearchOpts, PreprocessorOpts, CodeGenOpts, *M, Diags)); + Builder->getModuleDebugInfo()->setModuleMap(MMap); + } + + bool HandleTopLevelDecl(DeclGroupRef D) override { + if (Diags.hasErrorOccurred()) + return true; + + // Collect debug info for all decls in this group. + for (auto *I : D) + if (!I->isFromASTFile()) { + DebugTypeVisitor DTV(*Builder->getModuleDebugInfo(), *Ctx); + DTV.TraverseDecl(I); + } + return true; + } + + void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override { + HandleTopLevelDecl(D); + } + + void HandleTagDeclDefinition(TagDecl *D) override { + if (Diags.hasErrorOccurred()) + return; + + Builder->UpdateCompletedType(D); + } + + void HandleTagDeclRequiredDefinition(const TagDecl *D) override { + if (Diags.hasErrorOccurred()) + return; + + if (const RecordDecl *RD = dyn_cast<RecordDecl>(D)) + Builder->getModuleDebugInfo()->completeRequiredType(RD); } /// Emit a container holding the serialized AST. @@ -92,7 +202,8 @@ public: return; M->setTargetTriple(Ctx.getTargetInfo().getTriple().getTriple()); - M->setDataLayout(Ctx.getTargetInfo().getTargetDescription()); + M->setDataLayout(Ctx.getTargetInfo().getDataLayoutString()); + Builder->getModuleDebugInfo()->setDwoId(Buffer->Signature); // Finalize the Builder. if (Builder) @@ -133,15 +244,14 @@ public: llvm::SmallString<0> Buffer; llvm::raw_svector_ostream OS(Buffer); clang::EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts, - Ctx.getTargetInfo().getTargetDescription(), + Ctx.getTargetInfo().getDataLayoutString(), M.get(), BackendAction::Backend_EmitLL, &OS); - OS.flush(); llvm::dbgs() << Buffer; }); // Use the LLVM backend to emit the pch container. clang::EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts, - Ctx.getTargetInfo().getTargetDescription(), + Ctx.getTargetInfo().getDataLayoutString(), M.get(), BackendAction::Backend_EmitObj, OS); // Make sure the pch container hits disk. @@ -153,17 +263,15 @@ public: } }; -} // namespace +} // anonymous namespace std::unique_ptr<ASTConsumer> ObjectFilePCHContainerWriter::CreatePCHContainerGenerator( - DiagnosticsEngine &Diags, const HeaderSearchOptions &HSO, - const PreprocessorOptions &PPO, const TargetOptions &TO, - const LangOptions &LO, const std::string &MainFileName, + CompilerInstance &CI, const std::string &MainFileName, const std::string &OutputFileName, llvm::raw_pwrite_stream *OS, std::shared_ptr<PCHBuffer> Buffer) const { - return llvm::make_unique<PCHContainerGenerator>( - Diags, HSO, PPO, TO, LO, MainFileName, OutputFileName, OS, Buffer); + return llvm::make_unique<PCHContainerGenerator>(CI, MainFileName, + OutputFileName, OS, Buffer); } void ObjectFilePCHContainerReader::ExtractPCH( @@ -189,5 +297,4 @@ void ObjectFilePCHContainerReader::ExtractPCH( // As a fallback, treat the buffer as a raw AST. StreamFile.init((const unsigned char *)Buffer.getBufferStart(), (const unsigned char *)Buffer.getBufferEnd()); - return; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp index 25bd733..4566fdb 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp @@ -39,7 +39,7 @@ static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder, for (unsigned I = FirstIndex; I <= LastIndex; ++I) { llvm::Value *Cell = Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I); - Builder.CreateStore(Value, Cell); + Builder.CreateAlignedStore(Value, Cell, CharUnits::One()); } } @@ -48,6 +48,24 @@ static bool isAggregateTypeForABI(QualType T) { T->isMemberFunctionPointerType(); } +ABIArgInfo +ABIInfo::getNaturalAlignIndirect(QualType Ty, bool ByRef, bool Realign, + llvm::Type *Padding) const { + return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty), + ByRef, Realign, Padding); +} + +ABIArgInfo +ABIInfo::getNaturalAlignIndirectInReg(QualType Ty, bool Realign) const { + return ABIArgInfo::getIndirectInReg(getContext().getTypeAlignInChars(Ty), + /*ByRef*/ false, Realign); +} + +Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { + return Address::invalid(); +} + ABIInfo::~ABIInfo() {} static CGCXXABI::RecordArgABI getRecordArgABI(const RecordType *RT, @@ -133,7 +151,7 @@ void ABIArgInfo::dump() const { OS << "InAlloca Offset=" << getInAllocaFieldIndex(); break; case Indirect: - OS << "Indirect Align=" << getIndirectAlign() + OS << "Indirect Align=" << getIndirectAlign().getQuantity() << " ByVal=" << getIndirectByVal() << " Realign=" << getIndirectRealign(); break; @@ -144,6 +162,135 @@ void ABIArgInfo::dump() const { OS << ")\n"; } +// Dynamically round a pointer up to a multiple of the given alignment. +static llvm::Value *emitRoundPointerUpToAlignment(CodeGenFunction &CGF, + llvm::Value *Ptr, + CharUnits Align) { + llvm::Value *PtrAsInt = Ptr; + // OverflowArgArea = (OverflowArgArea + Align - 1) & -Align; + PtrAsInt = CGF.Builder.CreatePtrToInt(PtrAsInt, CGF.IntPtrTy); + PtrAsInt = CGF.Builder.CreateAdd(PtrAsInt, + llvm::ConstantInt::get(CGF.IntPtrTy, Align.getQuantity() - 1)); + PtrAsInt = CGF.Builder.CreateAnd(PtrAsInt, + llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity())); + PtrAsInt = CGF.Builder.CreateIntToPtr(PtrAsInt, + Ptr->getType(), + Ptr->getName() + ".aligned"); + return PtrAsInt; +} + +/// Emit va_arg for a platform using the common void* representation, +/// where arguments are simply emitted in an array of slots on the stack. +/// +/// This version implements the core direct-value passing rules. +/// +/// \param SlotSize - The size and alignment of a stack slot. +/// Each argument will be allocated to a multiple of this number of +/// slots, and all the slots will be aligned to this value. +/// \param AllowHigherAlign - The slot alignment is not a cap; +/// an argument type with an alignment greater than the slot size +/// will be emitted on a higher-alignment address, potentially +/// leaving one or more empty slots behind as padding. If this +/// is false, the returned address might be less-aligned than +/// DirectAlign. +static Address emitVoidPtrDirectVAArg(CodeGenFunction &CGF, + Address VAListAddr, + llvm::Type *DirectTy, + CharUnits DirectSize, + CharUnits DirectAlign, + CharUnits SlotSize, + bool AllowHigherAlign) { + // Cast the element type to i8* if necessary. Some platforms define + // va_list as a struct containing an i8* instead of just an i8*. + if (VAListAddr.getElementType() != CGF.Int8PtrTy) + VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy); + + llvm::Value *Ptr = CGF.Builder.CreateLoad(VAListAddr, "argp.cur"); + + // If the CC aligns values higher than the slot size, do so if needed. + Address Addr = Address::invalid(); + if (AllowHigherAlign && DirectAlign > SlotSize) { + Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign), + DirectAlign); + } else { + Addr = Address(Ptr, SlotSize); + } + + // Advance the pointer past the argument, then store that back. + CharUnits FullDirectSize = DirectSize.RoundUpToAlignment(SlotSize); + llvm::Value *NextPtr = + CGF.Builder.CreateConstInBoundsByteGEP(Addr.getPointer(), FullDirectSize, + "argp.next"); + CGF.Builder.CreateStore(NextPtr, VAListAddr); + + // If the argument is smaller than a slot, and this is a big-endian + // target, the argument will be right-adjusted in its slot. + if (DirectSize < SlotSize && CGF.CGM.getDataLayout().isBigEndian()) { + Addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, SlotSize - DirectSize); + } + + Addr = CGF.Builder.CreateElementBitCast(Addr, DirectTy); + return Addr; +} + +/// Emit va_arg for a platform using the common void* representation, +/// where arguments are simply emitted in an array of slots on the stack. +/// +/// \param IsIndirect - Values of this type are passed indirectly. +/// \param ValueInfo - The size and alignment of this type, generally +/// computed with getContext().getTypeInfoInChars(ValueTy). +/// \param SlotSizeAndAlign - The size and alignment of a stack slot. +/// Each argument will be allocated to a multiple of this number of +/// slots, and all the slots will be aligned to this value. +/// \param AllowHigherAlign - The slot alignment is not a cap; +/// an argument type with an alignment greater than the slot size +/// will be emitted on a higher-alignment address, potentially +/// leaving one or more empty slots behind as padding. +static Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType ValueTy, bool IsIndirect, + std::pair<CharUnits, CharUnits> ValueInfo, + CharUnits SlotSizeAndAlign, + bool AllowHigherAlign) { + // The size and alignment of the value that was passed directly. + CharUnits DirectSize, DirectAlign; + if (IsIndirect) { + DirectSize = CGF.getPointerSize(); + DirectAlign = CGF.getPointerAlign(); + } else { + DirectSize = ValueInfo.first; + DirectAlign = ValueInfo.second; + } + + // Cast the address we've calculated to the right type. + llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy); + if (IsIndirect) + DirectTy = DirectTy->getPointerTo(0); + + Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy, + DirectSize, DirectAlign, + SlotSizeAndAlign, + AllowHigherAlign); + + if (IsIndirect) { + Addr = Address(CGF.Builder.CreateLoad(Addr), ValueInfo.second); + } + + return Addr; + +} + +static Address emitMergePHI(CodeGenFunction &CGF, + Address Addr1, llvm::BasicBlock *Block1, + Address Addr2, llvm::BasicBlock *Block2, + const llvm::Twine &Name = "") { + assert(Addr1.getType() == Addr2.getType()); + llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name); + PHI->addIncoming(Addr1.getPointer(), Block1); + PHI->addIncoming(Addr2.getPointer(), Block2); + CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment()); + return Address(PHI, Align); +} + TargetCodeGenInfo::~TargetCodeGenInfo() { delete Info; } // If someone can figure out a general rule for this, that would be great. @@ -394,8 +541,8 @@ public: I.info = classifyArgumentType(I.type); } - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; }; class DefaultTargetCodeGenInfo : public TargetCodeGenInfo { @@ -404,9 +551,9 @@ public: : TargetCodeGenInfo(new DefaultABIInfo(CGT)) {} }; -llvm::Value *DefaultABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - return nullptr; +Address DefaultABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { + return Address::invalid(); } ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const { @@ -416,9 +563,9 @@ ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const { // Records with non-trivial destructors/copy-constructors should not be // passed by value. if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) - return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(Ty); } // Treat an enum type as its underlying type. @@ -434,7 +581,7 @@ ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const { return ABIArgInfo::getIgnore(); if (isAggregateTypeForABI(RetTy)) - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); // Treat an enum type as its underlying type. if (const EnumType *EnumTy = RetTy->getAs<EnumType>()) @@ -445,6 +592,80 @@ ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const { } //===----------------------------------------------------------------------===// +// WebAssembly ABI Implementation +// +// This is a very simple ABI that relies a lot on DefaultABIInfo. +//===----------------------------------------------------------------------===// + +class WebAssemblyABIInfo final : public DefaultABIInfo { +public: + explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT) + : DefaultABIInfo(CGT) {} + +private: + ABIArgInfo classifyReturnType(QualType RetTy) const; + ABIArgInfo classifyArgumentType(QualType Ty) const; + + // DefaultABIInfo's classifyReturnType and classifyArgumentType are + // non-virtual, but computeInfo is virtual, so we overload that. + void computeInfo(CGFunctionInfo &FI) const override { + if (!getCXXABI().classifyReturnType(FI)) + FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); + for (auto &Arg : FI.arguments()) + Arg.info = classifyArgumentType(Arg.type); + } +}; + +class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo { +public: + explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) + : TargetCodeGenInfo(new WebAssemblyABIInfo(CGT)) {} +}; + +/// \brief Classify argument of given type \p Ty. +ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const { + Ty = useFirstFieldIfTransparentUnion(Ty); + + if (isAggregateTypeForABI(Ty)) { + // Records with non-trivial destructors/copy-constructors should not be + // passed by value. + if (auto RAA = getRecordArgABI(Ty, getCXXABI())) + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); + // Ignore empty structs/unions. + if (isEmptyRecord(getContext(), Ty, true)) + return ABIArgInfo::getIgnore(); + // Lower single-element structs to just pass a regular value. TODO: We + // could do reasonable-size multiple-element structs too, using getExpand(), + // though watch out for things like bitfields. + if (const Type *SeltTy = isSingleElementStruct(Ty, getContext())) + return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0))); + } + + // Otherwise just do the default thing. + return DefaultABIInfo::classifyArgumentType(Ty); +} + +ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const { + if (isAggregateTypeForABI(RetTy)) { + // Records with non-trivial destructors/copy-constructors should not be + // returned by value. + if (!getRecordArgABI(RetTy, getCXXABI())) { + // Ignore empty structs/unions. + if (isEmptyRecord(getContext(), RetTy, true)) + return ABIArgInfo::getIgnore(); + // Lower single-element structs to just return a regular value. TODO: We + // could do reasonable-size multiple-element structs too, using + // ABIArgInfo::getDirect(). + if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext())) + return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0))); + } + } + + // Otherwise just do the default thing. + return DefaultABIInfo::classifyReturnType(RetTy); +} + +//===----------------------------------------------------------------------===// // le32/PNaCl bitcode ABI Implementation // // This is a simplified version of the x86_32 ABI. Arguments and return values @@ -459,8 +680,8 @@ class PNaClABIInfo : public ABIInfo { ABIArgInfo classifyArgumentType(QualType RetTy) const; void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, + Address VAListAddr, QualType Ty) const override; }; class PNaClTargetCodeGenInfo : public TargetCodeGenInfo { @@ -477,17 +698,17 @@ void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const { I.info = classifyArgumentType(I.type); } -llvm::Value *PNaClABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - return nullptr; +Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { + return Address::invalid(); } /// \brief Classify argument of given type \p Ty. ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const { if (isAggregateTypeForABI(Ty)) { if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) - return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); + return getNaturalAlignIndirect(Ty); } else if (const EnumType *EnumTy = Ty->getAs<EnumType>()) { // Treat an enum type as its underlying type. Ty = EnumTy->getDecl()->getIntegerType(); @@ -506,7 +727,7 @@ ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const { // In the PNaCl ABI we always return records/structures on the stack. if (isAggregateTypeForABI(RetTy)) - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); // Treat an enum type as its underlying type. if (const EnumType *EnumTy = RetTy->getAs<EnumType>()) @@ -585,8 +806,10 @@ class X86_32ABIInfo : public ABIInfo { static const unsigned MinABIStackAlignInBytes = 4; bool IsDarwinVectorABI; - bool IsSmallStructInRegABI; + bool IsRetSmallStructInRegABI; bool IsWin32StructABI; + bool IsSoftFloatABI; + bool IsMCUABI; unsigned DefaultNumRegisterParameters; static bool isRegisterSize(unsigned Size) { @@ -610,7 +833,7 @@ class X86_32ABIInfo : public ABIInfo { /// such that the argument will be passed in memory. ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const; - ABIArgInfo getIndirectReturnResult(CCState &State) const; + ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const; /// \brief Return the alignment to use for the given type on the stack. unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const; @@ -618,33 +841,47 @@ class X86_32ABIInfo : public ABIInfo { Class classify(QualType Ty) const; ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const; ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const; - bool shouldUseInReg(QualType Ty, CCState &State, bool &NeedsPadding) const; + /// \brief Updates the number of available free registers, returns + /// true if any registers were allocated. + bool updateFreeRegs(QualType Ty, CCState &State) const; + + bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg, + bool &NeedsPadding) const; + bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const; /// \brief Rewrite the function info so that all memory arguments use /// inalloca. void rewriteWithInAlloca(CGFunctionInfo &FI) const; void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields, - unsigned &StackOffset, ABIArgInfo &Info, + CharUnits &StackOffset, ABIArgInfo &Info, QualType Type) const; public: void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; - - X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool d, bool p, bool w, - unsigned r) - : ABIInfo(CGT), IsDarwinVectorABI(d), IsSmallStructInRegABI(p), - IsWin32StructABI(w), DefaultNumRegisterParameters(r) {} + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; + + X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI, + bool RetSmallStructInRegABI, bool Win32StructABI, + unsigned NumRegisterParameters, bool SoftFloatABI) + : ABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI), + IsRetSmallStructInRegABI(RetSmallStructInRegABI), + IsWin32StructABI(Win32StructABI), + IsSoftFloatABI(SoftFloatABI), + IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()), + DefaultNumRegisterParameters(NumRegisterParameters) {} }; class X86_32TargetCodeGenInfo : public TargetCodeGenInfo { public: - X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, - bool d, bool p, bool w, unsigned r) - :TargetCodeGenInfo(new X86_32ABIInfo(CGT, d, p, w, r)) {} + X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI, + bool RetSmallStructInRegABI, bool Win32StructABI, + unsigned NumRegisterParameters, bool SoftFloatABI) + : TargetCodeGenInfo(new X86_32ABIInfo( + CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI, + NumRegisterParameters, SoftFloatABI)) {} static bool isStructReturnInRegABI( const llvm::Triple &Triple, const CodeGenOptions &Opts); @@ -767,14 +1004,15 @@ void X86_32TargetCodeGenInfo::addReturnRegisterOutputs( } /// shouldReturnTypeInRegister - Determine if the given type should be -/// passed in a register (for the Darwin ABI). +/// returned in a register (for the Darwin and MCU ABI). bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const { uint64_t Size = Context.getTypeSize(Ty); - // Type must be register sized. - if (!isRegisterSize(Size)) - return false; + // For i386, type must be register sized. + // For the MCU ABI, it only needs to be <= 8-byte + if ((IsMCUABI && Size > 64) || (!IsMCUABI && !isRegisterSize(Size))) + return false; if (Ty->isVectorType()) { // 64- and 128- bit vectors inside structures are not returned in @@ -816,14 +1054,15 @@ bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty, return true; } -ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(CCState &State) const { +ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const { // If the return value is indirect, then the hidden argument is consuming one // integer register. if (State.FreeRegs) { --State.FreeRegs; - return ABIArgInfo::getIndirectInReg(/*Align=*/0, /*ByVal=*/false); + if (!IsMCUABI) + return getNaturalAlignIndirectInReg(RetTy); } - return ABIArgInfo::getIndirect(/*Align=*/0, /*ByVal=*/false); + return getNaturalAlignIndirect(RetTy, /*ByVal=*/false); } ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy, @@ -858,7 +1097,7 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy, return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size)); - return getIndirectReturnResult(State); + return getIndirectReturnResult(RetTy, State); } return ABIArgInfo::getDirect(); @@ -868,12 +1107,12 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy, if (const RecordType *RT = RetTy->getAs<RecordType>()) { // Structures with flexible arrays are always indirect. if (RT->getDecl()->hasFlexibleArrayMember()) - return getIndirectReturnResult(State); + return getIndirectReturnResult(RetTy, State); } // If specified, structs and unions are always indirect. - if (!IsSmallStructInRegABI && !RetTy->isAnyComplexType()) - return getIndirectReturnResult(State); + if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType()) + return getIndirectReturnResult(RetTy, State); // Small structures which are register sized are generally returned // in a register. @@ -895,7 +1134,7 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy, return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size)); } - return getIndirectReturnResult(State); + return getIndirectReturnResult(RetTy, State); } // Treat an enum type as its underlying type. @@ -961,21 +1200,23 @@ ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal, if (!ByVal) { if (State.FreeRegs) { --State.FreeRegs; // Non-byval indirects just use one pointer. - return ABIArgInfo::getIndirectInReg(0, false); + if (!IsMCUABI) + return getNaturalAlignIndirectInReg(Ty); } - return ABIArgInfo::getIndirect(0, false); + return getNaturalAlignIndirect(Ty, false); } // Compute the byval alignment. unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8; unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign); if (StackAlign == 0) - return ABIArgInfo::getIndirect(4, /*ByVal=*/true); + return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true); // If the stack alignment is less than the type alignment, realign the // argument. bool Realign = TypeAlign > StackAlign; - return ABIArgInfo::getIndirect(StackAlign, /*ByVal=*/true, Realign); + return ABIArgInfo::getIndirect(CharUnits::fromQuantity(StackAlign), + /*ByVal=*/true, Realign); } X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const { @@ -991,12 +1232,12 @@ X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const { return Integer; } -bool X86_32ABIInfo::shouldUseInReg(QualType Ty, CCState &State, - bool &NeedsPadding) const { - NeedsPadding = false; - Class C = classify(Ty); - if (C == Float) - return false; +bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const { + if (!IsSoftFloatABI) { + Class C = classify(Ty); + if (C == Float) + return false; + } unsigned Size = getContext().getTypeSize(Ty); unsigned SizeInRegs = (Size + 31) / 32; @@ -1004,31 +1245,61 @@ bool X86_32ABIInfo::shouldUseInReg(QualType Ty, CCState &State, if (SizeInRegs == 0) return false; - if (SizeInRegs > State.FreeRegs) { - State.FreeRegs = 0; - return false; + if (!IsMCUABI) { + if (SizeInRegs > State.FreeRegs) { + State.FreeRegs = 0; + return false; + } + } else { + // The MCU psABI allows passing parameters in-reg even if there are + // earlier parameters that are passed on the stack. Also, + // it does not allow passing >8-byte structs in-register, + // even if there are 3 free registers available. + if (SizeInRegs > State.FreeRegs || SizeInRegs > 2) + return false; } State.FreeRegs -= SizeInRegs; + return true; +} + +bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State, + bool &InReg, + bool &NeedsPadding) const { + NeedsPadding = false; + InReg = !IsMCUABI; + + if (!updateFreeRegs(Ty, State)) + return false; + + if (IsMCUABI) + return true; if (State.CC == llvm::CallingConv::X86_FastCall || State.CC == llvm::CallingConv::X86_VectorCall) { - if (Size > 32) - return false; - - if (Ty->isIntegralOrEnumerationType()) - return true; + if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs) + NeedsPadding = true; - if (Ty->isPointerType()) - return true; + return false; + } - if (Ty->isReferenceType()) - return true; + return true; +} - if (State.FreeRegs) - NeedsPadding = true; +bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const { + if (!updateFreeRegs(Ty, State)) + return false; + if (IsMCUABI) return false; + + if (State.CC == llvm::CallingConv::X86_FastCall || + State.CC == llvm::CallingConv::X86_VectorCall) { + if (getContext().getTypeSize(Ty) > 32) + return false; + + return (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() || + Ty->isReferenceType()); } return true; @@ -1084,12 +1355,15 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, llvm::LLVMContext &LLVMContext = getVMContext(); llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext); - bool NeedsPadding; - if (shouldUseInReg(Ty, State, NeedsPadding)) { + bool NeedsPadding, InReg; + if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) { unsigned SizeInRegs = (getContext().getTypeSize(Ty) + 31) / 32; SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32); llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements); - return ABIArgInfo::getDirectInReg(Result); + if (InReg) + return ABIArgInfo::getDirectInReg(Result); + else + return ABIArgInfo::getDirect(Result); } llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr; @@ -1097,8 +1371,11 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, // of those arguments will match the struct. This is important because the // LLVM backend isn't smart enough to remove byval, which inhibits many // optimizations. + // Don't do this for the MCU if there are still free integer registers + // (see X86_64 ABI for full explanation). if (getContext().getTypeSize(Ty) <= 4*32 && - canExpandIndirectArgument(Ty, getContext())) + canExpandIndirectArgument(Ty, getContext()) && + (!IsMCUABI || State.FreeRegs == 0)) return ABIArgInfo::getExpandWithPadding( State.CC == llvm::CallingConv::X86_FastCall || State.CC == llvm::CallingConv::X86_VectorCall, @@ -1128,14 +1405,14 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); - bool NeedsPadding; - bool InReg = shouldUseInReg(Ty, State, NeedsPadding); + bool InReg = shouldPrimitiveUseInReg(Ty, State); if (Ty->isPromotableIntegerType()) { if (InReg) return ABIArgInfo::getExtendInReg(); return ABIArgInfo::getExtend(); } + if (InReg) return ABIArgInfo::getDirectInReg(); return ABIArgInfo::getDirect(); @@ -1143,7 +1420,9 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const { CCState State(FI.getCallingConvention()); - if (State.CC == llvm::CallingConv::X86_FastCall) + if (IsMCUABI) + State.FreeRegs = 3; + else if (State.CC == llvm::CallingConv::X86_FastCall) State.FreeRegs = 2; else if (State.CC == llvm::CallingConv::X86_VectorCall) { State.FreeRegs = 2; @@ -1160,7 +1439,8 @@ void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const { // return value was sret and put it in a register ourselves if appropriate. if (State.FreeRegs) { --State.FreeRegs; // The sret parameter consumes a register. - FI.getReturnInfo().setInReg(true); + if (!IsMCUABI) + FI.getReturnInfo().setInReg(true); } } @@ -1182,22 +1462,23 @@ void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const { void X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields, - unsigned &StackOffset, - ABIArgInfo &Info, QualType Type) const { - assert(StackOffset % 4U == 0 && "unaligned inalloca struct"); + CharUnits &StackOffset, ABIArgInfo &Info, + QualType Type) const { + // Arguments are always 4-byte-aligned. + CharUnits FieldAlign = CharUnits::fromQuantity(4); + + assert(StackOffset.isMultipleOf(FieldAlign) && "unaligned inalloca struct"); Info = ABIArgInfo::getInAlloca(FrameFields.size()); FrameFields.push_back(CGT.ConvertTypeForMem(Type)); - StackOffset += getContext().getTypeSizeInChars(Type).getQuantity(); - - // Insert padding bytes to respect alignment. For x86_32, each argument is 4 - // byte aligned. - if (StackOffset % 4U) { - unsigned OldOffset = StackOffset; - StackOffset = llvm::RoundUpToAlignment(StackOffset, 4U); - unsigned NumBytes = StackOffset - OldOffset; - assert(NumBytes); + StackOffset += getContext().getTypeSizeInChars(Type); + + // Insert padding bytes to respect alignment. + CharUnits FieldEnd = StackOffset; + StackOffset = FieldEnd.RoundUpToAlignment(FieldAlign); + if (StackOffset != FieldEnd) { + CharUnits NumBytes = StackOffset - FieldEnd; llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext()); - Ty = llvm::ArrayType::get(Ty, NumBytes); + Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity()); FrameFields.push_back(Ty); } } @@ -1228,7 +1509,10 @@ void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const { // Build a packed struct type for all of the arguments in memory. SmallVector<llvm::Type *, 6> FrameFields; - unsigned StackOffset = 0; + // The stack alignment is always 4. + CharUnits StackAlign = CharUnits::fromQuantity(4); + + CharUnits StackOffset; CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end(); // Put 'this' into the struct before 'sret', if necessary. @@ -1260,47 +1544,25 @@ void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const { } FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields, - /*isPacked=*/true)); + /*isPacked=*/true), + StackAlign); } -llvm::Value *X86_32ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - llvm::Type *BPP = CGF.Int8PtrPtrTy; +Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF, + Address VAListAddr, QualType Ty) const { - CGBuilderTy &Builder = CGF.Builder; - llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, - "ap"); - llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); - - // Compute if the address needs to be aligned - unsigned Align = CGF.getContext().getTypeAlignInChars(Ty).getQuantity(); - Align = getTypeStackAlignInBytes(Ty, Align); - Align = std::max(Align, 4U); - if (Align > 4) { - // addr = (addr + align - 1) & -align; - llvm::Value *Offset = - llvm::ConstantInt::get(CGF.Int32Ty, Align - 1); - Addr = CGF.Builder.CreateGEP(Addr, Offset); - llvm::Value *AsInt = CGF.Builder.CreatePtrToInt(Addr, - CGF.Int32Ty); - llvm::Value *Mask = llvm::ConstantInt::get(CGF.Int32Ty, -Align); - Addr = CGF.Builder.CreateIntToPtr(CGF.Builder.CreateAnd(AsInt, Mask), - Addr->getType(), - "ap.cur.aligned"); - } - - llvm::Type *PTy = - llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); - llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy); - - uint64_t Offset = - llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, Align); - llvm::Value *NextAddr = - Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset), - "ap.next"); - Builder.CreateStore(NextAddr, VAListAddrAsBPP); - - return AddrTyped; + auto TypeInfo = getContext().getTypeInfoInChars(Ty); + + // x86-32 changes the alignment of certain arguments on the stack. + // + // Just messing with TypeInfo like this works because we never pass + // anything indirectly. + TypeInfo.second = CharUnits::fromQuantity( + getTypeStackAlignInBytes(Ty, TypeInfo.second.getQuantity())); + + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, + TypeInfo, CharUnits::fromQuantity(4), + /*AllowHigherAlign*/ true); } bool X86_32TargetCodeGenInfo::isStructReturnInRegABI( @@ -1316,7 +1578,7 @@ bool X86_32TargetCodeGenInfo::isStructReturnInRegABI( return true; } - if (Triple.isOSDarwin()) + if (Triple.isOSDarwin() || Triple.isOSIAMCU()) return true; switch (Triple.getOS()) { @@ -1334,7 +1596,7 @@ bool X86_32TargetCodeGenInfo::isStructReturnInRegABI( void X86_32TargetCodeGenInfo::setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const { - if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) { + if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) { if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) { // Get the LLVM function. llvm::Function *Fn = cast<llvm::Function>(GV); @@ -1372,8 +1634,9 @@ bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable( } else { // 9 is %eflags, which doesn't get a size on Darwin for some // reason. - Builder.CreateStore( - Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9)); + Builder.CreateAlignedStore( + Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9), + CharUnits::One()); // 11-16 are st(0..5). Not sure why we stop at 5. // These have size 12, which is sizeof(long double) on @@ -1542,8 +1805,10 @@ public: void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; + Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; bool has64BitPointers() const { return Has64BitPointers; @@ -1559,8 +1824,8 @@ public: void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; bool isHomogeneousAggregateBaseType(QualType Ty) const override { // FIXME: Assumes vectorcall is in use. @@ -1659,7 +1924,11 @@ public: void getDependentLibraryOption(llvm::StringRef Lib, llvm::SmallString<24> &Opt) const override { Opt = "\01"; - Opt += Lib; + // If the argument contains a space, enclose it in quotes. + if (Lib.find(" ") != StringRef::npos) + Opt += "\"" + Lib.str() + "\""; + else + Opt += Lib; } }; @@ -1679,8 +1948,10 @@ static std::string qualifyWindowsLibrary(llvm::StringRef Lib) { class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo { public: WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, - bool d, bool p, bool w, unsigned RegParms) - : X86_32TargetCodeGenInfo(CGT, d, p, w, RegParms) {} + bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI, + unsigned NumRegisterParameters) + : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI, + Win32StructABI, NumRegisterParameters, false) {} void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const override; @@ -1701,7 +1972,7 @@ public: static void addStackProbeSizeTargetAttribute(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) { - if (isa<FunctionDecl>(D)) { + if (D && isa<FunctionDecl>(D)) { if (CGM.getCodeGenOpts().StackProbeSize != 4096) { llvm::Function *Fn = cast<llvm::Function>(GV); @@ -1918,16 +2189,18 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, if (const VectorType *VT = Ty->getAs<VectorType>()) { uint64_t Size = getContext().getTypeSize(VT); - if (Size == 32) { - // gcc passes all <4 x char>, <2 x short>, <1 x int>, <1 x - // float> as integer. + if (Size == 1 || Size == 8 || Size == 16 || Size == 32) { + // gcc passes the following as integer: + // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float> + // 2 bytes - <2 x char>, <1 x short> + // 1 byte - <1 x char> Current = Integer; // If this type crosses an eightbyte boundary, it should be // split. - uint64_t EB_Real = (OffsetBase) / 64; - uint64_t EB_Imag = (OffsetBase + Size - 1) / 64; - if (EB_Real != EB_Imag) + uint64_t EB_Lo = (OffsetBase) / 64; + uint64_t EB_Hi = (OffsetBase + Size - 1) / 64; + if (EB_Lo != EB_Hi) Hi = Lo; } else if (Size == 64) { // gcc passes <1 x double> in memory. :( @@ -2178,7 +2451,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const { ABIArgInfo::getExtend() : ABIArgInfo::getDirect()); } - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(Ty); } bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const { @@ -2212,7 +2485,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty, } if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) - return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); // Compute the byval alignment. We specify the alignment of the byval in all // cases so that the mid-level optimizer knows the alignment of the byval. @@ -2249,7 +2522,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty, Size)); } - return ABIArgInfo::getIndirect(Align); + return ABIArgInfo::getIndirect(CharUnits::fromQuantity(Align)); } /// The ABI specifies that a value should be passed in a full vector XMM/YMM @@ -2833,11 +3106,10 @@ void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { } } -static llvm::Value *EmitVAArgFromMemory(llvm::Value *VAListAddr, - QualType Ty, - CodeGenFunction &CGF) { - llvm::Value *overflow_arg_area_p = CGF.Builder.CreateStructGEP( - nullptr, VAListAddr, 2, "overflow_arg_area_p"); +static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF, + Address VAListAddr, QualType Ty) { + Address overflow_arg_area_p = CGF.Builder.CreateStructGEP( + VAListAddr, 2, CharUnits::fromQuantity(8), "overflow_arg_area_p"); llvm::Value *overflow_arg_area = CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area"); @@ -2845,19 +3117,10 @@ static llvm::Value *EmitVAArgFromMemory(llvm::Value *VAListAddr, // byte boundary if alignment needed by type exceeds 8 byte boundary. // It isn't stated explicitly in the standard, but in practice we use // alignment greater than 16 where necessary. - uint64_t Align = CGF.getContext().getTypeAlign(Ty) / 8; - if (Align > 8) { - // overflow_arg_area = (overflow_arg_area + align - 1) & -align; - llvm::Value *Offset = - llvm::ConstantInt::get(CGF.Int64Ty, Align - 1); - overflow_arg_area = CGF.Builder.CreateGEP(overflow_arg_area, Offset); - llvm::Value *AsInt = CGF.Builder.CreatePtrToInt(overflow_arg_area, - CGF.Int64Ty); - llvm::Value *Mask = llvm::ConstantInt::get(CGF.Int64Ty, -(uint64_t)Align); - overflow_arg_area = - CGF.Builder.CreateIntToPtr(CGF.Builder.CreateAnd(AsInt, Mask), - overflow_arg_area->getType(), - "overflow_arg_area.align"); + CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty); + if (Align > CharUnits::fromQuantity(8)) { + overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area, + Align); } // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area. @@ -2879,11 +3142,11 @@ static llvm::Value *EmitVAArgFromMemory(llvm::Value *VAListAddr, CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p); // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type. - return Res; + return Address(Res, Align); } -llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { +Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { // Assume that va_list type is correct; should be pointer to LLVM type: // struct { // i32 gp_offset; @@ -2893,14 +3156,14 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, // }; unsigned neededInt, neededSSE; - Ty = CGF.getContext().getCanonicalType(Ty); + Ty = getContext().getCanonicalType(Ty); ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE, /*isNamedArg*/false); // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed // in the registers. If not go to step 7. if (!neededInt && !neededSSE) - return EmitVAArgFromMemory(VAListAddr, Ty, CGF); + return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty); // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of // general purpose registers needed to pass type and num_fp to hold @@ -2914,11 +3177,12 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, // register save space). llvm::Value *InRegs = nullptr; - llvm::Value *gp_offset_p = nullptr, *gp_offset = nullptr; - llvm::Value *fp_offset_p = nullptr, *fp_offset = nullptr; + Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid(); + llvm::Value *gp_offset = nullptr, *fp_offset = nullptr; if (neededInt) { gp_offset_p = - CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 0, "gp_offset_p"); + CGF.Builder.CreateStructGEP(VAListAddr, 0, CharUnits::Zero(), + "gp_offset_p"); gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset"); InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8); InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp"); @@ -2926,7 +3190,8 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, if (neededSSE) { fp_offset_p = - CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 1, "fp_offset_p"); + CGF.Builder.CreateStructGEP(VAListAddr, 1, CharUnits::fromQuantity(4), + "fp_offset_p"); fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset"); llvm::Value *FitsInFP = llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16); @@ -2954,14 +3219,17 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, // simple assembling of a structure from scattered addresses has many more // loads than necessary. Can we clean this up? llvm::Type *LTy = CGF.ConvertTypeForMem(Ty); - llvm::Value *RegAddr = CGF.Builder.CreateLoad( - CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3), "reg_save_area"); + llvm::Value *RegSaveArea = CGF.Builder.CreateLoad( + CGF.Builder.CreateStructGEP(VAListAddr, 3, CharUnits::fromQuantity(16)), + "reg_save_area"); + + Address RegAddr = Address::invalid(); if (neededInt && neededSSE) { // FIXME: Cleanup. assert(AI.isDirect() && "Unexpected ABI info for mixed regs"); llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType()); - llvm::Value *Tmp = CGF.CreateMemTemp(Ty); - Tmp = CGF.Builder.CreateBitCast(Tmp, ST->getPointerTo()); + Address Tmp = CGF.CreateMemTemp(Ty); + Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST); assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs"); llvm::Type *TyLo = ST->getElementType(0); llvm::Type *TyHi = ST->getElementType(1); @@ -2969,57 +3237,77 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, "Unexpected ABI info for mixed regs"); llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo); llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi); - llvm::Value *GPAddr = CGF.Builder.CreateGEP(RegAddr, gp_offset); - llvm::Value *FPAddr = CGF.Builder.CreateGEP(RegAddr, fp_offset); + llvm::Value *GPAddr = CGF.Builder.CreateGEP(RegSaveArea, gp_offset); + llvm::Value *FPAddr = CGF.Builder.CreateGEP(RegSaveArea, fp_offset); llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr; llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr; - llvm::Value *V = - CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegLoAddr, PTyLo)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 0)); - V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegHiAddr, PTyHi)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 1)); - RegAddr = CGF.Builder.CreateBitCast(Tmp, - llvm::PointerType::getUnqual(LTy)); + // Copy the first element. + llvm::Value *V = + CGF.Builder.CreateDefaultAlignedLoad( + CGF.Builder.CreateBitCast(RegLoAddr, PTyLo)); + CGF.Builder.CreateStore(V, + CGF.Builder.CreateStructGEP(Tmp, 0, CharUnits::Zero())); + + // Copy the second element. + V = CGF.Builder.CreateDefaultAlignedLoad( + CGF.Builder.CreateBitCast(RegHiAddr, PTyHi)); + CharUnits Offset = CharUnits::fromQuantity( + getDataLayout().getStructLayout(ST)->getElementOffset(1)); + CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1, Offset)); + + RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy); } else if (neededInt) { - RegAddr = CGF.Builder.CreateGEP(RegAddr, gp_offset); - RegAddr = CGF.Builder.CreateBitCast(RegAddr, - llvm::PointerType::getUnqual(LTy)); + RegAddr = Address(CGF.Builder.CreateGEP(RegSaveArea, gp_offset), + CharUnits::fromQuantity(8)); + RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy); // Copy to a temporary if necessary to ensure the appropriate alignment. std::pair<CharUnits, CharUnits> SizeAlign = - CGF.getContext().getTypeInfoInChars(Ty); + getContext().getTypeInfoInChars(Ty); uint64_t TySize = SizeAlign.first.getQuantity(); - unsigned TyAlign = SizeAlign.second.getQuantity(); - if (TyAlign > 8) { - llvm::Value *Tmp = CGF.CreateMemTemp(Ty); - CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, 8, false); + CharUnits TyAlign = SizeAlign.second; + + // Copy into a temporary if the type is more aligned than the + // register save area. + if (TyAlign.getQuantity() > 8) { + Address Tmp = CGF.CreateMemTemp(Ty); + CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false); RegAddr = Tmp; } + } else if (neededSSE == 1) { - RegAddr = CGF.Builder.CreateGEP(RegAddr, fp_offset); - RegAddr = CGF.Builder.CreateBitCast(RegAddr, - llvm::PointerType::getUnqual(LTy)); + RegAddr = Address(CGF.Builder.CreateGEP(RegSaveArea, fp_offset), + CharUnits::fromQuantity(16)); + RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy); } else { assert(neededSSE == 2 && "Invalid number of needed registers!"); // SSE registers are spaced 16 bytes apart in the register save // area, we need to collect the two eightbytes together. - llvm::Value *RegAddrLo = CGF.Builder.CreateGEP(RegAddr, fp_offset); - llvm::Value *RegAddrHi = CGF.Builder.CreateConstGEP1_32(RegAddrLo, 16); + // The ABI isn't explicit about this, but it seems reasonable + // to assume that the slots are 16-byte aligned, since the stack is + // naturally 16-byte aligned and the prologue is expected to store + // all the SSE registers to the RSA. + Address RegAddrLo = Address(CGF.Builder.CreateGEP(RegSaveArea, fp_offset), + CharUnits::fromQuantity(16)); + Address RegAddrHi = + CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo, + CharUnits::fromQuantity(16)); llvm::Type *DoubleTy = CGF.DoubleTy; - llvm::Type *DblPtrTy = - llvm::PointerType::getUnqual(DoubleTy); llvm::StructType *ST = llvm::StructType::get(DoubleTy, DoubleTy, nullptr); - llvm::Value *V, *Tmp = CGF.CreateMemTemp(Ty); - Tmp = CGF.Builder.CreateBitCast(Tmp, ST->getPointerTo()); - V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrLo, - DblPtrTy)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 0)); - V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrHi, - DblPtrTy)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 1)); - RegAddr = CGF.Builder.CreateBitCast(Tmp, - llvm::PointerType::getUnqual(LTy)); + llvm::Value *V; + Address Tmp = CGF.CreateMemTemp(Ty); + Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST); + V = CGF.Builder.CreateLoad( + CGF.Builder.CreateElementBitCast(RegAddrLo, DoubleTy)); + CGF.Builder.CreateStore(V, + CGF.Builder.CreateStructGEP(Tmp, 0, CharUnits::Zero())); + V = CGF.Builder.CreateLoad( + CGF.Builder.CreateElementBitCast(RegAddrHi, DoubleTy)); + CGF.Builder.CreateStore(V, + CGF.Builder.CreateStructGEP(Tmp, 1, CharUnits::fromQuantity(8))); + + RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy); } // AMD64-ABI 3.5.7p5: Step 5. Set: @@ -3040,18 +3328,24 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, // Emit code to load the value if it was passed in memory. CGF.EmitBlock(InMemBlock); - llvm::Value *MemAddr = EmitVAArgFromMemory(VAListAddr, Ty, CGF); + Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty); // Return the appropriate result. CGF.EmitBlock(ContBlock); - llvm::PHINode *ResAddr = CGF.Builder.CreatePHI(RegAddr->getType(), 2, - "vaarg.addr"); - ResAddr->addIncoming(RegAddr, InRegBlock); - ResAddr->addIncoming(MemAddr, InMemBlock); + Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock, + "vaarg.addr"); return ResAddr; } +Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false, + CGF.getContext().getTypeInfoInChars(Ty), + CharUnits::fromQuantity(8), + /*allowHigherAlign*/ false); +} + ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType) const { @@ -3063,17 +3357,18 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs, TypeInfo Info = getContext().getTypeInfo(Ty); uint64_t Width = Info.Width; - unsigned Align = getContext().toCharUnitsFromBits(Info.Align).getQuantity(); + CharUnits Align = getContext().toCharUnitsFromBits(Info.Align); const RecordType *RT = Ty->getAs<RecordType>(); if (RT) { if (!IsReturnType) { if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI())) - return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); } if (RT->getDecl()->hasFlexibleArrayMember()) - return ABIArgInfo::getIndirect(0, /*ByVal=*/false); + return getNaturalAlignIndirect(Ty, /*ByVal=*/false); + } // vectorcall adds the concept of a homogenous vector aggregate, similar to @@ -3103,7 +3398,7 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs, // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is // not 1, 2, 4, or 8 bytes, must be passed by reference." if (Width > 64 || !llvm::isPowerOf2_64(Width)) - return ABIArgInfo::getIndirect(0, /*ByVal=*/false); + return getNaturalAlignIndirect(Ty, /*ByVal=*/false); // Otherwise, coerce it to a small integer. return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width)); @@ -3141,43 +3436,31 @@ void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { I.info = classify(I.type, FreeSSERegs, false); } -llvm::Value *WinX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - llvm::Type *BPP = CGF.Int8PtrPtrTy; - - CGBuilderTy &Builder = CGF.Builder; - llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, - "ap"); - llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); - llvm::Type *PTy = - llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); - llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy); - - uint64_t Offset = - llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, 8); - llvm::Value *NextAddr = - Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset), - "ap.next"); - Builder.CreateStore(NextAddr, VAListAddrAsBPP); - - return AddrTyped; +Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false, + CGF.getContext().getTypeInfoInChars(Ty), + CharUnits::fromQuantity(8), + /*allowHigherAlign*/ false); } // PowerPC-32 namespace { /// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information. class PPC32_SVR4_ABIInfo : public DefaultABIInfo { +bool IsSoftFloatABI; public: - PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {} + PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI) + : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI) {} - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; }; class PPC32TargetCodeGenInfo : public TargetCodeGenInfo { public: - PPC32TargetCodeGenInfo(CodeGenTypes &CGT) - : TargetCodeGenInfo(new PPC32_SVR4_ABIInfo(CGT)) {} + PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI) + : TargetCodeGenInfo(new PPC32_SVR4_ABIInfo(CGT, SoftFloatABI)) {} int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { // This is recovered from gcc output. @@ -3190,64 +3473,51 @@ public: } -llvm::Value *PPC32_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, - QualType Ty, - CodeGenFunction &CGF) const { +Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, + QualType Ty) const { if (const ComplexType *CTy = Ty->getAs<ComplexType>()) { // TODO: Implement this. For now ignore. (void)CTy; - return nullptr; + return Address::invalid(); } + // struct __va_list_tag { + // unsigned char gpr; + // unsigned char fpr; + // unsigned short reserved; + // void *overflow_arg_area; + // void *reg_save_area; + // }; + bool isI64 = Ty->isIntegerType() && getContext().getTypeSize(Ty) == 64; bool isInt = Ty->isIntegerType() || Ty->isPointerType() || Ty->isAggregateType(); - llvm::Type *CharPtr = CGF.Int8PtrTy; - llvm::Type *CharPtrPtr = CGF.Int8PtrPtrTy; + bool isF64 = Ty->isFloatingType() && getContext().getTypeSize(Ty) == 64; + + // All aggregates are passed indirectly? That doesn't seem consistent + // with the argument-lowering code. + bool isIndirect = Ty->isAggregateType(); CGBuilderTy &Builder = CGF.Builder; - llvm::Value *GPRPtr = Builder.CreateBitCast(VAListAddr, CharPtr, "gprptr"); - llvm::Value *GPRPtrAsInt = Builder.CreatePtrToInt(GPRPtr, CGF.Int32Ty); - llvm::Value *FPRPtrAsInt = - Builder.CreateAdd(GPRPtrAsInt, Builder.getInt32(1)); - llvm::Value *FPRPtr = Builder.CreateIntToPtr(FPRPtrAsInt, CharPtr); - llvm::Value *OverflowAreaPtrAsInt = - Builder.CreateAdd(FPRPtrAsInt, Builder.getInt32(3)); - llvm::Value *OverflowAreaPtr = - Builder.CreateIntToPtr(OverflowAreaPtrAsInt, CharPtrPtr); - llvm::Value *RegsaveAreaPtrAsInt = - Builder.CreateAdd(OverflowAreaPtrAsInt, Builder.getInt32(4)); - llvm::Value *RegsaveAreaPtr = - Builder.CreateIntToPtr(RegsaveAreaPtrAsInt, CharPtrPtr); - llvm::Value *GPR = Builder.CreateLoad(GPRPtr, false, "gpr"); - // Align GPR when TY is i64. - if (isI64) { - llvm::Value *GPRAnd = Builder.CreateAnd(GPR, Builder.getInt8(1)); - llvm::Value *CC64 = Builder.CreateICmpEQ(GPRAnd, Builder.getInt8(1)); - llvm::Value *GPRPlusOne = Builder.CreateAdd(GPR, Builder.getInt8(1)); - GPR = Builder.CreateSelect(CC64, GPRPlusOne, GPR); - } - llvm::Value *FPR = Builder.CreateLoad(FPRPtr, false, "fpr"); - llvm::Value *OverflowArea = - Builder.CreateLoad(OverflowAreaPtr, false, "overflow_area"); - llvm::Value *OverflowAreaAsInt = - Builder.CreatePtrToInt(OverflowArea, CGF.Int32Ty); - llvm::Value *RegsaveArea = - Builder.CreateLoad(RegsaveAreaPtr, false, "regsave_area"); - llvm::Value *RegsaveAreaAsInt = - Builder.CreatePtrToInt(RegsaveArea, CGF.Int32Ty); - llvm::Value *CC = - Builder.CreateICmpULT(isInt ? GPR : FPR, Builder.getInt8(8), "cond"); + // The calling convention either uses 1-2 GPRs or 1 FPR. + Address NumRegsAddr = Address::invalid(); + if (isInt || IsSoftFloatABI) { + NumRegsAddr = Builder.CreateStructGEP(VAList, 0, CharUnits::Zero(), "gpr"); + } else { + NumRegsAddr = Builder.CreateStructGEP(VAList, 1, CharUnits::One(), "fpr"); + } - llvm::Value *RegConstant = - Builder.CreateMul(isInt ? GPR : FPR, Builder.getInt8(isInt ? 4 : 8)); + llvm::Value *NumRegs = Builder.CreateLoad(NumRegsAddr, "numUsedRegs"); - llvm::Value *OurReg = Builder.CreateAdd( - RegsaveAreaAsInt, Builder.CreateSExt(RegConstant, CGF.Int32Ty)); + // "Align" the register count when TY is i64. + if (isI64 || (isF64 && IsSoftFloatABI)) { + NumRegs = Builder.CreateAdd(NumRegs, Builder.getInt8(1)); + NumRegs = Builder.CreateAnd(NumRegs, Builder.getInt8((uint8_t) ~1U)); + } - if (Ty->isFloatingType()) - OurReg = Builder.CreateAdd(OurReg, Builder.getInt32(32)); + llvm::Value *CC = + Builder.CreateICmpULT(NumRegs, Builder.getInt8(8), "cond"); llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs"); llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow"); @@ -3255,39 +3525,91 @@ llvm::Value *PPC32_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, Builder.CreateCondBr(CC, UsingRegs, UsingOverflow); - CGF.EmitBlock(UsingRegs); + llvm::Type *DirectTy = CGF.ConvertType(Ty); + if (isIndirect) DirectTy = DirectTy->getPointerTo(0); - llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); - llvm::Value *Result1 = Builder.CreateIntToPtr(OurReg, PTy); - // Increase the GPR/FPR indexes. - if (isInt) { - GPR = Builder.CreateAdd(GPR, Builder.getInt8(isI64 ? 2 : 1)); - Builder.CreateStore(GPR, GPRPtr); - } else { - FPR = Builder.CreateAdd(FPR, Builder.getInt8(1)); - Builder.CreateStore(FPR, FPRPtr); - } - CGF.EmitBranch(Cont); + // Case 1: consume registers. + Address RegAddr = Address::invalid(); + { + CGF.EmitBlock(UsingRegs); + + Address RegSaveAreaPtr = + Builder.CreateStructGEP(VAList, 4, CharUnits::fromQuantity(8)); + RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr), + CharUnits::fromQuantity(8)); + assert(RegAddr.getElementType() == CGF.Int8Ty); + + // Floating-point registers start after the general-purpose registers. + if (!(isInt || IsSoftFloatABI)) { + RegAddr = Builder.CreateConstInBoundsByteGEP(RegAddr, + CharUnits::fromQuantity(32)); + } + + // Get the address of the saved value by scaling the number of + // registers we've used by the number of + CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8); + llvm::Value *RegOffset = + Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity())); + RegAddr = Address(Builder.CreateInBoundsGEP(CGF.Int8Ty, + RegAddr.getPointer(), RegOffset), + RegAddr.getAlignment().alignmentOfArrayElement(RegSize)); + RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy); + + // Increase the used-register count. + NumRegs = + Builder.CreateAdd(NumRegs, + Builder.getInt8((isI64 || (isF64 && IsSoftFloatABI)) ? 2 : 1)); + Builder.CreateStore(NumRegs, NumRegsAddr); + + CGF.EmitBranch(Cont); + } + + // Case 2: consume space in the overflow area. + Address MemAddr = Address::invalid(); + { + CGF.EmitBlock(UsingOverflow); + + // Everything in the overflow area is rounded up to a size of at least 4. + CharUnits OverflowAreaAlign = CharUnits::fromQuantity(4); + + CharUnits Size; + if (!isIndirect) { + auto TypeInfo = CGF.getContext().getTypeInfoInChars(Ty); + Size = TypeInfo.first.RoundUpToAlignment(OverflowAreaAlign); + } else { + Size = CGF.getPointerSize(); + } - CGF.EmitBlock(UsingOverflow); + Address OverflowAreaAddr = + Builder.CreateStructGEP(VAList, 3, CharUnits::fromQuantity(4)); + Address OverflowArea(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"), + OverflowAreaAlign); + // Round up address of argument to alignment + CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty); + if (Align > OverflowAreaAlign) { + llvm::Value *Ptr = OverflowArea.getPointer(); + OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align), + Align); + } + + MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy); - // Increase the overflow area. - llvm::Value *Result2 = Builder.CreateIntToPtr(OverflowAreaAsInt, PTy); - OverflowAreaAsInt = - Builder.CreateAdd(OverflowAreaAsInt, Builder.getInt32(isInt ? 4 : 8)); - Builder.CreateStore(Builder.CreateIntToPtr(OverflowAreaAsInt, CharPtr), - OverflowAreaPtr); - CGF.EmitBranch(Cont); + // Increase the overflow area. + OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size); + Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr); + CGF.EmitBranch(Cont); + } CGF.EmitBlock(Cont); - llvm::PHINode *Result = CGF.Builder.CreatePHI(PTy, 2, "vaarg.addr"); - Result->addIncoming(Result1, UsingRegs); - Result->addIncoming(Result2, UsingOverflow); + // Merge the cases with a phi. + Address Result = emitMergePHI(CGF, RegAddr, UsingRegs, MemAddr, UsingOverflow, + "vaarg.addr"); - if (Ty->isAggregateType()) { - llvm::Value *AGGPtr = Builder.CreateBitCast(Result, CharPtrPtr, "aggrptr"); - return Builder.CreateLoad(AGGPtr, false, "aggr"); + // Load the pointer if the argument was passed indirectly. + if (isIndirect) { + Result = Address(Builder.CreateLoad(Result, "aggr"), + getContext().getTypeAlignInChars(Ty)); } return Result; @@ -3383,7 +3705,7 @@ public: : DefaultABIInfo(CGT), Kind(Kind), HasQPX(HasQPX) {} bool isPromotableTypeForABI(QualType Ty) const; - bool isAlignedParamType(QualType Ty, bool &Align32) const; + CharUnits getParamTypeAlignment(QualType Ty) const; ABIArgInfo classifyReturnType(QualType RetTy) const; ABIArgInfo classifyArgumentType(QualType Ty) const; @@ -3420,8 +3742,8 @@ public: } } - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; }; class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo { @@ -3481,12 +3803,9 @@ PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const { return false; } -/// isAlignedParamType - Determine whether a type requires 16-byte -/// alignment in the parameter area. -bool -PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty, bool &Align32) const { - Align32 = false; - +/// isAlignedParamType - Determine whether a type requires 16-byte or +/// higher alignment in the parameter area. Always returns at least 8. +CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const { // Complex types are passed just like their elements. if (const ComplexType *CTy = Ty->getAs<ComplexType>()) Ty = CTy->getElementType(); @@ -3495,11 +3814,11 @@ PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty, bool &Align32) const { // passed via reference, smaller types are not aligned). if (IsQPXVectorTy(Ty)) { if (getContext().getTypeSize(Ty) > 128) - Align32 = true; + return CharUnits::fromQuantity(32); - return true; + return CharUnits::fromQuantity(16); } else if (Ty->isVectorType()) { - return getContext().getTypeSize(Ty) == 128; + return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8); } // For single-element float/vector structs, we consider the whole type @@ -3524,22 +3843,22 @@ PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty, bool &Align32) const { // With special case aggregates, only vector base types need alignment. if (AlignAsType && IsQPXVectorTy(AlignAsType)) { if (getContext().getTypeSize(AlignAsType) > 128) - Align32 = true; + return CharUnits::fromQuantity(32); - return true; + return CharUnits::fromQuantity(16); } else if (AlignAsType) { - return AlignAsType->isVectorType(); + return CharUnits::fromQuantity(AlignAsType->isVectorType() ? 16 : 8); } // Otherwise, we only need alignment for any aggregate type that // has an alignment requirement of >= 16 bytes. if (isAggregateTypeForABI(Ty) && getContext().getTypeAlign(Ty) >= 128) { if (HasQPX && getContext().getTypeAlign(Ty) >= 256) - Align32 = true; - return true; + return CharUnits::fromQuantity(32); + return CharUnits::fromQuantity(16); } - return false; + return CharUnits::fromQuantity(8); } /// isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous @@ -3672,7 +3991,7 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const { if (Ty->isVectorType() && !IsQPXVectorTy(Ty)) { uint64_t Size = getContext().getTypeSize(Ty); if (Size > 128) - return ABIArgInfo::getIndirect(0, /*ByVal=*/false); + return getNaturalAlignIndirect(Ty, /*ByVal=*/false); else if (Size < 128) { llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size); return ABIArgInfo::getDirect(CoerceTy); @@ -3681,12 +4000,10 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const { if (isAggregateTypeForABI(Ty)) { if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) - return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); - bool Align32; - uint64_t ABIAlign = isAlignedParamType(Ty, Align32) ? - (Align32 ? 32 : 16) : 8; - uint64_t TyAlign = getContext().getTypeAlign(Ty) / 8; + uint64_t ABIAlign = getParamTypeAlignment(Ty).getQuantity(); + uint64_t TyAlign = getContext().getTypeAlignInChars(Ty).getQuantity(); // ELFv2 homogeneous aggregates are passed as array types. const Type *Base = nullptr; @@ -3724,7 +4041,8 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const { } // All other aggregates are passed ByVal. - return ABIArgInfo::getIndirect(ABIAlign, /*ByVal=*/true, + return ABIArgInfo::getIndirect(CharUnits::fromQuantity(ABIAlign), + /*ByVal=*/true, /*Realign=*/TyAlign > ABIAlign); } @@ -3745,7 +4063,7 @@ PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const { if (RetTy->isVectorType() && !IsQPXVectorTy(RetTy)) { uint64_t Size = getContext().getTypeSize(RetTy); if (Size > 128) - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); else if (Size < 128) { llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size); return ABIArgInfo::getDirect(CoerceTy); @@ -3780,7 +4098,7 @@ PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const { } // All other aggregates are returned indirectly. - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); } return (isPromotableTypeForABI(RetTy) ? @@ -3788,47 +4106,12 @@ PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const { } // Based on ARMABIInfo::EmitVAArg, adjusted for 64-bit machine. -llvm::Value *PPC64_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, - QualType Ty, - CodeGenFunction &CGF) const { - llvm::Type *BP = CGF.Int8PtrTy; - llvm::Type *BPP = CGF.Int8PtrPtrTy; - - CGBuilderTy &Builder = CGF.Builder; - llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap"); - llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); - - // Handle types that require 16-byte alignment in the parameter save area. - bool Align32; - if (isAlignedParamType(Ty, Align32)) { - llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int64Ty); - AddrAsInt = Builder.CreateAdd(AddrAsInt, - Builder.getInt64(Align32 ? 31 : 15)); - AddrAsInt = Builder.CreateAnd(AddrAsInt, - Builder.getInt64(Align32 ? -32 : -16)); - Addr = Builder.CreateIntToPtr(AddrAsInt, BP, "ap.align"); - } - - // Update the va_list pointer. The pointer should be bumped by the - // size of the object. We can trust getTypeSize() except for a complex - // type whose base type is smaller than a doubleword. For these, the - // size of the object is 16 bytes; see below for further explanation. - unsigned SizeInBytes = CGF.getContext().getTypeSize(Ty) / 8; - QualType BaseTy; - unsigned CplxBaseSize = 0; +Address PPC64_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { + auto TypeInfo = getContext().getTypeInfoInChars(Ty); + TypeInfo.second = getParamTypeAlignment(Ty); - if (const ComplexType *CTy = Ty->getAs<ComplexType>()) { - BaseTy = CTy->getElementType(); - CplxBaseSize = CGF.getContext().getTypeSize(BaseTy) / 8; - if (CplxBaseSize < 8) - SizeInBytes = 16; - } - - unsigned Offset = llvm::RoundUpToAlignment(SizeInBytes, 8); - llvm::Value *NextAddr = - Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int64Ty, Offset), - "ap.next"); - Builder.CreateStore(NextAddr, VAListAddrAsBPP); + CharUnits SlotSize = CharUnits::fromQuantity(8); // If we have a complex type and the base type is smaller than 8 bytes, // the ABI calls for the real and imaginary parts to be right-adjusted @@ -3836,44 +4119,40 @@ llvm::Value *PPC64_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, // pointer to a structure with the two parts packed tightly. So generate // loads of the real and imaginary parts relative to the va_list pointer, // and store them to a temporary structure. - if (CplxBaseSize && CplxBaseSize < 8) { - llvm::Value *RealAddr = Builder.CreatePtrToInt(Addr, CGF.Int64Ty); - llvm::Value *ImagAddr = RealAddr; - if (CGF.CGM.getDataLayout().isBigEndian()) { - RealAddr = - Builder.CreateAdd(RealAddr, Builder.getInt64(8 - CplxBaseSize)); - ImagAddr = - Builder.CreateAdd(ImagAddr, Builder.getInt64(16 - CplxBaseSize)); - } else { - ImagAddr = Builder.CreateAdd(ImagAddr, Builder.getInt64(8)); + if (const ComplexType *CTy = Ty->getAs<ComplexType>()) { + CharUnits EltSize = TypeInfo.first / 2; + if (EltSize < SlotSize) { + Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, CGF.Int8Ty, + SlotSize * 2, SlotSize, + SlotSize, /*AllowHigher*/ true); + + Address RealAddr = Addr; + Address ImagAddr = RealAddr; + if (CGF.CGM.getDataLayout().isBigEndian()) { + RealAddr = CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, + SlotSize - EltSize); + ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(ImagAddr, + 2 * SlotSize - EltSize); + } else { + ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize); + } + + llvm::Type *EltTy = CGF.ConvertTypeForMem(CTy->getElementType()); + RealAddr = CGF.Builder.CreateElementBitCast(RealAddr, EltTy); + ImagAddr = CGF.Builder.CreateElementBitCast(ImagAddr, EltTy); + llvm::Value *Real = CGF.Builder.CreateLoad(RealAddr, ".vareal"); + llvm::Value *Imag = CGF.Builder.CreateLoad(ImagAddr, ".vaimag"); + + Address Temp = CGF.CreateMemTemp(Ty, "vacplx"); + CGF.EmitStoreOfComplex({Real, Imag}, CGF.MakeAddrLValue(Temp, Ty), + /*init*/ true); + return Temp; } - llvm::Type *PBaseTy = llvm::PointerType::getUnqual(CGF.ConvertType(BaseTy)); - RealAddr = Builder.CreateIntToPtr(RealAddr, PBaseTy); - ImagAddr = Builder.CreateIntToPtr(ImagAddr, PBaseTy); - llvm::Value *Real = Builder.CreateLoad(RealAddr, false, ".vareal"); - llvm::Value *Imag = Builder.CreateLoad(ImagAddr, false, ".vaimag"); - llvm::AllocaInst *Ptr = - CGF.CreateTempAlloca(CGT.ConvertTypeForMem(Ty), "vacplx"); - llvm::Value *RealPtr = - Builder.CreateStructGEP(Ptr->getAllocatedType(), Ptr, 0, ".real"); - llvm::Value *ImagPtr = - Builder.CreateStructGEP(Ptr->getAllocatedType(), Ptr, 1, ".imag"); - Builder.CreateStore(Real, RealPtr, false); - Builder.CreateStore(Imag, ImagPtr, false); - return Ptr; - } - - // If the argument is smaller than 8 bytes, it is right-adjusted in - // its doubleword slot. Adjust the pointer to pick it up from the - // correct offset. - if (SizeInBytes < 8 && CGF.CGM.getDataLayout().isBigEndian()) { - llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int64Ty); - AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt64(8 - SizeInBytes)); - Addr = Builder.CreateIntToPtr(AddrAsInt, BP); - } - - llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); - return Builder.CreateBitCast(Addr, PTy); + } + + // Otherwise, just use the general rule. + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, + TypeInfo, SlotSize, /*AllowHigher*/ true); } static bool @@ -3971,14 +4250,14 @@ private: it.info = classifyArgumentType(it.type); } - llvm::Value *EmitDarwinVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const; + Address EmitDarwinVAArg(Address VAListAddr, QualType Ty, + CodeGenFunction &CGF) const; - llvm::Value *EmitAAPCSVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const; + Address EmitAAPCSVAArg(Address VAListAddr, QualType Ty, + CodeGenFunction &CGF) const; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override { + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override { return isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF) : EmitAAPCSVAArg(VAListAddr, Ty, CGF); } @@ -4021,7 +4300,7 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty) const { llvm::VectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4); return ABIArgInfo::getDirect(ResType); } - return ABIArgInfo::getIndirect(0, /*ByVal=*/false); + return getNaturalAlignIndirect(Ty, /*ByVal=*/false); } if (!isAggregateTypeForABI(Ty)) { @@ -4037,8 +4316,8 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty) const { // Structures with either a non-trivial destructor or a non-trivial // copy constructor are always indirect. if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) { - return ABIArgInfo::getIndirect(0, /*ByVal=*/RAA == - CGCXXABI::RAA_DirectInMemory); + return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA == + CGCXXABI::RAA_DirectInMemory); } // Empty records are always ignored on Darwin, but actually passed in C++ mode @@ -4073,7 +4352,7 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty) const { return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size)); } - return ABIArgInfo::getIndirect(0, /*ByVal=*/false); + return getNaturalAlignIndirect(Ty, /*ByVal=*/false); } ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy) const { @@ -4082,7 +4361,7 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy) const { // Large vector types should be returned via memory. if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128) - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); if (!isAggregateTypeForABI(RetTy)) { // Treat an enum type as its underlying type. @@ -4118,7 +4397,7 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy) const { return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size)); } - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); } /// isIllegalVectorType - check whether the vector type is legal for AArch64. @@ -4156,7 +4435,7 @@ bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base, return Members <= 4; } -llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, +Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF) const { ABIArgInfo AI = classifyArgumentType(Ty); @@ -4190,24 +4469,32 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg"); llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack"); llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end"); - auto &Ctx = CGF.getContext(); - llvm::Value *reg_offs_p = nullptr, *reg_offs = nullptr; + auto TyInfo = getContext().getTypeInfoInChars(Ty); + CharUnits TyAlign = TyInfo.second; + + Address reg_offs_p = Address::invalid(); + llvm::Value *reg_offs = nullptr; int reg_top_index; - int RegSize = IsIndirect ? 8 : getContext().getTypeSize(Ty) / 8; + CharUnits reg_top_offset; + int RegSize = IsIndirect ? 8 : TyInfo.first.getQuantity(); if (!IsFPR) { // 3 is the field number of __gr_offs reg_offs_p = - CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3, "gr_offs_p"); + CGF.Builder.CreateStructGEP(VAListAddr, 3, CharUnits::fromQuantity(24), + "gr_offs_p"); reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs"); reg_top_index = 1; // field number for __gr_top + reg_top_offset = CharUnits::fromQuantity(8); RegSize = llvm::RoundUpToAlignment(RegSize, 8); } else { // 4 is the field number of __vr_offs. reg_offs_p = - CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 4, "vr_offs_p"); + CGF.Builder.CreateStructGEP(VAListAddr, 4, CharUnits::fromQuantity(28), + "vr_offs_p"); reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs"); reg_top_index = 2; // field number for __vr_top + reg_top_offset = CharUnits::fromQuantity(16); RegSize = 16 * NumRegs; } @@ -4232,8 +4519,8 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, // Integer arguments may need to correct register alignment (for example a // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we // align __gr_offs to calculate the potential address. - if (!IsFPR && !IsIndirect && Ctx.getTypeAlign(Ty) > 64) { - int Align = Ctx.getTypeAlign(Ty) / 8; + if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) { + int Align = TyAlign.getQuantity(); reg_offs = CGF.Builder.CreateAdd( reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, Align - 1), @@ -4244,6 +4531,9 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, } // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list. + // The fact that this is done unconditionally reflects the fact that + // allocating an argument to the stack also uses up all the remaining + // registers of the appropriate kind. llvm::Value *NewOffset = nullptr; NewOffset = CGF.Builder.CreateAdd( reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, RegSize), "new_reg_offs"); @@ -4265,13 +4555,14 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, // registers. First start the appropriate block: CGF.EmitBlock(InRegBlock); - llvm::Value *reg_top_p = nullptr, *reg_top = nullptr; - reg_top_p = CGF.Builder.CreateStructGEP(nullptr, VAListAddr, reg_top_index, - "reg_top_p"); + llvm::Value *reg_top = nullptr; + Address reg_top_p = CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index, + reg_top_offset, "reg_top_p"); reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top"); - llvm::Value *BaseAddr = CGF.Builder.CreateGEP(reg_top, reg_offs); - llvm::Value *RegAddr = nullptr; - llvm::Type *MemTy = llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty)); + Address BaseAddr(CGF.Builder.CreateInBoundsGEP(reg_top, reg_offs), + CharUnits::fromQuantity(IsFPR ? 16 : 8)); + Address RegAddr = Address::invalid(); + llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty); if (IsIndirect) { // If it's been passed indirectly (actually a struct), whatever we find from @@ -4288,43 +4579,45 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, // qN+1, ...). We reload and store into a temporary local variable // contiguously. assert(!IsIndirect && "Homogeneous aggregates should be passed directly"); + auto BaseTyInfo = getContext().getTypeInfoInChars(QualType(Base, 0)); llvm::Type *BaseTy = CGF.ConvertType(QualType(Base, 0)); llvm::Type *HFATy = llvm::ArrayType::get(BaseTy, NumMembers); - llvm::AllocaInst *Tmp = CGF.CreateTempAlloca(HFATy); + Address Tmp = CGF.CreateTempAlloca(HFATy, + std::max(TyAlign, BaseTyInfo.second)); + + // On big-endian platforms, the value will be right-aligned in its slot. int Offset = 0; + if (CGF.CGM.getDataLayout().isBigEndian() && + BaseTyInfo.first.getQuantity() < 16) + Offset = 16 - BaseTyInfo.first.getQuantity(); - if (CGF.CGM.getDataLayout().isBigEndian() && Ctx.getTypeSize(Base) < 128) - Offset = 16 - Ctx.getTypeSize(Base) / 8; for (unsigned i = 0; i < NumMembers; ++i) { - llvm::Value *BaseOffset = - llvm::ConstantInt::get(CGF.Int32Ty, 16 * i + Offset); - llvm::Value *LoadAddr = CGF.Builder.CreateGEP(BaseAddr, BaseOffset); - LoadAddr = CGF.Builder.CreateBitCast( - LoadAddr, llvm::PointerType::getUnqual(BaseTy)); - llvm::Value *StoreAddr = - CGF.Builder.CreateStructGEP(Tmp->getAllocatedType(), Tmp, i); + CharUnits BaseOffset = CharUnits::fromQuantity(16 * i + Offset); + Address LoadAddr = + CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, BaseOffset); + LoadAddr = CGF.Builder.CreateElementBitCast(LoadAddr, BaseTy); + + Address StoreAddr = + CGF.Builder.CreateConstArrayGEP(Tmp, i, BaseTyInfo.first); llvm::Value *Elem = CGF.Builder.CreateLoad(LoadAddr); CGF.Builder.CreateStore(Elem, StoreAddr); } - RegAddr = CGF.Builder.CreateBitCast(Tmp, MemTy); + RegAddr = CGF.Builder.CreateElementBitCast(Tmp, MemTy); } else { - // Otherwise the object is contiguous in memory - unsigned BeAlign = reg_top_index == 2 ? 16 : 8; - if (CGF.CGM.getDataLayout().isBigEndian() && - (IsHFA || !isAggregateTypeForABI(Ty)) && - Ctx.getTypeSize(Ty) < (BeAlign * 8)) { - int Offset = BeAlign - Ctx.getTypeSize(Ty) / 8; - BaseAddr = CGF.Builder.CreatePtrToInt(BaseAddr, CGF.Int64Ty); + // Otherwise the object is contiguous in memory. - BaseAddr = CGF.Builder.CreateAdd( - BaseAddr, llvm::ConstantInt::get(CGF.Int64Ty, Offset), "align_be"); - - BaseAddr = CGF.Builder.CreateIntToPtr(BaseAddr, CGF.Int8PtrTy); + // It might be right-aligned in its slot. + CharUnits SlotSize = BaseAddr.getAlignment(); + if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect && + (IsHFA || !isAggregateTypeForABI(Ty)) && + TyInfo.first < SlotSize) { + CharUnits Offset = SlotSize - TyInfo.first; + BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, Offset); } - RegAddr = CGF.Builder.CreateBitCast(BaseAddr, MemTy); + RegAddr = CGF.Builder.CreateElementBitCast(BaseAddr, MemTy); } CGF.EmitBranch(ContBlock); @@ -4334,55 +4627,51 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, //======================================= CGF.EmitBlock(OnStackBlock); - llvm::Value *stack_p = nullptr, *OnStackAddr = nullptr; - stack_p = CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 0, "stack_p"); - OnStackAddr = CGF.Builder.CreateLoad(stack_p, "stack"); + Address stack_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, + CharUnits::Zero(), "stack_p"); + llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(stack_p, "stack"); - // Again, stack arguments may need realigmnent. In this case both integer and + // Again, stack arguments may need realignment. In this case both integer and // floating-point ones might be affected. - if (!IsIndirect && Ctx.getTypeAlign(Ty) > 64) { - int Align = Ctx.getTypeAlign(Ty) / 8; + if (!IsIndirect && TyAlign.getQuantity() > 8) { + int Align = TyAlign.getQuantity(); - OnStackAddr = CGF.Builder.CreatePtrToInt(OnStackAddr, CGF.Int64Ty); + OnStackPtr = CGF.Builder.CreatePtrToInt(OnStackPtr, CGF.Int64Ty); - OnStackAddr = CGF.Builder.CreateAdd( - OnStackAddr, llvm::ConstantInt::get(CGF.Int64Ty, Align - 1), + OnStackPtr = CGF.Builder.CreateAdd( + OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, Align - 1), "align_stack"); - OnStackAddr = CGF.Builder.CreateAnd( - OnStackAddr, llvm::ConstantInt::get(CGF.Int64Ty, -Align), + OnStackPtr = CGF.Builder.CreateAnd( + OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, -Align), "align_stack"); - OnStackAddr = CGF.Builder.CreateIntToPtr(OnStackAddr, CGF.Int8PtrTy); + OnStackPtr = CGF.Builder.CreateIntToPtr(OnStackPtr, CGF.Int8PtrTy); } + Address OnStackAddr(OnStackPtr, + std::max(CharUnits::fromQuantity(8), TyAlign)); - uint64_t StackSize; + // All stack slots are multiples of 8 bytes. + CharUnits StackSlotSize = CharUnits::fromQuantity(8); + CharUnits StackSize; if (IsIndirect) - StackSize = 8; + StackSize = StackSlotSize; else - StackSize = Ctx.getTypeSize(Ty) / 8; - - // All stack slots are 8 bytes - StackSize = llvm::RoundUpToAlignment(StackSize, 8); + StackSize = TyInfo.first.RoundUpToAlignment(StackSlotSize); - llvm::Value *StackSizeC = llvm::ConstantInt::get(CGF.Int32Ty, StackSize); + llvm::Value *StackSizeC = CGF.Builder.getSize(StackSize); llvm::Value *NewStack = - CGF.Builder.CreateGEP(OnStackAddr, StackSizeC, "new_stack"); + CGF.Builder.CreateInBoundsGEP(OnStackPtr, StackSizeC, "new_stack"); // Write the new value of __stack for the next call to va_arg CGF.Builder.CreateStore(NewStack, stack_p); if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(Ty) && - Ctx.getTypeSize(Ty) < 64) { - int Offset = 8 - Ctx.getTypeSize(Ty) / 8; - OnStackAddr = CGF.Builder.CreatePtrToInt(OnStackAddr, CGF.Int64Ty); - - OnStackAddr = CGF.Builder.CreateAdd( - OnStackAddr, llvm::ConstantInt::get(CGF.Int64Ty, Offset), "align_be"); - - OnStackAddr = CGF.Builder.CreateIntToPtr(OnStackAddr, CGF.Int8PtrTy); + TyInfo.first < StackSlotSize) { + CharUnits Offset = StackSlotSize - TyInfo.first; + OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(OnStackAddr, Offset); } - OnStackAddr = CGF.Builder.CreateBitCast(OnStackAddr, MemTy); + OnStackAddr = CGF.Builder.CreateElementBitCast(OnStackAddr, MemTy); CGF.EmitBranch(ContBlock); @@ -4391,75 +4680,48 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, //======================================= CGF.EmitBlock(ContBlock); - llvm::PHINode *ResAddr = CGF.Builder.CreatePHI(MemTy, 2, "vaarg.addr"); - ResAddr->addIncoming(RegAddr, InRegBlock); - ResAddr->addIncoming(OnStackAddr, OnStackBlock); + Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, + OnStackAddr, OnStackBlock, "vaargs.addr"); if (IsIndirect) - return CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"); + return Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"), + TyInfo.second); return ResAddr; } -llvm::Value *AArch64ABIInfo::EmitDarwinVAArg(llvm::Value *VAListAddr, - QualType Ty, - CodeGenFunction &CGF) const { - // We do not support va_arg for aggregates or illegal vector types. - // Lower VAArg here for these cases and use the LLVM va_arg instruction for - // other cases. +Address AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty, + CodeGenFunction &CGF) const { + // The backend's lowering doesn't support va_arg for aggregates or + // illegal vector types. Lower VAArg here for these cases and use + // the LLVM va_arg instruction for everything else. if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty)) - return nullptr; - - uint64_t Size = CGF.getContext().getTypeSize(Ty) / 8; - uint64_t Align = CGF.getContext().getTypeAlign(Ty) / 8; - - const Type *Base = nullptr; - uint64_t Members = 0; - bool isHA = isHomogeneousAggregate(Ty, Base, Members); - - bool isIndirect = false; - // Arguments bigger than 16 bytes which aren't homogeneous aggregates should - // be passed indirectly. - if (Size > 16 && !isHA) { - isIndirect = true; - Size = 8; - Align = 8; - } + return Address::invalid(); - llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext()); - llvm::Type *BPP = llvm::PointerType::getUnqual(BP); - - CGBuilderTy &Builder = CGF.Builder; - llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap"); - llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); + CharUnits SlotSize = CharUnits::fromQuantity(8); + // Empty records are ignored for parameter passing purposes. if (isEmptyRecord(getContext(), Ty, true)) { - // These are ignored for parameter passing purposes. - llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); - return Builder.CreateBitCast(Addr, PTy); + Address Addr(CGF.Builder.CreateLoad(VAListAddr, "ap.cur"), SlotSize); + Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty)); + return Addr; } - const uint64_t MinABIAlign = 8; - if (Align > MinABIAlign) { - llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, Align - 1); - Addr = Builder.CreateGEP(Addr, Offset); - llvm::Value *AsInt = Builder.CreatePtrToInt(Addr, CGF.Int64Ty); - llvm::Value *Mask = llvm::ConstantInt::get(CGF.Int64Ty, ~(Align - 1)); - llvm::Value *Aligned = Builder.CreateAnd(AsInt, Mask); - Addr = Builder.CreateIntToPtr(Aligned, BP, "ap.align"); - } - - uint64_t Offset = llvm::RoundUpToAlignment(Size, MinABIAlign); - llvm::Value *NextAddr = Builder.CreateGEP( - Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset), "ap.next"); - Builder.CreateStore(NextAddr, VAListAddrAsBPP); + // The size of the actual thing passed, which might end up just + // being a pointer for indirect types. + auto TyInfo = getContext().getTypeInfoInChars(Ty); - if (isIndirect) - Addr = Builder.CreateLoad(Builder.CreateBitCast(Addr, BPP)); - llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); - llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy); + // Arguments bigger than 16 bytes which aren't homogeneous + // aggregates should be passed indirectly. + bool IsIndirect = false; + if (TyInfo.first.getQuantity() > 16) { + const Type *Base = nullptr; + uint64_t Members = 0; + IsIndirect = !isHomogeneousAggregate(Ty, Base, Members); + } - return AddrTyped; + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, + TyInfo, SlotSize, /*AllowHigherAlign*/ true); } //===----------------------------------------------------------------------===// @@ -4473,7 +4735,8 @@ public: enum ABIKind { APCS = 0, AAPCS = 1, - AAPCS_VFP + AAPCS_VFP = 2, + AAPCS16_VFP = 3, }; private: @@ -4507,6 +4770,11 @@ public: } } + bool isAndroid() const { + return (getTarget().getTriple().getEnvironment() == + llvm::Triple::Android); + } + ABIKind getABIKind() const { return Kind; } private: @@ -4520,8 +4788,8 @@ private: void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; llvm::CallingConv::ID getLLVMDefaultCC() const; llvm::CallingConv::ID getABIDefaultCC() const; @@ -4561,7 +4829,7 @@ public: void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const override { - const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); + const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D); if (!FD) return; @@ -4583,7 +4851,8 @@ public: Fn->addFnAttr("interrupt", Kind); - if (cast<ARMABIInfo>(getABIInfo()).getABIKind() == ARMABIInfo::APCS) + ARMABIInfo::ABIKind ABI = cast<ARMABIInfo>(getABIInfo()).getABIKind(); + if (ABI == ARMABIInfo::APCS) return; // AAPCS guarantees that sp will be 8-byte aligned on any public interface, @@ -4649,7 +4918,7 @@ void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const { /// Return the default calling convention that LLVM will use. llvm::CallingConv::ID ARMABIInfo::getLLVMDefaultCC() const { // The default calling convention that LLVM will infer. - if (isEABIHF()) + if (isEABIHF() || getTarget().getTriple().isWatchOS()) return llvm::CallingConv::ARM_AAPCS_VFP; else if (isEABI()) return llvm::CallingConv::ARM_AAPCS; @@ -4664,6 +4933,7 @@ llvm::CallingConv::ID ARMABIInfo::getABIDefaultCC() const { case APCS: return llvm::CallingConv::ARM_APCS; case AAPCS: return llvm::CallingConv::ARM_AAPCS; case AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP; + case AAPCS16_VFP: return llvm::CallingConv::ARM_AAPCS_VFP; } llvm_unreachable("bad ABI kind"); } @@ -4677,8 +4947,20 @@ void ARMABIInfo::setCCs() { if (abiCC != getLLVMDefaultCC()) RuntimeCC = abiCC; - BuiltinCC = (getABIKind() == APCS ? - llvm::CallingConv::ARM_APCS : llvm::CallingConv::ARM_AAPCS); + // AAPCS apparently requires runtime support functions to be soft-float, but + // that's almost certainly for historic reasons (Thumb1 not supporting VFP + // most likely). It's more convenient for AAPCS16_VFP to be hard-float. + switch (getABIKind()) { + case APCS: + case AAPCS16_VFP: + if (abiCC != getLLVMDefaultCC()) + BuiltinCC = abiCC; + break; + case AAPCS: + case AAPCS_VFP: + BuiltinCC = llvm::CallingConv::ARM_AAPCS; + break; + } } ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, @@ -4712,7 +4994,17 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, llvm::Type::getInt32Ty(getVMContext()), 4); return ABIArgInfo::getDirect(ResType); } - return ABIArgInfo::getIndirect(0, /*ByVal=*/false); + return getNaturalAlignIndirect(Ty, /*ByVal=*/false); + } + + // __fp16 gets passed as if it were an int or float, but with the top 16 bits + // unspecified. This is not done for OpenCL as it handles the half type + // natively, and does not need to interwork with AAPCS code. + if (Ty->isHalfType() && !getContext().getLangOpts().OpenCL) { + llvm::Type *ResType = IsEffectivelyAAPCS_VFP ? + llvm::Type::getFloatTy(getVMContext()) : + llvm::Type::getInt32Ty(getVMContext()); + return ABIArgInfo::getDirect(ResType); } if (!isAggregateTypeForABI(Ty)) { @@ -4726,7 +5018,7 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, } if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) { - return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); } // Ignore empty records. @@ -4743,6 +5035,27 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, // Base can be a floating-point or a vector. return ABIArgInfo::getDirect(nullptr, 0, nullptr, false); } + } else if (getABIKind() == ARMABIInfo::AAPCS16_VFP) { + // WatchOS does have homogeneous aggregates. Note that we intentionally use + // this convention even for a variadic function: the backend will use GPRs + // if needed. + const Type *Base = nullptr; + uint64_t Members = 0; + if (isHomogeneousAggregate(Ty, Base, Members)) { + assert(Base && Members <= 4 && "unexpected homogeneous aggregate"); + llvm::Type *Ty = + llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members); + return ABIArgInfo::getDirect(Ty, 0, nullptr, false); + } + } + + if (getABIKind() == ARMABIInfo::AAPCS16_VFP && + getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(16)) { + // WatchOS is adopting the 64-bit AAPCS rule on composite types: if they're + // bigger than 128-bits, they get placed in space allocated by the caller, + // and a pointer is passed. + return ABIArgInfo::getIndirect( + CharUnits::fromQuantity(getContext().getTypeAlign(Ty) / 8), false); } // Support byval for ARM. @@ -4756,8 +5069,10 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, ABIAlign = std::min(std::max(TyAlign, (uint64_t)4), (uint64_t)8); if (getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(64)) { - return ABIArgInfo::getIndirect(ABIAlign, /*ByVal=*/true, - /*Realign=*/TyAlign > ABIAlign); + assert(getABIKind() != ARMABIInfo::AAPCS16_VFP && "unexpected byval"); + return ABIArgInfo::getIndirect(CharUnits::fromQuantity(ABIAlign), + /*ByVal=*/true, + /*Realign=*/TyAlign > ABIAlign); } // Otherwise, pass by coercing to a structure of the appropriate size. @@ -4863,14 +5178,25 @@ static bool isIntegerLikeType(QualType Ty, ASTContext &Context, ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic) const { - bool IsEffectivelyAAPCS_VFP = getABIKind() == AAPCS_VFP && !isVariadic; + bool IsEffectivelyAAPCS_VFP = + (getABIKind() == AAPCS_VFP || getABIKind() == AAPCS16_VFP) && !isVariadic; if (RetTy->isVoidType()) return ABIArgInfo::getIgnore(); // Large vector types should be returned via memory. if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128) { - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); + } + + // __fp16 gets returned as if it were an int or float, but with the top 16 + // bits unspecified. This is not done for OpenCL as it handles the half type + // natively, and does not need to interwork with AAPCS code. + if (RetTy->isHalfType() && !getContext().getLangOpts().OpenCL) { + llvm::Type *ResType = IsEffectivelyAAPCS_VFP ? + llvm::Type::getFloatTy(getVMContext()) : + llvm::Type::getInt32Ty(getVMContext()); + return ABIArgInfo::getDirect(ResType); } if (!isAggregateTypeForABI(RetTy)) { @@ -4907,7 +5233,7 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, } // Otherwise return in memory. - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); } // Otherwise this is an AAPCS variant. @@ -4918,7 +5244,7 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, // Check for homogeneous aggregates with AAPCS-VFP. if (IsEffectivelyAAPCS_VFP) { const Type *Base = nullptr; - uint64_t Members; + uint64_t Members = 0; if (isHomogeneousAggregate(RetTy, Base, Members)) { assert(Base && "Base class should be set for homogeneous aggregate"); // Homogeneous Aggregates are returned directly. @@ -4940,22 +5266,39 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, if (Size <= 16) return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext())); return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext())); + } else if (Size <= 128 && getABIKind() == AAPCS16_VFP) { + llvm::Type *Int32Ty = llvm::Type::getInt32Ty(getVMContext()); + llvm::Type *CoerceTy = + llvm::ArrayType::get(Int32Ty, llvm::RoundUpToAlignment(Size, 32) / 32); + return ABIArgInfo::getDirect(CoerceTy); } - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); } /// isIllegalVector - check whether Ty is an illegal vector type. bool ARMABIInfo::isIllegalVectorType(QualType Ty) const { - if (const VectorType *VT = Ty->getAs<VectorType>()) { - // Check whether VT is legal. - unsigned NumElements = VT->getNumElements(); - uint64_t Size = getContext().getTypeSize(VT); - // NumElements should be power of 2. - if ((NumElements & (NumElements - 1)) != 0) - return true; - // Size should be greater than 32 bits. - return Size <= 32; + if (const VectorType *VT = Ty->getAs<VectorType> ()) { + if (isAndroid()) { + // Android shipped using Clang 3.1, which supported a slightly different + // vector ABI. The primary differences were that 3-element vector types + // were legal, and so were sub 32-bit vectors (i.e. <2 x i8>). This path + // accepts that legacy behavior for Android only. + // Check whether VT is legal. + unsigned NumElements = VT->getNumElements(); + // NumElements should be power of 2 or equal to 3. + if (!llvm::isPowerOf2_32(NumElements) && NumElements != 3) + return true; + } else { + // Check whether VT is legal. + unsigned NumElements = VT->getNumElements(); + uint64_t Size = getContext().getTypeSize(VT); + // NumElements should be power of 2. + if (!llvm::isPowerOf2_32(NumElements)) + return true; + // Size should be greater than 32 bits. + return Size <= 32; + } } return false; } @@ -4981,80 +5324,53 @@ bool ARMABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base, return Members <= 4; } -llvm::Value *ARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - llvm::Type *BP = CGF.Int8PtrTy; - llvm::Type *BPP = CGF.Int8PtrPtrTy; - - CGBuilderTy &Builder = CGF.Builder; - llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap"); - llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); +Address ARMABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { + CharUnits SlotSize = CharUnits::fromQuantity(4); + // Empty records are ignored for parameter passing purposes. if (isEmptyRecord(getContext(), Ty, true)) { - // These are ignored for parameter passing purposes. - llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); - return Builder.CreateBitCast(Addr, PTy); + Address Addr(CGF.Builder.CreateLoad(VAListAddr), SlotSize); + Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty)); + return Addr; } - uint64_t Size = CGF.getContext().getTypeSize(Ty) / 8; - uint64_t TyAlign = CGF.getContext().getTypeAlign(Ty) / 8; - bool IsIndirect = false; + auto TyInfo = getContext().getTypeInfoInChars(Ty); + CharUnits TyAlignForABI = TyInfo.second; - // The ABI alignment for 64-bit or 128-bit vectors is 8 for AAPCS and 4 for - // APCS. For AAPCS, the ABI alignment is at least 4-byte and at most 8-byte. - if (getABIKind() == ARMABIInfo::AAPCS_VFP || - getABIKind() == ARMABIInfo::AAPCS) - TyAlign = std::min(std::max(TyAlign, (uint64_t)4), (uint64_t)8); - else - TyAlign = 4; // Use indirect if size of the illegal vector is bigger than 16 bytes. - if (isIllegalVectorType(Ty) && Size > 16) { + bool IsIndirect = false; + const Type *Base = nullptr; + uint64_t Members = 0; + if (TyInfo.first > CharUnits::fromQuantity(16) && isIllegalVectorType(Ty)) { IsIndirect = true; - Size = 4; - TyAlign = 4; - } - // Handle address alignment for ABI alignment > 4 bytes. - if (TyAlign > 4) { - assert((TyAlign & (TyAlign - 1)) == 0 && - "Alignment is not power of 2!"); - llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int32Ty); - AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt32(TyAlign - 1)); - AddrAsInt = Builder.CreateAnd(AddrAsInt, Builder.getInt32(~(TyAlign - 1))); - Addr = Builder.CreateIntToPtr(AddrAsInt, BP, "ap.align"); - } + // ARMv7k passes structs bigger than 16 bytes indirectly, in space + // allocated by the caller. + } else if (TyInfo.first > CharUnits::fromQuantity(16) && + getABIKind() == ARMABIInfo::AAPCS16_VFP && + !isHomogeneousAggregate(Ty, Base, Members)) { + IsIndirect = true; - uint64_t Offset = - llvm::RoundUpToAlignment(Size, 4); - llvm::Value *NextAddr = - Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset), - "ap.next"); - Builder.CreateStore(NextAddr, VAListAddrAsBPP); + // Otherwise, bound the type's ABI alignment. + // The ABI alignment for 64-bit or 128-bit vectors is 8 for AAPCS and 4 for + // APCS. For AAPCS, the ABI alignment is at least 4-byte and at most 8-byte. + // Our callers should be prepared to handle an under-aligned address. + } else if (getABIKind() == ARMABIInfo::AAPCS_VFP || + getABIKind() == ARMABIInfo::AAPCS) { + TyAlignForABI = std::max(TyAlignForABI, CharUnits::fromQuantity(4)); + TyAlignForABI = std::min(TyAlignForABI, CharUnits::fromQuantity(8)); + } else if (getABIKind() == ARMABIInfo::AAPCS16_VFP) { + // ARMv7k allows type alignment up to 16 bytes. + TyAlignForABI = std::max(TyAlignForABI, CharUnits::fromQuantity(4)); + TyAlignForABI = std::min(TyAlignForABI, CharUnits::fromQuantity(16)); + } else { + TyAlignForABI = CharUnits::fromQuantity(4); + } + TyInfo.second = TyAlignForABI; - if (IsIndirect) - Addr = Builder.CreateLoad(Builder.CreateBitCast(Addr, BPP)); - else if (TyAlign < CGF.getContext().getTypeAlign(Ty) / 8) { - // We can't directly cast ap.cur to pointer to a vector type, since ap.cur - // may not be correctly aligned for the vector type. We create an aligned - // temporary space and copy the content over from ap.cur to the temporary - // space. This is necessary if the natural alignment of the type is greater - // than the ABI alignment. - llvm::Type *I8PtrTy = Builder.getInt8PtrTy(); - CharUnits CharSize = getContext().getTypeSizeInChars(Ty); - llvm::Value *AlignedTemp = CGF.CreateTempAlloca(CGF.ConvertType(Ty), - "var.align"); - llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy); - llvm::Value *Src = Builder.CreateBitCast(Addr, I8PtrTy); - Builder.CreateMemCpy(Dst, Src, - llvm::ConstantInt::get(CGF.IntPtrTy, CharSize.getQuantity()), - TyAlign, false); - Addr = AlignedTemp; //The content is in aligned location. - } - llvm::Type *PTy = - llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); - llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy); - - return AddrTyped; + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TyInfo, + SlotSize, /*AllowHigherAlign*/ true); } //===----------------------------------------------------------------------===// @@ -5071,8 +5387,8 @@ public: ABIArgInfo classifyArgumentType(QualType Ty) const; void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CFG) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; }; class NVPTXTargetCodeGenInfo : public TargetCodeGenInfo { @@ -5111,7 +5427,7 @@ ABIArgInfo NVPTXABIInfo::classifyArgumentType(QualType Ty) const { // Return aggregates type as indirect by value if (isAggregateTypeForABI(Ty)) - return ABIArgInfo::getIndirect(0, /* byval */ true); + return getNaturalAlignIndirect(Ty, /* byval */ true); return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend() : ABIArgInfo::getDirect()); @@ -5130,15 +5446,15 @@ void NVPTXABIInfo::computeInfo(CGFunctionInfo &FI) const { FI.setEffectiveCallingConvention(getRuntimeCC()); } -llvm::Value *NVPTXABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CFG) const { +Address NVPTXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { llvm_unreachable("NVPTX does not support varargs"); } void NVPTXTargetCodeGenInfo:: setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const{ - const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); + const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D); if (!FD) return; llvm::Function *F = cast<llvm::Function>(GV); @@ -5232,8 +5548,8 @@ public: I.info = classifyArgumentType(I.type); } - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; }; class SystemZTargetCodeGenInfo : public TargetCodeGenInfo { @@ -5334,8 +5650,8 @@ QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const { return Ty; } -llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { +Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { // Assume that va_list type is correct; should be pointer to LLVM type: // struct { // i64 __gpr; @@ -5347,59 +5663,69 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, // Every non-vector argument occupies 8 bytes and is passed by preference // in either GPRs or FPRs. Vector arguments occupy 8 or 16 bytes and are // always passed on the stack. - Ty = CGF.getContext().getCanonicalType(Ty); + Ty = getContext().getCanonicalType(Ty); + auto TyInfo = getContext().getTypeInfoInChars(Ty); llvm::Type *ArgTy = CGF.ConvertTypeForMem(Ty); - llvm::Type *APTy = llvm::PointerType::getUnqual(ArgTy); + llvm::Type *DirectTy = ArgTy; ABIArgInfo AI = classifyArgumentType(Ty); bool IsIndirect = AI.isIndirect(); bool InFPRs = false; bool IsVector = false; - unsigned UnpaddedBitSize; + CharUnits UnpaddedSize; + CharUnits DirectAlign; if (IsIndirect) { - APTy = llvm::PointerType::getUnqual(APTy); - UnpaddedBitSize = 64; + DirectTy = llvm::PointerType::getUnqual(DirectTy); + UnpaddedSize = DirectAlign = CharUnits::fromQuantity(8); } else { if (AI.getCoerceToType()) ArgTy = AI.getCoerceToType(); InFPRs = ArgTy->isFloatTy() || ArgTy->isDoubleTy(); IsVector = ArgTy->isVectorTy(); - UnpaddedBitSize = getContext().getTypeSize(Ty); + UnpaddedSize = TyInfo.first; + DirectAlign = TyInfo.second; } - unsigned PaddedBitSize = (IsVector && UnpaddedBitSize > 64) ? 128 : 64; - assert((UnpaddedBitSize <= PaddedBitSize) && "Invalid argument size."); + CharUnits PaddedSize = CharUnits::fromQuantity(8); + if (IsVector && UnpaddedSize > PaddedSize) + PaddedSize = CharUnits::fromQuantity(16); + assert((UnpaddedSize <= PaddedSize) && "Invalid argument size."); - unsigned PaddedSize = PaddedBitSize / 8; - unsigned Padding = (PaddedBitSize - UnpaddedBitSize) / 8; + CharUnits Padding = (PaddedSize - UnpaddedSize); llvm::Type *IndexTy = CGF.Int64Ty; - llvm::Value *PaddedSizeV = llvm::ConstantInt::get(IndexTy, PaddedSize); + llvm::Value *PaddedSizeV = + llvm::ConstantInt::get(IndexTy, PaddedSize.getQuantity()); if (IsVector) { // Work out the address of a vector argument on the stack. // Vector arguments are always passed in the high bits of a // single (8 byte) or double (16 byte) stack slot. - llvm::Value *OverflowArgAreaPtr = - CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 2, + Address OverflowArgAreaPtr = + CGF.Builder.CreateStructGEP(VAListAddr, 2, CharUnits::fromQuantity(16), "overflow_arg_area_ptr"); - llvm::Value *OverflowArgArea = - CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"); - llvm::Value *MemAddr = - CGF.Builder.CreateBitCast(OverflowArgArea, APTy, "mem_addr"); + Address OverflowArgArea = + Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"), + TyInfo.second); + Address MemAddr = + CGF.Builder.CreateElementBitCast(OverflowArgArea, DirectTy, "mem_addr"); // Update overflow_arg_area_ptr pointer llvm::Value *NewOverflowArgArea = - CGF.Builder.CreateGEP(OverflowArgArea, PaddedSizeV, "overflow_arg_area"); + CGF.Builder.CreateGEP(OverflowArgArea.getPointer(), PaddedSizeV, + "overflow_arg_area"); CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); return MemAddr; } - unsigned MaxRegs, RegCountField, RegSaveIndex, RegPadding; + assert(PaddedSize.getQuantity() == 8); + + unsigned MaxRegs, RegCountField, RegSaveIndex; + CharUnits RegPadding; if (InFPRs) { MaxRegs = 4; // Maximum of 4 FPR arguments RegCountField = 1; // __fpr RegSaveIndex = 16; // save offset for f0 - RegPadding = 0; // floats are passed in the high bits of an FPR + RegPadding = CharUnits(); // floats are passed in the high bits of an FPR } else { MaxRegs = 5; // Maximum of 5 GPR arguments RegCountField = 0; // __gpr @@ -5407,8 +5733,9 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, RegPadding = Padding; // values are passed in the low bits of a GPR } - llvm::Value *RegCountPtr = CGF.Builder.CreateStructGEP( - nullptr, VAListAddr, RegCountField, "reg_count_ptr"); + Address RegCountPtr = CGF.Builder.CreateStructGEP( + VAListAddr, RegCountField, RegCountField * CharUnits::fromQuantity(8), + "reg_count_ptr"); llvm::Value *RegCount = CGF.Builder.CreateLoad(RegCountPtr, "reg_count"); llvm::Value *MaxRegsV = llvm::ConstantInt::get(IndexTy, MaxRegs); llvm::Value *InRegs = CGF.Builder.CreateICmpULT(RegCount, MaxRegsV, @@ -5426,17 +5753,20 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, llvm::Value *ScaledRegCount = CGF.Builder.CreateMul(RegCount, PaddedSizeV, "scaled_reg_count"); llvm::Value *RegBase = - llvm::ConstantInt::get(IndexTy, RegSaveIndex * PaddedSize + RegPadding); + llvm::ConstantInt::get(IndexTy, RegSaveIndex * PaddedSize.getQuantity() + + RegPadding.getQuantity()); llvm::Value *RegOffset = CGF.Builder.CreateAdd(ScaledRegCount, RegBase, "reg_offset"); - llvm::Value *RegSaveAreaPtr = - CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3, "reg_save_area_ptr"); + Address RegSaveAreaPtr = + CGF.Builder.CreateStructGEP(VAListAddr, 3, CharUnits::fromQuantity(24), + "reg_save_area_ptr"); llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(RegSaveAreaPtr, "reg_save_area"); - llvm::Value *RawRegAddr = - CGF.Builder.CreateGEP(RegSaveArea, RegOffset, "raw_reg_addr"); - llvm::Value *RegAddr = - CGF.Builder.CreateBitCast(RawRegAddr, APTy, "reg_addr"); + Address RawRegAddr(CGF.Builder.CreateGEP(RegSaveArea, RegOffset, + "raw_reg_addr"), + PaddedSize); + Address RegAddr = + CGF.Builder.CreateElementBitCast(RawRegAddr, DirectTy, "reg_addr"); // Update the register count llvm::Value *One = llvm::ConstantInt::get(IndexTy, 1); @@ -5449,30 +5779,31 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, CGF.EmitBlock(InMemBlock); // Work out the address of a stack argument. - llvm::Value *OverflowArgAreaPtr = CGF.Builder.CreateStructGEP( - nullptr, VAListAddr, 2, "overflow_arg_area_ptr"); - llvm::Value *OverflowArgArea = - CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"); - llvm::Value *PaddingV = llvm::ConstantInt::get(IndexTy, Padding); - llvm::Value *RawMemAddr = - CGF.Builder.CreateGEP(OverflowArgArea, PaddingV, "raw_mem_addr"); - llvm::Value *MemAddr = - CGF.Builder.CreateBitCast(RawMemAddr, APTy, "mem_addr"); + Address OverflowArgAreaPtr = CGF.Builder.CreateStructGEP( + VAListAddr, 2, CharUnits::fromQuantity(16), "overflow_arg_area_ptr"); + Address OverflowArgArea = + Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"), + PaddedSize); + Address RawMemAddr = + CGF.Builder.CreateConstByteGEP(OverflowArgArea, Padding, "raw_mem_addr"); + Address MemAddr = + CGF.Builder.CreateElementBitCast(RawMemAddr, DirectTy, "mem_addr"); // Update overflow_arg_area_ptr pointer llvm::Value *NewOverflowArgArea = - CGF.Builder.CreateGEP(OverflowArgArea, PaddedSizeV, "overflow_arg_area"); + CGF.Builder.CreateGEP(OverflowArgArea.getPointer(), PaddedSizeV, + "overflow_arg_area"); CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); CGF.EmitBranch(ContBlock); // Return the appropriate result. CGF.EmitBlock(ContBlock); - llvm::PHINode *ResAddr = CGF.Builder.CreatePHI(APTy, 2, "va_arg.addr"); - ResAddr->addIncoming(RegAddr, InRegBlock); - ResAddr->addIncoming(MemAddr, InMemBlock); + Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, + MemAddr, InMemBlock, "va_arg.addr"); if (IsIndirect) - return CGF.Builder.CreateLoad(ResAddr, "indirect_arg"); + ResAddr = Address(CGF.Builder.CreateLoad(ResAddr, "indirect_arg"), + TyInfo.second); return ResAddr; } @@ -5483,7 +5814,7 @@ ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy) const { if (isVectorArgumentType(RetTy)) return ABIArgInfo::getDirect(); if (isCompoundType(RetTy) || getContext().getTypeSize(RetTy) > 64) - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); return (isPromotableIntegerType(RetTy) ? ABIArgInfo::getExtend() : ABIArgInfo::getDirect()); } @@ -5491,7 +5822,7 @@ ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy) const { ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { // Handle the generic C++ ABI. if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) - return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); // Integers and enums are extended to full register width. if (isPromotableIntegerType(Ty)) @@ -5508,7 +5839,7 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { // Values that are not 1, 2, 4 or 8 bytes in size are passed indirectly. if (Size != 8 && Size != 16 && Size != 32 && Size != 64) - return ABIArgInfo::getIndirect(0, /*ByVal=*/false); + return getNaturalAlignIndirect(Ty, /*ByVal=*/false); // Handle small structures. if (const RecordType *RT = Ty->getAs<RecordType>()) { @@ -5516,7 +5847,7 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { // fail the size test above. const RecordDecl *RD = RT->getDecl(); if (RD->hasFlexibleArrayMember()) - return ABIArgInfo::getIndirect(0, /*ByVal=*/false); + return getNaturalAlignIndirect(Ty, /*ByVal=*/false); // The structure is passed as an unextended integer, a float, or a double. llvm::Type *PassTy; @@ -5533,7 +5864,7 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { // Non-structure compounds are passed indirectly. if (isCompoundType(Ty)) - return ABIArgInfo::getIndirect(0, /*ByVal=*/false); + return getNaturalAlignIndirect(Ty, /*ByVal=*/false); return ABIArgInfo::getDirect(nullptr); } @@ -5557,7 +5888,7 @@ public: void MSP430TargetCodeGenInfo::setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { - if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) { + if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) { if (const MSP430InterruptAttr *attr = FD->getAttr<MSP430InterruptAttr>()) { // Handle 'interrupt' attribute: llvm::Function *F = cast<llvm::Function>(GV); @@ -5598,8 +5929,8 @@ public: ABIArgInfo classifyReturnType(QualType RetTy) const; ABIArgInfo classifyArgumentType(QualType RetTy, uint64_t &Offset) const; void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; bool shouldSignExtUnsignedType(QualType Ty) const override; }; @@ -5616,7 +5947,7 @@ public: void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const override { - const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); + const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D); if (!FD) return; llvm::Function *Fn = cast<llvm::Function>(GV); if (FD->hasAttr<Mips16Attr>()) { @@ -5625,6 +5956,26 @@ public: else if (FD->hasAttr<NoMips16Attr>()) { Fn->addFnAttr("nomips16"); } + + const MipsInterruptAttr *Attr = FD->getAttr<MipsInterruptAttr>(); + if (!Attr) + return; + + const char *Kind; + switch (Attr->getInterrupt()) { + case MipsInterruptAttr::eic: Kind = "eic"; break; + case MipsInterruptAttr::sw0: Kind = "sw0"; break; + case MipsInterruptAttr::sw1: Kind = "sw1"; break; + case MipsInterruptAttr::hw0: Kind = "hw0"; break; + case MipsInterruptAttr::hw1: Kind = "hw1"; break; + case MipsInterruptAttr::hw2: Kind = "hw2"; break; + case MipsInterruptAttr::hw3: Kind = "hw3"; break; + case MipsInterruptAttr::hw4: Kind = "hw4"; break; + case MipsInterruptAttr::hw5: Kind = "hw5"; break; + } + + Fn->addFnAttr("interrupt", Kind); + } bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, @@ -5738,7 +6089,7 @@ MipsABIInfo::classifyArgumentType(QualType Ty, uint64_t &Offset) const { if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) { Offset = OrigOffset + MinABIStackAlignInBytes; - return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); } // If we have reached here, aggregates are passed directly by coercing to @@ -5832,7 +6183,7 @@ ABIArgInfo MipsABIInfo::classifyReturnType(QualType RetTy) const { } } - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); } // Treat an enum type as its underlying type. @@ -5855,52 +6206,55 @@ void MipsABIInfo::computeInfo(CGFunctionInfo &FI) const { I.info = classifyArgumentType(I.type, Offset); } -llvm::Value* MipsABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - llvm::Type *BP = CGF.Int8PtrTy; - llvm::Type *BPP = CGF.Int8PtrPtrTy; +Address MipsABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType OrigTy) const { + QualType Ty = OrigTy; // Integer arguments are promoted to 32-bit on O32 and 64-bit on N32/N64. // Pointers are also promoted in the same way but this only matters for N32. unsigned SlotSizeInBits = IsO32 ? 32 : 64; unsigned PtrWidth = getTarget().getPointerWidth(0); + bool DidPromote = false; if ((Ty->isIntegerType() && - CGF.getContext().getIntWidth(Ty) < SlotSizeInBits) || + getContext().getIntWidth(Ty) < SlotSizeInBits) || (Ty->isPointerType() && PtrWidth < SlotSizeInBits)) { - Ty = CGF.getContext().getIntTypeForBitwidth(SlotSizeInBits, - Ty->isSignedIntegerType()); + DidPromote = true; + Ty = getContext().getIntTypeForBitwidth(SlotSizeInBits, + Ty->isSignedIntegerType()); } - CGBuilderTy &Builder = CGF.Builder; - llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap"); - llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); - int64_t TypeAlign = - std::min(getContext().getTypeAlign(Ty) / 8, StackAlignInBytes); - llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); - llvm::Value *AddrTyped; - llvm::IntegerType *IntTy = (PtrWidth == 32) ? CGF.Int32Ty : CGF.Int64Ty; - - if (TypeAlign > MinABIStackAlignInBytes) { - llvm::Value *AddrAsInt = CGF.Builder.CreatePtrToInt(Addr, IntTy); - llvm::Value *Inc = llvm::ConstantInt::get(IntTy, TypeAlign - 1); - llvm::Value *Mask = llvm::ConstantInt::get(IntTy, -TypeAlign); - llvm::Value *Add = CGF.Builder.CreateAdd(AddrAsInt, Inc); - llvm::Value *And = CGF.Builder.CreateAnd(Add, Mask); - AddrTyped = CGF.Builder.CreateIntToPtr(And, PTy); - } - else - AddrTyped = Builder.CreateBitCast(Addr, PTy); + auto TyInfo = getContext().getTypeInfoInChars(Ty); + + // The alignment of things in the argument area is never larger than + // StackAlignInBytes. + TyInfo.second = + std::min(TyInfo.second, CharUnits::fromQuantity(StackAlignInBytes)); + + // MinABIStackAlignInBytes is the size of argument slots on the stack. + CharUnits ArgSlotSize = CharUnits::fromQuantity(MinABIStackAlignInBytes); + + Address Addr = emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false, + TyInfo, ArgSlotSize, /*AllowHigherAlign*/ true); + - llvm::Value *AlignedAddr = Builder.CreateBitCast(AddrTyped, BP); - TypeAlign = std::max((unsigned)TypeAlign, MinABIStackAlignInBytes); - unsigned ArgSizeInBits = CGF.getContext().getTypeSize(Ty); - uint64_t Offset = llvm::RoundUpToAlignment(ArgSizeInBits / 8, TypeAlign); - llvm::Value *NextAddr = - Builder.CreateGEP(AlignedAddr, llvm::ConstantInt::get(IntTy, Offset), - "ap.next"); - Builder.CreateStore(NextAddr, VAListAddrAsBPP); + // If there was a promotion, "unpromote" into a temporary. + // TODO: can we just use a pointer into a subset of the original slot? + if (DidPromote) { + Address Temp = CGF.CreateMemTemp(OrigTy, "vaarg.promotion-temp"); + llvm::Value *Promoted = CGF.Builder.CreateLoad(Addr); - return AddrTyped; + // Truncate down to the right width. + llvm::Type *IntTy = (OrigTy->isIntegerType() ? Temp.getElementType() + : CGF.IntPtrTy); + llvm::Value *V = CGF.Builder.CreateTrunc(Promoted, IntTy); + if (OrigTy->isPointerType()) + V = CGF.Builder.CreateIntToPtr(V, Temp.getElementType()); + + CGF.Builder.CreateStore(V, Temp); + Addr = Temp; + } + + return Addr; } bool MipsABIInfo::shouldSignExtUnsignedType(QualType Ty) const { @@ -5960,7 +6314,7 @@ public: void TCETargetCodeGenInfo::setTargetAttributes( const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { - const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); + const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D); if (!FD) return; llvm::Function *F = cast<llvm::Function>(GV); @@ -6022,8 +6376,8 @@ private: void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; }; class HexagonTargetCodeGenInfo : public TargetCodeGenInfo { @@ -6060,11 +6414,11 @@ ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty) const { return ABIArgInfo::getIgnore(); if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) - return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); uint64_t Size = getContext().getTypeSize(Ty); if (Size > 64) - return ABIArgInfo::getIndirect(0, /*ByVal=*/true); + return getNaturalAlignIndirect(Ty, /*ByVal=*/true); // Pass in the smallest viable integer type. else if (Size > 32) return ABIArgInfo::getDirect(llvm::Type::getInt64Ty(getVMContext())); @@ -6082,7 +6436,7 @@ ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const { // Large vector types should be returned via memory. if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 64) - return ABIArgInfo::getIndirect(0); + return getNaturalAlignIndirect(RetTy); if (!isAggregateTypeForABI(RetTy)) { // Treat an enum type as its underlying type. @@ -6110,30 +6464,16 @@ ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const { return ABIArgInfo::getDirect(llvm::Type::getInt64Ty(getVMContext())); } - return ABIArgInfo::getIndirect(0, /*ByVal=*/true); + return getNaturalAlignIndirect(RetTy, /*ByVal=*/true); } -llvm::Value *HexagonABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - // FIXME: Need to handle alignment - llvm::Type *BPP = CGF.Int8PtrPtrTy; - - CGBuilderTy &Builder = CGF.Builder; - llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, - "ap"); - llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); - llvm::Type *PTy = - llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); - llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy); - - uint64_t Offset = - llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, 4); - llvm::Value *NextAddr = - Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset), - "ap.next"); - Builder.CreateStore(NextAddr, VAListAddrAsBPP); - - return AddrTyped; +Address HexagonABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { + // FIXME: Someone needs to audit that this handle alignment correctly. + return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false, + getContext().getTypeInfoInChars(Ty), + CharUnits::fromQuantity(4), + /*AllowHigherAlign*/ true); } //===----------------------------------------------------------------------===// @@ -6156,7 +6496,7 @@ void AMDGPUTargetCodeGenInfo::setTargetAttributes( const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { - const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); + const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D); if (!FD) return; @@ -6210,8 +6550,8 @@ public: private: ABIArgInfo classifyType(QualType RetTy, unsigned SizeLimit) const; void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; // Coercion type builder for structs passed in registers. The coercion type // serves two purposes: @@ -6331,7 +6671,7 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const { // Anything too big to fit in registers is passed with an explicit indirect // pointer / sret pointer. if (Size > SizeLimit) - return ABIArgInfo::getIndirect(0, /*ByVal=*/false); + return getNaturalAlignIndirect(Ty, /*ByVal=*/false); // Treat an enum type as its underlying type. if (const EnumType *EnumTy = Ty->getAs<EnumType>()) @@ -6348,7 +6688,7 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const { // If a C++ object has either a non-trivial copy constructor or a non-trivial // destructor, it is passed with an explicit indirect pointer / sret pointer. if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) - return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); + return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory); // This is a small aggregate type that should be passed in registers. // Build a coercion type from the LLVM struct type. @@ -6369,55 +6709,59 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const { return ABIArgInfo::getDirect(CoerceTy); } -llvm::Value *SparcV9ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { +Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { ABIArgInfo AI = classifyType(Ty, 16 * 8); llvm::Type *ArgTy = CGT.ConvertType(Ty); if (AI.canHaveCoerceToType() && !AI.getCoerceToType()) AI.setCoerceToType(ArgTy); - llvm::Type *BPP = CGF.Int8PtrPtrTy; + CharUnits SlotSize = CharUnits::fromQuantity(8); + CGBuilderTy &Builder = CGF.Builder; - llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap"); - llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); + Address Addr(Builder.CreateLoad(VAListAddr, "ap.cur"), SlotSize); llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy); - llvm::Value *ArgAddr; - unsigned Stride; + auto TypeInfo = getContext().getTypeInfoInChars(Ty); + + Address ArgAddr = Address::invalid(); + CharUnits Stride; switch (AI.getKind()) { case ABIArgInfo::Expand: case ABIArgInfo::InAlloca: llvm_unreachable("Unsupported ABI kind for va_arg"); - case ABIArgInfo::Extend: - Stride = 8; - ArgAddr = Builder - .CreateConstGEP1_32(Addr, 8 - getDataLayout().getTypeAllocSize(ArgTy), - "extend"); + case ABIArgInfo::Extend: { + Stride = SlotSize; + CharUnits Offset = SlotSize - TypeInfo.first; + ArgAddr = Builder.CreateConstInBoundsByteGEP(Addr, Offset, "extend"); break; + } - case ABIArgInfo::Direct: - Stride = getDataLayout().getTypeAllocSize(AI.getCoerceToType()); + case ABIArgInfo::Direct: { + auto AllocSize = getDataLayout().getTypeAllocSize(AI.getCoerceToType()); + Stride = CharUnits::fromQuantity(AllocSize).RoundUpToAlignment(SlotSize); ArgAddr = Addr; break; + } case ABIArgInfo::Indirect: - Stride = 8; - ArgAddr = Builder.CreateBitCast(Addr, - llvm::PointerType::getUnqual(ArgPtrTy), - "indirect"); - ArgAddr = Builder.CreateLoad(ArgAddr, "indirect.arg"); + Stride = SlotSize; + ArgAddr = Builder.CreateElementBitCast(Addr, ArgPtrTy, "indirect"); + ArgAddr = Address(Builder.CreateLoad(ArgAddr, "indirect.arg"), + TypeInfo.second); break; case ABIArgInfo::Ignore: - return llvm::UndefValue::get(ArgPtrTy); + return Address(llvm::UndefValue::get(ArgPtrTy), TypeInfo.second); } // Update VAList. - Addr = Builder.CreateConstGEP1_32(Addr, Stride, "ap.next"); - Builder.CreateStore(Addr, VAListAddrAsBPP); + llvm::Value *NextPtr = + Builder.CreateConstInBoundsByteGEP(Addr.getPointer(), Stride, "ap.next"); + Builder.CreateStore(NextPtr, VAListAddr); - return Builder.CreatePointerCast(ArgAddr, ArgPtrTy, "arg.addr"); + return Builder.CreateBitCast(ArgAddr, ArgPtrTy, "arg.addr"); } void SparcV9ABIInfo::computeInfo(CGFunctionInfo &FI) const { @@ -6552,7 +6896,7 @@ class TypeStringCache { unsigned IncompleteCount; // Number of Incomplete entries in the Map. unsigned IncompleteUsedCount; // Number of IncompleteUsed entries in the Map. public: - TypeStringCache() : IncompleteCount(0), IncompleteUsedCount(0) {}; + TypeStringCache() : IncompleteCount(0), IncompleteUsedCount(0) {} void addIncomplete(const IdentifierInfo *ID, std::string StubEnc); bool removeIncomplete(const IdentifierInfo *ID); void addIfComplete(const IdentifierInfo *ID, StringRef Str, @@ -6566,8 +6910,8 @@ class FieldEncoding { bool HasName; std::string Enc; public: - FieldEncoding(bool b, SmallStringEnc &e) : HasName(b), Enc(e.c_str()) {}; - StringRef str() {return Enc.c_str();}; + FieldEncoding(bool b, SmallStringEnc &e) : HasName(b), Enc(e.c_str()) {} + StringRef str() {return Enc.c_str();} bool operator<(const FieldEncoding &rhs) const { if (HasName != rhs.HasName) return HasName; return Enc < rhs.Enc; @@ -6577,8 +6921,8 @@ public: class XCoreABIInfo : public DefaultABIInfo { public: XCoreABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {} - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; + Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const override; }; class XCoreTargetCodeGenInfo : public TargetCodeGenInfo { @@ -6592,52 +6936,53 @@ public: } // End anonymous namespace. -llvm::Value *XCoreABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { +Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, + QualType Ty) const { CGBuilderTy &Builder = CGF.Builder; // Get the VAList. - llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, - CGF.Int8PtrPtrTy); - llvm::Value *AP = Builder.CreateLoad(VAListAddrAsBPP); + CharUnits SlotSize = CharUnits::fromQuantity(4); + Address AP(Builder.CreateLoad(VAListAddr), SlotSize); // Handle the argument. ABIArgInfo AI = classifyArgumentType(Ty); + CharUnits TypeAlign = getContext().getTypeAlignInChars(Ty); llvm::Type *ArgTy = CGT.ConvertType(Ty); if (AI.canHaveCoerceToType() && !AI.getCoerceToType()) AI.setCoerceToType(ArgTy); llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy); - llvm::Value *Val; - uint64_t ArgSize = 0; + + Address Val = Address::invalid(); + CharUnits ArgSize = CharUnits::Zero(); switch (AI.getKind()) { case ABIArgInfo::Expand: case ABIArgInfo::InAlloca: llvm_unreachable("Unsupported ABI kind for va_arg"); case ABIArgInfo::Ignore: - Val = llvm::UndefValue::get(ArgPtrTy); - ArgSize = 0; + Val = Address(llvm::UndefValue::get(ArgPtrTy), TypeAlign); + ArgSize = CharUnits::Zero(); break; case ABIArgInfo::Extend: case ABIArgInfo::Direct: - Val = Builder.CreatePointerCast(AP, ArgPtrTy); - ArgSize = getDataLayout().getTypeAllocSize(AI.getCoerceToType()); - if (ArgSize < 4) - ArgSize = 4; + Val = Builder.CreateBitCast(AP, ArgPtrTy); + ArgSize = CharUnits::fromQuantity( + getDataLayout().getTypeAllocSize(AI.getCoerceToType())); + ArgSize = ArgSize.RoundUpToAlignment(SlotSize); break; case ABIArgInfo::Indirect: - llvm::Value *ArgAddr; - ArgAddr = Builder.CreateBitCast(AP, llvm::PointerType::getUnqual(ArgPtrTy)); - ArgAddr = Builder.CreateLoad(ArgAddr); - Val = Builder.CreatePointerCast(ArgAddr, ArgPtrTy); - ArgSize = 4; + Val = Builder.CreateElementBitCast(AP, ArgPtrTy); + Val = Address(Builder.CreateLoad(Val), TypeAlign); + ArgSize = SlotSize; break; } // Increment the VAList. - if (ArgSize) { - llvm::Value *APN = Builder.CreateConstGEP1_32(AP, ArgSize); - Builder.CreateStore(APN, VAListAddrAsBPP); + if (!ArgSize.isZero()) { + llvm::Value *APN = + Builder.CreateConstInBoundsByteGEP(AP.getPointer(), ArgSize); + Builder.CreateStore(APN, VAListAddr); } + return Val; } @@ -6781,9 +7126,7 @@ static bool extractFieldType(SmallVectorImpl<FieldEncoding> &FE, if (Field->isBitField()) { Enc += "b("; llvm::raw_svector_ostream OS(Enc); - OS.resync(); OS << Field->getBitWidthValue(CGM.getContext()); - OS.flush(); Enc += ':'; } if (!appendType(Enc, Field->getType(), CGM, TSC)) @@ -6897,7 +7240,7 @@ static bool appendEnumType(SmallStringEnc &Enc, const EnumType *ET, /// This is done prior to appending the type's encoding. static void appendQualifier(SmallStringEnc &Enc, QualType QT) { // Qualifiers are emitted in alphabetical order. - static const char *Table[] = {"","c:","r:","cr:","v:","cv:","rv:","crv:"}; + static const char *const Table[]={"","c:","r:","cr:","v:","cv:","rv:","crv:"}; int Lookup = 0; if (QT.isConstQualified()) Lookup += 1<<0; @@ -7138,6 +7481,10 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { return *(TheTargetCodeGenInfo = new AArch64TargetCodeGenInfo(Types, Kind)); } + case llvm::Triple::wasm32: + case llvm::Triple::wasm64: + return *(TheTargetCodeGenInfo = new WebAssemblyTargetCodeGenInfo(Types)); + case llvm::Triple::arm: case llvm::Triple::armeb: case llvm::Triple::thumb: @@ -7150,8 +7497,11 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { } ARMABIInfo::ABIKind Kind = ARMABIInfo::AAPCS; - if (getTarget().getABI() == "apcs-gnu") + StringRef ABIStr = getTarget().getABI(); + if (ABIStr == "apcs-gnu") Kind = ARMABIInfo::APCS; + else if (ABIStr == "aapcs16") + Kind = ARMABIInfo::AAPCS16_VFP; else if (CodeGenOpts.FloatABI == "hard" || (CodeGenOpts.FloatABI != "soft" && Triple.getEnvironment() == llvm::Triple::GNUEABIHF)) @@ -7161,7 +7511,8 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { } case llvm::Triple::ppc: - return *(TheTargetCodeGenInfo = new PPC32TargetCodeGenInfo(Types)); + return *(TheTargetCodeGenInfo = + new PPC32TargetCodeGenInfo(Types, CodeGenOpts.FloatABI == "soft")); case llvm::Triple::ppc64: if (Triple.isOSBinFormatELF()) { PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv1; @@ -7202,18 +7553,19 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { case llvm::Triple::x86: { bool IsDarwinVectorABI = Triple.isOSDarwin(); - bool IsSmallStructInRegABI = + bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(Triple, CodeGenOpts); bool IsWin32FloatStructABI = Triple.isOSWindows() && !Triple.isOSCygMing(); if (Triple.getOS() == llvm::Triple::Win32) { return *(TheTargetCodeGenInfo = new WinX86_32TargetCodeGenInfo( - Types, IsDarwinVectorABI, IsSmallStructInRegABI, + Types, IsDarwinVectorABI, RetSmallStructInRegABI, IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters)); } else { return *(TheTargetCodeGenInfo = new X86_32TargetCodeGenInfo( - Types, IsDarwinVectorABI, IsSmallStructInRegABI, - IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters)); + Types, IsDarwinVectorABI, RetSmallStructInRegABI, + IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters, + CodeGenOpts.FloatABI == "soft")); } } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h index 95275d5..87b4704 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h @@ -47,7 +47,7 @@ class TargetCodeGenInfo { public: // WARNING: Acquires the ownership of ABIInfo. - TargetCodeGenInfo(ABIInfo *info = 0) : Info(info) {} + TargetCodeGenInfo(ABIInfo *info = nullptr) : Info(info) {} virtual ~TargetCodeGenInfo(); /// getABIInfo() - Returns ABI info helper for the target. @@ -219,6 +219,6 @@ public: llvm::StringRef Value, llvm::SmallString<32> &Opt) const {} }; -} +} // namespace clang -#endif +#endif // LLVM_CLANG_LIB_CODEGEN_TARGETINFO_H |
