diff options
Diffstat (limited to 'contrib/llvm/tools/clang/lib/CodeGen')
62 files changed, 15023 insertions, 6793 deletions
diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h index 7e7f7fa..cc8652e 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/ABIInfo.h @@ -87,6 +87,8 @@ namespace clang { virtual bool isHomogeneousAggregateSmallEnough(const Type *Base, uint64_t Members) const; + virtual bool shouldSignExtUnsignedType(QualType Ty) const; + bool isHomogeneousAggregate(QualType Ty, const Type *&Base, uint64_t &Members) const; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp index 25ecec5..afcb9e5 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/BackendUtil.cpp @@ -15,22 +15,22 @@ #include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Frontend/Utils.h" #include "llvm/ADT/StringSwitch.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Bitcode/BitcodeWriterPass.h" #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/IRPrintingPasses.h" +#include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" #include "llvm/MC/SubtargetFeature.h" -#include "llvm/PassManager.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/FormattedStream.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetLibraryInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetSubtargetInfo.h" @@ -55,44 +55,48 @@ class EmitAssemblyHelper { Timer CodeGenerationTime; - mutable PassManager *CodeGenPasses; - mutable PassManager *PerModulePasses; - mutable FunctionPassManager *PerFunctionPasses; + mutable legacy::PassManager *CodeGenPasses; + mutable legacy::PassManager *PerModulePasses; + mutable legacy::FunctionPassManager *PerFunctionPasses; private: - PassManager *getCodeGenPasses() const { + TargetIRAnalysis getTargetIRAnalysis() const { + if (TM) + return TM->getTargetIRAnalysis(); + + return TargetIRAnalysis(); + } + + legacy::PassManager *getCodeGenPasses() const { if (!CodeGenPasses) { - CodeGenPasses = new PassManager(); - CodeGenPasses->add(new DataLayoutPass()); - if (TM) - TM->addAnalysisPasses(*CodeGenPasses); + CodeGenPasses = new legacy::PassManager(); + CodeGenPasses->add( + createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); } return CodeGenPasses; } - PassManager *getPerModulePasses() const { + legacy::PassManager *getPerModulePasses() const { if (!PerModulePasses) { - PerModulePasses = new PassManager(); - PerModulePasses->add(new DataLayoutPass()); - if (TM) - TM->addAnalysisPasses(*PerModulePasses); + PerModulePasses = new legacy::PassManager(); + PerModulePasses->add( + createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); } return PerModulePasses; } - FunctionPassManager *getPerFunctionPasses() const { + legacy::FunctionPassManager *getPerFunctionPasses() const { if (!PerFunctionPasses) { - PerFunctionPasses = new FunctionPassManager(TheModule); - PerFunctionPasses->add(new DataLayoutPass()); - if (TM) - TM->addAnalysisPasses(*PerFunctionPasses); + PerFunctionPasses = new legacy::FunctionPassManager(TheModule); + PerFunctionPasses->add( + createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); } return PerFunctionPasses; } void CreatePasses(); - /// CreateTargetMachine - Generates the TargetMachine. + /// Generates the TargetMachine. /// Returns Null if it is unable to create the target machine. /// Some of our clang tests specify triples which are not built /// into clang. This is okay because these tests check the generated @@ -102,10 +106,10 @@ private: /// the requested target. TargetMachine *CreateTargetMachine(bool MustCreateTM); - /// AddEmitPasses - Add passes necessary to emit assembly or LLVM IR. + /// Add passes necessary to emit assembly or LLVM IR. /// /// \return True on success. - bool AddEmitPasses(BackendAction Action, formatted_raw_ostream &OS); + bool AddEmitPasses(BackendAction Action, raw_pwrite_stream &OS); public: EmitAssemblyHelper(DiagnosticsEngine &_Diags, @@ -128,7 +132,7 @@ public: std::unique_ptr<TargetMachine> TM; - void EmitAssembly(BackendAction Action, raw_ostream *OS); + void EmitAssembly(BackendAction Action, raw_pwrite_stream *OS); }; // We need this wrapper to access LangOpts and CGOpts from extension functions @@ -163,7 +167,7 @@ static void addObjCARCOptPass(const PassManagerBuilder &Builder, PassManagerBase } static void addSampleProfileLoaderPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast<const PassManagerBuilderWrapper &>(Builder); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); @@ -171,31 +175,44 @@ static void addSampleProfileLoaderPass(const PassManagerBuilder &Builder, } static void addAddDiscriminatorsPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { PM.add(createAddDiscriminatorsPass()); } static void addBoundsCheckingPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { PM.add(createBoundsCheckingPass()); } static void addSanitizerCoveragePass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast<const PassManagerBuilderWrapper&>(Builder); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); - PM.add(createSanitizerCoverageModulePass(CGOpts.SanitizeCoverage)); + SanitizerCoverageOptions Opts; + Opts.CoverageType = + static_cast<SanitizerCoverageOptions::Type>(CGOpts.SanitizeCoverageType); + Opts.IndirectCalls = CGOpts.SanitizeCoverageIndirectCalls; + Opts.TraceBB = CGOpts.SanitizeCoverageTraceBB; + Opts.TraceCmp = CGOpts.SanitizeCoverageTraceCmp; + Opts.Use8bitCounters = CGOpts.SanitizeCoverage8bitCounters; + PM.add(createSanitizerCoverageModulePass(Opts)); } static void addAddressSanitizerPasses(const PassManagerBuilder &Builder, - PassManagerBase &PM) { - PM.add(createAddressSanitizerFunctionPass()); - PM.add(createAddressSanitizerModulePass()); + legacy::PassManagerBase &PM) { + PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/false)); + PM.add(createAddressSanitizerModulePass(/*CompileKernel*/false)); +} + +static void addKernelAddressSanitizerPasses(const PassManagerBuilder &Builder, + legacy::PassManagerBase &PM) { + PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/true)); + PM.add(createAddressSanitizerModulePass(/*CompileKernel*/true)); } static void addMemorySanitizerPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast<const PassManagerBuilderWrapper&>(Builder); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); @@ -215,28 +232,36 @@ static void addMemorySanitizerPass(const PassManagerBuilder &Builder, } static void addThreadSanitizerPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { PM.add(createThreadSanitizerPass()); } static void addDataFlowSanitizerPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast<const PassManagerBuilderWrapper&>(Builder); const LangOptions &LangOpts = BuilderWrapper.getLangOpts(); - PM.add(createDataFlowSanitizerPass(LangOpts.SanitizerBlacklistFile)); + PM.add(createDataFlowSanitizerPass(LangOpts.SanitizerBlacklistFiles)); } -static TargetLibraryInfo *createTLI(llvm::Triple &TargetTriple, - const CodeGenOptions &CodeGenOpts) { - TargetLibraryInfo *TLI = new TargetLibraryInfo(TargetTriple); +static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple, + const CodeGenOptions &CodeGenOpts) { + TargetLibraryInfoImpl *TLII = new TargetLibraryInfoImpl(TargetTriple); if (!CodeGenOpts.SimplifyLibCalls) - TLI->disableAllFunctions(); - return TLI; + TLII->disableAllFunctions(); + + switch (CodeGenOpts.getVecLib()) { + case CodeGenOptions::Accelerate: + TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::Accelerate); + break; + default: + break; + } + return TLII; } static void addSymbolRewriterPass(const CodeGenOptions &Opts, - PassManager *MPM) { + legacy::PassManager *MPM) { llvm::SymbolRewriter::RewriteDescriptorList DL; llvm::SymbolRewriter::RewriteMapParser MapParser; @@ -264,10 +289,10 @@ void EmitAssemblyHelper::CreatePasses() { PMBuilder.SLPVectorize = CodeGenOpts.VectorizeSLP; PMBuilder.LoopVectorize = CodeGenOpts.VectorizeLoop; - PMBuilder.DisableTailCalls = CodeGenOpts.DisableTailCalls; PMBuilder.DisableUnitAtATime = !CodeGenOpts.UnitAtATime; PMBuilder.DisableUnrollLoops = !CodeGenOpts.UnrollLoops; PMBuilder.MergeFunctions = CodeGenOpts.MergeFunctions; + PMBuilder.PrepareForLTO = CodeGenOpts.PrepareForLTO; PMBuilder.RerollLoops = CodeGenOpts.RerollLoops; PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible, @@ -294,7 +319,9 @@ void EmitAssemblyHelper::CreatePasses() { addBoundsCheckingPass); } - if (CodeGenOpts.SanitizeCoverage) { + if (CodeGenOpts.SanitizeCoverageType || + CodeGenOpts.SanitizeCoverageIndirectCalls || + CodeGenOpts.SanitizeCoverageTraceCmp) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addSanitizerCoveragePass); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, @@ -308,6 +335,13 @@ void EmitAssemblyHelper::CreatePasses() { addAddressSanitizerPasses); } + if (LangOpts.Sanitize.has(SanitizerKind::KernelAddress)) { + PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, + addKernelAddressSanitizerPasses); + PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, + addKernelAddressSanitizerPasses); + } + if (LangOpts.Sanitize.has(SanitizerKind::Memory)) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addMemorySanitizerPass); @@ -331,7 +365,7 @@ void EmitAssemblyHelper::CreatePasses() { // Figure out TargetLibraryInfo. Triple TargetTriple(TheModule->getTargetTriple()); - PMBuilder.LibraryInfo = createTLI(TargetTriple, CodeGenOpts); + PMBuilder.LibraryInfo = createTLII(TargetTriple, CodeGenOpts); switch (Inlining) { case CodeGenOptions::NoInlining: break; @@ -351,17 +385,15 @@ void EmitAssemblyHelper::CreatePasses() { } // Set up the per-function pass manager. - FunctionPassManager *FPM = getPerFunctionPasses(); + legacy::FunctionPassManager *FPM = getPerFunctionPasses(); if (CodeGenOpts.VerifyModule) FPM->add(createVerifierPass()); PMBuilder.populateFunctionPassManager(*FPM); // Set up the per-module pass manager. - PassManager *MPM = getPerModulePasses(); + legacy::PassManager *MPM = getPerModulePasses(); if (!CodeGenOpts.RewriteMapFiles.empty()) addSymbolRewriterPass(CodeGenOpts, MPM); - if (CodeGenOpts.VerifyModule) - MPM->add(createDebugInfoVerifierPass()); if (!CodeGenOpts.DisableGCov && (CodeGenOpts.EmitGcovArcs || CodeGenOpts.EmitGcovNotes)) { @@ -375,6 +407,7 @@ void EmitAssemblyHelper::CreatePasses() { Options.NoRedZone = CodeGenOpts.DisableRedZone; Options.FunctionNamesInData = !CodeGenOpts.CoverageNoFunctionNamesInData; + Options.ExitBlockBeforeBody = CodeGenOpts.CoverageExitBlockBeforeBody; MPM->add(createGCOVProfilerPass(Options)); if (CodeGenOpts.getDebugInfo() == CodeGenOptions::NoDebugInfo) MPM->add(createStripSymbolsPass(true)); @@ -383,6 +416,7 @@ void EmitAssemblyHelper::CreatePasses() { if (CodeGenOpts.ProfileInstrGenerate) { InstrProfOptions Options; Options.NoRedZone = CodeGenOpts.DisableRedZone; + Options.InstrProfileOutput = CodeGenOpts.InstrProfileOutput; MPM->add(createInstrProfilingPass(Options)); } @@ -421,23 +455,17 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { BackendArgs.push_back("-limit-float-precision"); BackendArgs.push_back(CodeGenOpts.LimitFloatPrecision.c_str()); } - if (llvm::TimePassesIsEnabled) - BackendArgs.push_back("-time-passes"); for (unsigned i = 0, e = CodeGenOpts.BackendOptions.size(); i != e; ++i) BackendArgs.push_back(CodeGenOpts.BackendOptions[i].c_str()); - if (CodeGenOpts.NoGlobalMerge) - BackendArgs.push_back("-enable-global-merge=false"); BackendArgs.push_back(nullptr); llvm::cl::ParseCommandLineOptions(BackendArgs.size() - 1, BackendArgs.data()); std::string FeaturesStr; - if (TargetOpts.Features.size()) { + if (!TargetOpts.Features.empty()) { SubtargetFeatures Features; - for (std::vector<std::string>::const_iterator - it = TargetOpts.Features.begin(), - ie = TargetOpts.Features.end(); it != ie; ++it) - Features.AddFeature(*it); + for (const std::string &Feature : TargetOpts.Features) + Features.AddFeature(Feature); FeaturesStr = Features.getString(); } @@ -461,6 +489,9 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { llvm::TargetOptions Options; + if (!TargetOpts.Reciprocals.empty()) + Options.Reciprocals = TargetRecip(TargetOpts.Reciprocals); + Options.ThreadModel = llvm::StringSwitch<llvm::ThreadModel::Model>(CodeGenOpts.ThreadModel) .Case("posix", llvm::ThreadModel::POSIX) @@ -472,15 +503,6 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { if (CodeGenOpts.CompressDebugSections) Options.CompressDebugSections = true; - // Set frame pointer elimination mode. - if (!CodeGenOpts.DisableFPElim) { - Options.NoFramePointerElim = false; - } else if (CodeGenOpts.OmitLeafFramePointer) { - Options.NoFramePointerElim = false; - } else { - Options.NoFramePointerElim = true; - } - if (CodeGenOpts.UseInitArray) Options.UseInitArray = true; @@ -512,13 +534,11 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { Options.NoNaNsFPMath = CodeGenOpts.NoNaNsFPMath; Options.NoZerosInBSS = CodeGenOpts.NoZeroInitializedInBSS; Options.UnsafeFPMath = CodeGenOpts.UnsafeFPMath; - Options.UseSoftFloat = CodeGenOpts.SoftFloat; Options.StackAlignmentOverride = CodeGenOpts.StackAlignment; - Options.DisableTailCalls = CodeGenOpts.DisableTailCalls; - Options.TrapFuncName = CodeGenOpts.TrapFuncName; Options.PositionIndependentExecutable = LangOpts.PIELevel != 0; Options.FunctionSections = CodeGenOpts.FunctionSections; Options.DataSections = CodeGenOpts.DataSections; + Options.UniqueSectionNames = CodeGenOpts.UniqueSectionNames; Options.MCOptions.MCRelaxAll = CodeGenOpts.RelaxAll; Options.MCOptions.MCSaveTempLabels = CodeGenOpts.SaveTempLabels; @@ -536,17 +556,16 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { } bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action, - formatted_raw_ostream &OS) { + raw_pwrite_stream &OS) { // Create the code generator passes. - PassManager *PM = getCodeGenPasses(); + legacy::PassManager *PM = getCodeGenPasses(); // Add LibraryInfo. llvm::Triple TargetTriple(TheModule->getTargetTriple()); - PM->add(createTLI(TargetTriple, CodeGenOpts)); - - // Add Target specific analysis passes. - TM->addAnalysisPasses(*PM); + std::unique_ptr<TargetLibraryInfoImpl> TLII( + createTLII(TargetTriple, CodeGenOpts)); + PM->add(new TargetLibraryInfoWrapperPass(*TLII)); // Normal mode, emit a .s or .o file by running the code generator. Note, // this also adds codegenerator level optimization passes. @@ -561,8 +580,7 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action, // Add ObjC ARC final-cleanup optimizations. This is done as part of the // "codegen" passes so that it isn't run multiple times when there is // inlining happening. - if (LangOpts.ObjCAutoRefCount && - CodeGenOpts.OptimizationLevel > 0) + if (CodeGenOpts.OptimizationLevel > 0) PM->add(createObjCARCContractPass()); if (TM->addPassesToEmitFile(*PM, OS, CGFT, @@ -574,9 +592,9 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action, return true; } -void EmitAssemblyHelper::EmitAssembly(BackendAction Action, raw_ostream *OS) { +void EmitAssemblyHelper::EmitAssembly(BackendAction Action, + raw_pwrite_stream *OS) { TimeRegion Region(llvm::TimePassesIsEnabled ? &CodeGenerationTime : nullptr); - llvm::formatted_raw_ostream FormattedOS; bool UsesCodeGen = (Action != Backend_EmitNothing && Action != Backend_EmitBC && @@ -584,7 +602,10 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, raw_ostream *OS) { if (!TM) TM.reset(CreateTargetMachine(UsesCodeGen)); - if (UsesCodeGen && !TM) return; + if (UsesCodeGen && !TM) + return; + if (TM) + TheModule->setDataLayout(*TM->getDataLayout()); CreatePasses(); switch (Action) { @@ -592,17 +613,17 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, raw_ostream *OS) { break; case Backend_EmitBC: - getPerModulePasses()->add(createBitcodeWriterPass(*OS)); + getPerModulePasses()->add( + createBitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists)); break; case Backend_EmitLL: - FormattedOS.setStream(*OS, formatted_raw_ostream::PRESERVE_STREAM); - getPerModulePasses()->add(createPrintModulePass(FormattedOS)); + getPerModulePasses()->add( + createPrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists)); break; default: - FormattedOS.setStream(*OS, formatted_raw_ostream::PRESERVE_STREAM); - if (!AddEmitPasses(Action, FormattedOS)) + if (!AddEmitPasses(Action, *OS)) return; } @@ -616,10 +637,9 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, raw_ostream *OS) { PrettyStackTraceString CrashInfo("Per-function optimization"); PerFunctionPasses->doInitialization(); - for (Module::iterator I = TheModule->begin(), - E = TheModule->end(); I != E; ++I) - if (!I->isDeclaration()) - PerFunctionPasses->run(*I); + for (Function &F : *TheModule) + if (!F.isDeclaration()) + PerFunctionPasses->run(F); PerFunctionPasses->doFinalization(); } @@ -639,7 +659,7 @@ void clang::EmitBackendOutput(DiagnosticsEngine &Diags, const clang::TargetOptions &TOpts, const LangOptions &LOpts, StringRef TDesc, Module *M, BackendAction Action, - raw_ostream *OS) { + raw_pwrite_stream *OS) { EmitAssemblyHelper AsmHelper(Diags, CGOpts, TOpts, LOpts, M); AsmHelper.EmitAssembly(Action, OS); @@ -647,9 +667,8 @@ void clang::EmitBackendOutput(DiagnosticsEngine &Diags, // If an optional clang TargetInfo description string was passed in, use it to // verify the LLVM TargetMachine's DataLayout. if (AsmHelper.TM && !TDesc.empty()) { - std::string DLDesc = AsmHelper.TM->getSubtargetImpl() - ->getDataLayout() - ->getStringRepresentation(); + std::string DLDesc = + AsmHelper.TM->getDataLayout()->getStringRepresentation(); if (DLDesc != TDesc) { unsigned DiagID = Diags.getCustomDiagID( DiagnosticsEngine::Error, "backend data layout '%0' does not match " diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp index daac174..fc4b66b 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGAtomic.cpp @@ -13,6 +13,7 @@ #include "CodeGenFunction.h" #include "CGCall.h" +#include "CGRecordLayout.h" #include "CodeGenModule.h" #include "clang/AST/ASTContext.h" #include "clang/CodeGen/CGFunctionInfo.h" @@ -36,34 +37,95 @@ namespace { CharUnits LValueAlign; TypeEvaluationKind EvaluationKind; bool UseLibcall; + LValue LVal; + CGBitFieldInfo BFI; public: - AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) { - assert(lvalue.isSimple()); - - AtomicTy = lvalue.getType(); - ValueTy = AtomicTy->castAs<AtomicType>()->getValueType(); - EvaluationKind = CGF.getEvaluationKind(ValueTy); - + AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) + : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), + EvaluationKind(TEK_Scalar), UseLibcall(true) { + assert(!lvalue.isGlobalReg()); ASTContext &C = CGF.getContext(); - - uint64_t ValueAlignInBits; - uint64_t AtomicAlignInBits; - TypeInfo ValueTI = C.getTypeInfo(ValueTy); - ValueSizeInBits = ValueTI.Width; - ValueAlignInBits = ValueTI.Align; - - TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); - AtomicSizeInBits = AtomicTI.Width; - AtomicAlignInBits = AtomicTI.Align; - - assert(ValueSizeInBits <= AtomicSizeInBits); - assert(ValueAlignInBits <= AtomicAlignInBits); - - AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); - ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); - if (lvalue.getAlignment().isZero()) - lvalue.setAlignment(AtomicAlign); - + if (lvalue.isSimple()) { + AtomicTy = lvalue.getType(); + if (auto *ATy = AtomicTy->getAs<AtomicType>()) + ValueTy = ATy->getValueType(); + else + ValueTy = AtomicTy; + EvaluationKind = CGF.getEvaluationKind(ValueTy); + + uint64_t ValueAlignInBits; + uint64_t AtomicAlignInBits; + TypeInfo ValueTI = C.getTypeInfo(ValueTy); + ValueSizeInBits = ValueTI.Width; + ValueAlignInBits = ValueTI.Align; + + TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); + AtomicSizeInBits = AtomicTI.Width; + AtomicAlignInBits = AtomicTI.Align; + + assert(ValueSizeInBits <= AtomicSizeInBits); + assert(ValueAlignInBits <= AtomicAlignInBits); + + AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); + ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); + if (lvalue.getAlignment().isZero()) + lvalue.setAlignment(AtomicAlign); + + LVal = lvalue; + } else if (lvalue.isBitField()) { + ValueTy = lvalue.getType(); + ValueSizeInBits = C.getTypeSize(ValueTy); + auto &OrigBFI = lvalue.getBitFieldInfo(); + auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); + AtomicSizeInBits = C.toBits( + C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) + .RoundUpToAlignment(lvalue.getAlignment())); + auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldAddr()); + auto OffsetInChars = + (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * + lvalue.getAlignment(); + VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( + VoidPtrAddr, OffsetInChars.getQuantity()); + auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + VoidPtrAddr, + CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), + "atomic_bitfield_base"); + BFI = OrigBFI; + BFI.Offset = Offset; + BFI.StorageSize = AtomicSizeInBits; + BFI.StorageOffset += OffsetInChars; + LVal = LValue::MakeBitfield(Addr, BFI, lvalue.getType(), + lvalue.getAlignment()); + LVal.setTBAAInfo(lvalue.getTBAAInfo()); + AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); + if (AtomicTy.isNull()) { + llvm::APInt Size( + /*numBits=*/32, + C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); + AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal, + /*IndexTypeQuals=*/0); + } + AtomicAlign = ValueAlign = lvalue.getAlignment(); + } else if (lvalue.isVectorElt()) { + ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType(); + ValueSizeInBits = C.getTypeSize(ValueTy); + AtomicTy = lvalue.getType(); + AtomicSizeInBits = C.getTypeSize(AtomicTy); + AtomicAlign = ValueAlign = lvalue.getAlignment(); + LVal = lvalue; + } else { + assert(lvalue.isExtVectorElt()); + ValueTy = lvalue.getType(); + ValueSizeInBits = C.getTypeSize(ValueTy); + AtomicTy = ValueTy = CGF.getContext().getExtVectorType( + lvalue.getType(), lvalue.getExtVectorAddr() + ->getType() + ->getPointerElementType() + ->getVectorNumElements()); + AtomicSizeInBits = C.getTypeSize(AtomicTy); + AtomicAlign = ValueAlign = lvalue.getAlignment(); + LVal = lvalue; + } UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( AtomicSizeInBits, C.toBits(lvalue.getAlignment())); } @@ -76,6 +138,17 @@ namespace { uint64_t getValueSizeInBits() const { return ValueSizeInBits; } TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } bool shouldUseLibcall() const { return UseLibcall; } + const LValue &getAtomicLValue() const { return LVal; } + llvm::Value *getAtomicAddress() const { + if (LVal.isSimple()) + return LVal.getAddress(); + else if (LVal.isBitField()) + return LVal.getBitFieldAddr(); + else if (LVal.isVectorElt()) + return LVal.getVectorAddr(); + assert(LVal.isExtVectorElt()); + return LVal.getExtVectorAddr(); + } /// Is the atomic size larger than the underlying value type? /// @@ -87,7 +160,7 @@ namespace { return (ValueSizeInBits != AtomicSizeInBits); } - bool emitMemSetZeroIfNecessary(LValue dest) const; + bool emitMemSetZeroIfNecessary() const; llvm::Value *getAtomicSizeValue() const { CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); @@ -99,37 +172,141 @@ namespace { llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const; /// Turn an atomic-layout object into an r-value. - RValue convertTempToRValue(llvm::Value *addr, - AggValueSlot resultSlot, - SourceLocation loc) const; + RValue convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot, + SourceLocation loc, bool AsValue) const; /// \brief Converts a rvalue to integer value. llvm::Value *convertRValueToInt(RValue RVal) const; - RValue convertIntToValue(llvm::Value *IntVal, AggValueSlot ResultSlot, - SourceLocation Loc) const; + RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, + AggValueSlot ResultSlot, + SourceLocation Loc, bool AsValue) const; /// Copy an atomic r-value into atomic-layout memory. - void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const; + void emitCopyIntoMemory(RValue rvalue) const; /// Project an l-value down to the value field. - LValue projectValue(LValue lvalue) const { - llvm::Value *addr = lvalue.getAddress(); + LValue projectValue() const { + assert(LVal.isSimple()); + llvm::Value *addr = getAtomicAddress(); if (hasPadding()) - addr = CGF.Builder.CreateStructGEP(addr, 0); + addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0); - return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(), - CGF.getContext(), lvalue.getTBAAInfo()); + return LValue::MakeAddr(addr, getValueType(), LVal.getAlignment(), + CGF.getContext(), LVal.getTBAAInfo()); } + /// \brief Emits atomic load. + /// \returns Loaded value. + RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, + bool AsValue, llvm::AtomicOrdering AO, + bool IsVolatile); + + /// \brief Emits atomic compare-and-exchange sequence. + /// \param Expected Expected value. + /// \param Desired Desired value. + /// \param Success Atomic ordering for success operation. + /// \param Failure Atomic ordering for failed operation. + /// \param IsWeak true if atomic operation is weak, false otherwise. + /// \returns Pair of values: previous value from storage (value type) and + /// boolean flag (i1 type) with true if success and false otherwise. + std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange( + RValue Expected, RValue Desired, + llvm::AtomicOrdering Success = llvm::SequentiallyConsistent, + llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent, + bool IsWeak = false); + + /// \brief Emits atomic update. + /// \param AO Atomic ordering. + /// \param UpdateOp Update operation for the current lvalue. + void EmitAtomicUpdate(llvm::AtomicOrdering AO, + const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile); + /// \brief Emits atomic update. + /// \param AO Atomic ordering. + void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, + bool IsVolatile); + /// Materialize an atomic r-value in atomic-layout memory. llvm::Value *materializeRValue(RValue rvalue) const; + /// \brief Translates LLVM atomic ordering to GNU atomic ordering for + /// libcalls. + static AtomicExpr::AtomicOrderingKind + translateAtomicOrdering(const llvm::AtomicOrdering AO); + private: bool requiresMemSetZero(llvm::Type *type) const; + + /// \brief Creates temp alloca for intermediate operations on atomic value. + llvm::Value *CreateTempAlloca() const; + + /// \brief Emits atomic load as a libcall. + void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, + llvm::AtomicOrdering AO, bool IsVolatile); + /// \brief Emits atomic load as LLVM instruction. + llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); + /// \brief Emits atomic compare-and-exchange op as a libcall. + llvm::Value *EmitAtomicCompareExchangeLibcall( + llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, + llvm::AtomicOrdering Success = llvm::SequentiallyConsistent, + llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent); + /// \brief Emits atomic compare-and-exchange op as LLVM instruction. + std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( + llvm::Value *ExpectedVal, llvm::Value *DesiredVal, + llvm::AtomicOrdering Success = llvm::SequentiallyConsistent, + llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent, + bool IsWeak = false); + /// \brief Emit atomic update as libcalls. + void + EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, + const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile); + /// \brief Emit atomic update as LLVM instructions. + void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, + const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile); + /// \brief Emit atomic update as libcalls. + void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, + bool IsVolatile); + /// \brief Emit atomic update as LLVM instructions. + void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, + bool IsVolatile); }; } +AtomicExpr::AtomicOrderingKind +AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) { + switch (AO) { + case llvm::Unordered: + case llvm::NotAtomic: + case llvm::Monotonic: + return AtomicExpr::AO_ABI_memory_order_relaxed; + case llvm::Acquire: + return AtomicExpr::AO_ABI_memory_order_acquire; + case llvm::Release: + return AtomicExpr::AO_ABI_memory_order_release; + case llvm::AcquireRelease: + return AtomicExpr::AO_ABI_memory_order_acq_rel; + case llvm::SequentiallyConsistent: + return AtomicExpr::AO_ABI_memory_order_seq_cst; + } + llvm_unreachable("Unhandled AtomicOrdering"); +} + +llvm::Value *AtomicInfo::CreateTempAlloca() const { + auto *TempAlloca = CGF.CreateMemTemp( + (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy + : AtomicTy, + "atomic-temp"); + TempAlloca->setAlignment(getAtomicAlignment().getQuantity()); + // Cast to pointer to value type for bitfields. + if (LVal.isBitField()) + return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + TempAlloca, getAtomicAddress()->getType()); + return TempAlloca; +} + static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, @@ -172,14 +349,16 @@ bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { llvm_unreachable("bad evaluation kind"); } -bool AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const { - llvm::Value *addr = dest.getAddress(); +bool AtomicInfo::emitMemSetZeroIfNecessary() const { + assert(LVal.isSimple()); + llvm::Value *addr = LVal.getAddress(); if (!requiresMemSetZero(addr->getType()->getPointerElementType())) return false; - CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), - AtomicSizeInBits / 8, - dest.getAlignment().getQuantity()); + CGF.Builder.CreateMemSet( + addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), + CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), + LVal.getAlignment().getQuantity()); return true; } @@ -520,7 +699,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { switch (E->getOp()) { case AtomicExpr::AO__c11_atomic_init: - llvm_unreachable("Already handled!"); + llvm_unreachable("Already handled above with EmitAtomicInit!"); case AtomicExpr::AO__c11_atomic_load: case AtomicExpr::AO__atomic_load_n: @@ -606,20 +785,43 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { if (UseLibcall) { bool UseOptimizedLibcall = false; switch (E->getOp()) { + case AtomicExpr::AO__c11_atomic_init: + llvm_unreachable("Already handled above with EmitAtomicInit!"); + case AtomicExpr::AO__c11_atomic_fetch_add: case AtomicExpr::AO__atomic_fetch_add: case AtomicExpr::AO__c11_atomic_fetch_and: case AtomicExpr::AO__atomic_fetch_and: case AtomicExpr::AO__c11_atomic_fetch_or: case AtomicExpr::AO__atomic_fetch_or: + case AtomicExpr::AO__atomic_fetch_nand: case AtomicExpr::AO__c11_atomic_fetch_sub: case AtomicExpr::AO__atomic_fetch_sub: case AtomicExpr::AO__c11_atomic_fetch_xor: case AtomicExpr::AO__atomic_fetch_xor: + case AtomicExpr::AO__atomic_add_fetch: + case AtomicExpr::AO__atomic_and_fetch: + case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__atomic_or_fetch: + case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__atomic_xor_fetch: // For these, only library calls for certain sizes exist. UseOptimizedLibcall = true; break; - default: + + case AtomicExpr::AO__c11_atomic_load: + case AtomicExpr::AO__c11_atomic_store: + case AtomicExpr::AO__c11_atomic_exchange: + case AtomicExpr::AO__c11_atomic_compare_exchange_weak: + case AtomicExpr::AO__c11_atomic_compare_exchange_strong: + case AtomicExpr::AO__atomic_load_n: + case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__atomic_store_n: + case AtomicExpr::AO__atomic_store: + case AtomicExpr::AO__atomic_exchange_n: + case AtomicExpr::AO__atomic_exchange: + case AtomicExpr::AO__atomic_compare_exchange_n: + case AtomicExpr::AO__atomic_compare_exchange: // Only use optimized library calls for sizes for which they exist. if (Size == 1 || Size == 2 || Size == 4 || Size == 8) UseOptimizedLibcall = true; @@ -641,6 +843,9 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { QualType RetTy; bool HaveRetTy = false; switch (E->getOp()) { + case AtomicExpr::AO__c11_atomic_init: + llvm_unreachable("Already handled!"); + // There is only one libcall for compare an exchange, because there is no // optimisation benefit possible from a libcall version of a weak compare // and exchange. @@ -724,7 +929,49 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, E->getExprLoc(), sizeChars); break; - default: return EmitUnsupportedRValue(E, "atomic library call"); + // T __atomic_fetch_nand_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_fetch_nand: + LibCallName = "__atomic_fetch_nand"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, + E->getExprLoc(), sizeChars); + break; + + // T __atomic_add_fetch_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_add_fetch: + LibCallName = "__atomic_add_fetch"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy, + E->getExprLoc(), sizeChars); + break; + // T __atomic_and_fetch_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_and_fetch: + LibCallName = "__atomic_and_fetch"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, + E->getExprLoc(), sizeChars); + break; + // T __atomic_or_fetch_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_or_fetch: + LibCallName = "__atomic_or_fetch"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, + E->getExprLoc(), sizeChars); + break; + // T __atomic_sub_fetch_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_sub_fetch: + LibCallName = "__atomic_sub_fetch"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy, + E->getExprLoc(), sizeChars); + break; + // T __atomic_xor_fetch_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_xor_fetch: + LibCallName = "__atomic_xor_fetch"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, + E->getExprLoc(), sizeChars); + break; + // T __atomic_nand_fetch_N(T *mem, T val, int order) + case AtomicExpr::AO__atomic_nand_fetch: + LibCallName = "__atomic_nand_fetch"; + AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy, + E->getExprLoc(), sizeChars); + break; } // Optimized functions have the size in their name. @@ -901,29 +1148,53 @@ llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const { RValue AtomicInfo::convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot, - SourceLocation loc) const { - if (EvaluationKind == TEK_Aggregate) - return resultSlot.asRValue(); - - // Drill into the padding structure if we have one. - if (hasPadding()) - addr = CGF.Builder.CreateStructGEP(addr, 0); - - // Otherwise, just convert the temporary to an r-value using the - // normal conversion routine. - return CGF.convertTempToRValue(addr, getValueType(), loc); + SourceLocation loc, bool AsValue) const { + if (LVal.isSimple()) { + if (EvaluationKind == TEK_Aggregate) + return resultSlot.asRValue(); + + // Drill into the padding structure if we have one. + if (hasPadding()) + addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0); + + // Otherwise, just convert the temporary to an r-value using the + // normal conversion routine. + return CGF.convertTempToRValue(addr, getValueType(), loc); + } + if (!AsValue) + // Get RValue from temp memory as atomic for non-simple lvalues + return RValue::get( + CGF.Builder.CreateAlignedLoad(addr, AtomicAlign.getQuantity())); + if (LVal.isBitField()) + return CGF.EmitLoadOfBitfieldLValue(LValue::MakeBitfield( + addr, LVal.getBitFieldInfo(), LVal.getType(), LVal.getAlignment())); + if (LVal.isVectorElt()) + return CGF.EmitLoadOfLValue(LValue::MakeVectorElt(addr, LVal.getVectorIdx(), + LVal.getType(), + LVal.getAlignment()), + loc); + assert(LVal.isExtVectorElt()); + return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( + addr, LVal.getExtVectorElts(), LVal.getType(), LVal.getAlignment())); } -RValue AtomicInfo::convertIntToValue(llvm::Value *IntVal, - AggValueSlot ResultSlot, - SourceLocation Loc) const { +RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, + AggValueSlot ResultSlot, + SourceLocation Loc, + bool AsValue) const { // Try not to in some easy cases. assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); - if (getEvaluationKind() == TEK_Scalar && !hasPadding()) { - auto *ValTy = CGF.ConvertTypeForMem(ValueTy); + if (getEvaluationKind() == TEK_Scalar && + (((!LVal.isBitField() || + LVal.getBitFieldInfo().Size == ValueSizeInBits) && + !hasPadding()) || + !AsValue)) { + auto *ValTy = AsValue + ? CGF.ConvertTypeForMem(ValueTy) + : getAtomicAddress()->getType()->getPointerElementType(); if (ValTy->isIntegerTy()) { assert(IntVal->getType() == ValTy && "Different integer types."); - return RValue::get(IntVal); + return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); } else if (ValTy->isPointerTy()) return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) @@ -935,13 +1206,13 @@ RValue AtomicInfo::convertIntToValue(llvm::Value *IntVal, llvm::Value *Temp; bool TempIsVolatile = false; CharUnits TempAlignment; - if (getEvaluationKind() == TEK_Aggregate) { + if (AsValue && getEvaluationKind() == TEK_Aggregate) { assert(!ResultSlot.isIgnored()); Temp = ResultSlot.getAddr(); TempAlignment = getValueAlignment(); TempIsVolatile = ResultSlot.isVolatile(); } else { - Temp = CGF.CreateMemTemp(getAtomicType(), "atomic-temp"); + Temp = CreateTempAlloca(); TempAlignment = getAtomicAlignment(); } @@ -950,93 +1221,146 @@ RValue AtomicInfo::convertIntToValue(llvm::Value *IntVal, CGF.Builder.CreateAlignedStore(IntVal, CastTemp, TempAlignment.getQuantity()) ->setVolatile(TempIsVolatile); - return convertTempToRValue(Temp, ResultSlot, Loc); + return convertTempToRValue(Temp, ResultSlot, Loc, AsValue); } -/// Emit a load from an l-value of atomic type. Note that the r-value -/// we produce is an r-value of the atomic *value* type. -RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, - AggValueSlot resultSlot) { - AtomicInfo atomics(*this, src); +void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, + llvm::AtomicOrdering AO, bool) { + // void __atomic_load(size_t size, void *mem, void *return, int order); + CallArgList Args; + Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); + Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())), + CGF.getContext().VoidPtrTy); + Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), + CGF.getContext().VoidPtrTy); + Args.add(RValue::get( + llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))), + CGF.getContext().IntTy); + emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); +} - // Check whether we should use a library call. - if (atomics.shouldUseLibcall()) { - llvm::Value *tempAddr; - if (!resultSlot.isIgnored()) { - assert(atomics.getEvaluationKind() == TEK_Aggregate); - tempAddr = resultSlot.getAddr(); - } else { - tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp"); - } +llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, + bool IsVolatile) { + // Okay, we're doing this natively. + llvm::Value *Addr = emitCastToAtomicIntPointer(getAtomicAddress()); + llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); + Load->setAtomic(AO); - // void __atomic_load(size_t size, void *mem, void *return, int order); - CallArgList args; - args.add(RValue::get(atomics.getAtomicSizeValue()), - getContext().getSizeType()); - args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())), - getContext().VoidPtrTy); - args.add(RValue::get(EmitCastToVoidPtr(tempAddr)), - getContext().VoidPtrTy); - args.add(RValue::get(llvm::ConstantInt::get( - IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)), - getContext().IntTy); - emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args); + // Other decoration. + Load->setAlignment(getAtomicAlignment().getQuantity()); + if (IsVolatile) + Load->setVolatile(true); + if (LVal.getTBAAInfo()) + CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo()); + return Load; +} + +/// An LValue is a candidate for having its loads and stores be made atomic if +/// we are operating under /volatile:ms *and* the LValue itself is volatile and +/// performing such an operation can be performed without a libcall. +bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { + AtomicInfo AI(*this, LV); + bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); + // An atomic is inline if we don't need to use a libcall. + bool AtomicIsInline = !AI.shouldUseLibcall(); + return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline; +} + +/// An type is a candidate for having its loads and stores be made atomic if +/// we are operating under /volatile:ms *and* we know the access is volatile and +/// performing such an operation can be performed without a libcall. +bool CodeGenFunction::typeIsSuitableForInlineAtomic(QualType Ty, + bool IsVolatile) const { + // An atomic is inline if we don't need to use a libcall (e.g. it is builtin). + bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic( + getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty)); + return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline; +} - // Produce the r-value. - return atomics.convertTempToRValue(tempAddr, resultSlot, loc); +RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, + AggValueSlot Slot) { + llvm::AtomicOrdering AO; + bool IsVolatile = LV.isVolatileQualified(); + if (LV.getType()->isAtomicType()) { + AO = llvm::SequentiallyConsistent; + } else { + AO = llvm::Acquire; + IsVolatile = true; } + return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); +} - // Okay, we're doing this natively. - llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress()); - llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load"); - load->setAtomic(llvm::SequentiallyConsistent); +RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, + bool AsValue, llvm::AtomicOrdering AO, + bool IsVolatile) { + // Check whether we should use a library call. + if (shouldUseLibcall()) { + llvm::Value *TempAddr; + if (LVal.isSimple() && !ResultSlot.isIgnored()) { + assert(getEvaluationKind() == TEK_Aggregate); + TempAddr = ResultSlot.getAddr(); + } else + TempAddr = CreateTempAlloca(); + + EmitAtomicLoadLibcall(TempAddr, AO, IsVolatile); + + // Okay, turn that back into the original value or whole atomic (for + // non-simple lvalues) type. + return convertTempToRValue(TempAddr, ResultSlot, Loc, AsValue); + } - // Other decoration. - load->setAlignment(src.getAlignment().getQuantity()); - if (src.isVolatileQualified()) - load->setVolatile(true); - if (src.getTBAAInfo()) - CGM.DecorateInstruction(load, src.getTBAAInfo()); + // Okay, we're doing this natively. + auto *Load = EmitAtomicLoadOp(AO, IsVolatile); // If we're ignoring an aggregate return, don't do anything. - if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored()) + if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) return RValue::getAggregate(nullptr, false); - // Okay, turn that back into the original value type. - return atomics.convertIntToValue(load, resultSlot, loc); + // Okay, turn that back into the original value or atomic (for non-simple + // lvalues) type. + return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); } - +/// Emit a load from an l-value of atomic type. Note that the r-value +/// we produce is an r-value of the atomic *value* type. +RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, + llvm::AtomicOrdering AO, bool IsVolatile, + AggValueSlot resultSlot) { + AtomicInfo Atomics(*this, src); + return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, + IsVolatile); +} /// Copy an r-value into memory as part of storing to an atomic type. /// This needs to create a bit-pattern suitable for atomic operations. -void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const { +void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { + assert(LVal.isSimple()); // If we have an r-value, the rvalue should be of the atomic type, // which means that the caller is responsible for having zeroed // any padding. Just do an aggregate copy of that type. if (rvalue.isAggregate()) { - CGF.EmitAggregateCopy(dest.getAddress(), + CGF.EmitAggregateCopy(getAtomicAddress(), rvalue.getAggregateAddr(), getAtomicType(), (rvalue.isVolatileQualified() - || dest.isVolatileQualified()), - dest.getAlignment()); + || LVal.isVolatileQualified()), + LVal.getAlignment()); return; } // Okay, otherwise we're copying stuff. // Zero out the buffer if necessary. - emitMemSetZeroIfNecessary(dest); + emitMemSetZeroIfNecessary(); // Drill past the padding if present. - dest = projectValue(dest); + LValue TempLVal = projectValue(); // Okay, store the rvalue in. if (rvalue.isScalar()) { - CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true); + CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); } else { - CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true); + CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); } } @@ -1050,22 +1374,24 @@ llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const { return rvalue.getAggregateAddr(); // Otherwise, make a temporary and materialize into it. - llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp"); - LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment()); - emitCopyIntoMemory(rvalue, tempLV); - return temp; + LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType(), + getAtomicAlignment()); + AtomicInfo Atomics(CGF, TempLV); + Atomics.emitCopyIntoMemory(rvalue); + return TempLV.getAddress(); } llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { // If we've got a scalar value of the right size, try to avoid going // through memory. - if (RVal.isScalar() && !hasPadding()) { + if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { llvm::Value *Value = RVal.getScalarVal(); if (isa<llvm::IntegerType>(Value->getType())) - return Value; + return CGF.EmitToMemory(Value, ValueTy); else { - llvm::IntegerType *InputIntTy = - llvm::IntegerType::get(CGF.getLLVMContext(), getValueSizeInBits()); + llvm::IntegerType *InputIntTy = llvm::IntegerType::get( + CGF.getLLVMContext(), + LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); if (isa<llvm::PointerType>(Value->getType())) return CGF.Builder.CreatePtrToInt(Value, InputIntTy); else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) @@ -1082,12 +1408,324 @@ llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { getAtomicAlignment().getQuantity()); } +std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( + llvm::Value *ExpectedVal, llvm::Value *DesiredVal, + llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { + // Do the atomic store. + auto *Addr = emitCastToAtomicIntPointer(getAtomicAddress()); + auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal, + Success, Failure); + // Other decoration. + Inst->setVolatile(LVal.isVolatileQualified()); + Inst->setWeak(IsWeak); + + // Okay, turn that back into the original value type. + auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); + auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); + return std::make_pair(PreviousVal, SuccessFailureVal); +} + +llvm::Value * +AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, + llvm::Value *DesiredAddr, + llvm::AtomicOrdering Success, + llvm::AtomicOrdering Failure) { + // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, + // void *desired, int success, int failure); + CallArgList Args; + Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); + Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())), + CGF.getContext().VoidPtrTy); + Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), + CGF.getContext().VoidPtrTy); + Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), + CGF.getContext().VoidPtrTy); + Args.add(RValue::get(llvm::ConstantInt::get( + CGF.IntTy, translateAtomicOrdering(Success))), + CGF.getContext().IntTy); + Args.add(RValue::get(llvm::ConstantInt::get( + CGF.IntTy, translateAtomicOrdering(Failure))), + CGF.getContext().IntTy); + auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", + CGF.getContext().BoolTy, Args); + + return SuccessFailureRVal.getScalarVal(); +} + +std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( + RValue Expected, RValue Desired, llvm::AtomicOrdering Success, + llvm::AtomicOrdering Failure, bool IsWeak) { + if (Failure >= Success) + // Don't assert on undefined behavior. + Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); + + // Check whether we should use a library call. + if (shouldUseLibcall()) { + // Produce a source address. + auto *ExpectedAddr = materializeRValue(Expected); + auto *DesiredAddr = materializeRValue(Desired); + auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, + Success, Failure); + return std::make_pair( + convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false), + Res); + } + + // If we've got a scalar value of the right size, try to avoid going + // through memory. + auto *ExpectedVal = convertRValueToInt(Expected); + auto *DesiredVal = convertRValueToInt(Desired); + auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, + Failure, IsWeak); + return std::make_pair( + ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false), + Res.second); +} + +static void +EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, + const llvm::function_ref<RValue(RValue)> &UpdateOp, + llvm::Value *DesiredAddr) { + llvm::Value *Ptr = nullptr; + LValue UpdateLVal; + RValue UpRVal; + LValue AtomicLVal = Atomics.getAtomicLValue(); + LValue DesiredLVal; + if (AtomicLVal.isSimple()) { + UpRVal = OldRVal; + DesiredLVal = + LValue::MakeAddr(DesiredAddr, AtomicLVal.getType(), + AtomicLVal.getAlignment(), CGF.CGM.getContext()); + } else { + // Build new lvalue for temp address + Ptr = Atomics.materializeRValue(OldRVal); + if (AtomicLVal.isBitField()) { + UpdateLVal = + LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), + AtomicLVal.getType(), AtomicLVal.getAlignment()); + DesiredLVal = + LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), + AtomicLVal.getType(), AtomicLVal.getAlignment()); + } else if (AtomicLVal.isVectorElt()) { + UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), + AtomicLVal.getType(), + AtomicLVal.getAlignment()); + DesiredLVal = LValue::MakeVectorElt( + DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), + AtomicLVal.getAlignment()); + } else { + assert(AtomicLVal.isExtVectorElt()); + UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), + AtomicLVal.getType(), + AtomicLVal.getAlignment()); + DesiredLVal = LValue::MakeExtVectorElt( + DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), + AtomicLVal.getAlignment()); + } + UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); + DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); + UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); + } + // Store new value in the corresponding memory area + RValue NewRVal = UpdateOp(UpRVal); + if (NewRVal.isScalar()) { + CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); + } else { + assert(NewRVal.isComplex()); + CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, + /*isInit=*/false); + } +} + +void AtomicInfo::EmitAtomicUpdateLibcall( + llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile) { + auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); + + llvm::Value *ExpectedAddr = CreateTempAlloca(); + + EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile); + auto *ContBB = CGF.createBasicBlock("atomic_cont"); + auto *ExitBB = CGF.createBasicBlock("atomic_exit"); + CGF.EmitBlock(ContBB); + auto *DesiredAddr = CreateTempAlloca(); + if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || + requiresMemSetZero( + getAtomicAddress()->getType()->getPointerElementType())) { + auto *OldVal = CGF.Builder.CreateAlignedLoad( + ExpectedAddr, getAtomicAlignment().getQuantity()); + CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr, + getAtomicAlignment().getQuantity()); + } + auto OldRVal = convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false); + EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); + auto *Res = + EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure); + CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); + CGF.EmitBlock(ExitBB, /*IsFinished=*/true); +} + +void AtomicInfo::EmitAtomicUpdateOp( + llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile) { + auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); + + // Do the atomic load. + auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); + // For non-simple lvalues perform compare-and-swap procedure. + auto *ContBB = CGF.createBasicBlock("atomic_cont"); + auto *ExitBB = CGF.createBasicBlock("atomic_exit"); + auto *CurBB = CGF.Builder.GetInsertBlock(); + CGF.EmitBlock(ContBB); + llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), + /*NumReservedValues=*/2); + PHI->addIncoming(OldVal, CurBB); + auto *NewAtomicAddr = CreateTempAlloca(); + auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); + if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || + requiresMemSetZero( + getAtomicAddress()->getType()->getPointerElementType())) { + CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr, + getAtomicAlignment().getQuantity()); + } + auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false); + EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); + auto *DesiredVal = CGF.Builder.CreateAlignedLoad( + NewAtomicIntAddr, getAtomicAlignment().getQuantity()); + // Try to write new value using cmpxchg operation + auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); + PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); + CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); + CGF.EmitBlock(ExitBB, /*IsFinished=*/true); +} + +static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, + RValue UpdateRVal, llvm::Value *DesiredAddr) { + LValue AtomicLVal = Atomics.getAtomicLValue(); + LValue DesiredLVal; + // Build new lvalue for temp address + if (AtomicLVal.isBitField()) { + DesiredLVal = + LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), + AtomicLVal.getType(), AtomicLVal.getAlignment()); + } else if (AtomicLVal.isVectorElt()) { + DesiredLVal = + LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), + AtomicLVal.getType(), AtomicLVal.getAlignment()); + } else { + assert(AtomicLVal.isExtVectorElt()); + DesiredLVal = LValue::MakeExtVectorElt( + DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), + AtomicLVal.getAlignment()); + } + DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); + // Store new value in the corresponding memory area + assert(UpdateRVal.isScalar()); + CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); +} + +void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, + RValue UpdateRVal, bool IsVolatile) { + auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); + + llvm::Value *ExpectedAddr = CreateTempAlloca(); + + EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile); + auto *ContBB = CGF.createBasicBlock("atomic_cont"); + auto *ExitBB = CGF.createBasicBlock("atomic_exit"); + CGF.EmitBlock(ContBB); + auto *DesiredAddr = CreateTempAlloca(); + if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || + requiresMemSetZero( + getAtomicAddress()->getType()->getPointerElementType())) { + auto *OldVal = CGF.Builder.CreateAlignedLoad( + ExpectedAddr, getAtomicAlignment().getQuantity()); + CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr, + getAtomicAlignment().getQuantity()); + } + EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); + auto *Res = + EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure); + CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); + CGF.EmitBlock(ExitBB, /*IsFinished=*/true); +} + +void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, + bool IsVolatile) { + auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); + + // Do the atomic load. + auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); + // For non-simple lvalues perform compare-and-swap procedure. + auto *ContBB = CGF.createBasicBlock("atomic_cont"); + auto *ExitBB = CGF.createBasicBlock("atomic_exit"); + auto *CurBB = CGF.Builder.GetInsertBlock(); + CGF.EmitBlock(ContBB); + llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), + /*NumReservedValues=*/2); + PHI->addIncoming(OldVal, CurBB); + auto *NewAtomicAddr = CreateTempAlloca(); + auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); + if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || + requiresMemSetZero( + getAtomicAddress()->getType()->getPointerElementType())) { + CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr, + getAtomicAlignment().getQuantity()); + } + EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); + auto *DesiredVal = CGF.Builder.CreateAlignedLoad( + NewAtomicIntAddr, getAtomicAlignment().getQuantity()); + // Try to write new value using cmpxchg operation + auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); + PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); + CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); + CGF.EmitBlock(ExitBB, /*IsFinished=*/true); +} + +void AtomicInfo::EmitAtomicUpdate( + llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile) { + if (shouldUseLibcall()) { + EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); + } else { + EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); + } +} + +void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, + bool IsVolatile) { + if (shouldUseLibcall()) { + EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); + } else { + EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); + } +} + +void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, + bool isInit) { + bool IsVolatile = lvalue.isVolatileQualified(); + llvm::AtomicOrdering AO; + if (lvalue.getType()->isAtomicType()) { + AO = llvm::SequentiallyConsistent; + } else { + AO = llvm::Release; + IsVolatile = true; + } + return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); +} + /// Emit a store to an l-value of atomic type. /// /// Note that the r-value is expected to be an r-value *of the atomic /// type*; this means that for aggregate r-values, it should include /// storage for any padding that was necessary. -void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) { +void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, + llvm::AtomicOrdering AO, bool IsVolatile, + bool isInit) { // If this is an aggregate r-value, it should agree in type except // maybe for address-space qualification. assert(!rvalue.isAggregate() || @@ -1095,54 +1733,64 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) { == dest.getAddress()->getType()->getPointerElementType()); AtomicInfo atomics(*this, dest); + LValue LVal = atomics.getAtomicLValue(); // If this is an initialization, just put the value there normally. - if (isInit) { - atomics.emitCopyIntoMemory(rvalue, dest); - return; - } + if (LVal.isSimple()) { + if (isInit) { + atomics.emitCopyIntoMemory(rvalue); + return; + } - // Check whether we should use a library call. - if (atomics.shouldUseLibcall()) { - // Produce a source address. - llvm::Value *srcAddr = atomics.materializeRValue(rvalue); + // Check whether we should use a library call. + if (atomics.shouldUseLibcall()) { + // Produce a source address. + llvm::Value *srcAddr = atomics.materializeRValue(rvalue); - // void __atomic_store(size_t size, void *mem, void *val, int order) - CallArgList args; - args.add(RValue::get(atomics.getAtomicSizeValue()), - getContext().getSizeType()); - args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())), - getContext().VoidPtrTy); - args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), - getContext().VoidPtrTy); - args.add(RValue::get(llvm::ConstantInt::get( - IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)), - getContext().IntTy); - emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); + // void __atomic_store(size_t size, void *mem, void *val, int order) + CallArgList args; + args.add(RValue::get(atomics.getAtomicSizeValue()), + getContext().getSizeType()); + args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicAddress())), + getContext().VoidPtrTy); + args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), getContext().VoidPtrTy); + args.add(RValue::get(llvm::ConstantInt::get( + IntTy, AtomicInfo::translateAtomicOrdering(AO))), + getContext().IntTy); + emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); + return; + } + + // Okay, we're doing this natively. + llvm::Value *intValue = atomics.convertRValueToInt(rvalue); + + // Do the atomic store. + llvm::Value *addr = + atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); + intValue = Builder.CreateIntCast( + intValue, addr->getType()->getPointerElementType(), /*isSigned=*/false); + llvm::StoreInst *store = Builder.CreateStore(intValue, addr); + + // Initializations don't need to be atomic. + if (!isInit) + store->setAtomic(AO); + + // Other decoration. + store->setAlignment(dest.getAlignment().getQuantity()); + if (IsVolatile) + store->setVolatile(true); + if (dest.getTBAAInfo()) + CGM.DecorateInstruction(store, dest.getTBAAInfo()); return; } - // Okay, we're doing this natively. - llvm::Value *intValue = atomics.convertRValueToInt(rvalue); - - // Do the atomic store. - llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress()); - llvm::StoreInst *store = Builder.CreateStore(intValue, addr); - - // Initializations don't need to be atomic. - if (!isInit) store->setAtomic(llvm::SequentiallyConsistent); - - // Other decoration. - store->setAlignment(dest.getAlignment().getQuantity()); - if (dest.isVolatileQualified()) - store->setVolatile(true); - if (dest.getTBAAInfo()) - CGM.DecorateInstruction(store, dest.getTBAAInfo()); + // Emit simple atomic update operation. + atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); } /// Emit a compare-and-exchange op for atomic type. /// -std::pair<RValue, RValue> CodeGenFunction::EmitAtomicCompareExchange( +std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, AggValueSlot Slot) { @@ -1156,56 +1804,15 @@ std::pair<RValue, RValue> CodeGenFunction::EmitAtomicCompareExchange( Obj.getAddress()->getType()->getPointerElementType()); AtomicInfo Atomics(*this, Obj); - if (Failure >= Success) - // Don't assert on undefined behavior. - Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); - - auto Alignment = Atomics.getValueAlignment(); - // Check whether we should use a library call. - if (Atomics.shouldUseLibcall()) { - auto *ExpectedAddr = Atomics.materializeRValue(Expected); - // Produce a source address. - auto *DesiredAddr = Atomics.materializeRValue(Desired); - // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, - // void *desired, int success, int failure); - CallArgList Args; - Args.add(RValue::get(Atomics.getAtomicSizeValue()), - getContext().getSizeType()); - Args.add(RValue::get(EmitCastToVoidPtr(Obj.getAddress())), - getContext().VoidPtrTy); - Args.add(RValue::get(EmitCastToVoidPtr(ExpectedAddr)), - getContext().VoidPtrTy); - Args.add(RValue::get(EmitCastToVoidPtr(DesiredAddr)), - getContext().VoidPtrTy); - Args.add(RValue::get(llvm::ConstantInt::get(IntTy, Success)), - getContext().IntTy); - Args.add(RValue::get(llvm::ConstantInt::get(IntTy, Failure)), - getContext().IntTy); - auto SuccessFailureRVal = emitAtomicLibcall( - *this, "__atomic_compare_exchange", getContext().BoolTy, Args); - auto *PreviousVal = - Builder.CreateAlignedLoad(ExpectedAddr, Alignment.getQuantity()); - return std::make_pair(RValue::get(PreviousVal), SuccessFailureRVal); - } - - // If we've got a scalar value of the right size, try to avoid going - // through memory. - auto *ExpectedIntVal = Atomics.convertRValueToInt(Expected); - auto *DesiredIntVal = Atomics.convertRValueToInt(Desired); - - // Do the atomic store. - auto *Addr = Atomics.emitCastToAtomicIntPointer(Obj.getAddress()); - auto *Inst = Builder.CreateAtomicCmpXchg(Addr, ExpectedIntVal, DesiredIntVal, - Success, Failure); - // Other decoration. - Inst->setVolatile(Obj.isVolatileQualified()); - Inst->setWeak(IsWeak); + return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, + IsWeak); +} - // Okay, turn that back into the original value type. - auto *PreviousVal = Builder.CreateExtractValue(Inst, /*Idxs=*/0); - auto *SuccessFailureVal = Builder.CreateExtractValue(Inst, /*Idxs=*/1); - return std::make_pair(Atomics.convertIntToValue(PreviousVal, Slot, Loc), - RValue::get(SuccessFailureVal)); +void CodeGenFunction::EmitAtomicUpdate( + LValue LVal, llvm::AtomicOrdering AO, + const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { + AtomicInfo Atomics(*this, LVal); + Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); } void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { @@ -1214,13 +1821,13 @@ void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { switch (atomics.getEvaluationKind()) { case TEK_Scalar: { llvm::Value *value = EmitScalarExpr(init); - atomics.emitCopyIntoMemory(RValue::get(value), dest); + atomics.emitCopyIntoMemory(RValue::get(value)); return; } case TEK_Complex: { ComplexPairTy value = EmitComplexExpr(init); - atomics.emitCopyIntoMemory(RValue::getComplex(value), dest); + atomics.emitCopyIntoMemory(RValue::getComplex(value)); return; } @@ -1229,8 +1836,8 @@ void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { // of atomic type. bool Zeroed = false; if (!init->getType()->isAtomicType()) { - Zeroed = atomics.emitMemSetZeroIfNecessary(dest); - dest = atomics.projectValue(dest); + Zeroed = atomics.emitMemSetZeroIfNecessary(); + dest = atomics.projectValue(); } // Evaluate the expression directly into the destination. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp index b98460a..3fd344c 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBlocks.cpp @@ -621,8 +621,8 @@ static void enterBlockScope(CodeGenFunction &CGF, BlockDecl *block) { } // GEP down to the address. - llvm::Value *addr = CGF.Builder.CreateStructGEP(blockInfo.Address, - capture.getIndex()); + llvm::Value *addr = CGF.Builder.CreateStructGEP( + blockInfo.StructureType, blockInfo.Address, capture.getIndex()); // We can use that GEP as the dominating IP. if (!blockInfo.DominatingIP) @@ -721,6 +721,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // Build the block descriptor. llvm::Constant *descriptor = buildBlockDescriptor(CGM, blockInfo); + llvm::Type *blockTy = blockInfo.StructureType; llvm::AllocaInst *blockAddr = blockInfo.Address; assert(blockAddr && "block has no address!"); @@ -732,14 +733,17 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { if (blockInfo.UsesStret) flags |= BLOCK_USE_STRET; // Initialize the block literal. - Builder.CreateStore(isa, Builder.CreateStructGEP(blockAddr, 0, "block.isa")); - Builder.CreateStore(llvm::ConstantInt::get(IntTy, flags.getBitMask()), - Builder.CreateStructGEP(blockAddr, 1, "block.flags")); - Builder.CreateStore(llvm::ConstantInt::get(IntTy, 0), - Builder.CreateStructGEP(blockAddr, 2, "block.reserved")); - Builder.CreateStore(blockFn, Builder.CreateStructGEP(blockAddr, 3, - "block.invoke")); - Builder.CreateStore(descriptor, Builder.CreateStructGEP(blockAddr, 4, + Builder.CreateStore( + isa, Builder.CreateStructGEP(blockTy, blockAddr, 0, "block.isa")); + Builder.CreateStore( + llvm::ConstantInt::get(IntTy, flags.getBitMask()), + Builder.CreateStructGEP(blockTy, blockAddr, 1, "block.flags")); + Builder.CreateStore( + llvm::ConstantInt::get(IntTy, 0), + Builder.CreateStructGEP(blockTy, blockAddr, 2, "block.reserved")); + Builder.CreateStore( + blockFn, Builder.CreateStructGEP(blockTy, blockAddr, 3, "block.invoke")); + Builder.CreateStore(descriptor, Builder.CreateStructGEP(blockTy, blockAddr, 4, "block.descriptor")); // Finally, capture all the values into the block. @@ -747,9 +751,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // First, 'this'. if (blockDecl->capturesCXXThis()) { - llvm::Value *addr = Builder.CreateStructGEP(blockAddr, - blockInfo.CXXThisIndex, - "block.captured-this.addr"); + llvm::Value *addr = Builder.CreateStructGEP( + blockTy, blockAddr, blockInfo.CXXThisIndex, "block.captured-this.addr"); Builder.CreateStore(LoadCXXThis(), addr); } @@ -766,9 +769,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // This will be a [[type]]*, except that a byref entry will just be // an i8**. - llvm::Value *blockField = - Builder.CreateStructGEP(blockAddr, capture.getIndex(), - "block.captured"); + llvm::Value *blockField = Builder.CreateStructGEP( + blockTy, blockAddr, capture.getIndex(), "block.captured"); // Compute the address of the thing we're going to move into the // block literal. @@ -779,7 +781,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { BlockInfo->getCapture(variable); // This is a [[type]]*, except that a byref entry wil just be an i8**. - src = Builder.CreateStructGEP(LoadBlockStruct(), + src = Builder.CreateStructGEP(BlockInfo->StructureType, LoadBlockStruct(), enclosingCapture.getIndex(), "block.capture.addr"); } else if (blockDecl->isConversionFromLambda()) { @@ -964,7 +966,8 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, Builder.CreateBitCast(Callee, BlockLiteralTy, "block.literal"); // Get the function pointer from the literal. - llvm::Value *FuncPtr = Builder.CreateStructGEP(BlockLiteral, 3); + llvm::Value *FuncPtr = Builder.CreateStructGEP( + CGM.getGenericBlockLiteralType(), BlockLiteral, 3); BlockLiteral = Builder.CreateBitCast(BlockLiteral, VoidPtrTy); @@ -1004,26 +1007,27 @@ llvm::Value *CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable, if (capture.isConstant()) return LocalDeclMap[variable]; llvm::Value *addr = - Builder.CreateStructGEP(LoadBlockStruct(), capture.getIndex(), - "block.capture.addr"); + Builder.CreateStructGEP(BlockInfo->StructureType, LoadBlockStruct(), + capture.getIndex(), "block.capture.addr"); if (isByRef) { // addr should be a void** right now. Load, then cast the result // to byref*. addr = Builder.CreateLoad(addr); - llvm::PointerType *byrefPointerType - = llvm::PointerType::get(BuildByRefType(variable), 0); + auto *byrefType = BuildByRefType(variable); + llvm::PointerType *byrefPointerType = llvm::PointerType::get(byrefType, 0); addr = Builder.CreateBitCast(addr, byrefPointerType, "byref.addr"); // Follow the forwarding pointer. - addr = Builder.CreateStructGEP(addr, 1, "byref.forwarding"); + addr = Builder.CreateStructGEP(byrefType, addr, 1, "byref.forwarding"); addr = Builder.CreateLoad(addr, "byref.addr.forwarded"); // Cast back to byref* and GEP over to the actual object. addr = Builder.CreateBitCast(addr, byrefPointerType); - addr = Builder.CreateStructGEP(addr, getByRefValueLLVMField(variable), + addr = Builder.CreateStructGEP(byrefType, addr, + getByRefValueLLVMField(variable).second, variable->getNameAsString()); } @@ -1136,8 +1140,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, args.push_back(&selfDecl); // Now add the rest of the parameters. - for (auto i : blockDecl->params()) - args.push_back(i); + args.append(blockDecl->param_begin(), blockDecl->param_end()); // Create the function declaration. const FunctionProtoType *fnType = blockInfo.getBlockExpr()->getFunctionType(); @@ -1178,7 +1181,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, Alloca->setAlignment(Align); // Set the DebugLocation to empty, so the store is recognized as a // frame setup instruction by llvm::DwarfDebug::beginFunction(). - ApplyDebugLocation NL(*this); + auto NL = ApplyDebugLocation::CreateEmpty(*this); Builder.CreateAlignedStore(BlockPointer, Alloca, Align); BlockPointerDbgLoc = Alloca; } @@ -1186,9 +1189,9 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, // If we have a C++ 'this' reference, go ahead and force it into // existence now. if (blockDecl->capturesCXXThis()) { - llvm::Value *addr = Builder.CreateStructGEP(BlockPointer, - blockInfo.CXXThisIndex, - "block.captured-this"); + llvm::Value *addr = + Builder.CreateStructGEP(blockInfo.StructureType, BlockPointer, + blockInfo.CXXThisIndex, "block.captured-this"); CXXThisValue = Builder.CreateLoad(addr, "this"); } @@ -1218,8 +1221,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, EmitLambdaBlockInvokeBody(); else { PGO.assignRegionCounters(blockDecl, fn); - RegionCounter Cnt = getPGORegionCounter(blockDecl->getBody()); - Cnt.beginRegion(Builder); + incrementProfileCounter(blockDecl->getBody()); EmitStmt(blockDecl->getBody()); } @@ -1328,11 +1330,10 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { nullptr, SC_Static, false, false); - // Create a scope with an artificial location for the body of this function. - ApplyDebugLocation NL(*this); + auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(FD, C.VoidTy, Fn, FI, args); - ArtificialLocation AL(*this); - + // Create a scope with an artificial location for the body of this function. + auto AL = ApplyDebugLocation::CreateArtificial(*this); llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo(); llvm::Value *src = GetAddrOfLocalVar(&srcDecl); @@ -1404,8 +1405,10 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { } unsigned index = capture.getIndex(); - llvm::Value *srcField = Builder.CreateStructGEP(src, index); - llvm::Value *dstField = Builder.CreateStructGEP(dst, index); + llvm::Value *srcField = + Builder.CreateStructGEP(blockInfo.StructureType, src, index); + llvm::Value *dstField = + Builder.CreateStructGEP(blockInfo.StructureType, dst, index); // If there's an explicit copy expression, we do that. if (copyExpr) { @@ -1500,9 +1503,9 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) { nullptr, SC_Static, false, false); // Create a scope with an artificial location for the body of this function. - ApplyDebugLocation NL(*this); + auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(FD, C.VoidTy, Fn, FI, args); - ArtificialLocation AL(*this); + auto AL = ApplyDebugLocation::CreateArtificial(*this); llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo(); @@ -1562,7 +1565,8 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) { } unsigned index = capture.getIndex(); - llvm::Value *srcField = Builder.CreateStructGEP(src, index); + llvm::Value *srcField = + Builder.CreateStructGEP(blockInfo.StructureType, src, index); // If there's an explicit copy expression, we do that. if (dtor) { @@ -1801,13 +1805,15 @@ generateByrefCopyHelper(CodeGenFunction &CGF, llvm::Value *destField = CGF.GetAddrOfLocalVar(&dst); destField = CGF.Builder.CreateLoad(destField); destField = CGF.Builder.CreateBitCast(destField, byrefPtrType); - destField = CGF.Builder.CreateStructGEP(destField, valueFieldIndex, "x"); + destField = CGF.Builder.CreateStructGEP(&byrefType, destField, + valueFieldIndex, "x"); // src->x llvm::Value *srcField = CGF.GetAddrOfLocalVar(&src); srcField = CGF.Builder.CreateLoad(srcField); srcField = CGF.Builder.CreateBitCast(srcField, byrefPtrType); - srcField = CGF.Builder.CreateStructGEP(srcField, valueFieldIndex, "x"); + srcField = + CGF.Builder.CreateStructGEP(&byrefType, srcField, valueFieldIndex, "x"); byrefInfo.emitCopy(CGF, destField, srcField); } @@ -1868,7 +1874,7 @@ generateByrefDisposeHelper(CodeGenFunction &CGF, llvm::Value *V = CGF.GetAddrOfLocalVar(&src); V = CGF.Builder.CreateLoad(V); V = CGF.Builder.CreateBitCast(V, byrefType.getPointerTo(0)); - V = CGF.Builder.CreateStructGEP(V, byrefValueIndex, "x"); + V = CGF.Builder.CreateStructGEP(&byrefType, V, byrefValueIndex, "x"); byrefInfo.emitDispose(CGF, V); } @@ -1925,7 +1931,7 @@ CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType, const VarDecl &var = *emission.Variable; QualType type = var.getType(); - unsigned byrefValueIndex = getByRefValueLLVMField(&var); + unsigned byrefValueIndex = getByRefValueLLVMField(&var).second; if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) { const Expr *copyExpr = CGM.getContext().getBlockVarCopyInits(&var); @@ -1995,18 +2001,20 @@ CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType, return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo); } -unsigned CodeGenFunction::getByRefValueLLVMField(const ValueDecl *VD) const { +std::pair<llvm::Type *, unsigned> +CodeGenFunction::getByRefValueLLVMField(const ValueDecl *VD) const { assert(ByRefValueInfo.count(VD) && "Did not find value!"); - - return ByRefValueInfo.find(VD)->second.second; + + return ByRefValueInfo.find(VD)->second; } llvm::Value *CodeGenFunction::BuildBlockByrefAddress(llvm::Value *BaseAddr, const VarDecl *V) { - llvm::Value *Loc = Builder.CreateStructGEP(BaseAddr, 1, "forwarding"); + auto P = getByRefValueLLVMField(V); + llvm::Value *Loc = + Builder.CreateStructGEP(P.first, BaseAddr, 1, "forwarding"); Loc = Builder.CreateLoad(Loc); - Loc = Builder.CreateStructGEP(Loc, getByRefValueLLVMField(V), - V->getNameAsString()); + Loc = Builder.CreateStructGEP(P.first, Loc, P.second, V->getNameAsString()); return Loc; } @@ -2143,11 +2151,12 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { if (type.isObjCGCWeak()) isa = 1; V = Builder.CreateIntToPtr(Builder.getInt32(isa), Int8PtrTy, "isa"); - Builder.CreateStore(V, Builder.CreateStructGEP(addr, 0, "byref.isa")); + Builder.CreateStore(V, + Builder.CreateStructGEP(nullptr, addr, 0, "byref.isa")); // Store the address of the variable into its own forwarding pointer. - Builder.CreateStore(addr, - Builder.CreateStructGEP(addr, 1, "byref.forwarding")); + Builder.CreateStore( + addr, Builder.CreateStructGEP(nullptr, addr, 1, "byref.forwarding")); // Blocks ABI: // c) the flags field is set to either 0 if no helper functions are @@ -2193,25 +2202,26 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { printf("\n"); } } - + Builder.CreateStore(llvm::ConstantInt::get(IntTy, flags.getBitMask()), - Builder.CreateStructGEP(addr, 2, "byref.flags")); + Builder.CreateStructGEP(nullptr, addr, 2, "byref.flags")); CharUnits byrefSize = CGM.GetTargetTypeStoreSize(byrefType); V = llvm::ConstantInt::get(IntTy, byrefSize.getQuantity()); - Builder.CreateStore(V, Builder.CreateStructGEP(addr, 3, "byref.size")); + Builder.CreateStore(V, + Builder.CreateStructGEP(nullptr, addr, 3, "byref.size")); if (helpers) { - llvm::Value *copy_helper = Builder.CreateStructGEP(addr, 4); + llvm::Value *copy_helper = Builder.CreateStructGEP(nullptr, addr, 4); Builder.CreateStore(helpers->CopyHelper, copy_helper); - llvm::Value *destroy_helper = Builder.CreateStructGEP(addr, 5); + llvm::Value *destroy_helper = Builder.CreateStructGEP(nullptr, addr, 5); Builder.CreateStore(helpers->DisposeHelper, destroy_helper); } if (ByRefHasLifetime && HasByrefExtendedLayout) { llvm::Constant* ByrefLayoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, type); - llvm::Value *ByrefInfoAddr = Builder.CreateStructGEP(addr, helpers ? 6 : 4, - "byref.layout"); + llvm::Value *ByrefInfoAddr = + Builder.CreateStructGEP(nullptr, addr, helpers ? 6 : 4, "byref.layout"); // cast destination to pointer to source type. llvm::Type *DesTy = ByrefLayoutInfo->getType(); DesTy = DesTy->getPointerTo(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h b/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h index 72ba4fa..6610659 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBuilder.h @@ -33,7 +33,7 @@ protected: llvm::BasicBlock *BB, llvm::BasicBlock::iterator InsertPt) const; private: - void operator=(const CGBuilderInserter &) LLVM_DELETED_FUNCTION; + void operator=(const CGBuilderInserter &) = delete; CodeGenFunction *CGF; }; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp index 635e342..9b8694f 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "CodeGenFunction.h" +#include "CGCXXABI.h" #include "CGObjCRuntime.h" #include "CodeGenModule.h" #include "TargetInfo.h" @@ -21,9 +22,11 @@ #include "clang/Basic/TargetInfo.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/IR/CallSite.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Intrinsics.h" +#include <sstream> using namespace clang; using namespace CodeGen; @@ -79,9 +82,9 @@ static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, /// Utility to insert an atomic instruction based on Instrinsic::ID /// and the expression node. -static RValue EmitBinaryAtomic(CodeGenFunction &CGF, - llvm::AtomicRMWInst::BinOp Kind, - const CallExpr *E) { +static Value *MakeBinaryAtomicValue(CodeGenFunction &CGF, + llvm::AtomicRMWInst::BinOp Kind, + const CallExpr *E) { QualType T = E->getType(); assert(E->getArg(0)->getType()->isPointerType()); assert(CGF.getContext().hasSameUnqualifiedType(T, @@ -105,8 +108,13 @@ static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::Value *Result = CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1], llvm::SequentiallyConsistent); - Result = EmitFromInt(CGF, Result, T, ValueType); - return RValue::get(Result); + return EmitFromInt(CGF, Result, T, ValueType); +} + +static RValue EmitBinaryAtomic(CodeGenFunction &CGF, + llvm::AtomicRMWInst::BinOp Kind, + const CallExpr *E) { + return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E)); } /// Utility to insert an atomic instruction based Instrinsic::ID and @@ -148,6 +156,47 @@ static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, return RValue::get(Result); } +/// @brief Utility to insert an atomic cmpxchg instruction. +/// +/// @param CGF The current codegen function. +/// @param E Builtin call expression to convert to cmpxchg. +/// arg0 - address to operate on +/// arg1 - value to compare with +/// arg2 - new value +/// @param ReturnBool Specifies whether to return success flag of +/// cmpxchg result or the old value. +/// +/// @returns result of cmpxchg, according to ReturnBool +static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, + bool ReturnBool) { + QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType(); + llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); + unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); + + llvm::IntegerType *IntType = llvm::IntegerType::get( + CGF.getLLVMContext(), CGF.getContext().getTypeSize(T)); + llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); + + Value *Args[3]; + Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); + Args[1] = CGF.EmitScalarExpr(E->getArg(1)); + llvm::Type *ValueType = Args[1]->getType(); + Args[1] = EmitToInt(CGF, Args[1], T, IntType); + Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType); + + Value *Pair = CGF.Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2], + llvm::SequentiallyConsistent, + llvm::SequentiallyConsistent); + if (ReturnBool) + // Extract boolean success flag and zext it to int. + return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1), + CGF.ConvertType(E->getType())); + else + // Extract old value and emit it using the same type as compare value. + return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T, + ValueType); +} + /// EmitFAbs - Emit a call to @llvm.fabs(). static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) { Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType()); @@ -156,6 +205,27 @@ static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) { return Call; } +/// Emit the computation of the sign bit for a floating point value. Returns +/// the i1 sign bit value. +static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) { + LLVMContext &C = CGF.CGM.getLLVMContext(); + + llvm::Type *Ty = V->getType(); + int Width = Ty->getPrimitiveSizeInBits(); + llvm::Type *IntTy = llvm::IntegerType::get(C, Width); + V = CGF.Builder.CreateBitCast(V, IntTy); + if (Ty->isPPC_FP128Ty()) { + // The higher-order double comes first, and so we need to truncate the + // pair to extract the overall sign. The order of the pair is the same + // in both little- and big-Endian modes. + Width >>= 1; + IntTy = llvm::IntegerType::get(C, Width); + V = CGF.Builder.CreateTrunc(V, IntTy); + } + Value *Zero = llvm::Constant::getNullValue(IntTy); + return CGF.Builder.CreateICmpSLT(V, Zero); +} + static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, const CallExpr *E, llvm::Value *calleeValue) { return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E, @@ -181,7 +251,7 @@ static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, "arguments have the same integer width?)"); llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType()); - llvm::Value *Tmp = CGF.Builder.CreateCall2(Callee, X, Y); + llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y}); Carry = CGF.Builder.CreateExtractValue(Tmp, 1); return CGF.Builder.CreateExtractValue(Tmp, 0); } @@ -230,8 +300,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, DstPtr = Builder.CreateBitCast(DstPtr, Type); SrcPtr = Builder.CreateBitCast(SrcPtr, Type); - return RValue::get(Builder.CreateCall2(CGM.getIntrinsic(Intrinsic::vacopy), - DstPtr, SrcPtr)); + return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy), + {DstPtr, SrcPtr})); } case Builtin::BI__builtin_abs: case Builtin::BI__builtin_labs: @@ -309,7 +379,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::Type *ResultType = ConvertType(E->getType()); Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); - Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); + Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); if (Result->getType() != ResultType) Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, "cast"); @@ -326,7 +396,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::Type *ResultType = ConvertType(E->getType()); Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); - Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); + Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); if (Result->getType() != ResultType) Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, "cast"); @@ -342,9 +412,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); llvm::Type *ResultType = ConvertType(E->getType()); - Value *Tmp = Builder.CreateAdd(Builder.CreateCall2(F, ArgValue, - Builder.getTrue()), - llvm::ConstantInt::get(ArgType, 1)); + Value *Tmp = + Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}), + llvm::ConstantInt::get(ArgType, 1)); Value *Zero = llvm::Constant::getNullValue(ArgType); Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs"); @@ -389,11 +459,16 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *ArgValue = EmitScalarExpr(E->getArg(0)); llvm::Type *ArgType = ArgValue->getType(); - Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); Value *ExpectedValue = EmitScalarExpr(E->getArg(1)); + // Don't generate llvm.expect on -O0 as the backend won't use it for + // anything. + // Note, we still IRGen ExpectedValue because it could have side-effects. + if (CGM.getCodeGenOpts().OptimizationLevel == 0) + return RValue::get(ArgValue); - Value *Result = Builder.CreateCall2(FnExpect, ArgValue, ExpectedValue, - "expval"); + Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); + Value *Result = + Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval"); return RValue::get(Result); } case Builtin::BI__builtin_assume_aligned: { @@ -444,7 +519,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, // FIXME: Get right address space. llvm::Type *Tys[] = { ResType, Builder.getInt8PtrTy(0) }; Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); - return RValue::get(Builder.CreateCall2(F, EmitScalarExpr(E->getArg(0)),CI)); + return RValue::get( + Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0)), CI})); } case Builtin::BI__builtin_prefetch: { Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0)); @@ -455,7 +531,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::ConstantInt::get(Int32Ty, 3); Value *Data = llvm::ConstantInt::get(Int32Ty, 1); Value *F = CGM.getIntrinsic(Intrinsic::prefetch); - return RValue::get(Builder.CreateCall4(F, Address, RW, Locality, Data)); + return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data})); } case Builtin::BI__builtin_readcyclecounter: { Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); @@ -465,16 +541,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *Begin = EmitScalarExpr(E->getArg(0)); Value *End = EmitScalarExpr(E->getArg(1)); Value *F = CGM.getIntrinsic(Intrinsic::clear_cache); - return RValue::get(Builder.CreateCall2(F, Begin, End)); - } - case Builtin::BI__builtin_trap: { - Value *F = CGM.getIntrinsic(Intrinsic::trap); - return RValue::get(Builder.CreateCall(F)); - } - case Builtin::BI__debugbreak: { - Value *F = CGM.getIntrinsic(Intrinsic::debugtrap); - return RValue::get(Builder.CreateCall(F)); + return RValue::get(Builder.CreateCall(F, {Begin, End})); } + case Builtin::BI__builtin_trap: + return RValue::get(EmitTrapCall(Intrinsic::trap)); + case Builtin::BI__debugbreak: + return RValue::get(EmitTrapCall(Intrinsic::debugtrap)); case Builtin::BI__builtin_unreachable: { if (SanOpts.has(SanitizerKind::Unreachable)) { SanitizerScope SanScope(this); @@ -498,7 +570,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *Exponent = EmitScalarExpr(E->getArg(1)); llvm::Type *ArgType = Base->getType(); Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType); - return RValue::get(Builder.CreateCall2(F, Base, Exponent)); + return RValue::get(Builder.CreateCall(F, {Base, Exponent})); } case Builtin::BI__builtin_isgreater: @@ -551,8 +623,22 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); } - // TODO: BI__builtin_isinf_sign - // isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0 + case Builtin::BI__builtin_isinf_sign: { + // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0 + Value *Arg = EmitScalarExpr(E->getArg(0)); + Value *AbsArg = EmitFAbs(*this, Arg); + Value *IsInf = Builder.CreateFCmpOEQ( + AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf"); + Value *IsNeg = EmitSignBit(*this, Arg); + + llvm::Type *IntTy = ConvertType(E->getType()); + Value *Zero = Constant::getNullValue(IntTy); + Value *One = ConstantInt::get(IntTy, 1); + Value *NegativeOne = ConstantInt::get(IntTy, -1); + Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One); + Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero); + return RValue::get(Result); + } case Builtin::BI__builtin_isnormal: { // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min @@ -654,6 +740,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, std::pair<llvm::Value*, unsigned> Dest = EmitPointerWithAlignment(E->getArg(0)); Value *SizeVal = EmitScalarExpr(E->getArg(1)); + EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(), + E->getArg(0)->getExprLoc(), FD, 0); Builder.CreateMemSet(Dest.first, Builder.getInt8(0), SizeVal, Dest.second, false); return RValue::get(Dest.first); @@ -666,6 +754,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = EmitScalarExpr(E->getArg(2)); unsigned Align = std::min(Dest.second, Src.second); + EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(), + E->getArg(0)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Src.first), E->getArg(1)->getType(), + E->getArg(1)->getExprLoc(), FD, 1); Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false); return RValue::get(Dest.first); } @@ -723,6 +815,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = EmitScalarExpr(E->getArg(2)); unsigned Align = std::min(Dest.second, Src.second); + EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(), + E->getArg(0)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Src.first), E->getArg(1)->getType(), + E->getArg(1)->getExprLoc(), FD, 1); Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false); return RValue::get(Dest.first); } @@ -733,6 +829,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty()); Value *SizeVal = EmitScalarExpr(E->getArg(2)); + EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(), + E->getArg(0)->getExprLoc(), FD, 0); Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false); return RValue::get(Dest.first); } @@ -768,14 +866,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::ConstantInt::get(Int32Ty, Offset))); } case Builtin::BI__builtin_return_address: { - Value *Depth = EmitScalarExpr(E->getArg(0)); - Depth = Builder.CreateIntCast(Depth, Int32Ty, false); + Value *Depth = + CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this); Value *F = CGM.getIntrinsic(Intrinsic::returnaddress); return RValue::get(Builder.CreateCall(F, Depth)); } case Builtin::BI__builtin_frame_address: { - Value *Depth = EmitScalarExpr(E->getArg(0)); - Depth = Builder.CreateIntCast(Depth, Int32Ty, false); + Value *Depth = + CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this); Value *F = CGM.getIntrinsic(Intrinsic::frameaddress); return RValue::get(Builder.CreateCall(F, Depth)); } @@ -815,7 +913,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32 : Intrinsic::eh_return_i64); - Builder.CreateCall2(F, Int, Ptr); + Builder.CreateCall(F, {Int, Ptr}); Builder.CreateUnreachable(); // We do need to preserve an insertion point. @@ -864,7 +962,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, // Store the stack pointer to the setjmp buffer. Value *StackAddr = - Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); + Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); Value *StackSaveSlot = Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2)); Builder.CreateStore(StackAddr, StackSaveSlot); @@ -1001,58 +1099,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, case Builtin::BI__sync_val_compare_and_swap_2: case Builtin::BI__sync_val_compare_and_swap_4: case Builtin::BI__sync_val_compare_and_swap_8: - case Builtin::BI__sync_val_compare_and_swap_16: { - QualType T = E->getType(); - llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0)); - unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); - - llvm::IntegerType *IntType = - llvm::IntegerType::get(getLLVMContext(), - getContext().getTypeSize(T)); - llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); - - Value *Args[3]; - Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType); - Args[1] = EmitScalarExpr(E->getArg(1)); - llvm::Type *ValueType = Args[1]->getType(); - Args[1] = EmitToInt(*this, Args[1], T, IntType); - Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType); - - Value *Result = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2], - llvm::SequentiallyConsistent, - llvm::SequentiallyConsistent); - Result = Builder.CreateExtractValue(Result, 0); - Result = EmitFromInt(*this, Result, T, ValueType); - return RValue::get(Result); - } + case Builtin::BI__sync_val_compare_and_swap_16: + return RValue::get(MakeAtomicCmpXchgValue(*this, E, false)); case Builtin::BI__sync_bool_compare_and_swap_1: case Builtin::BI__sync_bool_compare_and_swap_2: case Builtin::BI__sync_bool_compare_and_swap_4: case Builtin::BI__sync_bool_compare_and_swap_8: - case Builtin::BI__sync_bool_compare_and_swap_16: { - QualType T = E->getArg(1)->getType(); - llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0)); - unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); - - llvm::IntegerType *IntType = - llvm::IntegerType::get(getLLVMContext(), - getContext().getTypeSize(T)); - llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); - - Value *Args[3]; - Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType); - Args[1] = EmitToInt(*this, EmitScalarExpr(E->getArg(1)), T, IntType); - Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType); - - Value *Pair = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2], - llvm::SequentiallyConsistent, - llvm::SequentiallyConsistent); - Value *Result = Builder.CreateExtractValue(Pair, 1); - // zext bool to int. - Result = Builder.CreateZExt(Result, ConvertType(E->getType())); - return RValue::get(Result); - } + case Builtin::BI__sync_bool_compare_and_swap_16: + return RValue::get(MakeAtomicCmpXchgValue(*this, E, true)); case Builtin::BI__sync_swap_1: case Builtin::BI__sync_swap_2: @@ -1357,6 +1412,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, return RValue::get(Builder.CreateCall(F, Arg0)); } + case Builtin::BI__builtin_pow: + case Builtin::BI__builtin_powf: + case Builtin::BI__builtin_powl: case Builtin::BIpow: case Builtin::BIpowf: case Builtin::BIpowl: { @@ -1367,7 +1425,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *Exponent = EmitScalarExpr(E->getArg(1)); llvm::Type *ArgType = Base->getType(); Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType); - return RValue::get(Builder.CreateCall2(F, Base, Exponent)); + return RValue::get(Builder.CreateCall(F, {Base, Exponent})); } case Builtin::BIfma: @@ -1380,32 +1438,17 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *FirstArg = EmitScalarExpr(E->getArg(0)); llvm::Type *ArgType = FirstArg->getType(); Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType); - return RValue::get(Builder.CreateCall3(F, FirstArg, - EmitScalarExpr(E->getArg(1)), - EmitScalarExpr(E->getArg(2)))); + return RValue::get( + Builder.CreateCall(F, {FirstArg, EmitScalarExpr(E->getArg(1)), + EmitScalarExpr(E->getArg(2))})); } case Builtin::BI__builtin_signbit: case Builtin::BI__builtin_signbitf: case Builtin::BI__builtin_signbitl: { - LLVMContext &C = CGM.getLLVMContext(); - - Value *Arg = EmitScalarExpr(E->getArg(0)); - llvm::Type *ArgTy = Arg->getType(); - int ArgWidth = ArgTy->getPrimitiveSizeInBits(); - llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth); - Value *BCArg = Builder.CreateBitCast(Arg, ArgIntTy); - if (ArgTy->isPPC_FP128Ty()) { - // The higher-order double comes first, and so we need to truncate the - // pair to extract the overall sign. The order of the pair is the same - // in both little- and big-Endian modes. - ArgWidth >>= 1; - ArgIntTy = llvm::IntegerType::get(C, ArgWidth); - BCArg = Builder.CreateTrunc(BCArg, ArgIntTy); - } - Value *ZeroCmp = llvm::Constant::getNullValue(ArgIntTy); - Value *Result = Builder.CreateICmpSLT(BCArg, ZeroCmp); - return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType()))); + return RValue::get( + Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))), + ConvertType(E->getType()))); } case Builtin::BI__builtin_annotation: { llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0)); @@ -1650,6 +1693,76 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Builder.CreateAlignedLoad(IntToPtr, /*Align=*/4, /*isVolatile=*/true); return RValue::get(Load); } + + case Builtin::BI__exception_code: + case Builtin::BI_exception_code: + return RValue::get(EmitSEHExceptionCode()); + case Builtin::BI__exception_info: + case Builtin::BI_exception_info: + return RValue::get(EmitSEHExceptionInfo()); + case Builtin::BI__abnormal_termination: + case Builtin::BI_abnormal_termination: + return RValue::get(EmitSEHAbnormalTermination()); + case Builtin::BI_setjmpex: { + if (getTarget().getTriple().isOSMSVCRT()) { + llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; + llvm::AttributeSet ReturnsTwiceAttr = + AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, + llvm::Attribute::ReturnsTwice); + llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), + "_setjmpex", ReturnsTwiceAttr); + llvm::Value *Buf = Builder.CreateBitOrPointerCast( + EmitScalarExpr(E->getArg(0)), Int8PtrTy); + llvm::Value *FrameAddr = + Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), + ConstantInt::get(Int32Ty, 0)); + llvm::Value *Args[] = {Buf, FrameAddr}; + llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args); + CS.setAttributes(ReturnsTwiceAttr); + return RValue::get(CS.getInstruction()); + } + break; + } + case Builtin::BI_setjmp: { + if (getTarget().getTriple().isOSMSVCRT()) { + llvm::AttributeSet ReturnsTwiceAttr = + AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, + llvm::Attribute::ReturnsTwice); + llvm::Value *Buf = Builder.CreateBitOrPointerCast( + EmitScalarExpr(E->getArg(0)), Int8PtrTy); + llvm::CallSite CS; + if (getTarget().getTriple().getArch() == llvm::Triple::x86) { + llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy}; + llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/true), + "_setjmp3", ReturnsTwiceAttr); + llvm::Value *Count = ConstantInt::get(IntTy, 0); + llvm::Value *Args[] = {Buf, Count}; + CS = EmitRuntimeCallOrInvoke(SetJmp3, Args); + } else { + llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; + llvm::Constant *SetJmp = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), + "_setjmp", ReturnsTwiceAttr); + llvm::Value *FrameAddr = + Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), + ConstantInt::get(Int32Ty, 0)); + llvm::Value *Args[] = {Buf, FrameAddr}; + CS = EmitRuntimeCallOrInvoke(SetJmp, Args); + } + CS.setAttributes(ReturnsTwiceAttr); + return RValue::get(CS.getInstruction()); + } + break; + } + + case Builtin::BI__GetExceptionInfo: { + if (llvm::GlobalVariable *GV = + CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType())) + return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy)); + break; + } } // If this is an alias for a lib function (e.g. __builtin_sin), emit @@ -1763,7 +1876,12 @@ Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, return EmitPPCBuiltinExpr(BuiltinID, E); case llvm::Triple::r600: case llvm::Triple::amdgcn: - return EmitR600BuiltinExpr(BuiltinID, E); + return EmitAMDGPUBuiltinExpr(BuiltinID, E); + case llvm::Triple::systemz: + return EmitSystemZBuiltinExpr(BuiltinID, E); + case llvm::Triple::nvptx: + case llvm::Triple::nvptx64: + return EmitNVPTXBuiltinExpr(BuiltinID, E); default: return nullptr; } @@ -2534,7 +2652,7 @@ Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID, // Return type. SmallVector<llvm::Type *, 3> Tys; if (Modifier & AddRetType) { - llvm::Type *Ty = ConvertType(E->getCallReturnType()); + llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); if (Modifier & VectorizeRetType) Ty = llvm::VectorType::get( Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1); @@ -2812,7 +2930,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Ops[2] = Builder.CreateBitCast(Ops[2], Ty); // NEON intrinsic puts accumulator first, unlike the LLVM fma. - return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); + return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); } case NEON::BI__builtin_neon_vld1_v: case NEON::BI__builtin_neon_vld1q_v: @@ -2825,7 +2943,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( case NEON::BI__builtin_neon_vld4_v: case NEON::BI__builtin_neon_vld4q_v: { Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); - Ops[1] = Builder.CreateCall2(F, Ops[1], Align, NameHint); + Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); return Builder.CreateStore(Ops[1], Ops[0]); @@ -3004,7 +3122,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Indices.push_back(Builder.getInt32(i+vi)); Indices.push_back(Builder.getInt32(i+e+vi)); } - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); SV = Builder.CreateStore(SV, Addr); @@ -3032,7 +3150,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); SV = Builder.CreateStore(SV, Addr); @@ -3052,7 +3170,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); } - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); SV = Builder.CreateStore(SV, Addr); @@ -3163,6 +3281,102 @@ Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) { } } +// Generates the IR for the read/write special register builtin, +// ValueType is the type of the value that is to be written or read, +// RegisterType is the type of the register being written to or read from. +static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, + const CallExpr *E, + llvm::Type *RegisterType, + llvm::Type *ValueType, bool IsRead) { + // write and register intrinsics only support 32 and 64 bit operations. + assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64)) + && "Unsupported size for register."); + + CodeGen::CGBuilderTy &Builder = CGF.Builder; + CodeGen::CodeGenModule &CGM = CGF.CGM; + LLVMContext &Context = CGM.getLLVMContext(); + + const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts(); + StringRef SysReg = cast<StringLiteral>(SysRegStrExpr)->getString(); + + llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; + llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); + llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); + + llvm::Type *Types[] = { RegisterType }; + + bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32); + assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64)) + && "Can't fit 64-bit value in 32-bit register"); + + if (IsRead) { + llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types); + llvm::Value *Call = Builder.CreateCall(F, Metadata); + + if (MixedTypes) + // Read into 64 bit register and then truncate result to 32 bit. + return Builder.CreateTrunc(Call, ValueType); + + if (ValueType->isPointerTy()) + // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*). + return Builder.CreateIntToPtr(Call, ValueType); + + return Call; + } + + llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types); + llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1)); + if (MixedTypes) { + // Extend 32 bit write value to 64 bit to pass to write. + ArgValue = Builder.CreateZExt(ArgValue, RegisterType); + return Builder.CreateCall(F, { Metadata, ArgValue }); + } + + if (ValueType->isPointerTy()) { + // Have VoidPtrTy ArgValue but want to return an i32/i64. + ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType); + return Builder.CreateCall(F, { Metadata, ArgValue }); + } + + return Builder.CreateCall(F, { Metadata, ArgValue }); +} + +/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra +/// argument that specifies the vector type. +static bool HasExtraNeonArgument(unsigned BuiltinID) { + switch (BuiltinID) { + default: break; + case NEON::BI__builtin_neon_vget_lane_i8: + case NEON::BI__builtin_neon_vget_lane_i16: + case NEON::BI__builtin_neon_vget_lane_i32: + case NEON::BI__builtin_neon_vget_lane_i64: + case NEON::BI__builtin_neon_vget_lane_f32: + case NEON::BI__builtin_neon_vgetq_lane_i8: + case NEON::BI__builtin_neon_vgetq_lane_i16: + case NEON::BI__builtin_neon_vgetq_lane_i32: + case NEON::BI__builtin_neon_vgetq_lane_i64: + case NEON::BI__builtin_neon_vgetq_lane_f32: + case NEON::BI__builtin_neon_vset_lane_i8: + case NEON::BI__builtin_neon_vset_lane_i16: + case NEON::BI__builtin_neon_vset_lane_i32: + case NEON::BI__builtin_neon_vset_lane_i64: + case NEON::BI__builtin_neon_vset_lane_f32: + case NEON::BI__builtin_neon_vsetq_lane_i8: + case NEON::BI__builtin_neon_vsetq_lane_i16: + case NEON::BI__builtin_neon_vsetq_lane_i32: + case NEON::BI__builtin_neon_vsetq_lane_i64: + case NEON::BI__builtin_neon_vsetq_lane_f32: + case NEON::BI__builtin_neon_vsha1h_u32: + case NEON::BI__builtin_neon_vsha1cq_u32: + case NEON::BI__builtin_neon_vsha1pq_u32: + case NEON::BI__builtin_neon_vsha1mq_u32: + case ARM::BI_MoveToCoprocessor: + case ARM::BI_MoveToCoprocessor2: + return false; + } + return true; +} + Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E) { if (auto Hint = GetValueForARMHint(BuiltinID)) @@ -3202,7 +3416,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, Value *Locality = llvm::ConstantInt::get(Int32Ty, 3); Value *F = CGM.getIntrinsic(Intrinsic::prefetch); - return Builder.CreateCall4(F, Address, RW, Locality, IsData); + return Builder.CreateCall(F, {Address, RW, Locality, IsData}); } if (BuiltinID == ARM::BI__builtin_arm_rbit) { @@ -3300,7 +3514,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, Value *Arg0 = Builder.CreateExtractValue(Val, 0); Value *Arg1 = Builder.CreateExtractValue(Val, 1); Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); - return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "strexd"); + return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd"); } if (BuiltinID == ARM::BI__builtin_arm_strex || @@ -3324,7 +3538,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, ? Intrinsic::arm_stlex : Intrinsic::arm_strex, StoreAddr->getType()); - return Builder.CreateCall2(F, StoreVal, StoreAddr, "strex"); + return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex"); } if (BuiltinID == ARM::BI__builtin_arm_clrex) { @@ -3365,19 +3579,59 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty); Function *F = CGM.getIntrinsic(CRCIntrinsicID); - Value *Res = Builder.CreateCall2(F, Arg0, Arg1a); - return Builder.CreateCall2(F, Res, Arg1b); + Value *Res = Builder.CreateCall(F, {Arg0, Arg1a}); + return Builder.CreateCall(F, {Res, Arg1b}); } else { Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty); Function *F = CGM.getIntrinsic(CRCIntrinsicID); - return Builder.CreateCall2(F, Arg0, Arg1); + return Builder.CreateCall(F, {Arg0, Arg1}); + } + } + + if (BuiltinID == ARM::BI__builtin_arm_rsr || + BuiltinID == ARM::BI__builtin_arm_rsr64 || + BuiltinID == ARM::BI__builtin_arm_rsrp || + BuiltinID == ARM::BI__builtin_arm_wsr || + BuiltinID == ARM::BI__builtin_arm_wsr64 || + BuiltinID == ARM::BI__builtin_arm_wsrp) { + + bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr || + BuiltinID == ARM::BI__builtin_arm_rsr64 || + BuiltinID == ARM::BI__builtin_arm_rsrp; + + bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp || + BuiltinID == ARM::BI__builtin_arm_wsrp; + + bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 || + BuiltinID == ARM::BI__builtin_arm_wsr64; + + llvm::Type *ValueType; + llvm::Type *RegisterType; + if (IsPointerBuiltin) { + ValueType = VoidPtrTy; + RegisterType = Int32Ty; + } else if (Is64Bit) { + ValueType = RegisterType = Int64Ty; + } else { + ValueType = RegisterType = Int32Ty; } + + return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); } + // Find out if any arguments are required to be integer constant + // expressions. + unsigned ICEArguments = 0; + ASTContext::GetBuiltinTypeError Error; + getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); + assert(Error == ASTContext::GE_None && "Should not codegen an error"); + SmallVector<Value*, 4> Ops; llvm::Value *Align = nullptr; - for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) { + bool HasExtraArg = HasExtraNeonArgument(BuiltinID); + unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0); + for (unsigned i = 0, e = NumArgs; i != e; i++) { if (i == 0) { switch (BuiltinID) { case NEON::BI__builtin_neon_vld1_v: @@ -3437,13 +3691,22 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, continue; } } - Ops.push_back(EmitScalarExpr(E->getArg(i))); + + if ((ICEArguments & (1 << i)) == 0) { + Ops.push_back(EmitScalarExpr(E->getArg(i))); + } else { + // If this is required to be a constant, constant fold it so that we know + // that the generated intrinsic gets a ConstantInt. + llvm::APSInt Result; + bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); + assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; + Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); + } } switch (BuiltinID) { default: break; - // vget_lane and vset_lane are not overloaded and do not have an extra - // argument that specifies the vector type. + case NEON::BI__builtin_neon_vget_lane_i8: case NEON::BI__builtin_neon_vget_lane_i16: case NEON::BI__builtin_neon_vget_lane_i32: @@ -3454,8 +3717,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vgetq_lane_i32: case NEON::BI__builtin_neon_vgetq_lane_i64: case NEON::BI__builtin_neon_vgetq_lane_f32: - return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), - "vget_lane"); + return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); + case NEON::BI__builtin_neon_vset_lane_i8: case NEON::BI__builtin_neon_vset_lane_i16: case NEON::BI__builtin_neon_vset_lane_i32: @@ -3466,29 +3729,34 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vsetq_lane_i32: case NEON::BI__builtin_neon_vsetq_lane_i64: case NEON::BI__builtin_neon_vsetq_lane_f32: - Ops.push_back(EmitScalarExpr(E->getArg(2))); return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); - // Non-polymorphic crypto instructions also not overloaded case NEON::BI__builtin_neon_vsha1h_u32: - Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, "vsha1h"); case NEON::BI__builtin_neon_vsha1cq_u32: - Ops.push_back(EmitScalarExpr(E->getArg(2))); return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, "vsha1h"); case NEON::BI__builtin_neon_vsha1pq_u32: - Ops.push_back(EmitScalarExpr(E->getArg(2))); return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, "vsha1h"); case NEON::BI__builtin_neon_vsha1mq_u32: - Ops.push_back(EmitScalarExpr(E->getArg(2))); return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, "vsha1h"); + + // The ARM _MoveToCoprocessor builtins put the input register value as + // the first argument, but the LLVM intrinsic expects it as the third one. + case ARM::BI_MoveToCoprocessor: + case ARM::BI_MoveToCoprocessor2: { + Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ? + Intrinsic::arm_mcr : Intrinsic::arm_mcr2); + return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], + Ops[3], Ops[4], Ops[5]}); + } } // Get the last argument, which specifies the vector type. + assert(HasExtraArg); llvm::APSInt Result; const Expr *Arg = E->getArg(E->getNumArgs()-1); if (!Arg->isIntegerConstantExpr(Result, getContext())) @@ -3547,7 +3815,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, // Load the value as a one-element vector. Ty = llvm::VectorType::get(VTy->getElementType(), 1); Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty); - Value *Ld = Builder.CreateCall2(F, Ops[0], Align); + Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); // Combine them. SmallVector<Constant*, 2> Indices; Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane)); @@ -3582,7 +3850,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, default: llvm_unreachable("unknown vld_dup intrinsic?"); } Function *F = CGM.getIntrinsic(Int, Ty); - Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup"); + Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup"); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); return Builder.CreateStore(Ops[1], Ops[0]); @@ -3651,7 +3919,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateBitCast(Ops[1], Ty); Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts; - Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]); + Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); case NEON::BI__builtin_neon_vsri_n_v: case NEON::BI__builtin_neon_vsriq_n_v: @@ -3897,38 +4165,6 @@ Value *CodeGenFunction::vectorWrapScalar8(Value *Op) { return Op; } -Value *CodeGenFunction:: -emitVectorWrappedScalar8Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops, - const char *Name) { - // i8 is not a legal types for AArch64, so we can't just use - // a normal overloaded intrinsic call for these scalar types. Instead - // we'll build 64-bit vectors w/ lane zero being our input values and - // perform the operation on that. The back end can pattern match directly - // to the scalar instruction. - Ops[0] = vectorWrapScalar8(Ops[0]); - Ops[1] = vectorWrapScalar8(Ops[1]); - llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8); - Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name); - Constant *CI = ConstantInt::get(SizeTy, 0); - return Builder.CreateExtractElement(V, CI, "lane0"); -} - -Value *CodeGenFunction:: -emitVectorWrappedScalar16Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops, - const char *Name) { - // i16 is not a legal types for AArch64, so we can't just use - // a normal overloaded intrinsic call for these scalar types. Instead - // we'll build 64-bit vectors w/ lane zero being our input values and - // perform the operation on that. The back end can pattern match directly - // to the scalar instruction. - Ops[0] = vectorWrapScalar16(Ops[0]); - Ops[1] = vectorWrapScalar16(Ops[1]); - llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); - Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name); - Constant *CI = ConstantInt::get(SizeTy, 0); - return Builder.CreateExtractElement(V, CI, "lane0"); -} - Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E) { unsigned HintID = static_cast<unsigned>(-1); @@ -3979,7 +4215,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify // PLDL3STRM or PLDL2STRM. Value *F = CGM.getIntrinsic(Intrinsic::prefetch); - return Builder.CreateCall4(F, Address, RW, Locality, IsData); + return Builder.CreateCall(F, {Address, RW, Locality, IsData}); } if (BuiltinID == AArch64::BI__builtin_arm_rbit) { @@ -4074,9 +4310,11 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Value *Arg1 = Builder.CreateExtractValue(Val, 1); Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); - return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "stxp"); - } else if (BuiltinID == AArch64::BI__builtin_arm_strex || - BuiltinID == AArch64::BI__builtin_arm_stlex) { + return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp"); + } + + if (BuiltinID == AArch64::BI__builtin_arm_strex || + BuiltinID == AArch64::BI__builtin_arm_stlex) { Value *StoreVal = EmitScalarExpr(E->getArg(0)); Value *StoreAddr = EmitScalarExpr(E->getArg(1)); @@ -4096,7 +4334,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr, StoreAddr->getType()); - return Builder.CreateCall2(F, StoreVal, StoreAddr, "stxr"); + return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr"); } if (BuiltinID == AArch64::BI__builtin_arm_clrex) { @@ -4133,12 +4371,60 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, llvm::Type *DataTy = F->getFunctionType()->getParamType(1); Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy); - return Builder.CreateCall2(F, Arg0, Arg1); + return Builder.CreateCall(F, {Arg0, Arg1}); + } + + if (BuiltinID == AArch64::BI__builtin_arm_rsr || + BuiltinID == AArch64::BI__builtin_arm_rsr64 || + BuiltinID == AArch64::BI__builtin_arm_rsrp || + BuiltinID == AArch64::BI__builtin_arm_wsr || + BuiltinID == AArch64::BI__builtin_arm_wsr64 || + BuiltinID == AArch64::BI__builtin_arm_wsrp) { + + bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr || + BuiltinID == AArch64::BI__builtin_arm_rsr64 || + BuiltinID == AArch64::BI__builtin_arm_rsrp; + + bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp || + BuiltinID == AArch64::BI__builtin_arm_wsrp; + + bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr && + BuiltinID != AArch64::BI__builtin_arm_wsr; + + llvm::Type *ValueType; + llvm::Type *RegisterType = Int64Ty; + if (IsPointerBuiltin) { + ValueType = VoidPtrTy; + } else if (Is64Bit) { + ValueType = Int64Ty; + } else { + ValueType = Int32Ty; + } + + return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); } + // Find out if any arguments are required to be integer constant + // expressions. + unsigned ICEArguments = 0; + ASTContext::GetBuiltinTypeError Error; + getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); + assert(Error == ASTContext::GE_None && "Should not codegen an error"); + llvm::SmallVector<Value*, 4> Ops; - for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) - Ops.push_back(EmitScalarExpr(E->getArg(i))); + for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) { + if ((ICEArguments & (1 << i)) == 0) { + Ops.push_back(EmitScalarExpr(E->getArg(i))); + } else { + // If this is required to be a constant, constant fold it so that we know + // that the generated intrinsic gets a ConstantInt. + llvm::APSInt Result; + bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); + assert(IsConst && "Constant arg isn't actually constant?"); + (void)IsConst; + Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); + } + } auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap); const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( @@ -4248,36 +4534,36 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vceqzs_f32: Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitAArch64CompareBuiltinExpr( - Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OEQ, - ICmpInst::ICMP_EQ, "vceqz"); + Ops[0], ConvertType(E->getCallReturnType(getContext())), + ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ, "vceqz"); case NEON::BI__builtin_neon_vcgezd_s64: case NEON::BI__builtin_neon_vcgezd_f64: case NEON::BI__builtin_neon_vcgezs_f32: Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitAArch64CompareBuiltinExpr( - Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGE, - ICmpInst::ICMP_SGE, "vcgez"); + Ops[0], ConvertType(E->getCallReturnType(getContext())), + ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE, "vcgez"); case NEON::BI__builtin_neon_vclezd_s64: case NEON::BI__builtin_neon_vclezd_f64: case NEON::BI__builtin_neon_vclezs_f32: Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitAArch64CompareBuiltinExpr( - Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLE, - ICmpInst::ICMP_SLE, "vclez"); + Ops[0], ConvertType(E->getCallReturnType(getContext())), + ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE, "vclez"); case NEON::BI__builtin_neon_vcgtzd_s64: case NEON::BI__builtin_neon_vcgtzd_f64: case NEON::BI__builtin_neon_vcgtzs_f32: Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitAArch64CompareBuiltinExpr( - Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGT, - ICmpInst::ICMP_SGT, "vcgtz"); + Ops[0], ConvertType(E->getCallReturnType(getContext())), + ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT, "vcgtz"); case NEON::BI__builtin_neon_vcltzd_s64: case NEON::BI__builtin_neon_vcltzd_f64: case NEON::BI__builtin_neon_vcltzs_f32: Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitAArch64CompareBuiltinExpr( - Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLT, - ICmpInst::ICMP_SLT, "vcltz"); + Ops[0], ConvertType(E->getCallReturnType(getContext())), + ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz"); case NEON::BI__builtin_neon_vceqzd_u64: { llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); @@ -4528,8 +4814,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, : Intrinsic::aarch64_neon_srshl; Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); - Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Int64Ty), Ops[1], - Builder.CreateSExt(Ops[2], Int64Ty)); + Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), + {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); } case NEON::BI__builtin_neon_vshld_n_s64: @@ -4699,7 +4985,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[2] = Builder.CreateBitCast(Ops[2], VTy); Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy); - Value *Result = Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); + Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); return Builder.CreateBitCast(Result, Ty); } Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); @@ -4713,7 +4999,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, cast<ConstantInt>(Ops[3])); Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); - return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); + return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); } case NEON::BI__builtin_neon_vfmaq_laneq_v: { Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); @@ -4722,17 +5008,17 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[2] = Builder.CreateBitCast(Ops[2], Ty); Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); - return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); + return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); } case NEON::BI__builtin_neon_vfmas_lane_f32: case NEON::BI__builtin_neon_vfmas_laneq_f32: case NEON::BI__builtin_neon_vfmad_lane_f64: case NEON::BI__builtin_neon_vfmad_laneq_f64: { Ops.push_back(EmitScalarExpr(E->getArg(3))); - llvm::Type *Ty = ConvertType(E->getCallReturnType()); + llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); - return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); + return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); } case NEON::BI__builtin_neon_vfms_v: case NEON::BI__builtin_neon_vfmsq_v: { // Only used for FP types @@ -5667,7 +5953,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Indices.push_back(ConstantInt::get(Int32Ty, i+vi)); Indices.push_back(ConstantInt::get(Int32Ty, i+e+vi)); } - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); SV = Builder.CreateStore(SV, Addr); @@ -5686,7 +5972,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); SV = Builder.CreateStore(SV, Addr); @@ -5706,7 +5992,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); } - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); SV = Builder.CreateStore(SV, Addr); @@ -5811,13 +6097,90 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, switch (BuiltinID) { default: return nullptr; + case X86::BI__builtin_cpu_supports: { + const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts(); + StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString(); + + // TODO: When/if this becomes more than x86 specific then use a TargetInfo + // based mapping. + // Processor features and mapping to processor feature value. + enum X86Features { + CMOV = 0, + MMX, + POPCNT, + SSE, + SSE2, + SSE3, + SSSE3, + SSE4_1, + SSE4_2, + AVX, + AVX2, + SSE4_A, + FMA4, + XOP, + FMA, + AVX512F, + BMI, + BMI2, + MAX + }; + + X86Features Feature = StringSwitch<X86Features>(FeatureStr) + .Case("cmov", X86Features::CMOV) + .Case("mmx", X86Features::MMX) + .Case("popcnt", X86Features::POPCNT) + .Case("sse", X86Features::SSE) + .Case("sse2", X86Features::SSE2) + .Case("sse3", X86Features::SSE3) + .Case("sse4.1", X86Features::SSE4_1) + .Case("sse4.2", X86Features::SSE4_2) + .Case("avx", X86Features::AVX) + .Case("avx2", X86Features::AVX2) + .Case("sse4a", X86Features::SSE4_A) + .Case("fma4", X86Features::FMA4) + .Case("xop", X86Features::XOP) + .Case("fma", X86Features::FMA) + .Case("avx512f", X86Features::AVX512F) + .Case("bmi", X86Features::BMI) + .Case("bmi2", X86Features::BMI2) + .Default(X86Features::MAX); + assert(Feature != X86Features::MAX && "Invalid feature!"); + + // Matching the struct layout from the compiler-rt/libgcc structure that is + // filled in: + // unsigned int __cpu_vendor; + // unsigned int __cpu_type; + // unsigned int __cpu_subtype; + // unsigned int __cpu_features[1]; + llvm::Type *STy = llvm::StructType::get( + Int32Ty, Int32Ty, Int32Ty, llvm::ArrayType::get(Int32Ty, 1), nullptr); + + // Grab the global __cpu_model. + llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model"); + + // Grab the first (0th) element from the field __cpu_features off of the + // global in the struct STy. + Value *Idxs[] = { + ConstantInt::get(Int32Ty, 0), + ConstantInt::get(Int32Ty, 3), + ConstantInt::get(Int32Ty, 0) + }; + Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs); + Value *Features = Builder.CreateLoad(CpuFeatures); + + // Check the value of the bit corresponding to the feature requested. + Value *Bitset = Builder.CreateAnd( + Features, llvm::ConstantInt::get(Int32Ty, 1 << Feature)); + return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0)); + } case X86::BI_mm_prefetch: { Value *Address = EmitScalarExpr(E->getArg(0)); Value *RW = ConstantInt::get(Int32Ty, 0); Value *Locality = EmitScalarExpr(E->getArg(1)); Value *Data = ConstantInt::get(Int32Ty, 1); Value *F = CGM.getIntrinsic(Intrinsic::prefetch); - return Builder.CreateCall4(F, Address, RW, Locality, Data); + return Builder.CreateCall(F, {Address, RW, Locality, Data}); } case X86::BI__builtin_ia32_vec_init_v8qi: case X86::BI__builtin_ia32_vec_init_v4hi: @@ -5856,104 +6219,96 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); return Builder.CreateStore(Ops[1], Ops[0]); } - case X86::BI__builtin_ia32_palignr: { - unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); - - // If palignr is shifting the pair of input vectors less than 9 bytes, - // emit a shuffle instruction. - if (shiftVal <= 8) { - SmallVector<llvm::Constant*, 8> Indices; - for (unsigned i = 0; i != 8; ++i) - Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); - - Value* SV = llvm::ConstantVector::get(Indices); - return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); + case X86::BI__builtin_ia32_palignr128: + case X86::BI__builtin_ia32_palignr256: { + unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); + + unsigned NumElts = + cast<llvm::VectorType>(Ops[0]->getType())->getNumElements(); + assert(NumElts % 16 == 0); + unsigned NumLanes = NumElts / 16; + unsigned NumLaneElts = NumElts / NumLanes; + + // If palignr is shifting the pair of vectors more than the size of two + // lanes, emit zero. + if (ShiftVal >= (2 * NumLaneElts)) + return llvm::Constant::getNullValue(ConvertType(E->getType())); + + // If palignr is shifting the pair of input vectors more than one lane, + // but less than two lanes, convert to shifting in zeroes. + if (ShiftVal > NumLaneElts) { + ShiftVal -= NumLaneElts; + Ops[1] = Ops[0]; + Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); } - // If palignr is shifting the pair of input vectors more than 8 but less - // than 16 bytes, emit a logical right shift of the destination. - if (shiftVal < 16) { - // MMX has these as 1 x i64 vectors for some odd optimization reasons. - llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1); - - Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); - Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8); - - // create i32 constant - llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q); - return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); + SmallVector<llvm::Constant*, 32> Indices; + // 256-bit palignr operates on 128-bit lanes so we need to handle that + for (unsigned l = 0; l != NumElts; l += NumLaneElts) { + for (unsigned i = 0; i != NumLaneElts; ++i) { + unsigned Idx = ShiftVal + i; + if (Idx >= NumLaneElts) + Idx += NumElts - NumLaneElts; // End of lane, switch operand. + Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l)); + } } - // If palignr is shifting the pair of vectors more than 16 bytes, emit zero. - return llvm::Constant::getNullValue(ConvertType(E->getType())); + Value* SV = llvm::ConstantVector::get(Indices); + return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); } - case X86::BI__builtin_ia32_palignr128: { - unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); + case X86::BI__builtin_ia32_pslldqi256: { + // Shift value is in bits so divide by 8. + unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; - // If palignr is shifting the pair of input vectors less than 17 bytes, - // emit a shuffle instruction. - if (shiftVal <= 16) { - SmallVector<llvm::Constant*, 16> Indices; - for (unsigned i = 0; i != 16; ++i) - Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); + // If pslldq is shifting the vector more than 15 bytes, emit zero. + if (shiftVal >= 16) + return llvm::Constant::getNullValue(ConvertType(E->getType())); - Value* SV = llvm::ConstantVector::get(Indices); - return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); + SmallVector<llvm::Constant*, 32> Indices; + // 256-bit pslldq operates on 128-bit lanes so we need to handle that + for (unsigned l = 0; l != 32; l += 16) { + for (unsigned i = 0; i != 16; ++i) { + unsigned Idx = 32 + i - shiftVal; + if (Idx < 32) Idx -= 16; // end of lane, switch operand. + Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l)); + } } - // If palignr is shifting the pair of input vectors more than 16 but less - // than 32 bytes, emit a logical right shift of the destination. - if (shiftVal < 32) { - llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); - - Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); - Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); - - // create i32 constant - llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq); - return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); - } + llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32); + Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); + Value *Zero = llvm::Constant::getNullValue(VecTy); - // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. - return llvm::Constant::getNullValue(ConvertType(E->getType())); - } - case X86::BI__builtin_ia32_palignr256: { - unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); - - // If palignr is shifting the pair of input vectors less than 17 bytes, - // emit a shuffle instruction. - if (shiftVal <= 16) { - SmallVector<llvm::Constant*, 32> Indices; - // 256-bit palignr operates on 128-bit lanes so we need to handle that - for (unsigned l = 0; l != 2; ++l) { - unsigned LaneStart = l * 16; - unsigned LaneEnd = (l+1) * 16; - for (unsigned i = 0; i != 16; ++i) { - unsigned Idx = shiftVal + i + LaneStart; - if (Idx >= LaneEnd) Idx += 16; // end of lane, switch operand - Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx)); - } + Value *SV = llvm::ConstantVector::get(Indices); + SV = Builder.CreateShuffleVector(Zero, Ops[0], SV, "pslldq"); + llvm::Type *ResultType = ConvertType(E->getType()); + return Builder.CreateBitCast(SV, ResultType, "cast"); + } + case X86::BI__builtin_ia32_psrldqi256: { + // Shift value is in bits so divide by 8. + unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; + + // If psrldq is shifting the vector more than 15 bytes, emit zero. + if (shiftVal >= 16) + return llvm::Constant::getNullValue(ConvertType(E->getType())); + + SmallVector<llvm::Constant*, 32> Indices; + // 256-bit psrldq operates on 128-bit lanes so we need to handle that + for (unsigned l = 0; l != 32; l += 16) { + for (unsigned i = 0; i != 16; ++i) { + unsigned Idx = i + shiftVal; + if (Idx >= 16) Idx += 16; // end of lane, switch operand. + Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l)); } - - Value* SV = llvm::ConstantVector::get(Indices); - return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); } - // If palignr is shifting the pair of input vectors more than 16 but less - // than 32 bytes, emit a logical right shift of the destination. - if (shiftVal < 32) { - llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 4); + llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32); + Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); + Value *Zero = llvm::Constant::getNullValue(VecTy); - Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); - Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); - - // create i32 constant - llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_avx2_psrl_dq); - return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); - } - - // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. - return llvm::Constant::getNullValue(ConvertType(E->getType())); + Value *SV = llvm::ConstantVector::get(Indices); + SV = Builder.CreateShuffleVector(Ops[0], Zero, SV, "psrldq"); + llvm::Type *ResultType = ConvertType(E->getType()); + return Builder.CreateBitCast(SV, ResultType, "cast"); } case X86::BI__builtin_ia32_movntps: case X86::BI__builtin_ia32_movntps256: @@ -5987,20 +6342,10 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, // 3DNow! case X86::BI__builtin_ia32_pswapdsf: case X86::BI__builtin_ia32_pswapdsi: { - const char *name; - Intrinsic::ID ID; - switch(BuiltinID) { - default: llvm_unreachable("Unsupported intrinsic!"); - case X86::BI__builtin_ia32_pswapdsf: - case X86::BI__builtin_ia32_pswapdsi: - name = "pswapd"; - ID = Intrinsic::x86_3dnowa_pswapd; - break; - } llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext()); Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); - llvm::Function *F = CGM.getIntrinsic(ID); - return Builder.CreateCall(F, Ops, name); + llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd); + return Builder.CreateCall(F, Ops, "pswapd"); } case X86::BI__builtin_ia32_rdrand16_step: case X86::BI__builtin_ia32_rdrand32_step: @@ -6035,13 +6380,6 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]); return Builder.CreateExtractValue(Call, 1); } - // AVX2 broadcast - case X86::BI__builtin_ia32_vbroadcastsi256: { - Value *VecTmp = CreateMemTemp(E->getArg(0)->getType()); - Builder.CreateStore(Ops[0], VecTmp); - Value *F = CGM.getIntrinsic(Intrinsic::x86_avx2_vbroadcasti128); - return Builder.CreateCall(F, Builder.CreateBitCast(VecTmp, Int8PtrTy)); - } // SSE comparison intrisics case X86::BI__builtin_ia32_cmpeqps: case X86::BI__builtin_ia32_cmpltps: @@ -6296,6 +6634,103 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, llvm::Function *F = CGM.getIntrinsic(ID); return Builder.CreateCall(F, Ops, ""); } + // Square root + case PPC::BI__builtin_vsx_xvsqrtsp: + case PPC::BI__builtin_vsx_xvsqrtdp: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + ID = Intrinsic::sqrt; + llvm::Function *F = CGM.getIntrinsic(ID, ResultType); + return Builder.CreateCall(F, X); + } + // Count leading zeros + case PPC::BI__builtin_altivec_vclzb: + case PPC::BI__builtin_altivec_vclzh: + case PPC::BI__builtin_altivec_vclzw: + case PPC::BI__builtin_altivec_vclzd: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); + Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); + return Builder.CreateCall(F, {X, Undef}); + } + // Copy sign + case PPC::BI__builtin_vsx_xvcpsgnsp: + case PPC::BI__builtin_vsx_xvcpsgndp: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Y = EmitScalarExpr(E->getArg(1)); + ID = Intrinsic::copysign; + llvm::Function *F = CGM.getIntrinsic(ID, ResultType); + return Builder.CreateCall(F, {X, Y}); + } + // Rounding/truncation + case PPC::BI__builtin_vsx_xvrspip: + case PPC::BI__builtin_vsx_xvrdpip: + case PPC::BI__builtin_vsx_xvrdpim: + case PPC::BI__builtin_vsx_xvrspim: + case PPC::BI__builtin_vsx_xvrdpi: + case PPC::BI__builtin_vsx_xvrspi: + case PPC::BI__builtin_vsx_xvrdpic: + case PPC::BI__builtin_vsx_xvrspic: + case PPC::BI__builtin_vsx_xvrdpiz: + case PPC::BI__builtin_vsx_xvrspiz: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim || + BuiltinID == PPC::BI__builtin_vsx_xvrspim) + ID = Intrinsic::floor; + else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi || + BuiltinID == PPC::BI__builtin_vsx_xvrspi) + ID = Intrinsic::round; + else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic || + BuiltinID == PPC::BI__builtin_vsx_xvrspic) + ID = Intrinsic::nearbyint; + else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip || + BuiltinID == PPC::BI__builtin_vsx_xvrspip) + ID = Intrinsic::ceil; + else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz || + BuiltinID == PPC::BI__builtin_vsx_xvrspiz) + ID = Intrinsic::trunc; + llvm::Function *F = CGM.getIntrinsic(ID, ResultType); + return Builder.CreateCall(F, X); + } + // FMA variations + case PPC::BI__builtin_vsx_xvmaddadp: + case PPC::BI__builtin_vsx_xvmaddasp: + case PPC::BI__builtin_vsx_xvnmaddadp: + case PPC::BI__builtin_vsx_xvnmaddasp: + case PPC::BI__builtin_vsx_xvmsubadp: + case PPC::BI__builtin_vsx_xvmsubasp: + case PPC::BI__builtin_vsx_xvnmsubadp: + case PPC::BI__builtin_vsx_xvnmsubasp: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Y = EmitScalarExpr(E->getArg(1)); + Value *Z = EmitScalarExpr(E->getArg(2)); + Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); + llvm::Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); + switch (BuiltinID) { + case PPC::BI__builtin_vsx_xvmaddadp: + case PPC::BI__builtin_vsx_xvmaddasp: + return Builder.CreateCall(F, {X, Y, Z}); + case PPC::BI__builtin_vsx_xvnmaddadp: + case PPC::BI__builtin_vsx_xvnmaddasp: + return Builder.CreateFSub(Zero, + Builder.CreateCall(F, {X, Y, Z}), "sub"); + case PPC::BI__builtin_vsx_xvmsubadp: + case PPC::BI__builtin_vsx_xvmsubasp: + return Builder.CreateCall(F, + {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); + case PPC::BI__builtin_vsx_xvnmsubadp: + case PPC::BI__builtin_vsx_xvnmsubasp: + Value *FsubRes = + Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); + return Builder.CreateFSub(Zero, FsubRes, "sub"); + } + llvm_unreachable("Unknown FMA operation"); + return nullptr; // Suppress no-return warning + } } } @@ -6318,7 +6753,7 @@ static Value *emitTernaryFPBuiltin(CodeGenFunction &CGF, llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2)); Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); - return CGF.Builder.CreateCall3(F, Src0, Src1, Src2); + return CGF.Builder.CreateCall(F, {Src0, Src1, Src2}); } // Emit an intrinsic that has 1 float or double operand, and 1 integer. @@ -6329,14 +6764,14 @@ static Value *emitFPIntBuiltin(CodeGenFunction &CGF, llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); - return CGF.Builder.CreateCall2(F, Src0, Src1); + return CGF.Builder.CreateCall(F, {Src0, Src1}); } -Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, - const CallExpr *E) { +Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, + const CallExpr *E) { switch (BuiltinID) { - case R600::BI__builtin_amdgpu_div_scale: - case R600::BI__builtin_amdgpu_div_scalef: { + case AMDGPU::BI__builtin_amdgpu_div_scale: + case AMDGPU::BI__builtin_amdgpu_div_scalef: { // Translate from the intrinsics's struct return to the builtin's out // argument. @@ -6350,7 +6785,7 @@ Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::AMDGPU_div_scale, X->getType()); - llvm::Value *Tmp = Builder.CreateCall3(Callee, X, Y, Z); + llvm::Value *Tmp = Builder.CreateCall(Callee, {X, Y, Z}); llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0); llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1); @@ -6363,8 +6798,8 @@ Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, FlagStore->setAlignment(FlagOutPtr.second); return Result; } - case R600::BI__builtin_amdgpu_div_fmas: - case R600::BI__builtin_amdgpu_div_fmasf: { + case AMDGPU::BI__builtin_amdgpu_div_fmas: + case AMDGPU::BI__builtin_amdgpu_div_fmasf: { llvm::Value *Src0 = EmitScalarExpr(E->getArg(0)); llvm::Value *Src1 = EmitScalarExpr(E->getArg(1)); llvm::Value *Src2 = EmitScalarExpr(E->getArg(2)); @@ -6373,30 +6808,342 @@ Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, llvm::Value *F = CGM.getIntrinsic(Intrinsic::AMDGPU_div_fmas, Src0->getType()); llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3); - return Builder.CreateCall4(F, Src0, Src1, Src2, Src3ToBool); + return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool}); } - case R600::BI__builtin_amdgpu_div_fixup: - case R600::BI__builtin_amdgpu_div_fixupf: + case AMDGPU::BI__builtin_amdgpu_div_fixup: + case AMDGPU::BI__builtin_amdgpu_div_fixupf: return emitTernaryFPBuiltin(*this, E, Intrinsic::AMDGPU_div_fixup); - case R600::BI__builtin_amdgpu_trig_preop: - case R600::BI__builtin_amdgpu_trig_preopf: + case AMDGPU::BI__builtin_amdgpu_trig_preop: + case AMDGPU::BI__builtin_amdgpu_trig_preopf: return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_trig_preop); - case R600::BI__builtin_amdgpu_rcp: - case R600::BI__builtin_amdgpu_rcpf: + case AMDGPU::BI__builtin_amdgpu_rcp: + case AMDGPU::BI__builtin_amdgpu_rcpf: return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rcp); - case R600::BI__builtin_amdgpu_rsq: - case R600::BI__builtin_amdgpu_rsqf: + case AMDGPU::BI__builtin_amdgpu_rsq: + case AMDGPU::BI__builtin_amdgpu_rsqf: return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq); - case R600::BI__builtin_amdgpu_rsq_clamped: - case R600::BI__builtin_amdgpu_rsq_clampedf: + case AMDGPU::BI__builtin_amdgpu_rsq_clamped: + case AMDGPU::BI__builtin_amdgpu_rsq_clampedf: return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq_clamped); - case R600::BI__builtin_amdgpu_ldexp: - case R600::BI__builtin_amdgpu_ldexpf: + case AMDGPU::BI__builtin_amdgpu_ldexp: + case AMDGPU::BI__builtin_amdgpu_ldexpf: return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_ldexp); - case R600::BI__builtin_amdgpu_class: - case R600::BI__builtin_amdgpu_classf: + case AMDGPU::BI__builtin_amdgpu_class: + case AMDGPU::BI__builtin_amdgpu_classf: return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_class); default: return nullptr; } } + +/// Handle a SystemZ function in which the final argument is a pointer +/// to an int that receives the post-instruction CC value. At the LLVM level +/// this is represented as a function that returns a {result, cc} pair. +static Value *EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, + unsigned IntrinsicID, + const CallExpr *E) { + unsigned NumArgs = E->getNumArgs() - 1; + SmallVector<Value *, 8> Args(NumArgs); + for (unsigned I = 0; I < NumArgs; ++I) + Args[I] = CGF.EmitScalarExpr(E->getArg(I)); + Value *CCPtr = CGF.EmitScalarExpr(E->getArg(NumArgs)); + Value *F = CGF.CGM.getIntrinsic(IntrinsicID); + Value *Call = CGF.Builder.CreateCall(F, Args); + Value *CC = CGF.Builder.CreateExtractValue(Call, 1); + CGF.Builder.CreateStore(CC, CCPtr); + return CGF.Builder.CreateExtractValue(Call, 0); +} + +Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID, + const CallExpr *E) { + switch (BuiltinID) { + case SystemZ::BI__builtin_tbegin: { + Value *TDB = EmitScalarExpr(E->getArg(0)); + Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); + Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin); + return Builder.CreateCall(F, {TDB, Control}); + } + case SystemZ::BI__builtin_tbegin_nofloat: { + Value *TDB = EmitScalarExpr(E->getArg(0)); + Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); + Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat); + return Builder.CreateCall(F, {TDB, Control}); + } + case SystemZ::BI__builtin_tbeginc: { + Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy); + Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08); + Value *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc); + return Builder.CreateCall(F, {TDB, Control}); + } + case SystemZ::BI__builtin_tabort: { + Value *Data = EmitScalarExpr(E->getArg(0)); + Value *F = CGM.getIntrinsic(Intrinsic::s390_tabort); + return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty, "tabort")); + } + case SystemZ::BI__builtin_non_tx_store: { + Value *Address = EmitScalarExpr(E->getArg(0)); + Value *Data = EmitScalarExpr(E->getArg(1)); + Value *F = CGM.getIntrinsic(Intrinsic::s390_ntstg); + return Builder.CreateCall(F, {Data, Address}); + } + + // Vector builtins. Note that most vector builtins are mapped automatically + // to target-specific LLVM intrinsics. The ones handled specially here can + // be represented via standard LLVM IR, which is preferable to enable common + // LLVM optimizations. + + case SystemZ::BI__builtin_s390_vpopctb: + case SystemZ::BI__builtin_s390_vpopcth: + case SystemZ::BI__builtin_s390_vpopctf: + case SystemZ::BI__builtin_s390_vpopctg: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType); + return Builder.CreateCall(F, X); + } + + case SystemZ::BI__builtin_s390_vclzb: + case SystemZ::BI__builtin_s390_vclzh: + case SystemZ::BI__builtin_s390_vclzf: + case SystemZ::BI__builtin_s390_vclzg: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); + Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); + return Builder.CreateCall(F, {X, Undef}); + } + + case SystemZ::BI__builtin_s390_vctzb: + case SystemZ::BI__builtin_s390_vctzh: + case SystemZ::BI__builtin_s390_vctzf: + case SystemZ::BI__builtin_s390_vctzg: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); + Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType); + return Builder.CreateCall(F, {X, Undef}); + } + + case SystemZ::BI__builtin_s390_vfsqdb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType); + return Builder.CreateCall(F, X); + } + case SystemZ::BI__builtin_s390_vfmadb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Y = EmitScalarExpr(E->getArg(1)); + Value *Z = EmitScalarExpr(E->getArg(2)); + Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); + return Builder.CreateCall(F, {X, Y, Z}); + } + case SystemZ::BI__builtin_s390_vfmsdb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Y = EmitScalarExpr(E->getArg(1)); + Value *Z = EmitScalarExpr(E->getArg(2)); + Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); + Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); + return Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); + } + case SystemZ::BI__builtin_s390_vflpdb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); + return Builder.CreateCall(F, X); + } + case SystemZ::BI__builtin_s390_vflndb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); + Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); + return Builder.CreateFSub(Zero, Builder.CreateCall(F, X), "sub"); + } + case SystemZ::BI__builtin_s390_vfidb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + // Constant-fold the M4 and M5 mask arguments. + llvm::APSInt M4, M5; + bool IsConstM4 = E->getArg(1)->isIntegerConstantExpr(M4, getContext()); + bool IsConstM5 = E->getArg(2)->isIntegerConstantExpr(M5, getContext()); + assert(IsConstM4 && IsConstM5 && "Constant arg isn't actually constant?"); + (void)IsConstM4; (void)IsConstM5; + // Check whether this instance of vfidb can be represented via a LLVM + // standard intrinsic. We only support some combinations of M4 and M5. + Intrinsic::ID ID = Intrinsic::not_intrinsic; + switch (M4.getZExtValue()) { + default: break; + case 0: // IEEE-inexact exception allowed + switch (M5.getZExtValue()) { + default: break; + case 0: ID = Intrinsic::rint; break; + } + break; + case 4: // IEEE-inexact exception suppressed + switch (M5.getZExtValue()) { + default: break; + case 0: ID = Intrinsic::nearbyint; break; + case 1: ID = Intrinsic::round; break; + case 5: ID = Intrinsic::trunc; break; + case 6: ID = Intrinsic::ceil; break; + case 7: ID = Intrinsic::floor; break; + } + break; + } + if (ID != Intrinsic::not_intrinsic) { + Function *F = CGM.getIntrinsic(ID, ResultType); + return Builder.CreateCall(F, X); + } + Function *F = CGM.getIntrinsic(Intrinsic::s390_vfidb); + Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4); + Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5); + return Builder.CreateCall(F, {X, M4Value, M5Value}); + } + + // Vector intrisincs that output the post-instruction CC value. + +#define INTRINSIC_WITH_CC(NAME) \ + case SystemZ::BI__builtin_##NAME: \ + return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E) + + INTRINSIC_WITH_CC(s390_vpkshs); + INTRINSIC_WITH_CC(s390_vpksfs); + INTRINSIC_WITH_CC(s390_vpksgs); + + INTRINSIC_WITH_CC(s390_vpklshs); + INTRINSIC_WITH_CC(s390_vpklsfs); + INTRINSIC_WITH_CC(s390_vpklsgs); + + INTRINSIC_WITH_CC(s390_vceqbs); + INTRINSIC_WITH_CC(s390_vceqhs); + INTRINSIC_WITH_CC(s390_vceqfs); + INTRINSIC_WITH_CC(s390_vceqgs); + + INTRINSIC_WITH_CC(s390_vchbs); + INTRINSIC_WITH_CC(s390_vchhs); + INTRINSIC_WITH_CC(s390_vchfs); + INTRINSIC_WITH_CC(s390_vchgs); + + INTRINSIC_WITH_CC(s390_vchlbs); + INTRINSIC_WITH_CC(s390_vchlhs); + INTRINSIC_WITH_CC(s390_vchlfs); + INTRINSIC_WITH_CC(s390_vchlgs); + + INTRINSIC_WITH_CC(s390_vfaebs); + INTRINSIC_WITH_CC(s390_vfaehs); + INTRINSIC_WITH_CC(s390_vfaefs); + + INTRINSIC_WITH_CC(s390_vfaezbs); + INTRINSIC_WITH_CC(s390_vfaezhs); + INTRINSIC_WITH_CC(s390_vfaezfs); + + INTRINSIC_WITH_CC(s390_vfeebs); + INTRINSIC_WITH_CC(s390_vfeehs); + INTRINSIC_WITH_CC(s390_vfeefs); + + INTRINSIC_WITH_CC(s390_vfeezbs); + INTRINSIC_WITH_CC(s390_vfeezhs); + INTRINSIC_WITH_CC(s390_vfeezfs); + + INTRINSIC_WITH_CC(s390_vfenebs); + INTRINSIC_WITH_CC(s390_vfenehs); + INTRINSIC_WITH_CC(s390_vfenefs); + + INTRINSIC_WITH_CC(s390_vfenezbs); + INTRINSIC_WITH_CC(s390_vfenezhs); + INTRINSIC_WITH_CC(s390_vfenezfs); + + INTRINSIC_WITH_CC(s390_vistrbs); + INTRINSIC_WITH_CC(s390_vistrhs); + INTRINSIC_WITH_CC(s390_vistrfs); + + INTRINSIC_WITH_CC(s390_vstrcbs); + INTRINSIC_WITH_CC(s390_vstrchs); + INTRINSIC_WITH_CC(s390_vstrcfs); + + INTRINSIC_WITH_CC(s390_vstrczbs); + INTRINSIC_WITH_CC(s390_vstrczhs); + INTRINSIC_WITH_CC(s390_vstrczfs); + + INTRINSIC_WITH_CC(s390_vfcedbs); + INTRINSIC_WITH_CC(s390_vfchdbs); + INTRINSIC_WITH_CC(s390_vfchedbs); + + INTRINSIC_WITH_CC(s390_vftcidb); + +#undef INTRINSIC_WITH_CC + + default: + return nullptr; + } +} + +Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, + const CallExpr *E) { + switch (BuiltinID) { + case NVPTX::BI__nvvm_atom_add_gen_i: + case NVPTX::BI__nvvm_atom_add_gen_l: + case NVPTX::BI__nvvm_atom_add_gen_ll: + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Add, E); + + case NVPTX::BI__nvvm_atom_sub_gen_i: + case NVPTX::BI__nvvm_atom_sub_gen_l: + case NVPTX::BI__nvvm_atom_sub_gen_ll: + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Sub, E); + + case NVPTX::BI__nvvm_atom_and_gen_i: + case NVPTX::BI__nvvm_atom_and_gen_l: + case NVPTX::BI__nvvm_atom_and_gen_ll: + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::And, E); + + case NVPTX::BI__nvvm_atom_or_gen_i: + case NVPTX::BI__nvvm_atom_or_gen_l: + case NVPTX::BI__nvvm_atom_or_gen_ll: + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Or, E); + + case NVPTX::BI__nvvm_atom_xor_gen_i: + case NVPTX::BI__nvvm_atom_xor_gen_l: + case NVPTX::BI__nvvm_atom_xor_gen_ll: + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xor, E); + + case NVPTX::BI__nvvm_atom_xchg_gen_i: + case NVPTX::BI__nvvm_atom_xchg_gen_l: + case NVPTX::BI__nvvm_atom_xchg_gen_ll: + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xchg, E); + + case NVPTX::BI__nvvm_atom_max_gen_i: + case NVPTX::BI__nvvm_atom_max_gen_l: + case NVPTX::BI__nvvm_atom_max_gen_ll: + case NVPTX::BI__nvvm_atom_max_gen_ui: + case NVPTX::BI__nvvm_atom_max_gen_ul: + case NVPTX::BI__nvvm_atom_max_gen_ull: + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E); + + case NVPTX::BI__nvvm_atom_min_gen_i: + case NVPTX::BI__nvvm_atom_min_gen_l: + case NVPTX::BI__nvvm_atom_min_gen_ll: + case NVPTX::BI__nvvm_atom_min_gen_ui: + case NVPTX::BI__nvvm_atom_min_gen_ul: + case NVPTX::BI__nvvm_atom_min_gen_ull: + return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E); + + case NVPTX::BI__nvvm_atom_cas_gen_i: + case NVPTX::BI__nvvm_atom_cas_gen_l: + case NVPTX::BI__nvvm_atom_cas_gen_ll: + return MakeAtomicCmpXchgValue(*this, E, true); + + case NVPTX::BI__nvvm_atom_add_gen_f: { + Value *Ptr = EmitScalarExpr(E->getArg(0)); + Value *Val = EmitScalarExpr(E->getArg(1)); + // atomicrmw only deals with integer arguments so we need to use + // LLVM's nvvm_atomic_load_add_f32 intrinsic for that. + Value *FnALAF32 = + CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType()); + return Builder.CreateCall(FnALAF32, {Ptr, Val}); + } + + default: + return nullptr; + } +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp index fb11751..67d0ab7 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDANV.cpp @@ -20,7 +20,6 @@ #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" -#include <vector> using namespace clang; using namespace CodeGen; @@ -30,29 +29,66 @@ namespace { class CGNVCUDARuntime : public CGCUDARuntime { private: - llvm::Type *IntTy, *SizeTy; - llvm::PointerType *CharPtrTy, *VoidPtrTy; + llvm::Type *IntTy, *SizeTy, *VoidTy; + llvm::PointerType *CharPtrTy, *VoidPtrTy, *VoidPtrPtrTy; + + /// Convenience reference to LLVM Context + llvm::LLVMContext &Context; + /// Convenience reference to the current module + llvm::Module &TheModule; + /// Keeps track of kernel launch stubs emitted in this module + llvm::SmallVector<llvm::Function *, 16> EmittedKernels; + /// Keeps track of variables containing handles of GPU binaries. Populated by + /// ModuleCtorFunction() and used to create corresponding cleanup calls in + /// ModuleDtorFunction() + llvm::SmallVector<llvm::GlobalVariable *, 16> GpuBinaryHandles; llvm::Constant *getSetupArgumentFn() const; llvm::Constant *getLaunchFn() const; + /// Creates a function to register all kernel stubs generated in this module. + llvm::Function *makeRegisterKernelsFn(); + + /// Helper function that generates a constant string and returns a pointer to + /// the start of the string. The result of this function can be used anywhere + /// where the C code specifies const char*. + llvm::Constant *makeConstantString(const std::string &Str, + const std::string &Name = "", + unsigned Alignment = 0) { + llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0), + llvm::ConstantInt::get(SizeTy, 0)}; + auto *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str()); + return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), + ConstStr, Zeros); + } + + void emitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args); + public: CGNVCUDARuntime(CodeGenModule &CGM); - void EmitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args) override; + void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) override; + /// Creates module constructor function + llvm::Function *makeModuleCtorFunction() override; + /// Creates module destructor function + llvm::Function *makeModuleDtorFunction() override; }; } -CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM) : CGCUDARuntime(CGM) { +CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM) + : CGCUDARuntime(CGM), Context(CGM.getLLVMContext()), + TheModule(CGM.getModule()) { CodeGen::CodeGenTypes &Types = CGM.getTypes(); ASTContext &Ctx = CGM.getContext(); IntTy = Types.ConvertType(Ctx.IntTy); SizeTy = Types.ConvertType(Ctx.getSizeType()); + VoidTy = llvm::Type::getVoidTy(Context); CharPtrTy = llvm::PointerType::getUnqual(Types.ConvertType(Ctx.CharTy)); VoidPtrTy = cast<llvm::PointerType>(Types.ConvertType(Ctx.VoidPtrTy)); + VoidPtrPtrTy = VoidPtrTy->getPointerTo(); } llvm::Constant *CGNVCUDARuntime::getSetupArgumentFn() const { @@ -68,14 +104,17 @@ llvm::Constant *CGNVCUDARuntime::getSetupArgumentFn() const { llvm::Constant *CGNVCUDARuntime::getLaunchFn() const { // cudaError_t cudaLaunch(char *) - std::vector<llvm::Type*> Params; - Params.push_back(CharPtrTy); - return CGM.CreateRuntimeFunction(llvm::FunctionType::get(IntTy, - Params, false), - "cudaLaunch"); + return CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, CharPtrTy, false), "cudaLaunch"); +} + +void CGNVCUDARuntime::emitDeviceStub(CodeGenFunction &CGF, + FunctionArgList &Args) { + EmittedKernels.push_back(CGF.CurFn); + emitDeviceStubBody(CGF, Args); } -void CGNVCUDARuntime::EmitDeviceStubBody(CodeGenFunction &CGF, +void CGNVCUDARuntime::emitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args) { // Build the argument value list and the argument stack struct type. SmallVector<llvm::Value *, 16> ArgValues; @@ -87,8 +126,7 @@ void CGNVCUDARuntime::EmitDeviceStubBody(CodeGenFunction &CGF, assert(isa<llvm::PointerType>(V->getType()) && "Arg type not PointerType"); ArgTypes.push_back(cast<llvm::PointerType>(V->getType())->getElementType()); } - llvm::StructType *ArgStackTy = llvm::StructType::get( - CGF.getLLVMContext(), ArgTypes); + llvm::StructType *ArgStackTy = llvm::StructType::get(Context, ArgTypes); llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); @@ -120,6 +158,160 @@ void CGNVCUDARuntime::EmitDeviceStubBody(CodeGenFunction &CGF, CGF.EmitBlock(EndBlock); } +/// Creates internal function to register all kernel stubs generated in this +/// module with the CUDA runtime. +/// \code +/// void __cuda_register_kernels(void** GpuBinaryHandle) { +/// __cudaRegisterFunction(GpuBinaryHandle,Kernel0,...); +/// ... +/// __cudaRegisterFunction(GpuBinaryHandle,KernelM,...); +/// } +/// \endcode +llvm::Function *CGNVCUDARuntime::makeRegisterKernelsFn() { + llvm::Function *RegisterKernelsFunc = llvm::Function::Create( + llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false), + llvm::GlobalValue::InternalLinkage, "__cuda_register_kernels", &TheModule); + llvm::BasicBlock *EntryBB = + llvm::BasicBlock::Create(Context, "entry", RegisterKernelsFunc); + CGBuilderTy Builder(Context); + Builder.SetInsertPoint(EntryBB); + + // void __cudaRegisterFunction(void **, const char *, char *, const char *, + // int, uint3*, uint3*, dim3*, dim3*, int*) + std::vector<llvm::Type *> RegisterFuncParams = { + VoidPtrPtrTy, CharPtrTy, CharPtrTy, CharPtrTy, IntTy, + VoidPtrTy, VoidPtrTy, VoidPtrTy, VoidPtrTy, IntTy->getPointerTo()}; + llvm::Constant *RegisterFunc = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, RegisterFuncParams, false), + "__cudaRegisterFunction"); + + // Extract GpuBinaryHandle passed as the first argument passed to + // __cuda_register_kernels() and generate __cudaRegisterFunction() call for + // each emitted kernel. + llvm::Argument &GpuBinaryHandlePtr = *RegisterKernelsFunc->arg_begin(); + for (llvm::Function *Kernel : EmittedKernels) { + llvm::Constant *KernelName = makeConstantString(Kernel->getName()); + llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(VoidPtrTy); + llvm::Value *args[] = { + &GpuBinaryHandlePtr, Builder.CreateBitCast(Kernel, VoidPtrTy), + KernelName, KernelName, llvm::ConstantInt::get(IntTy, -1), NullPtr, + NullPtr, NullPtr, NullPtr, + llvm::ConstantPointerNull::get(IntTy->getPointerTo())}; + Builder.CreateCall(RegisterFunc, args); + } + + Builder.CreateRetVoid(); + return RegisterKernelsFunc; +} + +/// Creates a global constructor function for the module: +/// \code +/// void __cuda_module_ctor(void*) { +/// Handle0 = __cudaRegisterFatBinary(GpuBinaryBlob0); +/// __cuda_register_kernels(Handle0); +/// ... +/// HandleN = __cudaRegisterFatBinary(GpuBinaryBlobN); +/// __cuda_register_kernels(HandleN); +/// } +/// \endcode +llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { + // void __cuda_register_kernels(void* handle); + llvm::Function *RegisterKernelsFunc = makeRegisterKernelsFn(); + // void ** __cudaRegisterFatBinary(void *); + llvm::Constant *RegisterFatbinFunc = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(VoidPtrPtrTy, VoidPtrTy, false), + "__cudaRegisterFatBinary"); + // struct { int magic, int version, void * gpu_binary, void * dont_care }; + llvm::StructType *FatbinWrapperTy = + llvm::StructType::get(IntTy, IntTy, VoidPtrTy, VoidPtrTy, nullptr); + + llvm::Function *ModuleCtorFunc = llvm::Function::Create( + llvm::FunctionType::get(VoidTy, VoidPtrTy, false), + llvm::GlobalValue::InternalLinkage, "__cuda_module_ctor", &TheModule); + llvm::BasicBlock *CtorEntryBB = + llvm::BasicBlock::Create(Context, "entry", ModuleCtorFunc); + CGBuilderTy CtorBuilder(Context); + + CtorBuilder.SetInsertPoint(CtorEntryBB); + + // For each GPU binary, register it with the CUDA runtime and store returned + // handle in a global variable and save the handle in GpuBinaryHandles vector + // to be cleaned up in destructor on exit. Then associate all known kernels + // with the GPU binary handle so CUDA runtime can figure out what to call on + // the GPU side. + for (const std::string &GpuBinaryFileName : + CGM.getCodeGenOpts().CudaGpuBinaryFileNames) { + llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> GpuBinaryOrErr = + llvm::MemoryBuffer::getFileOrSTDIN(GpuBinaryFileName); + if (std::error_code EC = GpuBinaryOrErr.getError()) { + CGM.getDiags().Report(diag::err_cannot_open_file) << GpuBinaryFileName + << EC.message(); + continue; + } + + // Create initialized wrapper structure that points to the loaded GPU binary + llvm::Constant *Values[] = { + llvm::ConstantInt::get(IntTy, 0x466243b1), // Fatbin wrapper magic. + llvm::ConstantInt::get(IntTy, 1), // Fatbin version. + makeConstantString(GpuBinaryOrErr.get()->getBuffer(), "", 16), // Data. + llvm::ConstantPointerNull::get(VoidPtrTy)}; // Unused in fatbin v1. + llvm::GlobalVariable *FatbinWrapper = new llvm::GlobalVariable( + TheModule, FatbinWrapperTy, true, llvm::GlobalValue::InternalLinkage, + llvm::ConstantStruct::get(FatbinWrapperTy, Values), + "__cuda_fatbin_wrapper"); + + // GpuBinaryHandle = __cudaRegisterFatBinary(&FatbinWrapper); + llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall( + RegisterFatbinFunc, + CtorBuilder.CreateBitCast(FatbinWrapper, VoidPtrTy)); + llvm::GlobalVariable *GpuBinaryHandle = new llvm::GlobalVariable( + TheModule, VoidPtrPtrTy, false, llvm::GlobalValue::InternalLinkage, + llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__cuda_gpubin_handle"); + CtorBuilder.CreateStore(RegisterFatbinCall, GpuBinaryHandle, false); + + // Call __cuda_register_kernels(GpuBinaryHandle); + CtorBuilder.CreateCall(RegisterKernelsFunc, RegisterFatbinCall); + + // Save GpuBinaryHandle so we can unregister it in destructor. + GpuBinaryHandles.push_back(GpuBinaryHandle); + } + + CtorBuilder.CreateRetVoid(); + return ModuleCtorFunc; +} + +/// Creates a global destructor function that unregisters all GPU code blobs +/// registered by constructor. +/// \code +/// void __cuda_module_dtor(void*) { +/// __cudaUnregisterFatBinary(Handle0); +/// ... +/// __cudaUnregisterFatBinary(HandleN); +/// } +/// \endcode +llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() { + // void __cudaUnregisterFatBinary(void ** handle); + llvm::Constant *UnregisterFatbinFunc = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false), + "__cudaUnregisterFatBinary"); + + llvm::Function *ModuleDtorFunc = llvm::Function::Create( + llvm::FunctionType::get(VoidTy, VoidPtrTy, false), + llvm::GlobalValue::InternalLinkage, "__cuda_module_dtor", &TheModule); + llvm::BasicBlock *DtorEntryBB = + llvm::BasicBlock::Create(Context, "entry", ModuleDtorFunc); + CGBuilderTy DtorBuilder(Context); + DtorBuilder.SetInsertPoint(DtorEntryBB); + + for (llvm::GlobalVariable *GpuBinaryHandle : GpuBinaryHandles) { + DtorBuilder.CreateCall(UnregisterFatbinFunc, + DtorBuilder.CreateLoad(GpuBinaryHandle, false)); + } + + DtorBuilder.CreateRetVoid(); + return ModuleDtorFunc; +} + CGCUDARuntime *CodeGen::CreateNVCUDARuntime(CodeGenModule &CGM) { return new CGNVCUDARuntime(CGM); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.h index 8c162fb..dcacf97 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCUDARuntime.h @@ -16,6 +16,10 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGCUDARUNTIME_H #define LLVM_CLANG_LIB_CODEGEN_CGCUDARUNTIME_H +namespace llvm { +class Function; +} + namespace clang { class CUDAKernelCallExpr; @@ -39,10 +43,17 @@ public: virtual RValue EmitCUDAKernelCallExpr(CodeGenFunction &CGF, const CUDAKernelCallExpr *E, ReturnValueSlot ReturnValue); - - virtual void EmitDeviceStubBody(CodeGenFunction &CGF, - FunctionArgList &Args) = 0; + /// Emits a kernel launch stub. + virtual void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) = 0; + + /// Constructs and returns a module initialization function or nullptr if it's + /// not needed. Must be called after all kernels have been emitted. + virtual llvm::Function *makeModuleCtorFunction() = 0; + + /// Returns a module cleanup function or nullptr if it's not needed. + /// Must be called after ModuleCtorFunction + virtual llvm::Function *makeModuleDtorFunction() = 0; }; /// Creates an instance of a CUDA runtime class. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp index 9f0e67e..7d7ed78 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXX.cpp @@ -182,8 +182,8 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl, return true; // Create the alias with no name. - auto *Alias = llvm::GlobalAlias::create(AliasType->getElementType(), 0, - Linkage, "", Aliasee, &getModule()); + auto *Alias = + llvm::GlobalAlias::create(AliasType, Linkage, "", Aliasee, &getModule()); // Switch any previous uses to the alias. if (Entry) { @@ -218,6 +218,8 @@ llvm::Function *CodeGenModule::codegenCXXStructor(const CXXMethodDecl *MD, } setFunctionLinkage(GD, Fn); + setFunctionDLLStorageClass(GD, Fn); + CodeGenFunction(*this).GenerateCode(GD, Fn, FnInfo); setFunctionDefinitionAttributes(MD, Fn); SetLLVMFunctionAttributesForDefinition(MD, Fn); @@ -231,8 +233,7 @@ llvm::GlobalValue *CodeGenModule::getAddrOfCXXStructor( if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) { GD = GlobalDecl(CD, toCXXCtorType(Type)); } else { - auto *DD = dyn_cast<CXXDestructorDecl>(MD); - GD = GlobalDecl(DD, toCXXDtorType(Type)); + GD = GlobalDecl(cast<CXXDestructorDecl>(MD), toCXXDtorType(Type)); } StringRef Name = getMangledName(GD); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp index d31331d..dc16616 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.cpp @@ -130,10 +130,9 @@ CGCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) { return GetBogusMemberPointer(QualType(MPT, 0)); } -llvm::Constant *CGCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) { - return GetBogusMemberPointer( - CGM.getContext().getMemberPointerType(MD->getType(), - MD->getParent()->getTypeForDecl())); +llvm::Constant *CGCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) { + return GetBogusMemberPointer(CGM.getContext().getMemberPointerType( + MD->getType(), MD->getParent()->getTypeForDecl())); } llvm::Constant *CGCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT, @@ -302,3 +301,10 @@ CGCXXABI::EmitCtorCompleteObjectHandler(CodeGenFunction &CGF, bool CGCXXABI::NeedsVTTParameter(GlobalDecl GD) { return false; } + +llvm::CallInst * +CGCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF, + llvm::Value *Exn) { + // Just call std::terminate and ignore the violating exception. + return CGF.EmitNounwindRuntimeCall(CGF.CGM.getTerminateFn()); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h index cc5c1b2..436b96a 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCXXABI.h @@ -22,6 +22,7 @@ namespace llvm { class Constant; class Type; class Value; +class CallInst; } namespace clang { @@ -171,7 +172,7 @@ public: virtual llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT); /// Create a member pointer for the given method. - virtual llvm::Constant *EmitMemberPointer(const CXXMethodDecl *MD); + virtual llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD); /// Create a member pointer for the given field. virtual llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT, @@ -214,8 +215,18 @@ public: llvm::Value *Ptr, QualType ElementType, const CXXDestructorDecl *Dtor) = 0; virtual void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) = 0; + virtual void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) = 0; + virtual llvm::GlobalVariable *getThrowInfo(QualType T) { return nullptr; } + + virtual void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) = 0; + + virtual llvm::CallInst * + emitTerminateForUnexpectedException(CodeGenFunction &CGF, + llvm::Value *Exn); virtual llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) = 0; + virtual llvm::Constant * + getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) = 0; virtual bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) = 0; @@ -355,7 +366,8 @@ public: virtual llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, llvm::Value *This, - llvm::Type *Ty) = 0; + llvm::Type *Ty, + SourceLocation Loc) = 0; /// Emit the ABI-specific virtual destructor call. virtual llvm::Value * diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp index 6403fa9..3e4d7f3 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.cpp @@ -30,6 +30,7 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/Transforms/Utils/Local.h" using namespace clang; using namespace CodeGen; @@ -51,6 +52,8 @@ static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) { case CC_X86Pascal: return llvm::CallingConv::C; // TODO: Add support for __vectorcall to LLVM. case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall; + case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC; + case CC_SpirKernel: return llvm::CallingConv::SPIR_KERNEL; } } @@ -96,8 +99,7 @@ arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod, CanQual<FunctionProtoType> FTP) { RequiredArgs required = RequiredArgs::forPrototypePlus(FTP, prefix.size()); // FIXME: Kill copy. - for (unsigned i = 0, e = FTP->getNumParams(); i != e; ++i) - prefix.push_back(FTP->getParamType(i)); + prefix.append(FTP->param_type_begin(), FTP->param_type_end()); CanQualType resultType = FTP->getReturnType().getUnqualifiedType(); return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod, /*chainCall=*/false, prefix, @@ -133,9 +135,6 @@ static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) { if (PcsAttr *PCS = D->getAttr<PcsAttr>()) return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP); - if (D->hasAttr<PnaclCallAttr>()) - return CC_PnaclCall; - if (D->hasAttr<IntelOclBiccAttr>()) return CC_IntelOclBicc; @@ -208,8 +207,7 @@ CodeGenTypes::arrangeCXXStructorDeclaration(const CXXMethodDecl *MD, CanQual<FunctionProtoType> FTP = GetFormalType(MD); // Add the formal parameters. - for (unsigned i = 0, e = FTP->getNumParams(); i != e; ++i) - argTypes.push_back(FTP->getParamType(i)); + argTypes.append(FTP->param_type_begin(), FTP->param_type_end()); TheCXXABI.buildStructorSignature(MD, Type, argTypes); @@ -349,6 +347,26 @@ CodeGenTypes::arrangeMSMemberPointerThunk(const CXXMethodDecl *MD) { FTP->getExtInfo(), RequiredArgs(1)); } +const CGFunctionInfo & +CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD, + CXXCtorType CT) { + assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure); + + CanQual<FunctionProtoType> FTP = GetFormalType(CD); + SmallVector<CanQualType, 2> ArgTys; + const CXXRecordDecl *RD = CD->getParent(); + ArgTys.push_back(GetThisType(Context, RD)); + if (CT == Ctor_CopyingClosure) + ArgTys.push_back(*FTP->param_type_begin()); + if (RD->getNumVBases() > 0) + ArgTys.push_back(Context.IntTy); + CallingConv CC = Context.getDefaultCallingConvention( + /*IsVariadic=*/false, /*IsCXXMethod=*/true); + return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true, + /*chainCall=*/false, ArgTys, + FunctionType::ExtInfo(CC), RequiredArgs::All); +} + /// Arrange a call as unto a free function, except possibly with an /// additional number of formal parameters considered required. static const CGFunctionInfo & @@ -716,7 +734,8 @@ void CodeGenFunction::ExpandTypeFromArgs( auto Exp = getTypeExpansion(Ty, getContext()); if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { for (int i = 0, n = CAExp->NumElts; i < n; i++) { - llvm::Value *EltAddr = Builder.CreateConstGEP2_32(LV.getAddress(), 0, i); + llvm::Value *EltAddr = + Builder.CreateConstGEP2_32(nullptr, LV.getAddress(), 0, i); LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy); ExpandTypeFromArgs(CAExp->EltTy, LV, AI); } @@ -738,10 +757,12 @@ void CodeGenFunction::ExpandTypeFromArgs( ExpandTypeFromArgs(FD->getType(), SubLV, AI); } } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) { - llvm::Value *RealAddr = Builder.CreateStructGEP(LV.getAddress(), 0, "real"); + llvm::Value *RealAddr = + Builder.CreateStructGEP(nullptr, LV.getAddress(), 0, "real"); EmitStoreThroughLValue(RValue::get(*AI++), MakeAddrLValue(RealAddr, CExp->EltTy)); - llvm::Value *ImagAddr = Builder.CreateStructGEP(LV.getAddress(), 1, "imag"); + llvm::Value *ImagAddr = + Builder.CreateStructGEP(nullptr, LV.getAddress(), 1, "imag"); EmitStoreThroughLValue(RValue::get(*AI++), MakeAddrLValue(ImagAddr, CExp->EltTy)); } else { @@ -757,7 +778,7 @@ void CodeGenFunction::ExpandTypeToArgs( if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { llvm::Value *Addr = RV.getAggregateAddr(); for (int i = 0, n = CAExp->NumElts; i < n; i++) { - llvm::Value *EltAddr = Builder.CreateConstGEP2_32(Addr, 0, i); + llvm::Value *EltAddr = Builder.CreateConstGEP2_32(nullptr, Addr, 0, i); RValue EltRV = convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()); ExpandTypeToArgs(CAExp->EltTy, EltRV, IRFuncTy, IRCallArgs, IRCallArgPos); @@ -825,7 +846,7 @@ EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr, return SrcPtr; // GEP into the first element. - SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive"); + SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcSTy, SrcPtr, 0, 0, "coerce.dive"); // If the first element is a struct, recurse. llvm::Type *SrcTy = @@ -891,20 +912,21 @@ static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, /// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as -/// a pointer to an object of type \arg Ty. +/// a pointer to an object of type \arg Ty, known to be aligned to +/// \arg SrcAlign bytes. /// /// This safely handles the case when the src type is smaller than the /// destination type; in this situation the values of bits which not /// present in the src are undefined. static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr, - llvm::Type *Ty, + llvm::Type *Ty, CharUnits SrcAlign, CodeGenFunction &CGF) { llvm::Type *SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType(); // If SrcTy and Ty are the same, just do a load. if (SrcTy == Ty) - return CGF.Builder.CreateLoad(SrcPtr); + return CGF.Builder.CreateAlignedLoad(SrcPtr, SrcAlign.getQuantity()); uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty); @@ -919,7 +941,8 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr, // extension or truncation to the desired type. if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) && (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) { - llvm::LoadInst *Load = CGF.Builder.CreateLoad(SrcPtr); + llvm::LoadInst *Load = + CGF.Builder.CreateAlignedLoad(SrcPtr, SrcAlign.getQuantity()); return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF); } @@ -933,23 +956,20 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr, // to that information. llvm::Value *Casted = CGF.Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(Ty)); - llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted); - // FIXME: Use better alignment / avoid requiring aligned load. - Load->setAlignment(1); - return Load; + return CGF.Builder.CreateAlignedLoad(Casted, SrcAlign.getQuantity()); } // Otherwise do coercion through memory. This is stupid, but // simple. - llvm::Value *Tmp = CGF.CreateTempAlloca(Ty); + llvm::AllocaInst *Tmp = CGF.CreateTempAlloca(Ty); + Tmp->setAlignment(SrcAlign.getQuantity()); llvm::Type *I8PtrTy = CGF.Builder.getInt8PtrTy(); llvm::Value *Casted = CGF.Builder.CreateBitCast(Tmp, I8PtrTy); llvm::Value *SrcCasted = CGF.Builder.CreateBitCast(SrcPtr, I8PtrTy); - // FIXME: Use better alignment. CGF.Builder.CreateMemCpy(Casted, SrcCasted, llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize), - 1, false); - return CGF.Builder.CreateLoad(Tmp); + SrcAlign.getQuantity(), false); + return CGF.Builder.CreateAlignedLoad(Tmp, SrcAlign.getQuantity()); } // Function to store a first-class aggregate into memory. We prefer to @@ -958,39 +978,45 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr, // FIXME: Do we need to recurse here? static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val, llvm::Value *DestPtr, bool DestIsVolatile, - bool LowAlignment) { + CharUnits DestAlign) { // Prefer scalar stores to first-class aggregate stores. if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) { + const llvm::StructLayout *Layout = + CGF.CGM.getDataLayout().getStructLayout(STy); + for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { - llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(DestPtr, 0, i); + llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(STy, DestPtr, 0, i); llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i); - llvm::StoreInst *SI = CGF.Builder.CreateStore(Elt, EltPtr, - DestIsVolatile); - if (LowAlignment) - SI->setAlignment(1); + uint64_t EltOffset = Layout->getElementOffset(i); + CharUnits EltAlign = + DestAlign.alignmentAtOffset(CharUnits::fromQuantity(EltOffset)); + CGF.Builder.CreateAlignedStore(Elt, EltPtr, EltAlign.getQuantity(), + DestIsVolatile); } } else { - llvm::StoreInst *SI = CGF.Builder.CreateStore(Val, DestPtr, DestIsVolatile); - if (LowAlignment) - SI->setAlignment(1); + CGF.Builder.CreateAlignedStore(Val, DestPtr, DestAlign.getQuantity(), + DestIsVolatile); } } /// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src, -/// where the source and destination may have different types. +/// where the source and destination may have different types. The +/// destination is known to be aligned to \arg DstAlign bytes. /// /// This safely handles the case when the src type is larger than the /// destination type; the upper bits of the src will be lost. static void CreateCoercedStore(llvm::Value *Src, llvm::Value *DstPtr, bool DstIsVolatile, + CharUnits DstAlign, CodeGenFunction &CGF) { llvm::Type *SrcTy = Src->getType(); llvm::Type *DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType(); if (SrcTy == DstTy) { - CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile); + CGF.Builder.CreateAlignedStore(Src, DstPtr, DstAlign.getQuantity(), + DstIsVolatile); return; } @@ -1006,7 +1032,8 @@ static void CreateCoercedStore(llvm::Value *Src, if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) && (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) { Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF); - CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile); + CGF.Builder.CreateAlignedStore(Src, DstPtr, DstAlign.getQuantity(), + DstIsVolatile); return; } @@ -1016,8 +1043,7 @@ static void CreateCoercedStore(llvm::Value *Src, if (SrcSize <= DstSize) { llvm::Value *Casted = CGF.Builder.CreateBitCast(DstPtr, llvm::PointerType::getUnqual(SrcTy)); - // FIXME: Use better alignment / avoid requiring aligned store. - BuildAggStore(CGF, Src, Casted, DstIsVolatile, true); + BuildAggStore(CGF, Src, Casted, DstIsVolatile, DstAlign); } else { // Otherwise do coercion through memory. This is stupid, but // simple. @@ -1028,15 +1054,15 @@ static void CreateCoercedStore(llvm::Value *Src, // // FIXME: Assert that we aren't truncating non-padding bits when have access // to that information. - llvm::Value *Tmp = CGF.CreateTempAlloca(SrcTy); - CGF.Builder.CreateStore(Src, Tmp); + llvm::AllocaInst *Tmp = CGF.CreateTempAlloca(SrcTy); + Tmp->setAlignment(DstAlign.getQuantity()); + CGF.Builder.CreateAlignedStore(Src, Tmp, DstAlign.getQuantity()); llvm::Type *I8PtrTy = CGF.Builder.getInt8PtrTy(); llvm::Value *Casted = CGF.Builder.CreateBitCast(Tmp, I8PtrTy); llvm::Value *DstCasted = CGF.Builder.CreateBitCast(DstPtr, I8PtrTy); - // FIXME: Use better alignment. CGF.Builder.CreateMemCpy(DstCasted, Casted, llvm::ConstantInt::get(CGF.IntPtrTy, DstSize), - 1, false); + DstAlign.getQuantity(), false); } } @@ -1403,7 +1429,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, FuncAttrs.addAttribute(llvm::Attribute::ReadOnly); FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); } - if (TargetDecl->hasAttr<MallocAttr>()) + if (TargetDecl->hasAttr<RestrictAttr>()) RetAttrs.addAttribute(llvm::Attribute::NoAlias); if (TargetDecl->hasAttr<ReturnsNonNullAttr>()) RetAttrs.addAttribute(llvm::Attribute::NonNull); @@ -1431,6 +1457,8 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, // Attributes that should go on the call site only. if (!CodeGenOpts.SimplifyLibCalls) FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin); + if (!CodeGenOpts.TrapFuncName.empty()) + FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName); } else { // Attributes that should go on the function, but not the call site. if (!CodeGenOpts.DisableFPElim) { @@ -1443,6 +1471,8 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, FuncAttrs.addAttribute("no-frame-pointer-elim-non-leaf"); } + FuncAttrs.addAttribute("disable-tail-calls", + llvm::toStringRef(CodeGenOpts.DisableTailCalls)); FuncAttrs.addAttribute("less-precise-fpmad", llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD)); FuncAttrs.addAttribute("no-infs-fp-math", @@ -1458,6 +1488,76 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, if (!CodeGenOpts.StackRealignment) FuncAttrs.addAttribute("no-realign-stack"); + + // Add target-cpu and target-features attributes to functions. If + // we have a decl for the function and it has a target attribute then + // parse that and add it to the feature set. + StringRef TargetCPU = getTarget().getTargetOpts().CPU; + + // TODO: Features gets us the features on the command line including + // feature dependencies. For canonicalization purposes we might want to + // avoid putting features in the target-features set if we know it'll be + // one of the default features in the backend, e.g. corei7-avx and +avx or + // figure out non-explicit dependencies. + // Canonicalize the existing features in a new feature map. + // TODO: Migrate the existing backends to keep the map around rather than + // the vector. + llvm::StringMap<bool> FeatureMap; + for (auto F : getTarget().getTargetOpts().Features) { + const char *Name = F.c_str(); + bool Enabled = Name[0] == '+'; + getTarget().setFeatureEnabled(FeatureMap, Name + 1, Enabled); + } + + const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl); + if (FD) { + if (const auto *TD = FD->getAttr<TargetAttr>()) { + StringRef FeaturesStr = TD->getFeatures(); + SmallVector<StringRef, 1> AttrFeatures; + FeaturesStr.split(AttrFeatures, ","); + + // Grab the various features and prepend a "+" to turn on the feature to + // the backend and add them to our existing set of features. + for (auto &Feature : AttrFeatures) { + // Go ahead and trim whitespace rather than either erroring or + // accepting it weirdly. + Feature = Feature.trim(); + + // While we're here iterating check for a different target cpu. + if (Feature.startswith("arch=")) + TargetCPU = Feature.split("=").second.trim(); + else if (Feature.startswith("tune=")) + // We don't support cpu tuning this way currently. + ; + else if (Feature.startswith("fpmath=")) + // TODO: Support the fpmath option this way. It will require checking + // overall feature validity for the function with the rest of the + // attributes on the function. + ; + else if (Feature.startswith("mno-")) + getTarget().setFeatureEnabled(FeatureMap, Feature.split("-").second, + false); + else + getTarget().setFeatureEnabled(FeatureMap, Feature, true); + } + } + } + + // Produce the canonical string for this set of features. + std::vector<std::string> Features; + for (llvm::StringMap<bool>::const_iterator it = FeatureMap.begin(), + ie = FeatureMap.end(); + it != ie; ++it) + Features.push_back((it->second ? "+" : "-") + it->first().str()); + + // Now add the target-cpu and target-features to the function. + if (TargetCPU != "") + FuncAttrs.addAttribute("target-cpu", TargetCPU); + if (!Features.empty()) { + std::sort(Features.begin(), Features.end()); + FuncAttrs.addAttribute("target-features", + llvm::join(Features.begin(), Features.end(), ",")); + } } ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI); @@ -1546,8 +1646,12 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, case ABIArgInfo::Extend: if (ParamType->isSignedIntegerOrEnumerationType()) Attrs.addAttribute(llvm::Attribute::SExt); - else if (ParamType->isUnsignedIntegerOrEnumerationType()) - Attrs.addAttribute(llvm::Attribute::ZExt); + else if (ParamType->isUnsignedIntegerOrEnumerationType()) { + if (getTypes().getABIInfo().shouldSignExtUnsignedType(ParamType)) + Attrs.addAttribute(llvm::Attribute::SExt); + else + Attrs.addAttribute(llvm::Attribute::ZExt); + } // FALL THROUGH case ABIArgInfo::Direct: if (ArgNo == 0 && FI.isChainCall()) @@ -1734,8 +1838,9 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, switch (ArgI.getKind()) { case ABIArgInfo::InAlloca: { assert(NumIRArgs == 0); - llvm::Value *V = Builder.CreateStructGEP( - ArgStruct, ArgI.getInAllocaFieldIndex(), Arg->getName()); + llvm::Value *V = + Builder.CreateStructGEP(FI.getArgStruct(), ArgStruct, + ArgI.getInAllocaFieldIndex(), Arg->getName()); ArgVals.push_back(ValueAndIsPtr(V, HavePointer)); break; } @@ -1770,8 +1875,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, ArgVals.push_back(ValueAndIsPtr(V, HavePointer)); } else { // Load scalar value from indirect argument. - CharUnits Alignment = getContext().getTypeAlignInChars(Ty); - V = EmitLoadOfScalar(V, false, Alignment.getQuantity(), Ty, + V = EmitLoadOfScalar(V, false, ArgI.getIndirectAlign(), Ty, Arg->getLocStart()); if (isPromoted) @@ -1897,13 +2001,15 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, Alloca->setAlignment(AlignmentToUse); llvm::Value *V = Alloca; llvm::Value *Ptr = V; // Pointer to store into. + CharUnits PtrAlign = CharUnits::fromQuantity(AlignmentToUse); // If the value is offset in memory, apply the offset now. if (unsigned Offs = ArgI.getDirectOffset()) { Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy()); - Ptr = Builder.CreateConstGEP1_32(Ptr, Offs); + Ptr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), Ptr, Offs); Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ArgI.getCoerceToType())); + PtrAlign = PtrAlign.alignmentAtOffset(CharUnits::fromQuantity(Offs)); } // Fast-isel and the optimizer generally like scalar values better than @@ -1923,7 +2029,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { auto AI = FnArgs[FirstIRArg + i]; AI->setName(Arg->getName() + ".coerce" + Twine(i)); - llvm::Value *EltPtr = Builder.CreateConstGEP2_32(Ptr, 0, i); + llvm::Value *EltPtr = Builder.CreateConstGEP2_32(STy, Ptr, 0, i); Builder.CreateStore(AI, EltPtr); } } else { @@ -1936,7 +2042,8 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { auto AI = FnArgs[FirstIRArg + i]; AI->setName(Arg->getName() + ".coerce" + Twine(i)); - llvm::Value *EltPtr = Builder.CreateConstGEP2_32(TempV, 0, i); + llvm::Value *EltPtr = + Builder.CreateConstGEP2_32(ArgI.getCoerceToType(), TempV, 0, i); Builder.CreateStore(AI, EltPtr); } @@ -1947,7 +2054,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, assert(NumIRArgs == 1); auto AI = FnArgs[FirstIRArg]; AI->setName(Arg->getName() + ".coerce"); - CreateCoercedStore(AI, Ptr, /*DestIsVolatile=*/false, *this); + CreateCoercedStore(AI, Ptr, /*DestIsVolatile=*/false, PtrAlign, *this); } @@ -2173,7 +2280,28 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { if (!CGF.ReturnValue->hasOneUse()) { llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock(); if (IP->empty()) return nullptr; - llvm::StoreInst *store = dyn_cast<llvm::StoreInst>(&IP->back()); + llvm::Instruction *I = &IP->back(); + + // Skip lifetime markers + for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(), + IE = IP->rend(); + II != IE; ++II) { + if (llvm::IntrinsicInst *Intrinsic = + dyn_cast<llvm::IntrinsicInst>(&*II)) { + if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) { + const llvm::Value *CastAddr = Intrinsic->getArgOperand(1); + ++II; + if (II == IE) + break; + if (isa<llvm::BitCastInst>(&*II) && (CastAddr == &*II)) + continue; + } + } + I = &*II; + break; + } + + llvm::StoreInst *store = dyn_cast<llvm::StoreInst>(I); if (!store) return nullptr; if (store->getPointerOperand() != CGF.ReturnValue) return nullptr; assert(!store->isAtomic() && !store->isVolatile()); // see below @@ -2231,8 +2359,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, llvm::Function::arg_iterator EI = CurFn->arg_end(); --EI; llvm::Value *ArgStruct = EI; - llvm::Value *SRet = - Builder.CreateStructGEP(ArgStruct, RetAI.getInAllocaFieldIndex()); + llvm::Value *SRet = Builder.CreateStructGEP( + nullptr, ArgStruct, RetAI.getInAllocaFieldIndex()); RV = Builder.CreateLoad(SRet, "sret"); } break; @@ -2271,7 +2399,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, // If there is a dominating store to ReturnValue, we can elide // the load, zap the store, and usually zap the alloca. - if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(*this)) { + if (llvm::StoreInst *SI = + findDominatingStoreToReturnValue(*this)) { // Reuse the debug location from the store unless there is // cleanup code to be emitted between the store and return // instruction. @@ -2293,15 +2422,17 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, } } else { llvm::Value *V = ReturnValue; + CharUnits Align = getContext().getTypeAlignInChars(RetTy); // If the value is offset in memory, apply the offset now. if (unsigned Offs = RetAI.getDirectOffset()) { V = Builder.CreateBitCast(V, Builder.getInt8PtrTy()); - V = Builder.CreateConstGEP1_32(V, Offs); + V = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), V, Offs); V = Builder.CreateBitCast(V, llvm::PointerType::getUnqual(RetAI.getCoerceToType())); + Align = Align.alignmentAtOffset(CharUnits::fromQuantity(Offs)); } - RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this); + RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), Align, *this); } // In ARC, end functions that return a retainable type with a call @@ -2342,8 +2473,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, Ret = Builder.CreateRetVoid(); } - if (!RetDbgLoc.isUnknown()) - Ret->setDebugLoc(RetDbgLoc); + if (RetDbgLoc) + Ret->setDebugLoc(std::move(RetDbgLoc)); } static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) { @@ -2501,7 +2632,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, LValue srcLV; // Make an optimistic effort to emit the address as an l-value. - // This can fail if the the argument expression is more complicated. + // This can fail if the argument expression is more complicated. if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) { srcLV = CGF.EmitLValue(lvExpr); @@ -2626,7 +2757,7 @@ void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) { // Save the stack. llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave); - StackBase = CGF.Builder.CreateCall(F, "inalloca.save"); + StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save"); // Control gets really tied up in landing pads, so we have to spill the // stacksave to an alloca to avoid violating SSA form. @@ -2649,27 +2780,28 @@ void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const { } } -static void emitNonNullArgCheck(CodeGenFunction &CGF, RValue RV, - QualType ArgType, SourceLocation ArgLoc, - const FunctionDecl *FD, unsigned ParmNum) { - if (!CGF.SanOpts.has(SanitizerKind::NonnullAttribute) || !FD) +void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType, + SourceLocation ArgLoc, + const FunctionDecl *FD, + unsigned ParmNum) { + if (!SanOpts.has(SanitizerKind::NonnullAttribute) || !FD) return; auto PVD = ParmNum < FD->getNumParams() ? FD->getParamDecl(ParmNum) : nullptr; unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum; auto NNAttr = getNonNullAttr(FD, PVD, ArgType, ArgNo); if (!NNAttr) return; - CodeGenFunction::SanitizerScope SanScope(&CGF); + SanitizerScope SanScope(this); assert(RV.isScalar()); llvm::Value *V = RV.getScalarVal(); llvm::Value *Cond = - CGF.Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType())); + Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType())); llvm::Constant *StaticData[] = { - CGF.EmitCheckSourceLocation(ArgLoc), - CGF.EmitCheckSourceLocation(NNAttr->getLocation()), - llvm::ConstantInt::get(CGF.Int32Ty, ArgNo + 1), + EmitCheckSourceLocation(ArgLoc), + EmitCheckSourceLocation(NNAttr->getLocation()), + llvm::ConstantInt::get(Int32Ty, ArgNo + 1), }; - CGF.EmitCheck(std::make_pair(Cond, SanitizerKind::NonnullAttribute), + EmitCheck(std::make_pair(Cond, SanitizerKind::NonnullAttribute), "nonnull_arg", StaticData, None); } @@ -2678,12 +2810,7 @@ void CodeGenFunction::EmitCallArgs(CallArgList &Args, CallExpr::const_arg_iterator ArgBeg, CallExpr::const_arg_iterator ArgEnd, const FunctionDecl *CalleeDecl, - unsigned ParamsToSkip, - bool ForceColumnInfo) { - CGDebugInfo *DI = getDebugInfo(); - SourceLocation CallLoc; - if (DI) CallLoc = DI->getLocation(); - + unsigned ParamsToSkip) { // We *have* to evaluate arguments from right to left in the MS C++ ABI, // because arguments are destroyed left to right in the callee. if (CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) { @@ -2702,10 +2829,8 @@ void CodeGenFunction::EmitCallArgs(CallArgList &Args, for (int I = ArgTypes.size() - 1; I >= 0; --I) { CallExpr::const_arg_iterator Arg = ArgBeg + I; EmitCallArg(Args, *Arg, ArgTypes[I]); - emitNonNullArgCheck(*this, Args.back().RV, ArgTypes[I], Arg->getExprLoc(), + EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], Arg->getExprLoc(), CalleeDecl, ParamsToSkip + I); - // Restore the debug location. - if (DI) DI->EmitLocation(Builder, CallLoc, ForceColumnInfo); } // Un-reverse the arguments we just evaluated so they match up with the LLVM @@ -2718,10 +2843,8 @@ void CodeGenFunction::EmitCallArgs(CallArgList &Args, CallExpr::const_arg_iterator Arg = ArgBeg + I; assert(Arg != ArgEnd); EmitCallArg(Args, *Arg, ArgTypes[I]); - emitNonNullArgCheck(*this, Args.back().RV, ArgTypes[I], Arg->getExprLoc(), + EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], Arg->getExprLoc(), CalleeDecl, ParamsToSkip + I); - // Restore the debug location. - if (DI) DI->EmitLocation(Builder, CallLoc, ForceColumnInfo); } } @@ -2744,8 +2867,22 @@ struct DestroyUnpassedArg : EHScopeStack::Cleanup { } +struct DisableDebugLocationUpdates { + CodeGenFunction &CGF; + bool disabledDebugInfo; + DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) { + if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && CGF.getDebugInfo())) + CGF.disableDebugInfo(); + } + ~DisableDebugLocationUpdates() { + if (disabledDebugInfo) + CGF.enableDebugInfo(); + } +}; + void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, QualType type) { + DisableDebugLocationUpdates Dis(*this, E); if (const ObjCIndirectCopyRestoreExpr *CRE = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) { assert(getLangOpts().ObjCAutoRefCount); @@ -2900,7 +3037,6 @@ void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(llvm::Value *callee, call->setCallingConv(getRuntimeCC()); Builder.CreateUnreachable(); } - PGO.setCurrentRegionUnreachable(); } /// Emits a call or invoke instruction to the given nullary runtime @@ -2949,7 +3085,7 @@ CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee, if (CGM.getLangOpts().ObjCAutoRefCount) AddObjCARCExceptionMetadata(Inst); - return Inst; + return llvm::CallSite(Inst); } /// \brief Store a non-aggregate value to an address to initialize it. For @@ -2986,7 +3122,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If we're using inalloca, insert the allocation after the stack save. // FIXME: Do this earlier rather than hacking it in here! - llvm::Value *ArgMemory = nullptr; + llvm::AllocaInst *ArgMemory = nullptr; if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) { llvm::Instruction *IP = CallArgs.getStackBase(); llvm::AllocaInst *AI; @@ -3007,15 +3143,24 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If the call returns a temporary with struct return, create a temporary // alloca to hold the result, unless one is given to us. llvm::Value *SRetPtr = nullptr; + size_t UnusedReturnSize = 0; if (RetAI.isIndirect() || RetAI.isInAlloca()) { SRetPtr = ReturnValue.getValue(); - if (!SRetPtr) + if (!SRetPtr) { SRetPtr = CreateMemTemp(RetTy); + if (HaveInsertPoint() && ReturnValue.isUnused()) { + uint64_t size = + CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy)); + if (EmitLifetimeStart(size, SRetPtr)) + UnusedReturnSize = size; + } + } if (IRFunctionArgs.hasSRetArg()) { IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr; } else { llvm::Value *Addr = - Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex()); + Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory, + RetAI.getInAllocaFieldIndex()); Builder.CreateStore(SRetPtr, Addr); } } @@ -3049,14 +3194,16 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, cast<llvm::Instruction>(RV.getAggregateAddr()); CGBuilderTy::InsertPoint IP = Builder.saveIP(); Builder.SetInsertPoint(Placeholder); - llvm::Value *Addr = Builder.CreateStructGEP( - ArgMemory, ArgInfo.getInAllocaFieldIndex()); + llvm::Value *Addr = + Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory, + ArgInfo.getInAllocaFieldIndex()); Builder.restoreIP(IP); deferPlaceholderReplacement(Placeholder, Addr); } else { // Store the RValue into the argument struct. llvm::Value *Addr = - Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex()); + Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory, + ArgInfo.getInAllocaFieldIndex()); unsigned AS = Addr->getType()->getPointerAddressSpace(); llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS); // There are some cases where a trivial bitcast is not avoidable. The @@ -3100,8 +3247,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, : 0); if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) || (ArgInfo.getIndirectByVal() && TypeAlign.getQuantity() < Align && - llvm::getOrEnforceKnownAlignment(Addr, Align, TD) < Align) || - (ArgInfo.getIndirectByVal() && (RVAddrSpace != ArgAddrSpace))) { + llvm::getOrEnforceKnownAlignment(Addr, Align, *TD) < Align) || + (ArgInfo.getIndirectByVal() && (RVAddrSpace != ArgAddrSpace))) { // Create an aligned temporary, and copy to it. llvm::AllocaInst *AI = CreateMemTemp(I->Ty); if (Align > AI->getAlignment()) @@ -3148,20 +3295,25 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // FIXME: Avoid the conversion through memory if possible. llvm::Value *SrcPtr; + CharUnits SrcAlign; if (RV.isScalar() || RV.isComplex()) { SrcPtr = CreateMemTemp(I->Ty, "coerce"); + SrcAlign = TypeAlign; LValue SrcLV = MakeAddrLValue(SrcPtr, I->Ty, TypeAlign); EmitInitStoreOfNonAggregate(*this, RV, SrcLV); - } else + } else { SrcPtr = RV.getAggregateAddr(); + // This alignment is guaranteed by EmitCallArg. + SrcAlign = TypeAlign; + } // If the value is offset in memory, apply the offset now. if (unsigned Offs = ArgInfo.getDirectOffset()) { SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy()); - SrcPtr = Builder.CreateConstGEP1_32(SrcPtr, Offs); + SrcPtr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), SrcPtr, Offs); SrcPtr = Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(ArgInfo.getCoerceToType())); - + SrcAlign = SrcAlign.alignmentAtOffset(CharUnits::fromQuantity(Offs)); } // Fast-isel and the optimizer generally like scalar values better than @@ -3190,7 +3342,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(NumIRArgs == STy->getNumElements()); for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { - llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i); + llvm::Value *EltPtr = Builder.CreateConstGEP2_32(STy, SrcPtr, 0, i); llvm::LoadInst *LI = Builder.CreateLoad(EltPtr); // We don't know what we're loading from. LI->setAlignment(1); @@ -3200,7 +3352,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // In the simple case, just pass the coerced loaded value. assert(NumIRArgs == 1); IRCallArgs[FirstIRArg] = - CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(), *this); + CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(), + SrcAlign, *this); } break; @@ -3300,7 +3453,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm::BasicBlock *InvokeDest = nullptr; if (!Attrs.hasAttribute(llvm::AttributeSet::FunctionIndex, - llvm::Attribute::NoUnwind)) + llvm::Attribute::NoUnwind) || + currentFunctionUsesSEHTry()) InvokeDest = getInvokeDest(); llvm::CallSite CS; @@ -3320,6 +3474,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex, llvm::Attribute::AlwaysInline); + // Disable inlining inside SEH __try blocks. + if (isSEHTryScope()) + Attrs = + Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex, + llvm::Attribute::NoInline); + CS.setAttributes(Attrs); CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv)); @@ -3332,6 +3492,10 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // insertion point; this allows the rest of IRgen to discard // unreachable code. if (CS.doesNotReturn()) { + if (UnusedReturnSize) + EmitLifetimeEnd(llvm::ConstantInt::get(Int64Ty, UnusedReturnSize), + SRetPtr); + Builder.CreateUnreachable(); Builder.ClearInsertionPoint(); @@ -3360,8 +3524,13 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, RValue Ret = [&] { switch (RetAI.getKind()) { case ABIArgInfo::InAlloca: - case ABIArgInfo::Indirect: - return convertTempToRValue(SRetPtr, RetTy, SourceLocation()); + case ABIArgInfo::Indirect: { + RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation()); + if (UnusedReturnSize) + EmitLifetimeEnd(llvm::ConstantInt::get(Int64Ty, UnusedReturnSize), + SRetPtr); + return ret; + } case ABIArgInfo::Ignore: // If we are ignoring an argument that had a result, make sure to @@ -3381,12 +3550,13 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, case TEK_Aggregate: { llvm::Value *DestPtr = ReturnValue.getValue(); bool DestIsVolatile = ReturnValue.isVolatile(); + CharUnits DestAlign = getContext().getTypeAlignInChars(RetTy); if (!DestPtr) { DestPtr = CreateMemTemp(RetTy, "agg.tmp"); DestIsVolatile = false; } - BuildAggStore(*this, CI, DestPtr, DestIsVolatile, false); + BuildAggStore(*this, CI, DestPtr, DestIsVolatile, DestAlign); return RValue::getAggregate(DestPtr); } case TEK_Scalar: { @@ -3403,6 +3573,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm::Value *DestPtr = ReturnValue.getValue(); bool DestIsVolatile = ReturnValue.isVolatile(); + CharUnits DestAlign = getContext().getTypeAlignInChars(RetTy); if (!DestPtr) { DestPtr = CreateMemTemp(RetTy, "coerce"); @@ -3411,13 +3582,17 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If the value is offset in memory, apply the offset now. llvm::Value *StorePtr = DestPtr; + CharUnits StoreAlign = DestAlign; if (unsigned Offs = RetAI.getDirectOffset()) { StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy()); - StorePtr = Builder.CreateConstGEP1_32(StorePtr, Offs); + StorePtr = + Builder.CreateConstGEP1_32(Builder.getInt8Ty(), StorePtr, Offs); StorePtr = Builder.CreateBitCast(StorePtr, llvm::PointerType::getUnqual(RetAI.getCoerceToType())); + StoreAlign = + StoreAlign.alignmentAtOffset(CharUnits::fromQuantity(Offs)); } - CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this); + CreateCoercedStore(CI, StorePtr, DestIsVolatile, StoreAlign, *this); return convertTempToRValue(DestPtr, RetTy, SourceLocation()); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h index b228733..7a4708e 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCall.h @@ -155,17 +155,25 @@ namespace CodeGen { /// ReturnValueSlot - Contains the address where the return value of a /// function can be stored, and whether the address is volatile or not. class ReturnValueSlot { - llvm::PointerIntPair<llvm::Value *, 1, bool> Value; + llvm::PointerIntPair<llvm::Value *, 2, unsigned int> Value; + + // Return value slot flags + enum Flags { + IS_VOLATILE = 0x1, + IS_UNUSED = 0x2, + }; public: ReturnValueSlot() {} - ReturnValueSlot(llvm::Value *Value, bool IsVolatile) - : Value(Value, IsVolatile) {} + ReturnValueSlot(llvm::Value *Value, bool IsVolatile, bool IsUnused = false) + : Value(Value, + (IsVolatile ? IS_VOLATILE : 0) | (IsUnused ? IS_UNUSED : 0)) {} bool isNull() const { return !getValue(); } - - bool isVolatile() const { return Value.getInt(); } + + bool isVolatile() const { return Value.getInt() & IS_VOLATILE; } llvm::Value *getValue() const { return Value.getPointer(); } + bool isUnused() const { return Value.getInt() & IS_UNUSED; } }; } // end namespace CodeGen diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp index d72eda9..c49f182 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGClass.cpp @@ -24,35 +24,35 @@ #include "clang/Basic/TargetBuiltins.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/CodeGenOptions.h" +#include "llvm/IR/Intrinsics.h" using namespace clang; using namespace CodeGen; -static CharUnits -ComputeNonVirtualBaseClassOffset(ASTContext &Context, - const CXXRecordDecl *DerivedClass, - CastExpr::path_const_iterator Start, - CastExpr::path_const_iterator End) { +CharUnits CodeGenModule::computeNonVirtualBaseClassOffset( + const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start, + CastExpr::path_const_iterator End) { CharUnits Offset = CharUnits::Zero(); - + + const ASTContext &Context = getContext(); const CXXRecordDecl *RD = DerivedClass; - + for (CastExpr::path_const_iterator I = Start; I != End; ++I) { const CXXBaseSpecifier *Base = *I; assert(!Base->isVirtual() && "Should not see virtual bases here!"); // Get the layout. const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD); - - const CXXRecordDecl *BaseDecl = + + const CXXRecordDecl *BaseDecl = cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl()); - + // Add the offset. Offset += Layout.getBaseClassOffset(BaseDecl); - + RD = BaseDecl; } - + return Offset; } @@ -62,15 +62,14 @@ CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl, CastExpr::path_const_iterator PathEnd) { assert(PathBegin != PathEnd && "Base path should not be empty!"); - CharUnits Offset = - ComputeNonVirtualBaseClassOffset(getContext(), ClassDecl, - PathBegin, PathEnd); + CharUnits Offset = + computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd); if (Offset.isZero()) return nullptr; - llvm::Type *PtrDiffTy = + llvm::Type *PtrDiffTy = Types.ConvertType(getContext().getPointerDiffType()); - + return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity()); } @@ -127,7 +126,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, llvm::Value *ptr, } else { baseOffset = virtualOffset; } - + // Apply the base offset. ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8PtrTy); ptr = CGF.Builder.CreateInBoundsGEP(ptr, baseOffset, "add.ptr"); @@ -149,7 +148,7 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( // *start* with a step down to the correct virtual base subobject, // and hence will not require any further steps. if ((*Start)->isVirtual()) { - VBase = + VBase = cast<CXXRecordDecl>((*Start)->getType()->getAs<RecordType>()->getDecl()); ++Start; } @@ -157,9 +156,8 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( // Compute the static offset of the ultimate destination within its // allocating subobject (the virtual base, if there is one, or else // the "complete" object that we see). - CharUnits NonVirtualOffset = - ComputeNonVirtualBaseClassOffset(getContext(), VBase ? VBase : Derived, - Start, PathEnd); + CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset( + VBase ? VBase : Derived, Start, PathEnd); // If there's a virtual step, we can sometimes "devirtualize" it. // For now, that's limited to when the derived type is final. @@ -172,7 +170,7 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( } // Get the base pointer type. - llvm::Type *BasePtrTy = + llvm::Type *BasePtrTy = ConvertType((PathEnd[-1])->getType())->getPointerTo(); QualType DerivedTy = getContext().getRecordType(Derived); @@ -197,7 +195,7 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( origBB = Builder.GetInsertBlock(); llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull"); endBB = createBasicBlock("cast.end"); - + llvm::Value *isNull = Builder.CreateIsNull(Value); Builder.CreateCondBr(isNull, endBB, notNullBB); EmitBlock(notNullBB); @@ -216,10 +214,10 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( } // Apply both offsets. - Value = ApplyNonVirtualAndVirtualOffset(*this, Value, + Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset, VirtualOffset); - + // Cast to the destination type. Value = Builder.CreateBitCast(Value, BasePtrTy); @@ -228,13 +226,13 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( llvm::BasicBlock *notNullBB = Builder.GetInsertBlock(); Builder.CreateBr(endBB); EmitBlock(endBB); - + llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result"); PHI->addIncoming(Value, notNullBB); PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB); Value = PHI; } - + return Value; } @@ -252,7 +250,7 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value, llvm::Value *NonVirtualOffset = CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd); - + if (!NonVirtualOffset) { // No offset, we can just cast back. return Builder.CreateBitCast(Value, DerivedPtrTy); @@ -266,12 +264,12 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value, CastNull = createBasicBlock("cast.null"); CastNotNull = createBasicBlock("cast.notnull"); CastEnd = createBasicBlock("cast.end"); - + llvm::Value *IsNull = Builder.CreateIsNull(Value); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } - + // Apply the offset. Value = Builder.CreateBitCast(Value, Int8PtrTy); Value = Builder.CreateGEP(Value, Builder.CreateNeg(NonVirtualOffset), @@ -285,14 +283,14 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value, EmitBlock(CastNull); Builder.CreateBr(CastEnd); EmitBlock(CastEnd); - + llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2); PHI->addIncoming(Value, CastNotNull); - PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), + PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull); Value = PHI; } - + return Value; } @@ -303,7 +301,7 @@ llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, // This constructor/destructor does not need a VTT parameter. return nullptr; } - + const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent(); const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent(); @@ -323,15 +321,15 @@ llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, SubVTTIndex = 0; } else { const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD); - CharUnits BaseOffset = ForVirtualBase ? - Layout.getVBaseClassOffset(Base) : + CharUnits BaseOffset = ForVirtualBase ? + Layout.getVBaseClassOffset(Base) : Layout.getBaseClassOffset(Base); - SubVTTIndex = + SubVTTIndex = CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset)); assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!"); } - + if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) { // A VTT parameter was passed to the constructor, use it. VTT = LoadCXXVTT(); @@ -358,7 +356,7 @@ namespace { cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent(); const CXXDestructorDecl *D = BaseClass->getDestructor(); - llvm::Value *Addr = + llvm::Value *Addr = CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThis(), DerivedClass, BaseClass, BaseIsVirtual); @@ -369,29 +367,29 @@ namespace { /// A visitor which checks whether an initializer uses 'this' in a /// way which requires the vtable to be properly set. - struct DynamicThisUseChecker : EvaluatedExprVisitor<DynamicThisUseChecker> { - typedef EvaluatedExprVisitor<DynamicThisUseChecker> super; + struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> { + typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super; bool UsesThis; - DynamicThisUseChecker(ASTContext &C) : super(C), UsesThis(false) {} + DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {} // Black-list all explicit and implicit references to 'this'. // // Do we need to worry about external references to 'this' derived // from arbitrary code? If so, then anything which runs arbitrary // external code might potentially access the vtable. - void VisitCXXThisExpr(CXXThisExpr *E) { UsesThis = true; } + void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; } }; } static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) { DynamicThisUseChecker Checker(C); - Checker.Visit(const_cast<Expr*>(Init)); + Checker.Visit(Init); return Checker.UsesThis; } -static void EmitBaseInitializer(CodeGenFunction &CGF, +static void EmitBaseInitializer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl, CXXCtorInitializer *BaseInit, CXXCtorType CtorType) { @@ -399,7 +397,7 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, "Must have base initializer!"); llvm::Value *ThisPtr = CGF.LoadCXXThis(); - + const Type *BaseType = BaseInit->getBaseClass(); CXXRecordDecl *BaseClassDecl = cast<CXXRecordDecl>(BaseType->getAs<RecordType>()->getDecl()); @@ -418,7 +416,7 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, // We can pretend to be a complete class because it only matters for // virtual bases, and we only do virtual bases for complete ctors. - llvm::Value *V = + llvm::Value *V = CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl, BaseClassDecl, isBaseVirtual); @@ -430,8 +428,8 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, AggValueSlot::IsNotAliased); CGF.EmitAggExpr(BaseInit->getInit(), AggSlot); - - if (CGF.CGM.getLangOpts().Exceptions && + + if (CGF.CGM.getLangOpts().Exceptions && !BaseClassDecl->hasTrivialDestructor()) CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl, isBaseVirtual); @@ -490,17 +488,17 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF, llvm::Value *IndexVar = CGF.GetAddrOfLocalVar(ArrayIndexes[Index]); assert(IndexVar && "Array index variable not loaded"); - + // Initialize this index variable to zero. llvm::Value* Zero = llvm::Constant::getNullValue( CGF.ConvertType(CGF.getContext().getSizeType())); CGF.Builder.CreateStore(Zero, IndexVar); - + // Start the loop with a block that tests the condition. llvm::BasicBlock *CondBlock = CGF.createBasicBlock("for.cond"); llvm::BasicBlock *AfterFor = CGF.createBasicBlock("for.end"); - + CGF.EmitBlock(CondBlock); llvm::BasicBlock *ForBody = CGF.createBasicBlock("for.body"); @@ -512,7 +510,7 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF, llvm::ConstantInt::get(Counter->getType(), NumElements); llvm::Value *IsLess = CGF.Builder.CreateICmpULT(Counter, NumElementsPtr, "isless"); - + // If the condition is true, execute the body. CGF.Builder.CreateCondBr(IsLess, ForBody, AfterFor); @@ -539,6 +537,37 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF, CGF.EmitBlock(AfterFor, true); } +static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) { + auto *CD = dyn_cast<CXXConstructorDecl>(D); + if (!(CD && CD->isCopyOrMoveConstructor()) && + !D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator()) + return false; + + // We can emit a memcpy for a trivial copy or move constructor/assignment. + if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding()) + return true; + + // We *must* emit a memcpy for a defaulted union copy or move op. + if (D->getParent()->isUnion() && D->isDefaulted()) + return true; + + return false; +} + +static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF, + CXXCtorInitializer *MemberInit, + LValue &LHS) { + FieldDecl *Field = MemberInit->getAnyMember(); + if (MemberInit->isIndirectMemberInitializer()) { + // If we are initializing an anonymous union field, drill down to the field. + IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember(); + for (const auto *I : IndirectField->chain()) + LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I)); + } else { + LHS = CGF.EmitLValueForFieldInitialization(LHS, Field); + } +} + static void EmitMemberInitializer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl, CXXCtorInitializer *MemberInit, @@ -548,7 +577,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF, assert(MemberInit->isAnyMemberInitializer() && "Must have member initializer!"); assert(MemberInit->getInit() && "Must have initializer!"); - + // non-static data member initializers. FieldDecl *Field = MemberInit->getAnyMember(); QualType FieldType = Field->getType(); @@ -557,16 +586,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF, QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl); LValue LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy); - if (MemberInit->isIndirectMemberInitializer()) { - // If we are initializing an anonymous union field, drill down to - // the field. - IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember(); - for (const auto *I : IndirectField->chain()) - LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I)); - FieldType = MemberInit->getIndirectMember()->getAnonField()->getType(); - } else { - LHS = CGF.EmitLValueForFieldInitialization(LHS, Field); - } + EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS); // Special case: if we are in a copy or move constructor, and we are copying // an array of PODs or classes with trivial copy constructors, ignore the @@ -580,17 +600,22 @@ static void EmitMemberInitializer(CodeGenFunction &CGF, QualType BaseElementTy = CGF.getContext().getBaseElementType(Array); CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit()); if (BaseElementTy.isPODType(CGF.getContext()) || - (CE && CE->getConstructor()->isTrivial())) { + (CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) { unsigned SrcArgIndex = CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args); llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex])); LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy); LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field); - + // Copy the aggregate. CGF.EmitAggregateCopy(LHS.getAddress(), Src.getAddress(), FieldType, LHS.isVolatileQualified()); + // Ensure that we destroy the objects if an exception is thrown later in + // the constructor. + QualType::DestructionKind dtorKind = FieldType.isDestructedType(); + if (CGF.needsEHCleanup(dtorKind)) + CGF.pushEHDestroy(dtorKind, LHS.getAddress(), FieldType); return; } } @@ -621,28 +646,28 @@ void CodeGenFunction::EmitInitializerForField( llvm::Value *ArrayIndexVar = nullptr; if (ArrayIndexes.size()) { llvm::Type *SizeTy = ConvertType(getContext().getSizeType()); - + // The LHS is a pointer to the first object we'll be constructing, as // a flat array. QualType BaseElementTy = getContext().getBaseElementType(FieldType); llvm::Type *BasePtr = ConvertType(BaseElementTy); BasePtr = llvm::PointerType::getUnqual(BasePtr); - llvm::Value *BaseAddrPtr = Builder.CreateBitCast(LHS.getAddress(), + llvm::Value *BaseAddrPtr = Builder.CreateBitCast(LHS.getAddress(), BasePtr); LHS = MakeAddrLValue(BaseAddrPtr, BaseElementTy); - + // Create an array index that will be used to walk over all of the // objects we're constructing. ArrayIndexVar = CreateTempAlloca(SizeTy, "object.index"); llvm::Value *Zero = llvm::Constant::getNullValue(SizeTy); Builder.CreateStore(Zero, ArrayIndexVar); - - + + // Emit the block variables for the array indices, if any. for (unsigned I = 0, N = ArrayIndexes.size(); I != N; ++I) EmitAutoVarDecl(*ArrayIndexes[I]); } - + EmitAggMemberInitializer(*this, LHS, Init, ArrayIndexVar, FieldType, ArrayIndexes, 0); } @@ -762,9 +787,9 @@ void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) { if (PoisonSize < AsanAlignment || !SSV[i].Size || (NextField % AsanAlignment) != 0) continue; - Builder.CreateCall2( - F, Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)), - Builder.getIntN(PtrSize, PoisonSize)); + Builder.CreateCall( + F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)), + Builder.getIntN(PtrSize, PoisonSize)}); } } @@ -796,8 +821,7 @@ void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) { if (IsTryBody) EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true); - RegionCounter Cnt = getPGORegionCounter(Body); - Cnt.beginRegion(Builder); + incrementProfileCounter(Body); RunCleanupsScope RunCleanups(*this); @@ -850,7 +874,7 @@ namespace { public: FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl, const VarDecl *SrcRec) - : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec), + : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec), RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)), FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0), LastFieldOffset(0), LastAddedFieldIndex(0) {} @@ -876,7 +900,7 @@ namespace { unsigned LastFieldSize = LastField->isBitField() ? LastField->getBitWidthValue(CGF.getContext()) : - CGF.getContext().getTypeSize(LastField->getType()); + CGF.getContext().getTypeSize(LastField->getType()); uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize - FirstByteOffset + CGF.getContext().getCharWidth() - 1; @@ -892,32 +916,18 @@ namespace { return; } - CharUnits Alignment; - uint64_t FirstByteOffset; if (FirstField->isBitField()) { const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(FirstField->getParent()); const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField); - Alignment = CharUnits::fromQuantity(BFInfo.StorageAlignment); // FirstFieldOffset is not appropriate for bitfields, - // it won't tell us what the storage offset should be and thus might not - // be properly aligned. - // - // Instead calculate the storage offset using the offset of the field in - // the struct type. - const llvm::DataLayout &DL = CGF.CGM.getDataLayout(); - FirstByteOffset = - DL.getStructLayout(RL.getLLVMType()) - ->getElementOffsetInBits(RL.getLLVMFieldNo(FirstField)); + // we need to use the storage offset instead. + FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset); } else { - Alignment = CGF.getContext().getDeclAlign(FirstField); FirstByteOffset = FirstFieldOffset; } - assert((CGF.getContext().toCharUnitsFromBits(FirstByteOffset) % - Alignment) == 0 && "Bad field alignment."); - CharUnits MemcpySize = getMemcpySize(FirstByteOffset); QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl); llvm::Value *ThisPtr = CGF.LoadCXXThis(); @@ -927,6 +937,9 @@ namespace { LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy); LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField); + CharUnits Offset = CGF.getContext().toCharUnitsFromBits(FirstByteOffset); + CharUnits Alignment = DestLV.getAlignment().alignmentAtOffset(Offset); + emitMemcpyIR(Dest.isBitField() ? Dest.getBitFieldAddr() : Dest.getAddress(), Src.isBitField() ? Src.getBitFieldAddr() : Src.getAddress(), MemcpySize, Alignment); @@ -1021,8 +1034,8 @@ namespace { QualType FieldType = Field->getType(); CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit()); - // Bail out on non-POD, not-trivially-constructable members. - if (!(CE && CE->getConstructor()->isTrivial()) && + // Bail out on non-memcpyable, not-trivially-copyable members. + if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) && !(FieldType.isTriviallyCopyableType(CGF.getContext()) || FieldType->isReferenceType())) return false; @@ -1064,6 +1077,7 @@ namespace { CopyingValueRepresentation CVR(CGF); EmitMemberInitializer(CGF, ConstructorDecl->getParent(), AggregatedInits[0], ConstructorDecl, Args); + AggregatedInits.clear(); } reset(); return; @@ -1080,10 +1094,14 @@ namespace { LValue LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy); for (unsigned i = 0; i < AggregatedInits.size(); ++i) { - QualType FieldType = AggregatedInits[i]->getMember()->getType(); + CXXCtorInitializer *MemberInit = AggregatedInits[i]; + QualType FieldType = MemberInit->getAnyMember()->getType(); QualType::DestructionKind dtorKind = FieldType.isDestructedType(); - if (CGF.needsEHCleanup(dtorKind)) - CGF.pushEHDestroy(dtorKind, LHS.getAddress(), FieldType); + if (!CGF.needsEHCleanup(dtorKind)) + continue; + LValue FieldLHS = LHS; + EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS); + CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(), FieldType); } } @@ -1127,9 +1145,7 @@ namespace { return Field; } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) { CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl()); - if (!(MD && (MD->isCopyAssignmentOperator() || - MD->isMoveAssignmentOperator()) && - MD->isTrivial())) + if (!(MD && isMemcpyEquivalentSpecialMember(MD))) return nullptr; MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument()); if (!IOA) @@ -1189,7 +1205,7 @@ namespace { if (F) { addMemcpyableField(F); AggregatedStmts.push_back(S); - } else { + } else { emitAggregatedStmts(); CGF.EmitStmt(S); } @@ -1274,7 +1290,7 @@ static bool FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field); static bool -HasTrivialDestructorBody(ASTContext &Context, +HasTrivialDestructorBody(ASTContext &Context, const CXXRecordDecl *BaseClassDecl, const CXXRecordDecl *MostDerivedClassDecl) { @@ -1309,7 +1325,7 @@ HasTrivialDestructorBody(ASTContext &Context, cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); if (!HasTrivialDestructorBody(Context, VirtualBase, MostDerivedClassDecl)) - return false; + return false; } } @@ -1325,8 +1341,13 @@ FieldHasTrivialDestructorBody(ASTContext &Context, const RecordType *RT = FieldBaseElementType->getAs<RecordType>(); if (!RT) return true; - + CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl()); + + // The destructor for an implicit anonymous union member is never invoked. + if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion()) + return false; + return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl); } @@ -1346,11 +1367,34 @@ static bool CanSkipVTablePointerInitialization(ASTContext &Context, return true; } +// Generates function call for handling object poisoning, passing in +// references to 'this' and its size as arguments. +static void EmitDtorSanitizerCallback(CodeGenFunction &CGF, + const CXXDestructorDecl *Dtor) { + const ASTRecordLayout &Layout = + CGF.getContext().getASTRecordLayout(Dtor->getParent()); + + llvm::Value *Args[] = { + CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.VoidPtrTy), + llvm::ConstantInt::get(CGF.SizeTy, Layout.getSize().getQuantity())}; + llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy}; + + llvm::FunctionType *FnType = + llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false); + llvm::Value *Fn = + CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback"); + CGF.EmitNounwindRuntimeCall(Fn, Args); +} + /// EmitDestructorBody - Emits the body of the current destructor. void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl()); CXXDtorType DtorType = CurGD.getDtorType(); + Stmt *Body = Dtor->getBody(); + if (Body) + incrementProfileCounter(Body); + // The call to operator delete in a deleting destructor happens // outside of the function-try-block, which means it's always // possible to delegate the destructor body to the complete @@ -1363,8 +1407,6 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { return; } - Stmt *Body = Dtor->getBody(); - // If the body is a function-try-block, enter the try before // anything else. bool isTryBody = (Body && isa<CXXTryStmt>(Body)); @@ -1374,11 +1416,11 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { // Enter the epilogue cleanups. RunCleanupsScope DtorEpilogue(*this); - + // If this is the complete variant, just invoke the base variant; // the epilogue will destruct the virtual bases. But we can't do // this optimization if the body is a function-try-block, because - // we'd introduce *two* handler blocks. In the Microsoft ABI, we + // we'd introduce *two* handler blocks. In the Microsoft ABI, we // always delegate because we might not have a definition in this TU. switch (DtorType) { case Dtor_Comdat: @@ -1399,13 +1441,10 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { break; } // Fallthrough: act like we're in the base variant. - + case Dtor_Base: assert(Body); - RegionCounter Cnt = getPGORegionCounter(Body); - Cnt.beginRegion(Builder); - // Enter the cleanup scopes for fields and non-virtual bases. EnterDtorCleanups(Dtor, Dtor_Base); @@ -1434,6 +1473,10 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { // Exit the try if applicable. if (isTryBody) ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true); + + // Insert memory-poisoning instrumentation. + if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor) + EmitDtorSanitizerCallback(*this, Dtor); } void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) { @@ -1447,7 +1490,7 @@ void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) AssignmentMemcpyizer AM(*this, AssignOp, Args); for (auto *I : RootCS->body()) - AM.emitAssignment(I); + AM.emitAssignment(I); AM.finish(); } @@ -1508,7 +1551,7 @@ namespace { LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy); LValue LV = CGF.EmitLValueForField(ThisLV, field); assert(LV.isSimple()); - + CGF.emitDestroy(LV.getAddress(), field->getType(), destroyer, flags.isForNormalCleanup() && useEHCleanupForArray); } @@ -1526,7 +1569,7 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD, // The deleting-destructor phase just needs to call the appropriate // operator delete that Sema picked up. if (DtorType == Dtor_Deleting) { - assert(DD->getOperatorDelete() && + assert(DD->getOperatorDelete() && "operator delete missing - EnterDtorCleanups"); if (CXXStructorImplicitParamValue) { // If there is an implicit param to the deleting dtor, it's a boolean @@ -1553,7 +1596,7 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD, for (const auto &Base : ClassDecl->vbases()) { CXXRecordDecl *BaseClassDecl = cast<CXXRecordDecl>(Base.getType()->getAs<RecordType>()->getDecl()); - + // Ignore trivial destructors. if (BaseClassDecl->hasTrivialDestructor()) continue; @@ -1567,15 +1610,15 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD, } assert(DtorType == Dtor_Base); - + // Destroy non-virtual bases. for (const auto &Base : ClassDecl->bases()) { // Ignore virtual bases. if (Base.isVirtual()) continue; - + CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl(); - + // Ignore trivial destructors. if (BaseClassDecl->hasTrivialDestructor()) continue; @@ -1656,7 +1699,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB); EmitBlock(loopBB); } - + // Find the end of the array. llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(arrayBegin, numElements, "arrayctor.end"); @@ -1676,15 +1719,15 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, // Zero initialize the storage, if requested. if (zeroInitialize) EmitNullInitialization(cur, type); - - // C++ [class.temporary]p4: + + // C++ [class.temporary]p4: // There are two contexts in which temporaries are destroyed at a different // point than the end of the full-expression. The first context is when a - // default constructor is called to initialize an element of an array. - // If the constructor has one or more default arguments, the destruction of - // every temporary created in a default argument expression is sequenced + // default constructor is called to initialize an element of an array. + // If the constructor has one or more default arguments, the destruction of + // every temporary created in a default argument expression is sequenced // before the construction of the next array element, if any. - + { RunCleanupsScope Scope(*this); @@ -1733,33 +1776,32 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, bool ForVirtualBase, bool Delegating, llvm::Value *This, const CXXConstructExpr *E) { - // If this is a trivial constructor, just emit what's needed. - if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding()) { - if (E->getNumArgs() == 0) { - // Trivial default constructor, no codegen required. - assert(D->isDefaultConstructor() && - "trivial 0-arg ctor not a default ctor"); - return; - } + // C++11 [class.mfct.non-static]p2: + // If a non-static member function of a class X is called for an object that + // is not of type X, or of a type derived from X, the behavior is undefined. + // FIXME: Provide a source location here. + EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, SourceLocation(), This, + getContext().getRecordType(D->getParent())); + + if (D->isTrivial() && D->isDefaultConstructor()) { + assert(E->getNumArgs() == 0 && "trivial default ctor with args"); + return; + } + // If this is a trivial constructor, just emit what's needed. If this is a + // union copy constructor, we must emit a memcpy, because the AST does not + // model that copy. + if (isMemcpyEquivalentSpecialMember(D)) { assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor"); - assert(D->isCopyOrMoveConstructor() && - "trivial 1-arg ctor not a copy/move ctor"); const Expr *Arg = E->getArg(0); - QualType Ty = Arg->getType(); + QualType SrcTy = Arg->getType(); llvm::Value *Src = EmitLValue(Arg).getAddress(); - EmitAggregateCopy(This, Src, Ty); + QualType DestTy = getContext().getTypeDeclType(D->getParent()); + EmitAggregateCopyCtor(This, Src, DestTy, SrcTy); return; } - // C++11 [class.mfct.non-static]p2: - // If a non-static member function of a class X is called for an object that - // is not of type X, or of a type derived from X, the behavior is undefined. - // FIXME: Provide a source location here. - EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, SourceLocation(), This, - getContext().getRecordType(D->getParent())); - CallArgList Args; // Push the this ptr. @@ -1784,25 +1826,26 @@ void CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, llvm::Value *This, llvm::Value *Src, const CXXConstructExpr *E) { - if (D->isTrivial() && - !D->getParent()->mayInsertExtraPadding()) { + if (isMemcpyEquivalentSpecialMember(D)) { assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor"); assert(D->isCopyOrMoveConstructor() && "trivial 1-arg ctor not a copy/move ctor"); - EmitAggregateCopy(This, Src, E->arg_begin()->getType()); + EmitAggregateCopyCtor(This, Src, + getContext().getTypeDeclType(D->getParent()), + E->arg_begin()->getType()); return; } llvm::Value *Callee = CGM.getAddrOfCXXStructor(D, StructorType::Complete); assert(D->isInstance() && "Trying to emit a member call expr on a static method!"); - + const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>(); - + CallArgList Args; - + // Push the this ptr. Args.add(RValue::get(This), D->getThisType(getContext())); - + // Push the src ptr. QualType QT = *(FPT->param_type_begin()); llvm::Type *t = CGM.getTypes().ConvertType(QT); @@ -1945,10 +1988,18 @@ void CodeGenFunction::PushDestructorCleanup(QualType T, llvm::Value *Addr) { } void -CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, +CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, const CXXRecordDecl *NearestVBase, CharUnits OffsetFromNearestVBase, const CXXRecordDecl *VTableClass) { + const CXXRecordDecl *RD = Base.getBase(); + + // Don't initialize the vtable pointer if the class is marked with the + // 'novtable' attribute. + if ((RD == VTableClass || RD == NearestVBase) && + VTableClass->hasAttr<MSNoVTableAttr>()) + return; + // Compute the address point. bool NeedsVirtualOffset; llvm::Value *VTableAddressPoint = @@ -1960,7 +2011,7 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, // Compute where to store the address point. llvm::Value *VirtualOffset = nullptr; CharUnits NonVirtualOffset = CharUnits::Zero(); - + if (NeedsVirtualOffset) { // We need to use the virtual base offset offset because the virtual base // might have a different offset in the most derived class. @@ -1973,12 +2024,12 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, // We can just use the base offset in the complete class. NonVirtualOffset = Base.getBaseOffset(); } - + // Apply the offsets. llvm::Value *VTableField = LoadCXXThis(); - + if (!NonVirtualOffset.isZero() || VirtualOffset) - VTableField = ApplyNonVirtualAndVirtualOffset(*this, VTableField, + VTableField = ApplyNonVirtualAndVirtualOffset(*this, VTableField, NonVirtualOffset, VirtualOffset); @@ -1995,7 +2046,7 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, } void -CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, +CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase, CharUnits OffsetFromNearestVBase, bool BaseIsNonVirtualPrimaryBase, @@ -2008,7 +2059,7 @@ CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, InitializeVTablePointer(Base, NearestVBase, OffsetFromNearestVBase, VTableClass); } - + const CXXRecordDecl *RD = Base.getBase(); // Traverse bases. @@ -2029,7 +2080,7 @@ CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, if (!VBases.insert(BaseDecl).second) continue; - const ASTRecordLayout &Layout = + const ASTRecordLayout &Layout = getContext().getASTRecordLayout(VTableClass); BaseOffset = Layout.getVBaseClassOffset(BaseDecl); @@ -2039,15 +2090,15 @@ CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD); BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl); - BaseOffsetFromNearestVBase = + BaseOffsetFromNearestVBase = OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl); BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl; } - - InitializeVTablePointers(BaseSubobject(BaseDecl, BaseOffset), + + InitializeVTablePointers(BaseSubobject(BaseDecl, BaseOffset), I.isVirtual() ? BaseDecl : NearestVBase, BaseOffsetFromNearestVBase, - BaseDeclIsNonVirtualPrimaryBase, + BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases); } } @@ -2059,7 +2110,7 @@ void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) { // Initialize the vtable pointers for this class and all of its bases. VisitedVirtualBasesSetTy VBases; - InitializeVTablePointers(BaseSubobject(RD, CharUnits::Zero()), + InitializeVTablePointers(BaseSubobject(RD, CharUnits::Zero()), /*NearestVBase=*/nullptr, /*OffsetFromNearestVBase=*/CharUnits::Zero(), /*BaseIsNonVirtualPrimaryBase=*/false, RD, VBases); @@ -2076,6 +2127,149 @@ llvm::Value *CodeGenFunction::GetVTablePtr(llvm::Value *This, return VTable; } +// If a class has a single non-virtual base and does not introduce or override +// virtual member functions or fields, it will have the same layout as its base. +// This function returns the least derived such class. +// +// Casting an instance of a base class to such a derived class is technically +// undefined behavior, but it is a relatively common hack for introducing member +// functions on class instances with specific properties (e.g. llvm::Operator) +// that works under most compilers and should not have security implications, so +// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict. +static const CXXRecordDecl * +LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) { + if (!RD->field_empty()) + return RD; + + if (RD->getNumVBases() != 0) + return RD; + + if (RD->getNumBases() != 1) + return RD; + + for (const CXXMethodDecl *MD : RD->methods()) { + if (MD->isVirtual()) { + // Virtual member functions are only ok if they are implicit destructors + // because the implicit destructor will have the same semantics as the + // base class's destructor if no fields are added. + if (isa<CXXDestructorDecl>(MD) && MD->isImplicit()) + continue; + return RD; + } + } + + return LeastDerivedClassWithSameLayout( + RD->bases_begin()->getType()->getAsCXXRecordDecl()); +} + +void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXMethodDecl *MD, + llvm::Value *VTable, + CFITypeCheckKind TCK, + SourceLocation Loc) { + const CXXRecordDecl *ClassDecl = MD->getParent(); + if (!SanOpts.has(SanitizerKind::CFICastStrict)) + ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl); + + EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc); +} + +void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, + llvm::Value *Derived, + bool MayBeNull, + CFITypeCheckKind TCK, + SourceLocation Loc) { + if (!getLangOpts().CPlusPlus) + return; + + auto *ClassTy = T->getAs<RecordType>(); + if (!ClassTy) + return; + + const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl()); + + if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass()) + return; + + SmallString<64> MangledName; + llvm::raw_svector_ostream Out(MangledName); + CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T.getUnqualifiedType(), + Out); + + // Blacklist based on the mangled type. + if (CGM.getContext().getSanitizerBlacklist().isBlacklistedType(Out.str())) + return; + + if (!SanOpts.has(SanitizerKind::CFICastStrict)) + ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl); + + llvm::BasicBlock *ContBlock = 0; + + if (MayBeNull) { + llvm::Value *DerivedNotNull = + Builder.CreateIsNotNull(Derived, "cast.nonnull"); + + llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check"); + ContBlock = createBasicBlock("cast.cont"); + + Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock); + + EmitBlock(CheckBlock); + } + + llvm::Value *VTable = GetVTablePtr(Derived, Int8PtrTy); + EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc); + + if (MayBeNull) { + Builder.CreateBr(ContBlock); + EmitBlock(ContBlock); + } +} + +void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD, + llvm::Value *VTable, + CFITypeCheckKind TCK, + SourceLocation Loc) { + if (CGM.IsCFIBlacklistedRecord(RD)) + return; + + SanitizerScope SanScope(this); + + std::string OutName; + llvm::raw_string_ostream Out(OutName); + CGM.getCXXABI().getMangleContext().mangleCXXVTableBitSet(RD, Out); + + llvm::Value *BitSetName = llvm::MetadataAsValue::get( + getLLVMContext(), llvm::MDString::get(getLLVMContext(), Out.str())); + + llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy); + llvm::Value *BitSetTest = + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::bitset_test), + {CastedVTable, BitSetName}); + + SanitizerMask M; + switch (TCK) { + case CFITCK_VCall: + M = SanitizerKind::CFIVCall; + break; + case CFITCK_NVCall: + M = SanitizerKind::CFINVCall; + break; + case CFITCK_DerivedCast: + M = SanitizerKind::CFIDerivedCast; + break; + case CFITCK_UnrelatedCast: + M = SanitizerKind::CFIUnrelatedCast; + break; + } + + llvm::Constant *StaticData[] = { + EmitCheckSourceLocation(Loc), + EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)), + llvm::ConstantInt::get(Int8Ty, TCK), + }; + EmitCheck(std::make_pair(BitSetTest, M), "cfi_bad_type", StaticData, + CastedVTable); +} // FIXME: Ideally Expr::IgnoreParenNoopCasts should do this, but it doesn't do // quite what we want. @@ -2140,7 +2334,7 @@ CodeGenFunction::CanDevirtualizeMemberFunctionCall(const Expr *Base, // This is a record decl. We know the type and can devirtualize it. return VD->getType()->isRecordType(); } - + return false; } @@ -2154,14 +2348,14 @@ CodeGenFunction::CanDevirtualizeMemberFunctionCall(const Expr *Base, // We can always devirtualize calls on temporary object expressions. if (isa<CXXConstructExpr>(Base)) return true; - + // And calls on bound temporaries. if (isa<CXXBindTemporaryExpr>(Base)) return true; - + // Check if this is a call expr that returns a record type. if (const CallExpr *CE = dyn_cast<CallExpr>(Base)) - return CE->getCallReturnType()->isRecordType(); + return CE->getCallReturnType(getContext())->isRecordType(); // We can't devirtualize the call. return false; @@ -2190,7 +2384,7 @@ void CodeGenFunction::EmitForwardingCallToLambda( // We don't need to separately arrange the call arguments because // the call can't be variadic anyway --- it's impossible to forward // variadic arguments. - + // Now emit our call. RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs, callOperator); @@ -2218,7 +2412,7 @@ void CodeGenFunction::EmitLambdaBlockInvokeBody() { for (auto param : BD->params()) EmitDelegateCallArg(CallArgs, param, param->getLocStart()); - assert(!Lambda->isGenericLambda() && + assert(!Lambda->isGenericLambda() && "generic lambda interconversion to block not implemented"); EmitForwardingCallToLambda(Lambda->getLambdaCallOperator(), CallArgs); } @@ -2256,7 +2450,7 @@ void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) { const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs(); FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate(); void *InsertPos = nullptr; - FunctionDecl *CorrespondingCallOpSpecialization = + FunctionDecl *CorrespondingCallOpSpecialization = CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos); assert(CorrespondingCallOpSpecialization); CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp index 18ed3e5..d97e405 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.cpp @@ -52,8 +52,10 @@ DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) { llvm::StructType::get(V.first->getType(), V.second->getType(), (void*) nullptr); llvm::Value *addr = CGF.CreateTempAlloca(ComplexTy, "saved-complex"); - CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0)); - CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1)); + CGF.Builder.CreateStore(V.first, + CGF.Builder.CreateStructGEP(ComplexTy, addr, 0)); + CGF.Builder.CreateStore(V.second, + CGF.Builder.CreateStructGEP(ComplexTy, addr, 1)); return saved_type(addr, ComplexAddress); } @@ -82,9 +84,9 @@ RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) { return RValue::getAggregate(CGF.Builder.CreateLoad(Value)); case ComplexAddress: { llvm::Value *real = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(Value, 0)); + CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(nullptr, Value, 0)); llvm::Value *imag = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(Value, 1)); + CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(nullptr, Value, 1)); return RValue::getComplex(real, imag); } } @@ -123,6 +125,17 @@ char *EHScopeStack::allocate(size_t Size) { return StartOfData; } +bool EHScopeStack::containsOnlyLifetimeMarkers( + EHScopeStack::stable_iterator Old) const { + for (EHScopeStack::iterator it = begin(); stabilize(it) != Old; it++) { + EHCleanupScope *cleanup = dyn_cast<EHCleanupScope>(&*it); + if (!cleanup || !cleanup->isLifetimeMarker()) + return false; + } + + return true; +} + EHScopeStack::stable_iterator EHScopeStack::getInnermostActiveNormalCleanup() const { for (stable_iterator si = getInnermostNormalCleanup(), se = stable_end(); @@ -469,8 +482,14 @@ static void EmitCleanup(CodeGenFunction &CGF, EHScopeStack::Cleanup *Fn, EHScopeStack::Cleanup::Flags flags, llvm::Value *ActiveFlag) { - // EH cleanups always occur within a terminate scope. - if (flags.isForEHCleanup()) CGF.EHStack.pushTerminate(); + // Itanium EH cleanups occur within a terminate scope. Microsoft SEH doesn't + // have this behavior, and the Microsoft C++ runtime will call terminate for + // us if the cleanup throws. + bool PushedTerminate = false; + if (flags.isForEHCleanup() && !CGF.getTarget().getCXXABI().isMicrosoft()) { + CGF.EHStack.pushTerminate(); + PushedTerminate = true; + } // If there's an active flag, load it and skip the cleanup if it's // false. @@ -493,7 +512,8 @@ static void EmitCleanup(CodeGenFunction &CGF, CGF.EmitBlock(ContBB); // Leave the terminate scope. - if (flags.isForEHCleanup()) CGF.EHStack.popTerminate(); + if (PushedTerminate) + CGF.EHStack.popTerminate(); } static void ForwardPrebranchedFallthrough(llvm::BasicBlock *Exit, @@ -739,7 +759,15 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { Scope.getNumBranchAfters() == 1) { assert(!BranchThroughDest || !IsActive); - // TODO: clean up the possibly dead stores to the cleanup dest slot. + // Clean up the possibly dead store to the cleanup dest slot. + llvm::Instruction *NormalCleanupDestSlot = + cast<llvm::Instruction>(getNormalCleanupDestSlot()); + if (NormalCleanupDestSlot->hasOneUse()) { + NormalCleanupDestSlot->user_back()->eraseFromParent(); + NormalCleanupDestSlot->eraseFromParent(); + NormalCleanupDest = nullptr; + } + llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0); InstsToAppend.push_back(llvm::BranchInst::Create(BranchAfter)); @@ -861,8 +889,6 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { // Emit the EH cleanup if required. if (RequiresEHCleanup) { - ApplyDebugLocation AutoRestoreLocation(*this, CurEHLocation); - CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP(); EmitBlock(EHEntry); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h index dd156c6..81c6412 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGCleanup.h @@ -62,6 +62,9 @@ protected: /// Whether this cleanup is currently active. unsigned IsActive : 1; + /// Whether this cleanup is a lifetime marker + unsigned IsLifetimeMarker : 1; + /// Whether the normal cleanup should test the activation flag. unsigned TestFlagInNormalCleanup : 1; @@ -75,7 +78,7 @@ protected: /// The number of fixups required by enclosing scopes (not including /// this one). If this is the top cleanup scope, all the fixups /// from this index onwards belong to this scope. - unsigned FixupDepth : 32 - 17 - NumCommonBits; // currently 13 + unsigned FixupDepth : 32 - 18 - NumCommonBits; // currently 13 }; class FilterBitFields { @@ -272,6 +275,7 @@ public: CleanupBits.IsNormalCleanup = isNormal; CleanupBits.IsEHCleanup = isEH; CleanupBits.IsActive = isActive; + CleanupBits.IsLifetimeMarker = false; CleanupBits.TestFlagInNormalCleanup = false; CleanupBits.TestFlagInEHCleanup = false; CleanupBits.CleanupSize = cleanupSize; @@ -284,19 +288,20 @@ public: delete ExtInfo; } // Objects of EHCleanupScope are not destructed. Use Destroy(). - ~EHCleanupScope() LLVM_DELETED_FUNCTION; + ~EHCleanupScope() = delete; bool isNormalCleanup() const { return CleanupBits.IsNormalCleanup; } llvm::BasicBlock *getNormalBlock() const { return NormalBlock; } void setNormalBlock(llvm::BasicBlock *BB) { NormalBlock = BB; } bool isEHCleanup() const { return CleanupBits.IsEHCleanup; } - llvm::BasicBlock *getEHBlock() const { return getCachedEHDispatchBlock(); } - void setEHBlock(llvm::BasicBlock *BB) { setCachedEHDispatchBlock(BB); } bool isActive() const { return CleanupBits.IsActive; } void setActive(bool A) { CleanupBits.IsActive = A; } + bool isLifetimeMarker() const { return CleanupBits.IsLifetimeMarker; } + void setLifetimeMarker() { CleanupBits.IsLifetimeMarker = true; } + llvm::AllocaInst *getActiveFlag() const { return ActiveFlag; } void setActiveFlag(llvm::AllocaInst *Var) { ActiveFlag = Var; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp index 978e1bb..93a2287 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.cpp @@ -27,6 +27,8 @@ #include "clang/Basic/SourceManager.h" #include "clang/Basic/Version.h" #include "clang/Frontend/CodeGenOptions.h" +#include "clang/Lex/HeaderSearchOptions.h" +#include "clang/Lex/PreprocessorOptions.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/IR/Constants.h" @@ -52,35 +54,48 @@ CGDebugInfo::~CGDebugInfo() { "Region stack mismatch, stack not empty!"); } -ArtificialLocation::ArtificialLocation(CodeGenFunction &CGF) - : ApplyDebugLocation(CGF) { - if (auto *DI = CGF.getDebugInfo()) { - // Construct a location that has a valid scope, but no line info. - assert(!DI->LexicalBlockStack.empty()); - llvm::DIDescriptor Scope(DI->LexicalBlockStack.back()); - CGF.Builder.SetCurrentDebugLocation(llvm::DebugLoc::get(0, 0, Scope)); - } +ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, + SourceLocation TemporaryLocation) + : CGF(CGF) { + init(TemporaryLocation); } ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, - SourceLocation TemporaryLocation, - bool ForceColumnInfo) + bool DefaultToEmpty, + SourceLocation TemporaryLocation) : CGF(CGF) { + init(TemporaryLocation, DefaultToEmpty); +} + +void ApplyDebugLocation::init(SourceLocation TemporaryLocation, + bool DefaultToEmpty) { if (auto *DI = CGF.getDebugInfo()) { OriginalLocation = CGF.Builder.getCurrentDebugLocation(); - if (TemporaryLocation.isInvalid()) - CGF.Builder.SetCurrentDebugLocation(llvm::DebugLoc()); - else - DI->EmitLocation(CGF.Builder, TemporaryLocation, ForceColumnInfo); + if (TemporaryLocation.isInvalid()) { + if (DefaultToEmpty) + CGF.Builder.SetCurrentDebugLocation(llvm::DebugLoc()); + else { + // Construct a location that has a valid scope, but no line info. + assert(!DI->LexicalBlockStack.empty()); + CGF.Builder.SetCurrentDebugLocation( + llvm::DebugLoc::get(0, 0, DI->LexicalBlockStack.back())); + } + } else + DI->EmitLocation(CGF.Builder, TemporaryLocation); } } +ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, const Expr *E) + : CGF(CGF) { + init(E->getExprLoc()); +} + ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, llvm::DebugLoc Loc) : CGF(CGF) { if (CGF.getDebugInfo()) { OriginalLocation = CGF.Builder.getCurrentDebugLocation(); - if (!Loc.isUnknown()) - CGF.Builder.SetCurrentDebugLocation(Loc); + if (Loc) + CGF.Builder.SetCurrentDebugLocation(std::move(Loc)); } } @@ -88,11 +103,9 @@ ApplyDebugLocation::~ApplyDebugLocation() { // Query CGF so the location isn't overwritten when location updates are // temporarily disabled (for C++ default function arguments) if (CGF.getDebugInfo()) - CGF.Builder.SetCurrentDebugLocation(OriginalLocation); + CGF.Builder.SetCurrentDebugLocation(std::move(OriginalLocation)); } -/// ArtificialLocation - An RAII object that temporarily switches to -/// an artificial debug location that has a valid scope, but no line void CGDebugInfo::setLocation(SourceLocation Loc) { // If the new location isn't valid return. if (Loc.isInvalid()) @@ -107,37 +120,32 @@ void CGDebugInfo::setLocation(SourceLocation Loc) { return; SourceManager &SM = CGM.getContext().getSourceManager(); - llvm::DIScope Scope(LexicalBlockStack.back()); + auto *Scope = cast<llvm::DIScope>(LexicalBlockStack.back()); PresumedLoc PCLoc = SM.getPresumedLoc(CurLoc); - if (PCLoc.isInvalid() || Scope.getFilename() == PCLoc.getFilename()) + if (PCLoc.isInvalid() || Scope->getFilename() == PCLoc.getFilename()) return; - if (Scope.isLexicalBlockFile()) { - llvm::DILexicalBlockFile LBF = llvm::DILexicalBlockFile(Scope); - llvm::DIDescriptor D = DBuilder.createLexicalBlockFile( - LBF.getScope(), getOrCreateFile(CurLoc)); - llvm::MDNode *N = D; + if (auto *LBF = dyn_cast<llvm::DILexicalBlockFile>(Scope)) { LexicalBlockStack.pop_back(); - LexicalBlockStack.emplace_back(N); - } else if (Scope.isLexicalBlock() || Scope.isSubprogram()) { - llvm::DIDescriptor D = - DBuilder.createLexicalBlockFile(Scope, getOrCreateFile(CurLoc)); - llvm::MDNode *N = D; + LexicalBlockStack.emplace_back(DBuilder.createLexicalBlockFile( + LBF->getScope(), getOrCreateFile(CurLoc))); + } else if (isa<llvm::DILexicalBlock>(Scope) || + isa<llvm::DISubprogram>(Scope)) { LexicalBlockStack.pop_back(); - LexicalBlockStack.emplace_back(N); + LexicalBlockStack.emplace_back( + DBuilder.createLexicalBlockFile(Scope, getOrCreateFile(CurLoc))); } } -/// getContextDescriptor - Get context info for the decl. -llvm::DIScope CGDebugInfo::getContextDescriptor(const Decl *Context) { +llvm::DIScope *CGDebugInfo::getContextDescriptor(const Decl *Context) { if (!Context) return TheCU; auto I = RegionMap.find(Context); if (I != RegionMap.end()) { llvm::Metadata *V = I->second; - return llvm::DIScope(dyn_cast_or_null<llvm::MDNode>(V)); + return dyn_cast_or_null<llvm::DIScope>(V); } // Check namespace. @@ -151,9 +159,6 @@ llvm::DIScope CGDebugInfo::getContextDescriptor(const Decl *Context) { return TheCU; } -/// getFunctionName - Get function name for the given FunctionDecl. If the -/// name is constructed on demand (e.g. C++ destructor) then the name -/// is stored on the side. StringRef CGDebugInfo::getFunctionName(const FunctionDecl *FD) { assert(FD && "Invalid FunctionDecl!"); IdentifierInfo *FII = FD->getIdentifier(); @@ -209,13 +214,10 @@ StringRef CGDebugInfo::getObjCMethodName(const ObjCMethodDecl *OMD) { return internString(OS.str()); } -/// getSelectorName - Return selector name. This is used for debugging -/// info. StringRef CGDebugInfo::getSelectorName(Selector S) { return internString(S.getAsString()); } -/// getClassName - Get class name including template argument list. StringRef CGDebugInfo::getClassName(const RecordDecl *RD) { // quick optimization to avoid having to intern strings that are already // stored reliably elsewhere @@ -233,18 +235,17 @@ StringRef CGDebugInfo::getClassName(const RecordDecl *RD) { return internString(Name); } -/// getOrCreateFile - Get the file debug info descriptor for the input location. -llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) { +llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) { if (!Loc.isValid()) // If Location is not valid then use main input file. - return DBuilder.createFile(TheCU.getFilename(), TheCU.getDirectory()); + return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory()); SourceManager &SM = CGM.getContext().getSourceManager(); PresumedLoc PLoc = SM.getPresumedLoc(Loc); if (PLoc.isInvalid() || StringRef(PLoc.getFilename()).empty()) // If the location is not valid then use main input file. - return DBuilder.createFile(TheCU.getFilename(), TheCU.getDirectory()); + return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory()); // Cache the results. const char *fname = PLoc.getFilename(); @@ -253,22 +254,20 @@ llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) { if (it != DIFileCache.end()) { // Verify that the information still exists. if (llvm::Metadata *V = it->second) - return llvm::DIFile(cast<llvm::MDNode>(V)); + return cast<llvm::DIFile>(V); } - llvm::DIFile F = DBuilder.createFile(PLoc.getFilename(), getCurrentDirname()); + llvm::DIFile *F = + DBuilder.createFile(PLoc.getFilename(), getCurrentDirname()); DIFileCache[fname].reset(F); return F; } -/// getOrCreateMainFile - Get the file info for main compile unit. -llvm::DIFile CGDebugInfo::getOrCreateMainFile() { - return DBuilder.createFile(TheCU.getFilename(), TheCU.getDirectory()); +llvm::DIFile *CGDebugInfo::getOrCreateMainFile() { + return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory()); } -/// getLineNumber - Get line number for the location. If location is invalid -/// then use current location. unsigned CGDebugInfo::getLineNumber(SourceLocation Loc) { if (Loc.isInvalid() && CurLoc.isInvalid()) return 0; @@ -277,7 +276,6 @@ unsigned CGDebugInfo::getLineNumber(SourceLocation Loc) { return PLoc.isValid() ? PLoc.getLine() : 0; } -/// getColumnNumber - Get column number for the location. unsigned CGDebugInfo::getColumnNumber(SourceLocation Loc, bool Force) { // We may not want column information at all. if (!Force && !CGM.getCodeGenOpts().DebugColumnInfo) @@ -302,7 +300,6 @@ StringRef CGDebugInfo::getCurrentDirname() { return CWDName = internString(CWD); } -/// CreateCompileUnit - Create new compile unit. void CGDebugInfo::CreateCompileUnit() { // Should we be asking the SourceManager for the main file name, instead of @@ -369,12 +366,11 @@ void CGDebugInfo::CreateCompileUnit() { DebugKind <= CodeGenOptions::DebugLineTablesOnly ? llvm::DIBuilder::LineTablesOnly : llvm::DIBuilder::FullDebug, + 0 /* DWOid */, DebugKind != CodeGenOptions::LocTrackingOnly); } -/// CreateType - Get the Basic type from the cache or create a new -/// one if necessary. -llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) { +llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { llvm::dwarf::TypeKind Encoding; StringRef BTName; switch (BT->getKind()) { @@ -386,7 +382,7 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) { case BuiltinType::NullPtr: return DBuilder.createNullPtrType(); case BuiltinType::Void: - return llvm::DIType(); + return nullptr; case BuiltinType::ObjCClass: if (!ClassTy) ClassTy = DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type, @@ -409,11 +405,11 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) { unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy); - llvm::DIType ISATy = DBuilder.createPointerType(ClassTy, Size); + auto *ISATy = DBuilder.createPointerType(ClassTy, Size); ObjTy = DBuilder.createStructType(TheCU, "objc_object", getOrCreateMainFile(), - 0, 0, 0, 0, llvm::DIType(), llvm::DIArray()); + 0, 0, 0, 0, nullptr, llvm::DINodeArray()); DBuilder.replaceArrays( ObjTy, @@ -510,11 +506,10 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) { // Bit size, align and offset of the type. uint64_t Size = CGM.getContext().getTypeSize(BT); uint64_t Align = CGM.getContext().getTypeAlign(BT); - llvm::DIType DbgTy = DBuilder.createBasicType(BTName, Size, Align, Encoding); - return DbgTy; + return DBuilder.createBasicType(BTName, Size, Align, Encoding); } -llvm::DIType CGDebugInfo::CreateType(const ComplexType *Ty) { +llvm::DIType *CGDebugInfo::CreateType(const ComplexType *Ty) { // Bit size, align and offset of the type. llvm::dwarf::TypeKind Encoding = llvm::dwarf::DW_ATE_complex_float; if (Ty->isComplexIntegerType()) @@ -522,15 +517,11 @@ llvm::DIType CGDebugInfo::CreateType(const ComplexType *Ty) { uint64_t Size = CGM.getContext().getTypeSize(Ty); uint64_t Align = CGM.getContext().getTypeAlign(Ty); - llvm::DIType DbgTy = - DBuilder.createBasicType("complex", Size, Align, Encoding); - - return DbgTy; + return DBuilder.createBasicType("complex", Size, Align, Encoding); } -/// CreateCVRType - Get the qualified type from the cache or create -/// a new one if necessary. -llvm::DIType CGDebugInfo::CreateQualifiedType(QualType Ty, llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateQualifiedType(QualType Ty, + llvm::DIFile *Unit) { QualifierCollector Qc; const Type *T = Qc.strip(Ty); @@ -556,17 +547,15 @@ llvm::DIType CGDebugInfo::CreateQualifiedType(QualType Ty, llvm::DIFile Unit) { return getOrCreateType(QualType(T, 0), Unit); } - llvm::DIType FromTy = getOrCreateType(Qc.apply(CGM.getContext(), T), Unit); + auto *FromTy = getOrCreateType(Qc.apply(CGM.getContext(), T), Unit); // No need to fill in the Name, Line, Size, Alignment, Offset in case of // CVR derived types. - llvm::DIType DbgTy = DBuilder.createQualifiedType(Tag, FromTy); - - return DbgTy; + return DBuilder.createQualifiedType(Tag, FromTy); } -llvm::DIType CGDebugInfo::CreateType(const ObjCObjectPointerType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const ObjCObjectPointerType *Ty, + llvm::DIFile *Unit) { // The frontend treats 'id' as a typedef to an ObjCObjectType, // whereas 'id<protocol>' is treated as an ObjCPointerType. For the @@ -574,28 +563,39 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCObjectPointerType *Ty, if (Ty->isObjCQualifiedIdType()) return getOrCreateType(CGM.getContext().getObjCIdType(), Unit); - llvm::DIType DbgTy = CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type, - Ty, Ty->getPointeeType(), Unit); - return DbgTy; + return CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type, Ty, + Ty->getPointeeType(), Unit); } -llvm::DIType CGDebugInfo::CreateType(const PointerType *Ty, llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const PointerType *Ty, + llvm::DIFile *Unit) { return CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type, Ty, Ty->getPointeeType(), Unit); } +/// \return whether a C++ mangling exists for the type defined by TD. +static bool hasCXXMangling(const TagDecl *TD, llvm::DICompileUnit *TheCU) { + switch (TheCU->getSourceLanguage()) { + case llvm::dwarf::DW_LANG_C_plus_plus: + return true; + case llvm::dwarf::DW_LANG_ObjC_plus_plus: + return isa<CXXRecordDecl>(TD) || isa<EnumDecl>(TD); + default: + return false; + } +} + /// In C++ mode, types have linkage, so we can rely on the ODR and /// on their mangled names, if they're external. static SmallString<256> getUniqueTagTypeName(const TagType *Ty, CodeGenModule &CGM, - llvm::DICompileUnit TheCU) { + llvm::DICompileUnit *TheCU) { SmallString<256> FullName; - // FIXME: ODR should apply to ObjC++ exactly the same wasy it does to C++. - // For now, only apply ODR with C++. const TagDecl *TD = Ty->getDecl(); - if (TheCU.getLanguage() != llvm::dwarf::DW_LANG_C_plus_plus || - !TD->isExternallyVisible()) + + if (!hasCXXMangling(TD, TheCU) || !TD->isExternallyVisible()) return FullName; + // Microsoft Mangler does not have support for mangleCXXRTTIName yet. if (CGM.getTarget().getCXXABI().isMicrosoft()) return FullName; @@ -608,41 +608,56 @@ static SmallString<256> getUniqueTagTypeName(const TagType *Ty, return FullName; } -// Creates a forward declaration for a RecordDecl in the given context. -llvm::DICompositeType -CGDebugInfo::getOrCreateRecordFwdDecl(const RecordType *Ty, - llvm::DIDescriptor Ctx) { - const RecordDecl *RD = Ty->getDecl(); - if (llvm::DIType T = getTypeOrNull(CGM.getContext().getRecordType(RD))) - return llvm::DICompositeType(T); - llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation()); - unsigned Line = getLineNumber(RD->getLocation()); - StringRef RDName = getClassName(RD); - - llvm::dwarf::Tag Tag; +/// \return the approproate DWARF tag for a composite type. +static llvm::dwarf::Tag getTagForRecord(const RecordDecl *RD) { + llvm::dwarf::Tag Tag; if (RD->isStruct() || RD->isInterface()) Tag = llvm::dwarf::DW_TAG_structure_type; else if (RD->isUnion()) Tag = llvm::dwarf::DW_TAG_union_type; else { + // FIXME: This could be a struct type giving a default visibility different + // than C++ class type, but needs llvm metadata changes first. assert(RD->isClass()); Tag = llvm::dwarf::DW_TAG_class_type; } + return Tag; +} + +llvm::DICompositeType * +CGDebugInfo::getOrCreateRecordFwdDecl(const RecordType *Ty, + llvm::DIScope *Ctx) { + const RecordDecl *RD = Ty->getDecl(); + if (llvm::DIType *T = getTypeOrNull(CGM.getContext().getRecordType(RD))) + return cast<llvm::DICompositeType>(T); + llvm::DIFile *DefUnit = getOrCreateFile(RD->getLocation()); + unsigned Line = getLineNumber(RD->getLocation()); + StringRef RDName = getClassName(RD); + + uint64_t Size = 0; + uint64_t Align = 0; + + const RecordDecl *D = RD->getDefinition(); + if (D && D->isCompleteDefinition()) { + Size = CGM.getContext().getTypeSize(Ty); + Align = CGM.getContext().getTypeAlign(Ty); + } // Create the type. SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU); - llvm::DICompositeType RetTy = DBuilder.createReplaceableForwardDecl( - Tag, RDName, Ctx, DefUnit, Line, 0, 0, 0, FullName); + llvm::DICompositeType *RetTy = DBuilder.createReplaceableCompositeType( + getTagForRecord(RD), RDName, Ctx, DefUnit, Line, 0, Size, Align, + llvm::DINode::FlagFwdDecl, FullName); ReplaceMap.emplace_back( std::piecewise_construct, std::make_tuple(Ty), std::make_tuple(static_cast<llvm::Metadata *>(RetTy))); return RetTy; } -llvm::DIType CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag, - const Type *Ty, - QualType PointeeTy, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag, + const Type *Ty, + QualType PointeeTy, + llvm::DIFile *Unit) { if (Tag == llvm::dwarf::DW_TAG_reference_type || Tag == llvm::dwarf::DW_TAG_rvalue_reference_type) return DBuilder.createReferenceType(Tag, getOrCreateType(PointeeTy, Unit)); @@ -658,8 +673,8 @@ llvm::DIType CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag, Align); } -llvm::DIType CGDebugInfo::getOrCreateStructPtrType(StringRef Name, - llvm::DIType &Cache) { +llvm::DIType *CGDebugInfo::getOrCreateStructPtrType(StringRef Name, + llvm::DIType *&Cache) { if (Cache) return Cache; Cache = DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type, Name, @@ -669,18 +684,13 @@ llvm::DIType CGDebugInfo::getOrCreateStructPtrType(StringRef Name, return Cache; } -llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty, - llvm::DIFile Unit) { - if (BlockLiteralGeneric) - return BlockLiteralGeneric; - +llvm::DIType *CGDebugInfo::CreateType(const BlockPointerType *Ty, + llvm::DIFile *Unit) { SmallVector<llvm::Metadata *, 8> EltTys; - llvm::DIType FieldTy; QualType FType; uint64_t FieldSize, FieldOffset; unsigned FieldAlign; - llvm::DIArray Elements; - llvm::DIType EltTy, DescTy; + llvm::DINodeArray Elements; FieldOffset = 0; FType = CGM.getContext().UnsignedLongTy; @@ -690,17 +700,17 @@ llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty, Elements = DBuilder.getOrCreateArray(EltTys); EltTys.clear(); - unsigned Flags = llvm::DIDescriptor::FlagAppleBlock; - unsigned LineNo = getLineNumber(CurLoc); + unsigned Flags = llvm::DINode::FlagAppleBlock; + unsigned LineNo = 0; - EltTy = DBuilder.createStructType(Unit, "__block_descriptor", Unit, LineNo, - FieldOffset, 0, Flags, llvm::DIType(), - Elements); + auto *EltTy = + DBuilder.createStructType(Unit, "__block_descriptor", nullptr, LineNo, + FieldOffset, 0, Flags, nullptr, Elements); // Bit size, align and offset of the type. uint64_t Size = CGM.getContext().getTypeSize(Ty); - DescTy = DBuilder.createPointerType(EltTy, Size); + auto *DescTy = DBuilder.createPointerType(EltTy, Size); FieldOffset = 0; FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy); @@ -712,29 +722,30 @@ llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty, EltTys.push_back(CreateMemberType(Unit, FType, "__FuncPtr", &FieldOffset)); FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy); - FieldTy = DescTy; FieldSize = CGM.getContext().getTypeSize(Ty); FieldAlign = CGM.getContext().getTypeAlign(Ty); - FieldTy = - DBuilder.createMemberType(Unit, "__descriptor", Unit, LineNo, FieldSize, - FieldAlign, FieldOffset, 0, FieldTy); - EltTys.push_back(FieldTy); + EltTys.push_back(DBuilder.createMemberType(Unit, "__descriptor", nullptr, LineNo, + FieldSize, FieldAlign, FieldOffset, + 0, DescTy)); FieldOffset += FieldSize; Elements = DBuilder.getOrCreateArray(EltTys); - EltTy = DBuilder.createStructType(Unit, "__block_literal_generic", Unit, - LineNo, FieldOffset, 0, Flags, - llvm::DIType(), Elements); + // The __block_literal_generic structs are marked with a special + // DW_AT_APPLE_BLOCK attribute and are an implementation detail only + // the debugger needs to know about. To allow type uniquing, emit + // them without a name or a location. + EltTy = + DBuilder.createStructType(Unit, "", nullptr, LineNo, + FieldOffset, 0, Flags, nullptr, Elements); - BlockLiteralGeneric = DBuilder.createPointerType(EltTy, Size); - return BlockLiteralGeneric; + return DBuilder.createPointerType(EltTy, Size); } -llvm::DIType CGDebugInfo::CreateType(const TemplateSpecializationType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const TemplateSpecializationType *Ty, + llvm::DIFile *Unit) { assert(Ty->isTypeAlias()); - llvm::DIType Src = getOrCreateType(Ty->getAliasedType(), Unit); + llvm::DIType *Src = getOrCreateType(Ty->getAliasedType(), Unit); SmallString<128> NS; llvm::raw_svector_ostream OS(NS); @@ -749,35 +760,26 @@ llvm::DIType CGDebugInfo::CreateType(const TemplateSpecializationType *Ty, Ty->getTemplateName().getAsTemplateDecl())->getTemplatedDecl(); SourceLocation Loc = AliasDecl->getLocation(); - llvm::DIFile File = getOrCreateFile(Loc); - unsigned Line = getLineNumber(Loc); - - llvm::DIDescriptor Ctxt = - getContextDescriptor(cast<Decl>(AliasDecl->getDeclContext())); - - return DBuilder.createTypedef(Src, internString(OS.str()), File, Line, Ctxt); + return DBuilder.createTypedef( + Src, internString(OS.str()), getOrCreateFile(Loc), getLineNumber(Loc), + getContextDescriptor(cast<Decl>(AliasDecl->getDeclContext()))); } -llvm::DIType CGDebugInfo::CreateType(const TypedefType *Ty, llvm::DIFile Unit) { - // Typedefs are derived from some other type. If we have a typedef of a - // typedef, make sure to emit the whole chain. - llvm::DIType Src = getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit); +llvm::DIType *CGDebugInfo::CreateType(const TypedefType *Ty, + llvm::DIFile *Unit) { // We don't set size information, but do specify where the typedef was // declared. SourceLocation Loc = Ty->getDecl()->getLocation(); - llvm::DIFile File = getOrCreateFile(Loc); - unsigned Line = getLineNumber(Loc); - const TypedefNameDecl *TyDecl = Ty->getDecl(); - - llvm::DIDescriptor TypedefContext = - getContextDescriptor(cast<Decl>(Ty->getDecl()->getDeclContext())); - return DBuilder.createTypedef(Src, TyDecl->getName(), File, Line, - TypedefContext); + // Typedefs are derived from some other type. + return DBuilder.createTypedef( + getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit), + Ty->getDecl()->getName(), getOrCreateFile(Loc), getLineNumber(Loc), + getContextDescriptor(cast<Decl>(Ty->getDecl()->getDeclContext()))); } -llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const FunctionType *Ty, + llvm::DIFile *Unit) { SmallVector<llvm::Metadata *, 16> EltTys; // Add the result type at least. @@ -794,11 +796,11 @@ llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty, EltTys.push_back(DBuilder.createUnspecifiedParameter()); } - llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); + llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); return DBuilder.createSubroutineType(Unit, EltTypeArray); } -/// Convert an AccessSpecifier into the corresponding DIDescriptor flag. +/// Convert an AccessSpecifier into the corresponding DINode flag. /// As an optimization, return 0 if the access specifier equals the /// default for the containing type. static unsigned getAccessFlag(AccessSpecifier Access, const RecordDecl *RD) { @@ -813,25 +815,25 @@ static unsigned getAccessFlag(AccessSpecifier Access, const RecordDecl *RD) { switch (Access) { case clang::AS_private: - return llvm::DIDescriptor::FlagPrivate; + return llvm::DINode::FlagPrivate; case clang::AS_protected: - return llvm::DIDescriptor::FlagProtected; + return llvm::DINode::FlagProtected; case clang::AS_public: - return llvm::DIDescriptor::FlagPublic; + return llvm::DINode::FlagPublic; case clang::AS_none: return 0; } llvm_unreachable("unexpected access enumerator"); } -llvm::DIType CGDebugInfo::createFieldType( +llvm::DIType *CGDebugInfo::createFieldType( StringRef name, QualType type, uint64_t sizeInBitsOverride, SourceLocation loc, AccessSpecifier AS, uint64_t offsetInBits, - llvm::DIFile tunit, llvm::DIScope scope, const RecordDecl *RD) { - llvm::DIType debugType = getOrCreateType(type, tunit); + llvm::DIFile *tunit, llvm::DIScope *scope, const RecordDecl *RD) { + llvm::DIType *debugType = getOrCreateType(type, tunit); // Get the location for the field. - llvm::DIFile file = getOrCreateFile(loc); + llvm::DIFile *file = getOrCreateFile(loc); unsigned line = getLineNumber(loc); uint64_t SizeInBits = 0; @@ -850,10 +852,9 @@ llvm::DIType CGDebugInfo::createFieldType( AlignInBits, offsetInBits, flags, debugType); } -/// CollectRecordLambdaFields - Helper for CollectRecordFields. void CGDebugInfo::CollectRecordLambdaFields( const CXXRecordDecl *CXXDecl, SmallVectorImpl<llvm::Metadata *> &elements, - llvm::DIType RecordTy) { + llvm::DIType *RecordTy) { // For C++11 Lambdas a Field will be the same as a Capture, but the Capture // has the name and the location of the variable so we should iterate over // both concurrently. @@ -866,14 +867,14 @@ void CGDebugInfo::CollectRecordLambdaFields( const LambdaCapture &C = *I; if (C.capturesVariable()) { VarDecl *V = C.getCapturedVar(); - llvm::DIFile VUnit = getOrCreateFile(C.getLocation()); + llvm::DIFile *VUnit = getOrCreateFile(C.getLocation()); StringRef VName = V->getName(); uint64_t SizeInBitsOverride = 0; if (Field->isBitField()) { SizeInBitsOverride = Field->getBitWidthValue(CGM.getContext()); assert(SizeInBitsOverride && "found named 0-width bitfield"); } - llvm::DIType fieldType = createFieldType( + llvm::DIType *fieldType = createFieldType( VName, Field->getType(), SizeInBitsOverride, C.getLocation(), Field->getAccess(), layout.getFieldOffset(fieldno), VUnit, RecordTy, CXXDecl); @@ -884,9 +885,9 @@ void CGDebugInfo::CollectRecordLambdaFields( // by using AT_object_pointer for the function and having that be // used as 'this' for semantic references. FieldDecl *f = *Field; - llvm::DIFile VUnit = getOrCreateFile(f->getLocation()); + llvm::DIFile *VUnit = getOrCreateFile(f->getLocation()); QualType type = f->getType(); - llvm::DIType fieldType = createFieldType( + llvm::DIType *fieldType = createFieldType( "this", type, 0, f->getLocation(), f->getAccess(), layout.getFieldOffset(fieldno), VUnit, RecordTy, CXXDecl); @@ -895,15 +896,14 @@ void CGDebugInfo::CollectRecordLambdaFields( } } -/// Helper for CollectRecordFields. -llvm::DIDerivedType CGDebugInfo::CreateRecordStaticField(const VarDecl *Var, - llvm::DIType RecordTy, - const RecordDecl *RD) { +llvm::DIDerivedType * +CGDebugInfo::CreateRecordStaticField(const VarDecl *Var, llvm::DIType *RecordTy, + const RecordDecl *RD) { // Create the descriptor for the static variable, with or without // constant initializers. Var = Var->getCanonicalDecl(); - llvm::DIFile VUnit = getOrCreateFile(Var->getLocation()); - llvm::DIType VTy = getOrCreateType(Var->getType(), VUnit); + llvm::DIFile *VUnit = getOrCreateFile(Var->getLocation()); + llvm::DIType *VTy = getOrCreateType(Var->getType(), VUnit); unsigned LineNumber = getLineNumber(Var->getLocation()); StringRef VName = Var->getName(); @@ -919,16 +919,15 @@ llvm::DIDerivedType CGDebugInfo::CreateRecordStaticField(const VarDecl *Var, } unsigned Flags = getAccessFlag(Var->getAccess(), RD); - llvm::DIDerivedType GV = DBuilder.createStaticMemberType( + llvm::DIDerivedType *GV = DBuilder.createStaticMemberType( RecordTy, VName, VUnit, LineNumber, VTy, Flags, C); StaticDataMemberCache[Var->getCanonicalDecl()].reset(GV); return GV; } -/// CollectRecordNormalField - Helper for CollectRecordFields. void CGDebugInfo::CollectRecordNormalField( - const FieldDecl *field, uint64_t OffsetInBits, llvm::DIFile tunit, - SmallVectorImpl<llvm::Metadata *> &elements, llvm::DIType RecordTy, + const FieldDecl *field, uint64_t OffsetInBits, llvm::DIFile *tunit, + SmallVectorImpl<llvm::Metadata *> &elements, llvm::DIType *RecordTy, const RecordDecl *RD) { StringRef name = field->getName(); QualType type = field->getType(); @@ -943,19 +942,17 @@ void CGDebugInfo::CollectRecordNormalField( assert(SizeInBitsOverride && "found named 0-width bitfield"); } - llvm::DIType fieldType = + llvm::DIType *fieldType = createFieldType(name, type, SizeInBitsOverride, field->getLocation(), field->getAccess(), OffsetInBits, tunit, RecordTy, RD); elements.push_back(fieldType); } -/// CollectRecordFields - A helper function to collect debug info for -/// record fields. This is used while creating debug info entry for a Record. void CGDebugInfo::CollectRecordFields( - const RecordDecl *record, llvm::DIFile tunit, + const RecordDecl *record, llvm::DIFile *tunit, SmallVectorImpl<llvm::Metadata *> &elements, - llvm::DICompositeType RecordTy) { + llvm::DICompositeType *RecordTy) { const CXXRecordDecl *CXXDecl = dyn_cast<CXXRecordDecl>(record); if (CXXDecl && CXXDecl->isLambda()) @@ -975,8 +972,7 @@ void CGDebugInfo::CollectRecordFields( if (MI != StaticDataMemberCache.end()) { assert(MI->second && "Static data member declaration should still exist"); - elements.push_back( - llvm::DIDerivedType(cast<llvm::MDNode>(MI->second))); + elements.push_back(cast<llvm::DIDerivedTypeBase>(MI->second)); } else { auto Field = CreateRecordStaticField(V, RecordTy, record); elements.push_back(Field); @@ -991,30 +987,29 @@ void CGDebugInfo::CollectRecordFields( } } -/// getOrCreateMethodType - CXXMethodDecl's type is a FunctionType. This -/// function type is not updated to include implicit "this" pointer. Use this -/// routine to get a method type which includes "this" pointer. -llvm::DICompositeType +llvm::DISubroutineType * CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method, - llvm::DIFile Unit) { + llvm::DIFile *Unit) { const FunctionProtoType *Func = Method->getType()->getAs<FunctionProtoType>(); if (Method->isStatic()) - return llvm::DICompositeType(getOrCreateType(QualType(Func, 0), Unit)); + return cast_or_null<llvm::DISubroutineType>( + getOrCreateType(QualType(Func, 0), Unit)); return getOrCreateInstanceMethodType(Method->getThisType(CGM.getContext()), Func, Unit); } -llvm::DICompositeType CGDebugInfo::getOrCreateInstanceMethodType( - QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile Unit) { +llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType( + QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile *Unit) { // Add "this" pointer. - llvm::DITypeArray Args = llvm::DISubroutineType( - getOrCreateType(QualType(Func, 0), Unit)).getTypeArray(); - assert(Args.getNumElements() && "Invalid number of arguments!"); + llvm::DITypeRefArray Args( + cast<llvm::DISubroutineType>(getOrCreateType(QualType(Func, 0), Unit)) + ->getTypeArray()); + assert(Args.size() && "Invalid number of arguments!"); SmallVector<llvm::Metadata *, 16> Elts; // First element is always return type. For 'void' functions it is NULL. - Elts.push_back(Args.getElement(0)); + Elts.push_back(Args[0]); // "this" pointer is always first argument. const CXXRecordDecl *RD = ThisPtr->getPointeeCXXRecordDecl(); @@ -1025,8 +1020,8 @@ llvm::DICompositeType CGDebugInfo::getOrCreateInstanceMethodType( unsigned AS = CGM.getContext().getTargetAddressSpace(PointeeTy); uint64_t Size = CGM.getTarget().getPointerWidth(AS); uint64_t Align = CGM.getContext().getTypeAlign(ThisPtrTy); - llvm::DIType PointeeType = getOrCreateType(PointeeTy, Unit); - llvm::DIType ThisPtrType = + llvm::DIType *PointeeType = getOrCreateType(PointeeTy, Unit); + llvm::DIType *ThisPtrType = DBuilder.createPointerType(PointeeType, Size, Align); TypeCache[ThisPtr.getAsOpaquePtr()].reset(ThisPtrType); // TODO: This and the artificial type below are misleading, the @@ -1035,23 +1030,23 @@ llvm::DICompositeType CGDebugInfo::getOrCreateInstanceMethodType( ThisPtrType = DBuilder.createObjectPointerType(ThisPtrType); Elts.push_back(ThisPtrType); } else { - llvm::DIType ThisPtrType = getOrCreateType(ThisPtr, Unit); + llvm::DIType *ThisPtrType = getOrCreateType(ThisPtr, Unit); TypeCache[ThisPtr.getAsOpaquePtr()].reset(ThisPtrType); ThisPtrType = DBuilder.createObjectPointerType(ThisPtrType); Elts.push_back(ThisPtrType); } // Copy rest of the arguments. - for (unsigned i = 1, e = Args.getNumElements(); i != e; ++i) - Elts.push_back(Args.getElement(i)); + for (unsigned i = 1, e = Args.size(); i != e; ++i) + Elts.push_back(Args[i]); - llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts); + llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts); unsigned Flags = 0; if (Func->getExtProtoInfo().RefQualifier == RQ_LValue) - Flags |= llvm::DIDescriptor::FlagLValueReference; + Flags |= llvm::DINode::FlagLValueReference; if (Func->getExtProtoInfo().RefQualifier == RQ_RValue) - Flags |= llvm::DIDescriptor::FlagRValueReference; + Flags |= llvm::DINode::FlagRValueReference; return DBuilder.createSubroutineType(Unit, EltTypeArray, Flags); } @@ -1066,16 +1061,13 @@ static bool isFunctionLocalClass(const CXXRecordDecl *RD) { return false; } -/// CreateCXXMemberFunction - A helper function to create a DISubprogram for -/// a single member function GlobalDecl. -llvm::DISubprogram -CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method, - llvm::DIFile Unit, llvm::DIType RecordTy) { +llvm::DISubprogram *CGDebugInfo::CreateCXXMemberFunction( + const CXXMethodDecl *Method, llvm::DIFile *Unit, llvm::DIType *RecordTy) { bool IsCtorOrDtor = isa<CXXConstructorDecl>(Method) || isa<CXXDestructorDecl>(Method); StringRef MethodName = getFunctionName(Method); - llvm::DICompositeType MethodTy = getOrCreateMethodType(Method, Unit); + llvm::DISubroutineType *MethodTy = getOrCreateMethodType(Method, Unit); // Since a single ctor/dtor corresponds to multiple functions, it doesn't // make sense to give a single ctor/dtor a linkage name. @@ -1084,7 +1076,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method, MethodLinkageName = CGM.getMangledName(Method); // Get the location for the method. - llvm::DIFile MethodDefUnit; + llvm::DIFile *MethodDefUnit = nullptr; unsigned MethodLine = 0; if (!Method->isImplicit()) { MethodDefUnit = getOrCreateFile(Method->getLocation()); @@ -1092,7 +1084,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method, } // Collect virtual method info. - llvm::DIType ContainingType; + llvm::DIType *ContainingType = nullptr; unsigned Virtuality = 0; unsigned VIndex = 0; @@ -1115,41 +1107,38 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method, unsigned Flags = 0; if (Method->isImplicit()) - Flags |= llvm::DIDescriptor::FlagArtificial; + Flags |= llvm::DINode::FlagArtificial; Flags |= getAccessFlag(Method->getAccess(), Method->getParent()); if (const CXXConstructorDecl *CXXC = dyn_cast<CXXConstructorDecl>(Method)) { if (CXXC->isExplicit()) - Flags |= llvm::DIDescriptor::FlagExplicit; + Flags |= llvm::DINode::FlagExplicit; } else if (const CXXConversionDecl *CXXC = dyn_cast<CXXConversionDecl>(Method)) { if (CXXC->isExplicit()) - Flags |= llvm::DIDescriptor::FlagExplicit; + Flags |= llvm::DINode::FlagExplicit; } if (Method->hasPrototype()) - Flags |= llvm::DIDescriptor::FlagPrototyped; + Flags |= llvm::DINode::FlagPrototyped; if (Method->getRefQualifier() == RQ_LValue) - Flags |= llvm::DIDescriptor::FlagLValueReference; + Flags |= llvm::DINode::FlagLValueReference; if (Method->getRefQualifier() == RQ_RValue) - Flags |= llvm::DIDescriptor::FlagRValueReference; + Flags |= llvm::DINode::FlagRValueReference; - llvm::DIArray TParamsArray = CollectFunctionTemplateParams(Method, Unit); - llvm::DISubprogram SP = DBuilder.createMethod( + llvm::DINodeArray TParamsArray = CollectFunctionTemplateParams(Method, Unit); + llvm::DISubprogram *SP = DBuilder.createMethod( RecordTy, MethodName, MethodLinkageName, MethodDefUnit, MethodLine, MethodTy, /*isLocalToUnit=*/false, /* isDefinition=*/false, Virtuality, VIndex, ContainingType, Flags, - CGM.getLangOpts().Optimize, nullptr, TParamsArray); + CGM.getLangOpts().Optimize, nullptr, TParamsArray.get()); SPCache[Method->getCanonicalDecl()].reset(SP); return SP; } -/// CollectCXXMemberFunctions - A helper function to collect debug info for -/// C++ member functions. This is used while creating debug info entry for -/// a Record. void CGDebugInfo::CollectCXXMemberFunctions( - const CXXRecordDecl *RD, llvm::DIFile Unit, - SmallVectorImpl<llvm::Metadata *> &EltTys, llvm::DIType RecordTy) { + const CXXRecordDecl *RD, llvm::DIFile *Unit, + SmallVectorImpl<llvm::Metadata *> &EltTys, llvm::DIType *RecordTy) { // Since we want more than just the individual member decls if we // have templated functions iterate over every declaration to gather @@ -1160,12 +1149,13 @@ void CGDebugInfo::CollectCXXMemberFunctions( // the member being added to type units by LLVM, while still allowing it // to be emitted into the type declaration/reference inside the compile // unit. + // Ditto 'nodebug' methods, for consistency with CodeGenFunction.cpp. // FIXME: Handle Using(Shadow?)Decls here to create // DW_TAG_imported_declarations inside the class for base decls brought into // derived classes. GDB doesn't seem to notice/leverage these when I tried // it, so I'm not rushing to fix this. (GCC seems to produce them, if // referenced) - if (!Method || Method->isImplicit()) + if (!Method || Method->isImplicit() || Method->hasAttr<NoDebugAttr>()) continue; if (Method->getType()->getAs<FunctionProtoType>()->getContainedAutoType()) @@ -1184,13 +1174,9 @@ void CGDebugInfo::CollectCXXMemberFunctions( } } -/// CollectCXXBases - A helper function to collect debug info for -/// C++ base classes. This is used while creating debug info entry for -/// a Record. -void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile Unit, +void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile *Unit, SmallVectorImpl<llvm::Metadata *> &EltTys, - llvm::DIType RecordTy) { - + llvm::DIType *RecordTy) { const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD); for (const auto &BI : RD->bases()) { unsigned BFlags = 0; @@ -1212,24 +1198,23 @@ void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile Unit, BaseOffset = 4 * CGM.getMicrosoftVTableContext().getVBTableIndex(RD, Base); } - BFlags = llvm::DIDescriptor::FlagVirtual; + BFlags = llvm::DINode::FlagVirtual; } else BaseOffset = CGM.getContext().toBits(RL.getBaseClassOffset(Base)); // FIXME: Inconsistent units for BaseOffset. It is in bytes when // BI->isVirtual() and bits when not. BFlags |= getAccessFlag(BI.getAccessSpecifier(), RD); - llvm::DIType DTy = DBuilder.createInheritance( + llvm::DIType *DTy = DBuilder.createInheritance( RecordTy, getOrCreateType(BI.getType(), Unit), BaseOffset, BFlags); EltTys.push_back(DTy); } } -/// CollectTemplateParams - A helper function to collect template parameters. -llvm::DIArray +llvm::DINodeArray CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, ArrayRef<TemplateArgument> TAList, - llvm::DIFile Unit) { + llvm::DIFile *Unit) { SmallVector<llvm::Metadata *, 16> TemplateParams; for (unsigned i = 0, e = TAList.size(); i != e; ++i) { const TemplateArgument &TA = TAList[i]; @@ -1238,23 +1223,20 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, Name = TPList->getParam(i)->getName(); switch (TA.getKind()) { case TemplateArgument::Type: { - llvm::DIType TTy = getOrCreateType(TA.getAsType(), Unit); - llvm::DITemplateTypeParameter TTP = - DBuilder.createTemplateTypeParameter(TheCU, Name, TTy); - TemplateParams.push_back(TTP); + llvm::DIType *TTy = getOrCreateType(TA.getAsType(), Unit); + TemplateParams.push_back( + DBuilder.createTemplateTypeParameter(TheCU, Name, TTy)); } break; case TemplateArgument::Integral: { - llvm::DIType TTy = getOrCreateType(TA.getIntegralType(), Unit); - llvm::DITemplateValueParameter TVP = - DBuilder.createTemplateValueParameter( - TheCU, Name, TTy, - llvm::ConstantInt::get(CGM.getLLVMContext(), TA.getAsIntegral())); - TemplateParams.push_back(TVP); + llvm::DIType *TTy = getOrCreateType(TA.getIntegralType(), Unit); + TemplateParams.push_back(DBuilder.createTemplateValueParameter( + TheCU, Name, TTy, + llvm::ConstantInt::get(CGM.getLLVMContext(), TA.getAsIntegral()))); } break; case TemplateArgument::Declaration: { const ValueDecl *D = TA.getAsDecl(); QualType T = TA.getParamTypeForDecl().getDesugaredType(CGM.getContext()); - llvm::DIType TTy = getOrCreateType(T, Unit); + llvm::DIType *TTy = getOrCreateType(T, Unit); llvm::Constant *V = nullptr; const CXXMethodDecl *MD; // Variable pointer template parameters have a value that is the address @@ -1264,7 +1246,7 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, // Member function pointers have special support for building them, though // this is currently unsupported in LLVM CodeGen. else if ((MD = dyn_cast<CXXMethodDecl>(D)) && MD->isInstance()) - V = CGM.getCXXABI().EmitMemberPointer(MD); + V = CGM.getCXXABI().EmitMemberFunctionPointer(MD); else if (const auto *FD = dyn_cast<FunctionDecl>(D)) V = CGM.GetAddrOfFunction(FD); // Member data pointers have special handling too to compute the fixed @@ -1278,15 +1260,13 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, CGM.getContext().toCharUnitsFromBits((int64_t)fieldOffset); V = CGM.getCXXABI().EmitMemberDataPointer(MPT, chars); } - llvm::DITemplateValueParameter TVP = - DBuilder.createTemplateValueParameter( - TheCU, Name, TTy, - cast_or_null<llvm::Constant>(V->stripPointerCasts())); - TemplateParams.push_back(TVP); + TemplateParams.push_back(DBuilder.createTemplateValueParameter( + TheCU, Name, TTy, + cast_or_null<llvm::Constant>(V->stripPointerCasts()))); } break; case TemplateArgument::NullPtr: { QualType T = TA.getNullPtrType(); - llvm::DIType TTy = getOrCreateType(T, Unit); + llvm::DIType *TTy = getOrCreateType(T, Unit); llvm::Constant *V = nullptr; // Special case member data pointer null values since they're actually -1 // instead of zero. @@ -1301,24 +1281,19 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, V = CGM.getCXXABI().EmitNullMemberPointer(MPT); if (!V) V = llvm::ConstantInt::get(CGM.Int8Ty, 0); - llvm::DITemplateValueParameter TVP = - DBuilder.createTemplateValueParameter(TheCU, Name, TTy, - cast<llvm::Constant>(V)); - TemplateParams.push_back(TVP); - } break; - case TemplateArgument::Template: { - llvm::DITemplateValueParameter - TVP = DBuilder.createTemplateTemplateParameter( - TheCU, Name, llvm::DIType(), - TA.getAsTemplate().getAsTemplateDecl()->getQualifiedNameAsString()); - TemplateParams.push_back(TVP); - } break; - case TemplateArgument::Pack: { - llvm::DITemplateValueParameter TVP = DBuilder.createTemplateParameterPack( - TheCU, Name, llvm::DIType(), - CollectTemplateParams(nullptr, TA.getPackAsArray(), Unit)); - TemplateParams.push_back(TVP); + TemplateParams.push_back(DBuilder.createTemplateValueParameter( + TheCU, Name, TTy, cast<llvm::Constant>(V))); } break; + case TemplateArgument::Template: + TemplateParams.push_back(DBuilder.createTemplateTemplateParameter( + TheCU, Name, nullptr, + TA.getAsTemplate().getAsTemplateDecl()->getQualifiedNameAsString())); + break; + case TemplateArgument::Pack: + TemplateParams.push_back(DBuilder.createTemplateParameterPack( + TheCU, Name, nullptr, + CollectTemplateParams(nullptr, TA.getPackAsArray(), Unit))); + break; case TemplateArgument::Expression: { const Expr *E = TA.getAsExpr(); QualType T = E->getType(); @@ -1326,11 +1301,9 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, T = CGM.getContext().getLValueReferenceType(T); llvm::Constant *V = CGM.EmitConstantExpr(E, T); assert(V && "Expression in template argument isn't constant"); - llvm::DIType TTy = getOrCreateType(T, Unit); - llvm::DITemplateValueParameter TVP = - DBuilder.createTemplateValueParameter( - TheCU, Name, TTy, cast<llvm::Constant>(V->stripPointerCasts())); - TemplateParams.push_back(TVP); + llvm::DIType *TTy = getOrCreateType(T, Unit); + TemplateParams.push_back(DBuilder.createTemplateValueParameter( + TheCU, Name, TTy, cast<llvm::Constant>(V->stripPointerCasts()))); } break; // And the following should never occur: case TemplateArgument::TemplateExpansion: @@ -1342,10 +1315,9 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, return DBuilder.getOrCreateArray(TemplateParams); } -/// CollectFunctionTemplateParams - A helper function to collect debug -/// info for function template parameters. -llvm::DIArray CGDebugInfo::CollectFunctionTemplateParams(const FunctionDecl *FD, - llvm::DIFile Unit) { +llvm::DINodeArray +CGDebugInfo::CollectFunctionTemplateParams(const FunctionDecl *FD, + llvm::DIFile *Unit) { if (FD->getTemplatedKind() == FunctionDecl::TK_FunctionTemplateSpecialization) { const TemplateParameterList *TList = FD->getTemplateSpecializationInfo() @@ -1354,13 +1326,11 @@ llvm::DIArray CGDebugInfo::CollectFunctionTemplateParams(const FunctionDecl *FD, return CollectTemplateParams( TList, FD->getTemplateSpecializationArgs()->asArray(), Unit); } - return llvm::DIArray(); + return llvm::DINodeArray(); } -/// CollectCXXTemplateParams - A helper function to collect debug info for -/// template parameters. -llvm::DIArray CGDebugInfo::CollectCXXTemplateParams( - const ClassTemplateSpecializationDecl *TSpecial, llvm::DIFile Unit) { +llvm::DINodeArray CGDebugInfo::CollectCXXTemplateParams( + const ClassTemplateSpecializationDecl *TSpecial, llvm::DIFile *Unit) { // Always get the full list of parameters, not just the ones from // the specialization. TemplateParameterList *TPList = @@ -1369,33 +1339,29 @@ llvm::DIArray CGDebugInfo::CollectCXXTemplateParams( return CollectTemplateParams(TPList, TAList.asArray(), Unit); } -/// getOrCreateVTablePtrType - Return debug info descriptor for vtable. -llvm::DIType CGDebugInfo::getOrCreateVTablePtrType(llvm::DIFile Unit) { - if (VTablePtrType.isValid()) +llvm::DIType *CGDebugInfo::getOrCreateVTablePtrType(llvm::DIFile *Unit) { + if (VTablePtrType) return VTablePtrType; ASTContext &Context = CGM.getContext(); /* Function type */ llvm::Metadata *STy = getOrCreateType(Context.IntTy, Unit); - llvm::DITypeArray SElements = DBuilder.getOrCreateTypeArray(STy); - llvm::DIType SubTy = DBuilder.createSubroutineType(Unit, SElements); + llvm::DITypeRefArray SElements = DBuilder.getOrCreateTypeArray(STy); + llvm::DIType *SubTy = DBuilder.createSubroutineType(Unit, SElements); unsigned Size = Context.getTypeSize(Context.VoidPtrTy); - llvm::DIType vtbl_ptr_type = + llvm::DIType *vtbl_ptr_type = DBuilder.createPointerType(SubTy, Size, 0, "__vtbl_ptr_type"); VTablePtrType = DBuilder.createPointerType(vtbl_ptr_type, Size); return VTablePtrType; } -/// getVTableName - Get vtable name for the given Class. StringRef CGDebugInfo::getVTableName(const CXXRecordDecl *RD) { // Copy the gdb compatible name on the side and use its reference. return internString("_vptr$", RD->getNameAsString()); } -/// CollectVTableInfo - If the C++ class has vtable info then insert appropriate -/// debug info entry in EltTys vector. -void CGDebugInfo::CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile Unit, +void CGDebugInfo::CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile *Unit, SmallVectorImpl<llvm::Metadata *> &EltTys) { const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD); @@ -1408,26 +1374,23 @@ void CGDebugInfo::CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile Unit, return; unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy); - llvm::DIType VPTR = DBuilder.createMemberType( + llvm::DIType *VPTR = DBuilder.createMemberType( Unit, getVTableName(RD), Unit, 0, Size, 0, 0, - llvm::DIDescriptor::FlagArtificial, getOrCreateVTablePtrType(Unit)); + llvm::DINode::FlagArtificial, getOrCreateVTablePtrType(Unit)); EltTys.push_back(VPTR); } -/// getOrCreateRecordType - Emit record type's standalone debug info. -llvm::DIType CGDebugInfo::getOrCreateRecordType(QualType RTy, - SourceLocation Loc) { +llvm::DIType *CGDebugInfo::getOrCreateRecordType(QualType RTy, + SourceLocation Loc) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); - llvm::DIType T = getOrCreateType(RTy, getOrCreateFile(Loc)); + llvm::DIType *T = getOrCreateType(RTy, getOrCreateFile(Loc)); return T; } -/// getOrCreateInterfaceType - Emit an objective c interface type standalone -/// debug info. -llvm::DIType CGDebugInfo::getOrCreateInterfaceType(QualType D, - SourceLocation Loc) { +llvm::DIType *CGDebugInfo::getOrCreateInterfaceType(QualType D, + SourceLocation Loc) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); - llvm::DIType T = getOrCreateType(D, getOrCreateFile(Loc)); + llvm::DIType *T = getOrCreateType(D, getOrCreateFile(Loc)); RetainedTypes.push_back(D.getAsOpaquePtr()); return T; } @@ -1438,11 +1401,10 @@ void CGDebugInfo::completeType(const EnumDecl *ED) { QualType Ty = CGM.getContext().getEnumType(ED); void *TyPtr = Ty.getAsOpaquePtr(); auto I = TypeCache.find(TyPtr); - if (I == TypeCache.end() || - !llvm::DIType(cast<llvm::MDNode>(I->second)).isForwardDecl()) + if (I == TypeCache.end() || !cast<llvm::DIType>(I->second)->isForwardDecl()) return; - llvm::DIType Res = CreateTypeDefinition(Ty->castAs<EnumType>()); - assert(!Res.isForwardDecl()); + llvm::DIType *Res = CreateTypeDefinition(Ty->castAs<EnumType>()); + assert(!Res->isForwardDecl()); TypeCache[TyPtr].reset(Res); } @@ -1461,8 +1423,8 @@ void CGDebugInfo::completeRequiredType(const RecordDecl *RD) { return; QualType Ty = CGM.getContext().getRecordType(RD); - llvm::DIType T = getTypeOrNull(Ty); - if (T && T.isForwardDecl()) + llvm::DIType *T = getTypeOrNull(Ty); + if (T && T->isForwardDecl()) completeClassData(RD); } @@ -1472,11 +1434,10 @@ void CGDebugInfo::completeClassData(const RecordDecl *RD) { QualType Ty = CGM.getContext().getRecordType(RD); void *TyPtr = Ty.getAsOpaquePtr(); auto I = TypeCache.find(TyPtr); - if (I != TypeCache.end() && - !llvm::DIType(cast<llvm::MDNode>(I->second)).isForwardDecl()) + if (I != TypeCache.end() && !cast<llvm::DIType>(I->second)->isForwardDecl()) return; - llvm::DIType Res = CreateTypeDefinition(Ty->castAs<RecordType>()); - assert(!Res.isForwardDecl()); + llvm::DIType *Res = CreateTypeDefinition(Ty->castAs<RecordType>()); + assert(!Res->isForwardDecl()); TypeCache[TyPtr].reset(Res); } @@ -1523,10 +1484,9 @@ static bool shouldOmitDefinition(CodeGenOptions::DebugInfoKind DebugKind, return false; } -/// CreateType - get structure or union type. -llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) { +llvm::DIType *CGDebugInfo::CreateType(const RecordType *Ty) { RecordDecl *RD = Ty->getDecl(); - llvm::DICompositeType T(getTypeOrNull(QualType(Ty, 0))); + llvm::DIType *T = cast_or_null<llvm::DIType>(getTypeOrNull(QualType(Ty, 0))); if (T || shouldOmitDefinition(DebugKind, RD, CGM.getLangOpts())) { if (!T) T = getOrCreateRecordFwdDecl( @@ -1537,11 +1497,11 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) { return CreateTypeDefinition(Ty); } -llvm::DIType CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) { +llvm::DIType *CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) { RecordDecl *RD = Ty->getDecl(); // Get overall information about the record type for the debug info. - llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation()); + llvm::DIFile *DefUnit = getOrCreateFile(RD->getLocation()); // Records and classes and unions can all be recursive. To handle them, we // first generate a debug descriptor for the struct as a forward declaration. @@ -1550,11 +1510,11 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) { // may refer to the forward decl if the struct is recursive) and replace all // uses of the forward declaration with the final definition. - llvm::DICompositeType FwdDecl(getOrCreateLimitedType(Ty, DefUnit)); - assert(FwdDecl.isCompositeType() && - "The debug type of a RecordType should be a llvm::DICompositeType"); + auto *FwdDecl = + cast<llvm::DICompositeType>(getOrCreateLimitedType(Ty, DefUnit)); - if (FwdDecl.isForwardDecl()) + const RecordDecl *D = RD->getDefinition(); + if (!D || !D->isCompleteDefinition()) return FwdDecl; if (const CXXRecordDecl *CXXDecl = dyn_cast<CXXRecordDecl>(RD)) @@ -1586,16 +1546,19 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) { LexicalBlockStack.pop_back(); RegionMap.erase(Ty->getDecl()); - llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys); + llvm::DINodeArray Elements = DBuilder.getOrCreateArray(EltTys); DBuilder.replaceArrays(FwdDecl, Elements); + if (FwdDecl->isTemporary()) + FwdDecl = + llvm::MDNode::replaceWithPermanent(llvm::TempDICompositeType(FwdDecl)); + RegionMap[Ty->getDecl()].reset(FwdDecl); return FwdDecl; } -/// CreateType - get objective-c object type. -llvm::DIType CGDebugInfo::CreateType(const ObjCObjectType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const ObjCObjectType *Ty, + llvm::DIFile *Unit) { // Ignore protocols. return getOrCreateType(Ty->getBaseType(), Unit); } @@ -1624,23 +1587,23 @@ static bool hasDefaultSetterName(const ObjCPropertyDecl *PD, Setter->getDeclName().getObjCSelector().getNameForSlot(0); } -/// CreateType - get objective-c interface type. -llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const ObjCInterfaceType *Ty, + llvm::DIFile *Unit) { ObjCInterfaceDecl *ID = Ty->getDecl(); if (!ID) - return llvm::DIType(); + return nullptr; // Get overall information about the record type for the debug info. - llvm::DIFile DefUnit = getOrCreateFile(ID->getLocation()); + llvm::DIFile *DefUnit = getOrCreateFile(ID->getLocation()); unsigned Line = getLineNumber(ID->getLocation()); - llvm::dwarf::SourceLanguage RuntimeLang = TheCU.getLanguage(); + auto RuntimeLang = + static_cast<llvm::dwarf::SourceLanguage>(TheCU->getSourceLanguage()); // If this is just a forward declaration return a special forward-declaration // debug type since we won't be able to lay out the entire type. ObjCInterfaceDecl *Def = ID->getDefinition(); if (!Def || !Def->getImplementation()) { - llvm::DIType FwdDecl = DBuilder.createReplaceableForwardDecl( + llvm::DIType *FwdDecl = DBuilder.createReplaceableCompositeType( llvm::dwarf::DW_TAG_structure_type, ID->getName(), TheCU, DefUnit, Line, RuntimeLang); ObjCInterfaceCache.push_back(ObjCInterfaceCacheEntry(Ty, FwdDecl, Unit)); @@ -1650,12 +1613,53 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty, return CreateTypeDefinition(Ty, Unit); } -llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, - llvm::DIFile Unit) { +llvm::DIModule * +CGDebugInfo::getOrCreateModuleRef(ExternalASTSource::ASTSourceDescriptor Mod) { + auto it = ModuleRefCache.find(Mod.Signature); + if (it != ModuleRefCache.end()) + return it->second; + + // Macro definitions that were defined with "-D" on the command line. + SmallString<128> ConfigMacros; + { + llvm::raw_svector_ostream OS(ConfigMacros); + const auto &PPOpts = CGM.getPreprocessorOpts(); + unsigned I = 0; + // Translate the macro definitions back into a commmand line. + for (auto &M : PPOpts.Macros) { + if (++I > 1) + OS << " "; + const std::string &Macro = M.first; + bool Undef = M.second; + OS << "\"-" << (Undef ? 'U' : 'D'); + for (char c : Macro) + switch (c) { + case '\\' : OS << "\\\\"; break; + case '"' : OS << "\\\""; break; + default: OS << c; + } + OS << '\"'; + } + } + llvm::DIBuilder DIB(CGM.getModule()); + auto *CU = DIB.createCompileUnit( + TheCU->getSourceLanguage(), internString(Mod.ModuleName), + internString(Mod.Path), TheCU->getProducer(), true, StringRef(), 0, + internString(Mod.ASTFile), llvm::DIBuilder::FullDebug, Mod.Signature); + llvm::DIModule *ModuleRef = + DIB.createModule(CU, Mod.ModuleName, ConfigMacros, internString(Mod.Path), + internString(CGM.getHeaderSearchOpts().Sysroot)); + DIB.finalize(); + ModuleRefCache.insert(std::make_pair(Mod.Signature, ModuleRef)); + return ModuleRef; +} + +llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, + llvm::DIFile *Unit) { ObjCInterfaceDecl *ID = Ty->getDecl(); - llvm::DIFile DefUnit = getOrCreateFile(ID->getLocation()); + llvm::DIFile *DefUnit = getOrCreateFile(ID->getLocation()); unsigned Line = getLineNumber(ID->getLocation()); - unsigned RuntimeLang = TheCU.getLanguage(); + unsigned RuntimeLang = TheCU->getSourceLanguage(); // Bit size, align and offset of the type. uint64_t Size = CGM.getContext().getTypeSize(Ty); @@ -1663,17 +1667,17 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, unsigned Flags = 0; if (ID->getImplementation()) - Flags |= llvm::DIDescriptor::FlagObjcClassComplete; + Flags |= llvm::DINode::FlagObjcClassComplete; - llvm::DICompositeType RealDecl = DBuilder.createStructType( - Unit, ID->getName(), DefUnit, Line, Size, Align, Flags, llvm::DIType(), - llvm::DIArray(), RuntimeLang); + llvm::DICompositeType *RealDecl = DBuilder.createStructType( + Unit, ID->getName(), DefUnit, Line, Size, Align, Flags, nullptr, + llvm::DINodeArray(), RuntimeLang); QualType QTy(Ty, 0); TypeCache[QTy.getAsOpaquePtr()].reset(RealDecl); // Push the struct on region stack. - LexicalBlockStack.emplace_back(static_cast<llvm::MDNode *>(RealDecl)); + LexicalBlockStack.emplace_back(RealDecl); RegionMap[Ty->getDecl()].reset(RealDecl); // Convert all the elements. @@ -1681,19 +1685,19 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, ObjCInterfaceDecl *SClass = ID->getSuperClass(); if (SClass) { - llvm::DIType SClassTy = + llvm::DIType *SClassTy = getOrCreateType(CGM.getContext().getObjCInterfaceType(SClass), Unit); - if (!SClassTy.isValid()) - return llvm::DIType(); + if (!SClassTy) + return nullptr; - llvm::DIType InhTag = DBuilder.createInheritance(RealDecl, SClassTy, 0, 0); + llvm::DIType *InhTag = DBuilder.createInheritance(RealDecl, SClassTy, 0, 0); EltTys.push_back(InhTag); } // Create entries for all of the properties. for (const auto *PD : ID->properties()) { SourceLocation Loc = PD->getLocation(); - llvm::DIFile PUnit = getOrCreateFile(Loc); + llvm::DIFile *PUnit = getOrCreateFile(Loc); unsigned PLine = getLineNumber(Loc); ObjCMethodDecl *Getter = PD->getGetterMethodDecl(); ObjCMethodDecl *Setter = PD->getSetterMethodDecl(); @@ -1711,9 +1715,9 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, unsigned FieldNo = 0; for (ObjCIvarDecl *Field = ID->all_declared_ivar_begin(); Field; Field = Field->getNextIvar(), ++FieldNo) { - llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit); - if (!FieldTy.isValid()) - return llvm::DIType(); + llvm::DIType *FieldTy = getOrCreateType(Field->getType(), Unit); + if (!FieldTy) + return nullptr; StringRef FieldName = Field->getName(); @@ -1722,7 +1726,7 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, continue; // Get the location for the field. - llvm::DIFile FieldDefUnit = getOrCreateFile(Field->getLocation()); + llvm::DIFile *FieldDefUnit = getOrCreateFile(Field->getLocation()); unsigned FieldLine = getLineNumber(Field->getLocation()); QualType FType = Field->getType(); uint64_t FieldSize = 0; @@ -1755,11 +1759,11 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, unsigned Flags = 0; if (Field->getAccessControl() == ObjCIvarDecl::Protected) - Flags = llvm::DIDescriptor::FlagProtected; + Flags = llvm::DINode::FlagProtected; else if (Field->getAccessControl() == ObjCIvarDecl::Private) - Flags = llvm::DIDescriptor::FlagPrivate; + Flags = llvm::DINode::FlagPrivate; else if (Field->getAccessControl() == ObjCIvarDecl::Public) - Flags = llvm::DIDescriptor::FlagPublic; + Flags = llvm::DINode::FlagPublic; llvm::MDNode *PropertyNode = nullptr; if (ObjCImplementationDecl *ImpD = ID->getImplementation()) { @@ -1767,7 +1771,7 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, ImpD->FindPropertyImplIvarDecl(Field->getIdentifier())) { if (ObjCPropertyDecl *PD = PImpD->getPropertyDecl()) { SourceLocation Loc = PD->getLocation(); - llvm::DIFile PUnit = getOrCreateFile(Loc); + llvm::DIFile *PUnit = getOrCreateFile(Loc); unsigned PLine = getLineNumber(Loc); ObjCMethodDecl *Getter = PD->getGetterMethodDecl(); ObjCMethodDecl *Setter = PD->getSetterMethodDecl(); @@ -1788,15 +1792,16 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, EltTys.push_back(FieldTy); } - llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys); + llvm::DINodeArray Elements = DBuilder.getOrCreateArray(EltTys); DBuilder.replaceArrays(RealDecl, Elements); LexicalBlockStack.pop_back(); return RealDecl; } -llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty, llvm::DIFile Unit) { - llvm::DIType ElementTy = getOrCreateType(Ty->getElementType(), Unit); +llvm::DIType *CGDebugInfo::CreateType(const VectorType *Ty, + llvm::DIFile *Unit) { + llvm::DIType *ElementTy = getOrCreateType(Ty->getElementType(), Unit); int64_t Count = Ty->getNumElements(); if (Count == 0) // If number of elements are not known then this is an unbounded array. @@ -1804,7 +1809,7 @@ llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty, llvm::DIFile Unit) { Count = -1; llvm::Metadata *Subscript = DBuilder.getOrCreateSubrange(0, Count); - llvm::DIArray SubscriptArray = DBuilder.getOrCreateArray(Subscript); + llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscript); uint64_t Size = CGM.getContext().getTypeSize(Ty); uint64_t Align = CGM.getContext().getTypeAlign(Ty); @@ -1812,7 +1817,7 @@ llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty, llvm::DIFile Unit) { return DBuilder.createVectorType(Size, Align, ElementTy, SubscriptArray); } -llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) { uint64_t Size; uint64_t Align; @@ -1858,32 +1863,33 @@ llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile Unit) { EltTy = Ty->getElementType(); } - llvm::DIArray SubscriptArray = DBuilder.getOrCreateArray(Subscripts); + llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscripts); - llvm::DIType DbgTy = DBuilder.createArrayType( - Size, Align, getOrCreateType(EltTy, Unit), SubscriptArray); - return DbgTy; + return DBuilder.createArrayType(Size, Align, getOrCreateType(EltTy, Unit), + SubscriptArray); } -llvm::DIType CGDebugInfo::CreateType(const LValueReferenceType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const LValueReferenceType *Ty, + llvm::DIFile *Unit) { return CreatePointerLikeType(llvm::dwarf::DW_TAG_reference_type, Ty, Ty->getPointeeType(), Unit); } -llvm::DIType CGDebugInfo::CreateType(const RValueReferenceType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const RValueReferenceType *Ty, + llvm::DIFile *Unit) { return CreatePointerLikeType(llvm::dwarf::DW_TAG_rvalue_reference_type, Ty, Ty->getPointeeType(), Unit); } -llvm::DIType CGDebugInfo::CreateType(const MemberPointerType *Ty, - llvm::DIFile U) { - llvm::DIType ClassType = getOrCreateType(QualType(Ty->getClass(), 0), U); - if (!Ty->getPointeeType()->isFunctionType()) +llvm::DIType *CGDebugInfo::CreateType(const MemberPointerType *Ty, + llvm::DIFile *U) { + uint64_t Size = CGM.getCXXABI().isTypeInfoCalculable(QualType(Ty, 0)) + ? CGM.getContext().getTypeSize(Ty) + : 0; + llvm::DIType *ClassType = getOrCreateType(QualType(Ty->getClass(), 0), U); + if (Ty->isMemberDataPointerType()) return DBuilder.createMemberPointerType( - getOrCreateType(Ty->getPointeeType(), U), ClassType, - CGM.getContext().getTypeSize(Ty)); + getOrCreateType(Ty->getPointeeType(), U), ClassType, Size); const FunctionProtoType *FPT = Ty->getPointeeType()->getAs<FunctionProtoType>(); @@ -1891,17 +1897,16 @@ llvm::DIType CGDebugInfo::CreateType(const MemberPointerType *Ty, getOrCreateInstanceMethodType(CGM.getContext().getPointerType(QualType( Ty->getClass(), FPT->getTypeQuals())), FPT, U), - ClassType, CGM.getContext().getTypeSize(Ty)); + ClassType, Size); } -llvm::DIType CGDebugInfo::CreateType(const AtomicType *Ty, llvm::DIFile U) { +llvm::DIType *CGDebugInfo::CreateType(const AtomicType *Ty, llvm::DIFile *U) { // Ignore the atomic wrapping // FIXME: What is the correct representation? return getOrCreateType(Ty->getValueType(), U); } -/// CreateEnumType - get enumeration type. -llvm::DIType CGDebugInfo::CreateEnumType(const EnumType *Ty) { +llvm::DIType *CGDebugInfo::CreateEnumType(const EnumType *Ty) { const EnumDecl *ED = Ty->getDecl(); uint64_t Size = 0; uint64_t Align = 0; @@ -1915,14 +1920,14 @@ llvm::DIType CGDebugInfo::CreateEnumType(const EnumType *Ty) { // If this is just a forward declaration, construct an appropriately // marked node and just return it. if (!ED->getDefinition()) { - llvm::DIDescriptor EDContext; - EDContext = getContextDescriptor(cast<Decl>(ED->getDeclContext())); - llvm::DIFile DefUnit = getOrCreateFile(ED->getLocation()); + llvm::DIScope *EDContext = + getContextDescriptor(cast<Decl>(ED->getDeclContext())); + llvm::DIFile *DefUnit = getOrCreateFile(ED->getLocation()); unsigned Line = getLineNumber(ED->getLocation()); StringRef EDName = ED->getName(); - llvm::DIType RetTy = DBuilder.createReplaceableForwardDecl( + llvm::DIType *RetTy = DBuilder.createReplaceableCompositeType( llvm::dwarf::DW_TAG_enumeration_type, EDName, EDContext, DefUnit, Line, - 0, Size, Align, FullName); + 0, Size, Align, llvm::DINode::FlagFwdDecl, FullName); ReplaceMap.emplace_back( std::piecewise_construct, std::make_tuple(Ty), std::make_tuple(static_cast<llvm::Metadata *>(RetTy))); @@ -1932,7 +1937,7 @@ llvm::DIType CGDebugInfo::CreateEnumType(const EnumType *Ty) { return CreateTypeDefinition(Ty); } -llvm::DIType CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) { +llvm::DIType *CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) { const EnumDecl *ED = Ty->getDecl(); uint64_t Size = 0; uint64_t Align = 0; @@ -1943,7 +1948,7 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) { SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU); - // Create DIEnumerator elements for each enumerator. + // Create elements for each enumerator. SmallVector<llvm::Metadata *, 16> Enumerators; ED = ED->getDefinition(); for (const auto *Enum : ED->enumerators()) { @@ -1952,19 +1957,17 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) { } // Return a CompositeType for the enum itself. - llvm::DIArray EltArray = DBuilder.getOrCreateArray(Enumerators); + llvm::DINodeArray EltArray = DBuilder.getOrCreateArray(Enumerators); - llvm::DIFile DefUnit = getOrCreateFile(ED->getLocation()); + llvm::DIFile *DefUnit = getOrCreateFile(ED->getLocation()); unsigned Line = getLineNumber(ED->getLocation()); - llvm::DIDescriptor EnumContext = + llvm::DIScope *EnumContext = getContextDescriptor(cast<Decl>(ED->getDeclContext())); - llvm::DIType ClassTy = ED->isFixed() - ? getOrCreateType(ED->getIntegerType(), DefUnit) - : llvm::DIType(); - llvm::DIType DbgTy = - DBuilder.createEnumerationType(EnumContext, ED->getName(), DefUnit, Line, - Size, Align, EltArray, ClassTy, FullName); - return DbgTy; + llvm::DIType *ClassTy = + ED->isFixed() ? getOrCreateType(ED->getIntegerType(), DefUnit) : nullptr; + return DBuilder.createEnumerationType(EnumContext, ED->getName(), DefUnit, + Line, Size, Align, EltArray, ClassTy, + FullName); } static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) { @@ -2022,9 +2025,7 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) { } while (true); } -/// getType - Get the type from the cache or return null type if it doesn't -/// exist. -llvm::DIType CGDebugInfo::getTypeOrNull(QualType Ty) { +llvm::DIType *CGDebugInfo::getTypeOrNull(QualType Ty) { // Unwrap the type as needed for debug information. Ty = UnwrapTypeForDebugInfo(Ty, CGM.getContext()); @@ -2033,10 +2034,10 @@ llvm::DIType CGDebugInfo::getTypeOrNull(QualType Ty) { if (it != TypeCache.end()) { // Verify that the debug info still exists. if (llvm::Metadata *V = it->second) - return llvm::DIType(cast<llvm::MDNode>(V)); + return cast<llvm::DIType>(V); } - return llvm::DIType(); + return nullptr; } void CGDebugInfo::completeTemplateDefinition( @@ -2050,20 +2051,18 @@ void CGDebugInfo::completeTemplateDefinition( RetainedTypes.push_back(CGM.getContext().getRecordType(&SD).getAsOpaquePtr()); } -/// getOrCreateType - Get the type from the cache or create a new -/// one if necessary. -llvm::DIType CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile *Unit) { if (Ty.isNull()) - return llvm::DIType(); + return nullptr; // Unwrap the type as needed for debug information. Ty = UnwrapTypeForDebugInfo(Ty, CGM.getContext()); - if (llvm::DIType T = getTypeOrNull(Ty)) + if (auto *T = getTypeOrNull(Ty)) return T; // Otherwise create the type. - llvm::DIType Res = CreateTypeNode(Ty, Unit); + llvm::DIType *Res = CreateTypeNode(Ty, Unit); void *TyPtr = Ty.getAsOpaquePtr(); // And update the type cache. @@ -2072,8 +2071,6 @@ llvm::DIType CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile Unit) { return Res; } -/// Currently the checksum of an interface includes the number of -/// ivars and property accessors. unsigned CGDebugInfo::Checksum(const ObjCInterfaceDecl *ID) { // The assumption is that the number of ivars can only increase // monotonically, so it is safe to just use their current number as @@ -2098,8 +2095,7 @@ ObjCInterfaceDecl *CGDebugInfo::getObjCInterfaceDecl(QualType Ty) { } } -/// CreateTypeNode - Create a new debug type node. -llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) { // Handle qualifiers, which recursively handles what they refer to. if (Ty.hasLocalQualifiers()) return CreateQualifiedType(Ty, Unit); @@ -2179,27 +2175,25 @@ llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile Unit) { llvm_unreachable("type should have been unwrapped!"); } -/// getOrCreateLimitedType - Get the type from the cache or create a new -/// limited type if necessary. -llvm::DIType CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty, + llvm::DIFile *Unit) { QualType QTy(Ty, 0); - llvm::DICompositeType T(getTypeOrNull(QTy)); + auto *T = cast_or_null<llvm::DICompositeTypeBase>(getTypeOrNull(QTy)); // We may have cached a forward decl when we could have created // a non-forward decl. Go ahead and create a non-forward decl // now. - if (T && !T.isForwardDecl()) + if (T && !T->isForwardDecl()) return T; // Otherwise create the type. - llvm::DICompositeType Res = CreateLimitedType(Ty); + llvm::DICompositeType *Res = CreateLimitedType(Ty); // Propagate members from the declaration to the definition // CreateType(const RecordType*) will overwrite this with the members in the // correct order if the full type is needed. - DBuilder.replaceArrays(Res, T.getElements()); + DBuilder.replaceArrays(Res, T ? T->getElements() : llvm::DINodeArray()); // And update the type cache. TypeCache[QTy.getAsOpaquePtr()].reset(Res); @@ -2207,21 +2201,22 @@ llvm::DIType CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty, } // TODO: Currently used for context chains when limiting debug info. -llvm::DICompositeType CGDebugInfo::CreateLimitedType(const RecordType *Ty) { +llvm::DICompositeType *CGDebugInfo::CreateLimitedType(const RecordType *Ty) { RecordDecl *RD = Ty->getDecl(); // Get overall information about the record type for the debug info. - llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation()); + llvm::DIFile *DefUnit = getOrCreateFile(RD->getLocation()); unsigned Line = getLineNumber(RD->getLocation()); StringRef RDName = getClassName(RD); - llvm::DIDescriptor RDContext = + llvm::DIScope *RDContext = getContextDescriptor(cast<Decl>(RD->getDeclContext())); // If we ended up creating the type during the context chain construction, // just return that. - llvm::DICompositeType T(getTypeOrNull(CGM.getContext().getRecordType(RD))); - if (T && (!T.isForwardDecl() || !RD->getDefinition())) + auto *T = cast_or_null<llvm::DICompositeType>( + getTypeOrNull(CGM.getContext().getRecordType(RD))); + if (T && (!T->isForwardDecl() || !RD->getDefinition())) return T; // If this is just a forward or incomplete declaration, construct an @@ -2232,38 +2227,27 @@ llvm::DICompositeType CGDebugInfo::CreateLimitedType(const RecordType *Ty) { uint64_t Size = CGM.getContext().getTypeSize(Ty); uint64_t Align = CGM.getContext().getTypeAlign(Ty); - llvm::DICompositeType RealDecl; SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU); - if (RD->isUnion()) - RealDecl = DBuilder.createUnionType(RDContext, RDName, DefUnit, Line, Size, - Align, 0, llvm::DIArray(), 0, FullName); - else if (RD->isClass()) { - // FIXME: This could be a struct type giving a default visibility different - // than C++ class type, but needs llvm metadata changes first. - RealDecl = DBuilder.createClassType( - RDContext, RDName, DefUnit, Line, Size, Align, 0, 0, llvm::DIType(), - llvm::DIArray(), llvm::DIType(), llvm::DIArray(), FullName); - } else - RealDecl = DBuilder.createStructType( - RDContext, RDName, DefUnit, Line, Size, Align, 0, llvm::DIType(), - llvm::DIArray(), 0, llvm::DIType(), FullName); + llvm::DICompositeType *RealDecl = DBuilder.createReplaceableCompositeType( + getTagForRecord(RD), RDName, RDContext, DefUnit, Line, 0, Size, Align, 0, + FullName); RegionMap[Ty->getDecl()].reset(RealDecl); TypeCache[QualType(Ty, 0).getAsOpaquePtr()].reset(RealDecl); if (const ClassTemplateSpecializationDecl *TSpecial = dyn_cast<ClassTemplateSpecializationDecl>(RD)) - DBuilder.replaceArrays(RealDecl, llvm::DIArray(), + DBuilder.replaceArrays(RealDecl, llvm::DINodeArray(), CollectCXXTemplateParams(TSpecial, DefUnit)); return RealDecl; } void CGDebugInfo::CollectContainingType(const CXXRecordDecl *RD, - llvm::DICompositeType RealDecl) { + llvm::DICompositeType *RealDecl) { // A class's primary base or the class itself contains the vtable. - llvm::DICompositeType ContainingType; + llvm::DICompositeType *ContainingType = nullptr; const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD); if (const CXXRecordDecl *PBase = RL.getPrimaryBase()) { // Seek non-virtual primary base root. @@ -2275,7 +2259,7 @@ void CGDebugInfo::CollectContainingType(const CXXRecordDecl *RD, else break; } - ContainingType = llvm::DICompositeType( + ContainingType = cast<llvm::DICompositeType>( getOrCreateType(QualType(PBase->getTypeForDecl(), 0), getOrCreateFile(RD->getLocation()))); } else if (RD->isDynamicClass()) @@ -2284,30 +2268,29 @@ void CGDebugInfo::CollectContainingType(const CXXRecordDecl *RD, DBuilder.replaceVTableHolder(RealDecl, ContainingType); } -/// CreateMemberType - Create new member and increase Offset by FType's size. -llvm::DIType CGDebugInfo::CreateMemberType(llvm::DIFile Unit, QualType FType, - StringRef Name, uint64_t *Offset) { - llvm::DIType FieldTy = CGDebugInfo::getOrCreateType(FType, Unit); +llvm::DIType *CGDebugInfo::CreateMemberType(llvm::DIFile *Unit, QualType FType, + StringRef Name, uint64_t *Offset) { + llvm::DIType *FieldTy = CGDebugInfo::getOrCreateType(FType, Unit); uint64_t FieldSize = CGM.getContext().getTypeSize(FType); unsigned FieldAlign = CGM.getContext().getTypeAlign(FType); - llvm::DIType Ty = DBuilder.createMemberType(Unit, Name, Unit, 0, FieldSize, - FieldAlign, *Offset, 0, FieldTy); + llvm::DIType *Ty = DBuilder.createMemberType(Unit, Name, Unit, 0, FieldSize, + FieldAlign, *Offset, 0, FieldTy); *Offset += FieldSize; return Ty; } -void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD, - llvm::DIFile Unit, - StringRef &Name, StringRef &LinkageName, - llvm::DIDescriptor &FDContext, - llvm::DIArray &TParamsArray, +void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD, llvm::DIFile *Unit, + StringRef &Name, + StringRef &LinkageName, + llvm::DIScope *&FDContext, + llvm::DINodeArray &TParamsArray, unsigned &Flags) { const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); Name = getFunctionName(FD); // Use mangled name as linkage name for C/C++ functions. if (FD->hasPrototype()) { LinkageName = CGM.getMangledName(GD); - Flags |= llvm::DIDescriptor::FlagPrototyped; + Flags |= llvm::DINode::FlagPrototyped; } // No need to replicate the linkage name if it isn't different from the // subprogram name, no need to have it at all unless coverage is enabled or @@ -2330,10 +2313,10 @@ void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD, } } -void CGDebugInfo::collectVarDeclProps(const VarDecl *VD, llvm::DIFile &Unit, +void CGDebugInfo::collectVarDeclProps(const VarDecl *VD, llvm::DIFile *&Unit, unsigned &LineNo, QualType &T, StringRef &Name, StringRef &LinkageName, - llvm::DIDescriptor &VDContext) { + llvm::DIScope *&VDContext) { Unit = getOrCreateFile(VD->getLocation()); LineNo = getLineNumber(VD->getLocation()); @@ -2362,19 +2345,27 @@ void CGDebugInfo::collectVarDeclProps(const VarDecl *VD, llvm::DIFile &Unit, // FIXME: Generalize this for even non-member global variables where the // declaration and definition may have different lexical decl contexts, once // we have support for emitting declarations of (non-member) global variables. - VDContext = getContextDescriptor( - dyn_cast<Decl>(VD->isStaticDataMember() ? VD->getLexicalDeclContext() - : VD->getDeclContext())); -} - -llvm::DISubprogram + const DeclContext *DC = VD->isStaticDataMember() ? VD->getLexicalDeclContext() + : VD->getDeclContext(); + // When a record type contains an in-line initialization of a static data + // member, and the record type is marked as __declspec(dllexport), an implicit + // definition of the member will be created in the record context. DWARF + // doesn't seem to have a nice way to describe this in a form that consumers + // are likely to understand, so fake the "normal" situation of a definition + // outside the class by putting it in the global scope. + if (DC->isRecord()) + DC = CGM.getContext().getTranslationUnitDecl(); + VDContext = getContextDescriptor(dyn_cast<Decl>(DC)); +} + +llvm::DISubprogram * CGDebugInfo::getFunctionForwardDeclaration(const FunctionDecl *FD) { - llvm::DIArray TParamsArray; + llvm::DINodeArray TParamsArray; StringRef Name, LinkageName; unsigned Flags = 0; SourceLocation Loc = FD->getLocation(); - llvm::DIFile Unit = getOrCreateFile(Loc); - llvm::DIDescriptor DContext(Unit); + llvm::DIFile *Unit = getOrCreateFile(Loc); + llvm::DIScope *DContext = Unit; unsigned Line = getLineNumber(Loc); collectFunctionDeclProps(FD, Unit, Name, LinkageName, DContext, @@ -2386,35 +2377,31 @@ CGDebugInfo::getFunctionForwardDeclaration(const FunctionDecl *FD) { QualType FnType = CGM.getContext().getFunctionType(FD->getReturnType(), ArgTypes, FunctionProtoType::ExtProtoInfo()); - llvm::DISubprogram SP = - DBuilder.createTempFunctionFwdDecl(DContext, Name, LinkageName, Unit, Line, - getOrCreateFunctionType(FD, FnType, Unit), - !FD->isExternallyVisible(), - false /*declaration*/, 0, Flags, - CGM.getLangOpts().Optimize, nullptr, - TParamsArray, getFunctionDeclaration(FD)); + llvm::DISubprogram *SP = DBuilder.createTempFunctionFwdDecl( + DContext, Name, LinkageName, Unit, Line, + getOrCreateFunctionType(FD, FnType, Unit), !FD->isExternallyVisible(), + false /*declaration*/, 0, Flags, CGM.getLangOpts().Optimize, nullptr, + TParamsArray.get(), getFunctionDeclaration(FD)); const FunctionDecl *CanonDecl = cast<FunctionDecl>(FD->getCanonicalDecl()); - FwdDeclReplaceMap.emplace_back( - std::piecewise_construct, std::make_tuple(CanonDecl), - std::make_tuple(static_cast<llvm::Metadata *>(SP))); + FwdDeclReplaceMap.emplace_back(std::piecewise_construct, + std::make_tuple(CanonDecl), + std::make_tuple(SP)); return SP; } -llvm::DIGlobalVariable +llvm::DIGlobalVariable * CGDebugInfo::getGlobalVariableForwardDeclaration(const VarDecl *VD) { QualType T; StringRef Name, LinkageName; SourceLocation Loc = VD->getLocation(); - llvm::DIFile Unit = getOrCreateFile(Loc); - llvm::DIDescriptor DContext(Unit); + llvm::DIFile *Unit = getOrCreateFile(Loc); + llvm::DIScope *DContext = Unit; unsigned Line = getLineNumber(Loc); collectVarDeclProps(VD, Unit, Line, T, Name, LinkageName, DContext); - llvm::DIGlobalVariable GV = - DBuilder.createTempGlobalVariableFwdDecl(DContext, Name, LinkageName, Unit, - Line, getOrCreateType(T, Unit), - !VD->isExternallyVisible(), - nullptr, nullptr); + auto *GV = DBuilder.createTempGlobalVariableFwdDecl( + DContext, Name, LinkageName, Unit, Line, getOrCreateType(T, Unit), + !VD->isExternallyVisible(), nullptr, nullptr); FwdDeclReplaceMap.emplace_back( std::piecewise_construct, std::make_tuple(cast<VarDecl>(VD->getCanonicalDecl())), @@ -2422,7 +2409,7 @@ CGDebugInfo::getGlobalVariableForwardDeclaration(const VarDecl *VD) { return GV; } -llvm::DIDescriptor CGDebugInfo::getDeclarationOrDefinition(const Decl *D) { +llvm::DINode *CGDebugInfo::getDeclarationOrDefinition(const Decl *D) { // We only need a declaration (not a definition) of the type - so use whatever // we would otherwise do to get a type for a pointee. (forward declarations in // limited debug info, full definitions (if the type definition is available) @@ -2433,7 +2420,7 @@ llvm::DIDescriptor CGDebugInfo::getDeclarationOrDefinition(const Decl *D) { auto I = DeclCache.find(D->getCanonicalDecl()); if (I != DeclCache.end()) - return llvm::DIDescriptor(dyn_cast_or_null<llvm::MDNode>(I->second)); + return dyn_cast_or_null<llvm::DINode>(I->second); // No definition for now. Emit a forward definition that might be // merged with a potential upcoming definition. @@ -2442,59 +2429,53 @@ llvm::DIDescriptor CGDebugInfo::getDeclarationOrDefinition(const Decl *D) { else if (const auto *VD = dyn_cast<VarDecl>(D)) return getGlobalVariableForwardDeclaration(VD); - return llvm::DIDescriptor(); + return nullptr; } -/// getFunctionDeclaration - Return debug info descriptor to describe method -/// declaration for the given method definition. -llvm::DISubprogram CGDebugInfo::getFunctionDeclaration(const Decl *D) { +llvm::DISubprogram *CGDebugInfo::getFunctionDeclaration(const Decl *D) { if (!D || DebugKind <= CodeGenOptions::DebugLineTablesOnly) - return llvm::DISubprogram(); + return nullptr; const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); if (!FD) - return llvm::DISubprogram(); + return nullptr; // Setup context. - llvm::DIScope S = getContextDescriptor(cast<Decl>(D->getDeclContext())); + auto *S = getContextDescriptor(cast<Decl>(D->getDeclContext())); auto MI = SPCache.find(FD->getCanonicalDecl()); if (MI == SPCache.end()) { if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD->getCanonicalDecl())) { - llvm::DICompositeType T(S); - llvm::DISubprogram SP = - CreateCXXMemberFunction(MD, getOrCreateFile(MD->getLocation()), T); - return SP; + return CreateCXXMemberFunction(MD, getOrCreateFile(MD->getLocation()), + cast<llvm::DICompositeType>(S)); } } if (MI != SPCache.end()) { - llvm::DISubprogram SP(dyn_cast_or_null<llvm::MDNode>(MI->second)); - if (SP.isSubprogram() && !SP.isDefinition()) + auto *SP = dyn_cast_or_null<llvm::DISubprogram>(MI->second); + if (SP && !SP->isDefinition()) return SP; } for (auto NextFD : FD->redecls()) { auto MI = SPCache.find(NextFD->getCanonicalDecl()); if (MI != SPCache.end()) { - llvm::DISubprogram SP(dyn_cast_or_null<llvm::MDNode>(MI->second)); - if (SP.isSubprogram() && !SP.isDefinition()) + auto *SP = dyn_cast_or_null<llvm::DISubprogram>(MI->second); + if (SP && !SP->isDefinition()) return SP; } } - return llvm::DISubprogram(); + return nullptr; } -// getOrCreateFunctionType - Construct DIType. If it is a c++ method, include +// getOrCreateFunctionType - Construct type. If it is a c++ method, include // implicit parameter "this". -llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D, - QualType FnType, - llvm::DIFile F) { +llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D, + QualType FnType, + llvm::DIFile *F) { if (!D || DebugKind <= CodeGenOptions::DebugLineTablesOnly) - // Create fake but valid subroutine type. Otherwise - // llvm::DISubprogram::Verify() would return false, and - // subprogram DIE will miss DW_AT_decl_file and - // DW_AT_decl_line fields. + // Create fake but valid subroutine type. Otherwise -verify would fail, and + // subprogram DIE will miss DW_AT_decl_file and DW_AT_decl_line fields. return DBuilder.createSubroutineType(F, DBuilder.getOrCreateTypeArray(None)); @@ -2515,11 +2496,10 @@ llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D, Elts.push_back(getOrCreateType(ResultTy, F)); // "self" pointer is always first argument. QualType SelfDeclTy = OMethod->getSelfDecl()->getType(); - llvm::DIType SelfTy = getOrCreateType(SelfDeclTy, F); - Elts.push_back(CreateSelfType(SelfDeclTy, SelfTy)); + Elts.push_back(CreateSelfType(SelfDeclTy, getOrCreateType(SelfDeclTy, F))); // "_cmd" pointer is always second argument. - llvm::DIType CmdTy = getOrCreateType(OMethod->getCmdDecl()->getType(), F); - Elts.push_back(DBuilder.createArtificialType(CmdTy)); + Elts.push_back(DBuilder.createArtificialType( + getOrCreateType(OMethod->getCmdDecl()->getType(), F))); // Get rest of the arguments. for (const auto *PI : OMethod->params()) Elts.push_back(getOrCreateType(PI->getType(), F)); @@ -2527,7 +2507,7 @@ llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D, if (OMethod->isVariadic()) Elts.push_back(DBuilder.createUnspecifiedParameter()); - llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts); + llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts); return DBuilder.createSubroutineType(F, EltTypeArray); } @@ -2541,14 +2521,13 @@ llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D, for (unsigned i = 0, e = FPT->getNumParams(); i != e; ++i) EltTys.push_back(getOrCreateType(FPT->getParamType(i), F)); EltTys.push_back(DBuilder.createUnspecifiedParameter()); - llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); + llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); return DBuilder.createSubroutineType(F, EltTypeArray); } - return llvm::DICompositeType(getOrCreateType(FnType, F)); + return cast<llvm::DISubroutineType>(getOrCreateType(FnType, F)); } -/// EmitFunctionStart - Constructs the debug code for entering a function. void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, SourceLocation ScopeLoc, QualType FnType, llvm::Function *Fn, CGBuilderTy &Builder) { @@ -2562,20 +2541,19 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, bool HasDecl = (D != nullptr); unsigned Flags = 0; - llvm::DIFile Unit = getOrCreateFile(Loc); - llvm::DIDescriptor FDContext(Unit); - llvm::DIArray TParamsArray; + llvm::DIFile *Unit = getOrCreateFile(Loc); + llvm::DIScope *FDContext = Unit; + llvm::DINodeArray TParamsArray; if (!HasDecl) { // Use llvm function name. LinkageName = Fn->getName(); } else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) { - // If there is a DISubprogram for this function available then use it. + // If there is a subprogram for this function available then use it. auto FI = SPCache.find(FD->getCanonicalDecl()); if (FI != SPCache.end()) { - llvm::DIDescriptor SP(dyn_cast_or_null<llvm::MDNode>(FI->second)); - if (SP.isSubprogram() && llvm::DISubprogram(SP).isDefinition()) { - llvm::MDNode *SPN = SP; - LexicalBlockStack.emplace_back(SPN); + auto *SP = dyn_cast_or_null<llvm::DISubprogram>(FI->second); + if (SP && SP->isDefinition()) { + LexicalBlockStack.emplace_back(SP); RegionMap[D].reset(SP); return; } @@ -2584,17 +2562,17 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, TParamsArray, Flags); } else if (const ObjCMethodDecl *OMD = dyn_cast<ObjCMethodDecl>(D)) { Name = getObjCMethodName(OMD); - Flags |= llvm::DIDescriptor::FlagPrototyped; + Flags |= llvm::DINode::FlagPrototyped; } else { // Use llvm function name. Name = Fn->getName(); - Flags |= llvm::DIDescriptor::FlagPrototyped; + Flags |= llvm::DINode::FlagPrototyped; } if (!Name.empty() && Name[0] == '\01') Name = Name.substr(1); if (!HasDecl || D->isImplicit()) { - Flags |= llvm::DIDescriptor::FlagArtificial; + Flags |= llvm::DINode::FlagArtificial; // Artificial functions without a location should not silently reuse CurLoc. if (Loc.isInvalid()) CurLoc = SourceLocation(); @@ -2607,11 +2585,11 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, // FunctionDecls. When/if we fix this we can have FDContext be TheCU/null for // all subprograms instead of the actual context since subprogram definitions // are emitted as CU level entities by the backend. - llvm::DISubprogram SP = DBuilder.createFunction( + llvm::DISubprogram *SP = DBuilder.createFunction( FDContext, Name, LinkageName, Unit, LineNo, getOrCreateFunctionType(D, FnType, Unit), Fn->hasInternalLinkage(), true /*definition*/, ScopeLine, Flags, CGM.getLangOpts().Optimize, Fn, - TParamsArray, getFunctionDeclaration(D)); + TParamsArray.get(), getFunctionDeclaration(D)); // We might get here with a VarDecl in the case we're generating // code for the initialization of globals. Do not record these decls // as they will overwrite the actual VarDecl Decl in the cache. @@ -2619,18 +2597,13 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, DeclCache[D->getCanonicalDecl()].reset(static_cast<llvm::Metadata *>(SP)); // Push the function onto the lexical block stack. - llvm::MDNode *SPN = SP; - LexicalBlockStack.emplace_back(SPN); + LexicalBlockStack.emplace_back(SP); if (HasDecl) RegionMap[D].reset(SP); } -/// EmitLocation - Emit metadata to indicate a change in line/column -/// information in the source file. If the location is invalid, the -/// previous location will be reused. -void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc, - bool ForceColumnInfo) { +void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc) { // Update our current location setLocation(Loc); @@ -2639,24 +2612,18 @@ void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc, llvm::MDNode *Scope = LexicalBlockStack.back(); Builder.SetCurrentDebugLocation(llvm::DebugLoc::get( - getLineNumber(CurLoc), getColumnNumber(CurLoc, ForceColumnInfo), Scope)); + getLineNumber(CurLoc), getColumnNumber(CurLoc), Scope)); } -/// CreateLexicalBlock - Creates a new lexical block node and pushes it on -/// the stack. void CGDebugInfo::CreateLexicalBlock(SourceLocation Loc) { llvm::MDNode *Back = nullptr; if (!LexicalBlockStack.empty()) Back = LexicalBlockStack.back().get(); - llvm::DIDescriptor D = DBuilder.createLexicalBlock( - llvm::DIDescriptor(Back), getOrCreateFile(CurLoc), getLineNumber(CurLoc), - getColumnNumber(CurLoc)); - llvm::MDNode *DN = D; - LexicalBlockStack.emplace_back(DN); + LexicalBlockStack.emplace_back(DBuilder.createLexicalBlock( + cast<llvm::DIScope>(Back), getOrCreateFile(CurLoc), getLineNumber(CurLoc), + getColumnNumber(CurLoc))); } -/// EmitLexicalBlockStart - Constructs the debug code for entering a declarative -/// region - beginning of a DW_TAG_lexical_block. void CGDebugInfo::EmitLexicalBlockStart(CGBuilderTy &Builder, SourceLocation Loc) { // Set our current location. @@ -2673,8 +2640,6 @@ void CGDebugInfo::EmitLexicalBlockStart(CGBuilderTy &Builder, CreateLexicalBlock(Loc); } -/// EmitLexicalBlockEnd - Constructs the debug code for exiting a declarative -/// region - end of a DW_TAG_lexical_block. void CGDebugInfo::EmitLexicalBlockEnd(CGBuilderTy &Builder, SourceLocation Loc) { assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!"); @@ -2688,7 +2653,6 @@ void CGDebugInfo::EmitLexicalBlockEnd(CGBuilderTy &Builder, LexicalBlockStack.pop_back(); } -/// EmitFunctionEnd - Constructs the debug code for exiting a function. void CGDebugInfo::EmitFunctionEnd(CGBuilderTy &Builder) { assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!"); unsigned RCount = FnBeginRegionCount.back(); @@ -2703,17 +2667,15 @@ void CGDebugInfo::EmitFunctionEnd(CGBuilderTy &Builder) { FnBeginRegionCount.pop_back(); } -// EmitTypeForVarWithBlocksAttr - Build up structure info for the byref. -// See BuildByRefType. -llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD, - uint64_t *XOffset) { +llvm::DIType *CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD, + uint64_t *XOffset) { SmallVector<llvm::Metadata *, 5> EltTys; QualType FType; uint64_t FieldSize, FieldOffset; unsigned FieldAlign; - llvm::DIFile Unit = getOrCreateFile(VD->getLocation()); + llvm::DIFile *Unit = getOrCreateFile(VD->getLocation()); QualType Type = VD->getType(); FieldOffset = 0; @@ -2760,7 +2722,7 @@ llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD, } FType = Type; - llvm::DIType FieldTy = getOrCreateType(FType, Unit); + llvm::DIType *FieldTy = getOrCreateType(FType, Unit); FieldSize = CGM.getContext().getTypeSize(FType); FieldAlign = CGM.getContext().toBits(Align); @@ -2770,16 +2732,15 @@ llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD, EltTys.push_back(FieldTy); FieldOffset += FieldSize; - llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys); + llvm::DINodeArray Elements = DBuilder.getOrCreateArray(EltTys); - unsigned Flags = llvm::DIDescriptor::FlagBlockByrefStruct; + unsigned Flags = llvm::DINode::FlagBlockByrefStruct; return DBuilder.createStructType(Unit, "", Unit, 0, FieldOffset, 0, Flags, - llvm::DIType(), Elements); + nullptr, Elements); } -/// EmitDeclare - Emit local variable declaration debug info. -void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag, +void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::Tag Tag, llvm::Value *Storage, unsigned ArgNo, CGBuilderTy &Builder) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); @@ -2788,10 +2749,10 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag, bool Unwritten = VD->isImplicit() || (isa<Decl>(VD->getDeclContext()) && cast<Decl>(VD->getDeclContext())->isImplicit()); - llvm::DIFile Unit; + llvm::DIFile *Unit = nullptr; if (!Unwritten) Unit = getOrCreateFile(VD->getLocation()); - llvm::DIType Ty; + llvm::DIType *Ty; uint64_t XOffset = 0; if (VD->hasAttr<BlocksAttr>()) Ty = EmitTypeForVarWithBlocksAttr(VD, &XOffset); @@ -2810,58 +2771,64 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag, Line = getLineNumber(VD->getLocation()); Column = getColumnNumber(VD->getLocation()); } + SmallVector<int64_t, 9> Expr; unsigned Flags = 0; if (VD->isImplicit()) - Flags |= llvm::DIDescriptor::FlagArtificial; + Flags |= llvm::DINode::FlagArtificial; // If this is the first argument and it is implicit then // give it an object pointer flag. // FIXME: There has to be a better way to do this, but for static // functions there won't be an implicit param at arg1 and // otherwise it is 'self' or 'this'. if (isa<ImplicitParamDecl>(VD) && ArgNo == 1) - Flags |= llvm::DIDescriptor::FlagObjectPointer; + Flags |= llvm::DINode::FlagObjectPointer; if (llvm::Argument *Arg = dyn_cast<llvm::Argument>(Storage)) if (Arg->getType()->isPointerTy() && !Arg->hasByValAttr() && !VD->getType()->isPointerType()) - Flags |= llvm::DIDescriptor::FlagIndirectVariable; + Expr.push_back(llvm::dwarf::DW_OP_deref); - llvm::MDNode *Scope = LexicalBlockStack.back(); + auto *Scope = cast<llvm::DIScope>(LexicalBlockStack.back()); StringRef Name = VD->getName(); if (!Name.empty()) { if (VD->hasAttr<BlocksAttr>()) { CharUnits offset = CharUnits::fromQuantity(32); - SmallVector<int64_t, 9> addr; - addr.push_back(llvm::dwarf::DW_OP_plus); + Expr.push_back(llvm::dwarf::DW_OP_plus); // offset of __forwarding field offset = CGM.getContext().toCharUnitsFromBits( CGM.getTarget().getPointerWidth(0)); - addr.push_back(offset.getQuantity()); - addr.push_back(llvm::dwarf::DW_OP_deref); - addr.push_back(llvm::dwarf::DW_OP_plus); + Expr.push_back(offset.getQuantity()); + Expr.push_back(llvm::dwarf::DW_OP_deref); + Expr.push_back(llvm::dwarf::DW_OP_plus); // offset of x field offset = CGM.getContext().toCharUnitsFromBits(XOffset); - addr.push_back(offset.getQuantity()); + Expr.push_back(offset.getQuantity()); // Create the descriptor for the variable. - llvm::DIVariable D = DBuilder.createLocalVariable( - Tag, llvm::DIDescriptor(Scope), VD->getName(), Unit, Line, Ty, ArgNo); + auto *D = DBuilder.createLocalVariable(Tag, Scope, VD->getName(), Unit, + Line, Ty, ArgNo); // Insert an llvm.dbg.declare into the current block. - llvm::Instruction *Call = - DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), - Builder.GetInsertBlock()); - Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope)); + DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), + llvm::DebugLoc::get(Line, Column, Scope), + Builder.GetInsertBlock()); return; } else if (isa<VariableArrayType>(VD->getType())) - Flags |= llvm::DIDescriptor::FlagIndirectVariable; + Expr.push_back(llvm::dwarf::DW_OP_deref); } else if (const RecordType *RT = dyn_cast<RecordType>(VD->getType())) { // If VD is an anonymous union then Storage represents value for // all union fields. const RecordDecl *RD = cast<RecordDecl>(RT->getDecl()); if (RD->isUnion() && RD->isAnonymousStructOrUnion()) { + // GDB has trouble finding local variables in anonymous unions, so we emit + // artifical local variables for each of the members. + // + // FIXME: Remove this code as soon as GDB supports this. + // The debug info verifier in LLVM operates based on the assumption that a + // variable has the same size as its storage and we had to disable the check + // for artificial variables. for (const auto *Field : RD->fields()) { - llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit); + llvm::DIType *FieldTy = getOrCreateType(Field->getType(), Unit); StringRef FieldName = Field->getName(); // Ignore unnamed fields. Do not ignore unnamed records. @@ -2869,28 +2836,28 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag, continue; // Use VarDecl's Tag, Scope and Line number. - llvm::DIVariable D = DBuilder.createLocalVariable( - Tag, llvm::DIDescriptor(Scope), FieldName, Unit, Line, FieldTy, - CGM.getLangOpts().Optimize, Flags, ArgNo); + auto *D = DBuilder.createLocalVariable( + Tag, Scope, FieldName, Unit, Line, FieldTy, + CGM.getLangOpts().Optimize, Flags | llvm::DINode::FlagArtificial, + ArgNo); // Insert an llvm.dbg.declare into the current block. - llvm::Instruction *Call = DBuilder.insertDeclare( - Storage, D, DBuilder.createExpression(), Builder.GetInsertBlock()); - Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope)); + DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), + llvm::DebugLoc::get(Line, Column, Scope), + Builder.GetInsertBlock()); } - return; } } // Create the descriptor for the variable. - llvm::DIVariable D = DBuilder.createLocalVariable( - Tag, llvm::DIDescriptor(Scope), Name, Unit, Line, Ty, - CGM.getLangOpts().Optimize, Flags, ArgNo); + auto *D = + DBuilder.createLocalVariable(Tag, Scope, Name, Unit, Line, Ty, + CGM.getLangOpts().Optimize, Flags, ArgNo); // Insert an llvm.dbg.declare into the current block. - llvm::Instruction *Call = DBuilder.insertDeclare( - Storage, D, DBuilder.createExpression(), Builder.GetInsertBlock()); - Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope)); + DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), + llvm::DebugLoc::get(Line, Column, Scope), + Builder.GetInsertBlock()); } void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD, @@ -2900,15 +2867,9 @@ void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD, EmitDeclare(VD, llvm::dwarf::DW_TAG_auto_variable, Storage, 0, Builder); } -/// Look up the completed type for a self pointer in the TypeCache and -/// create a copy of it with the ObjectPointer and Artificial flags -/// set. If the type is not cached, a new one is created. This should -/// never happen though, since creating a type for the implicit self -/// argument implies that we already parsed the interface definition -/// and the ivar declarations in the implementation. -llvm::DIType CGDebugInfo::CreateSelfType(const QualType &QualTy, - llvm::DIType Ty) { - llvm::DIType CachedTy = getTypeOrNull(QualTy); +llvm::DIType *CGDebugInfo::CreateSelfType(const QualType &QualTy, + llvm::DIType *Ty) { + llvm::DIType *CachedTy = getTypeOrNull(QualTy); if (CachedTy) Ty = CachedTy; return DBuilder.createObjectPointerType(Ty); @@ -2926,8 +2887,8 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable( bool isByRef = VD->hasAttr<BlocksAttr>(); uint64_t XOffset = 0; - llvm::DIFile Unit = getOrCreateFile(VD->getLocation()); - llvm::DIType Ty; + llvm::DIFile *Unit = getOrCreateFile(VD->getLocation()); + llvm::DIType *Ty; if (isByRef) Ty = EmitTypeForVarWithBlocksAttr(VD, &XOffset); else @@ -2968,23 +2929,21 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable( } // Create the descriptor for the variable. - llvm::DIVariable D = - DBuilder.createLocalVariable(llvm::dwarf::DW_TAG_auto_variable, - llvm::DIDescriptor(LexicalBlockStack.back()), - VD->getName(), Unit, Line, Ty); + auto *D = DBuilder.createLocalVariable( + llvm::dwarf::DW_TAG_auto_variable, + cast<llvm::DILocalScope>(LexicalBlockStack.back()), VD->getName(), Unit, + Line, Ty); // Insert an llvm.dbg.declare into the current block. - llvm::Instruction *Call = InsertPoint ? - DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), - InsertPoint) - : DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), - Builder.GetInsertBlock()); - Call->setDebugLoc( - llvm::DebugLoc::get(Line, Column, LexicalBlockStack.back())); + auto DL = llvm::DebugLoc::get(Line, Column, LexicalBlockStack.back()); + if (InsertPoint) + DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), DL, + InsertPoint); + else + DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), DL, + Builder.GetInsertBlock()); } -/// EmitDeclareOfArgVariable - Emit call to llvm.dbg.declare for an argument -/// variable declaration. void CGDebugInfo::EmitDeclareOfArgVariable(const VarDecl *VD, llvm::Value *AI, unsigned ArgNo, CGBuilderTy &Builder) { @@ -3013,7 +2972,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, // Collect some general information about the block's location. SourceLocation loc = blockDecl->getCaretLocation(); - llvm::DIFile tunit = getOrCreateFile(loc); + llvm::DIFile *tunit = getOrCreateFile(loc); unsigned line = getLineNumber(loc); unsigned column = getColumnNumber(loc); @@ -3096,7 +3055,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, const VarDecl *variable = capture->getVariable(); StringRef name = variable->getName(); - llvm::DIType fieldType; + llvm::DIType *fieldType; if (capture->isByRef()) { TypeInfo PtrInfo = C.getTypeInfo(C.VoidPtrTy); @@ -3118,70 +3077,62 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, llvm::raw_svector_ostream(typeName) << "__block_literal_" << CGM.getUniqueBlockCount(); - llvm::DIArray fieldsArray = DBuilder.getOrCreateArray(fields); + llvm::DINodeArray fieldsArray = DBuilder.getOrCreateArray(fields); - llvm::DIType type = - DBuilder.createStructType(tunit, typeName.str(), tunit, line, - CGM.getContext().toBits(block.BlockSize), - CGM.getContext().toBits(block.BlockAlign), 0, - llvm::DIType(), fieldsArray); + llvm::DIType *type = DBuilder.createStructType( + tunit, typeName.str(), tunit, line, + CGM.getContext().toBits(block.BlockSize), + CGM.getContext().toBits(block.BlockAlign), 0, nullptr, fieldsArray); type = DBuilder.createPointerType(type, CGM.PointerWidthInBits); // Get overall information about the block. - unsigned flags = llvm::DIDescriptor::FlagArtificial; - llvm::MDNode *scope = LexicalBlockStack.back(); + unsigned flags = llvm::DINode::FlagArtificial; + auto *scope = cast<llvm::DILocalScope>(LexicalBlockStack.back()); // Create the descriptor for the parameter. - llvm::DIVariable debugVar = DBuilder.createLocalVariable( - llvm::dwarf::DW_TAG_arg_variable, llvm::DIDescriptor(scope), - Arg->getName(), tunit, line, type, CGM.getLangOpts().Optimize, flags, - ArgNo); + auto *debugVar = DBuilder.createLocalVariable( + llvm::dwarf::DW_TAG_arg_variable, scope, Arg->getName(), tunit, line, + type, CGM.getLangOpts().Optimize, flags, ArgNo); if (LocalAddr) { // Insert an llvm.dbg.value into the current block. - llvm::Instruction *DbgVal = DBuilder.insertDbgValueIntrinsic( + DBuilder.insertDbgValueIntrinsic( LocalAddr, 0, debugVar, DBuilder.createExpression(), - Builder.GetInsertBlock()); - DbgVal->setDebugLoc(llvm::DebugLoc::get(line, column, scope)); + llvm::DebugLoc::get(line, column, scope), Builder.GetInsertBlock()); } // Insert an llvm.dbg.declare into the current block. - llvm::Instruction *DbgDecl = DBuilder.insertDeclare( - Arg, debugVar, DBuilder.createExpression(), Builder.GetInsertBlock()); - DbgDecl->setDebugLoc(llvm::DebugLoc::get(line, column, scope)); + DBuilder.insertDeclare(Arg, debugVar, DBuilder.createExpression(), + llvm::DebugLoc::get(line, column, scope), + Builder.GetInsertBlock()); } -/// If D is an out-of-class definition of a static data member of a class, find -/// its corresponding in-class declaration. -llvm::DIDerivedType +llvm::DIDerivedType * CGDebugInfo::getOrCreateStaticDataMemberDeclarationOrNull(const VarDecl *D) { if (!D->isStaticDataMember()) - return llvm::DIDerivedType(); + return nullptr; + auto MI = StaticDataMemberCache.find(D->getCanonicalDecl()); if (MI != StaticDataMemberCache.end()) { assert(MI->second && "Static data member declaration should still exist"); - return llvm::DIDerivedType(cast<llvm::MDNode>(MI->second)); + return cast<llvm::DIDerivedType>(MI->second); } // If the member wasn't found in the cache, lazily construct and add it to the // type (used when a limited form of the type is emitted). auto DC = D->getDeclContext(); - llvm::DICompositeType Ctxt(getContextDescriptor(cast<Decl>(DC))); + auto *Ctxt = + cast<llvm::DICompositeType>(getContextDescriptor(cast<Decl>(DC))); return CreateRecordStaticField(D, Ctxt, cast<RecordDecl>(DC)); } -/// Recursively collect all of the member fields of a global anonymous decl and -/// create static variables for them. The first time this is called it needs -/// to be on a union and then from there we can have additional unnamed fields. -llvm::DIGlobalVariable -CGDebugInfo::CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile Unit, - unsigned LineNo, StringRef LinkageName, - llvm::GlobalVariable *Var, - llvm::DIDescriptor DContext) { - llvm::DIGlobalVariable GV; +llvm::DIGlobalVariable *CGDebugInfo::CollectAnonRecordDecls( + const RecordDecl *RD, llvm::DIFile *Unit, unsigned LineNo, + StringRef LinkageName, llvm::GlobalVariable *Var, llvm::DIScope *DContext) { + llvm::DIGlobalVariable *GV = nullptr; for (const auto *Field : RD->fields()) { - llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit); + llvm::DIType *FieldTy = getOrCreateType(Field->getType(), Unit); StringRef FieldName = Field->getName(); // Ignore unnamed fields, but recurse into anonymous records. @@ -3193,20 +3144,19 @@ CGDebugInfo::CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile Unit, continue; } // Use VarDecl's Tag, Scope and Line number. - GV = DBuilder.createGlobalVariable( - DContext, FieldName, LinkageName, Unit, LineNo, FieldTy, - Var->hasInternalLinkage(), Var, llvm::DIDerivedType()); + GV = DBuilder.createGlobalVariable(DContext, FieldName, LinkageName, Unit, + LineNo, FieldTy, + Var->hasInternalLinkage(), Var, nullptr); } return GV; } -/// EmitGlobalVariable - Emit information about a global variable. void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var, const VarDecl *D) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); // Create global variable debug descriptor. - llvm::DIFile Unit; - llvm::DIDescriptor DContext; + llvm::DIFile *Unit = nullptr; + llvm::DIScope *DContext = nullptr; unsigned LineNo; StringRef DeclName, LinkageName; QualType T; @@ -3214,7 +3164,7 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var, // Attempt to store one global variable for the declaration - even if we // emit a lot of fields. - llvm::DIGlobalVariable GV; + llvm::DIGlobalVariable *GV = nullptr; // If this is an anonymous union then we'll want to emit a global // variable for each member of the anonymous union so that it's possible @@ -3233,21 +3183,22 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var, DeclCache[D->getCanonicalDecl()].reset(static_cast<llvm::Metadata *>(GV)); } -/// EmitGlobalVariable - Emit global variable's debug info. void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, llvm::Constant *Init) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); // Create the descriptor for the variable. - llvm::DIFile Unit = getOrCreateFile(VD->getLocation()); + llvm::DIFile *Unit = getOrCreateFile(VD->getLocation()); StringRef Name = VD->getName(); - llvm::DIType Ty = getOrCreateType(VD->getType(), Unit); + llvm::DIType *Ty = getOrCreateType(VD->getType(), Unit); if (const EnumConstantDecl *ECD = dyn_cast<EnumConstantDecl>(VD)) { const EnumDecl *ED = cast<EnumDecl>(ECD->getDeclContext()); assert(isa<EnumType>(ED->getTypeForDecl()) && "Enum without EnumType?"); Ty = getOrCreateType(QualType(ED->getTypeForDecl(), 0), Unit); } - // Do not use DIGlobalVariable for enums. - if (Ty.getTag() == llvm::dwarf::DW_TAG_enumeration_type) + // Do not use global variables for enums. + // + // FIXME: why not? + if (Ty->getTag() == llvm::dwarf::DW_TAG_enumeration_type) return; // Do not emit separate definitions for function local const/statics. if (isa<FunctionDecl>(VD->getDeclContext())) @@ -3263,7 +3214,7 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, return; } - llvm::DIDescriptor DContext = + llvm::DIScope *DContext = getContextDescriptor(dyn_cast<Decl>(VD->getDeclContext())); auto &GV = DeclCache[VD]; @@ -3274,9 +3225,9 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, true, Init, getOrCreateStaticDataMemberDeclarationOrNull(VarD))); } -llvm::DIScope CGDebugInfo::getCurrentContextDescriptor(const Decl *D) { +llvm::DIScope *CGDebugInfo::getCurrentContextDescriptor(const Decl *D) { if (!LexicalBlockStack.empty()) - return llvm::DIScope(LexicalBlockStack.back()); + return LexicalBlockStack.back(); return getContextDescriptor(D); } @@ -3297,21 +3248,30 @@ void CGDebugInfo::EmitUsingDecl(const UsingDecl &UD) { // Emitting one decl is sufficient - debuggers can detect that this is an // overloaded name & provide lookup for all the overloads. const UsingShadowDecl &USD = **UD.shadow_begin(); - if (llvm::DIDescriptor Target = + if (llvm::DINode *Target = getDeclarationOrDefinition(USD.getUnderlyingDecl())) DBuilder.createImportedDeclaration( getCurrentContextDescriptor(cast<Decl>(USD.getDeclContext())), Target, getLineNumber(USD.getLocation())); } -llvm::DIImportedEntity +void CGDebugInfo::EmitImportDecl(const ImportDecl &ID) { + auto *Reader = CGM.getContext().getExternalSource(); + auto Info = Reader->getSourceDescriptor(*ID.getImportedModule()); + DBuilder.createImportedDeclaration( + getCurrentContextDescriptor(cast<Decl>(ID.getDeclContext())), + getOrCreateModuleRef(Info), + getLineNumber(ID.getLocation())); +} + +llvm::DIImportedEntity * CGDebugInfo::EmitNamespaceAlias(const NamespaceAliasDecl &NA) { if (CGM.getCodeGenOpts().getDebugInfo() < CodeGenOptions::LimitedDebugInfo) - return llvm::DIImportedEntity(nullptr); + return nullptr; auto &VH = NamespaceAliasCache[&NA]; if (VH) - return llvm::DIImportedEntity(cast<llvm::MDNode>(VH)); - llvm::DIImportedEntity R(nullptr); + return cast<llvm::DIImportedEntity>(VH); + llvm::DIImportedEntity *R; if (const NamespaceAliasDecl *Underlying = dyn_cast<NamespaceAliasDecl>(NA.getAliasedNamespace())) // This could cache & dedup here rather than relying on metadata deduping. @@ -3328,21 +3288,19 @@ CGDebugInfo::EmitNamespaceAlias(const NamespaceAliasDecl &NA) { return R; } -/// getOrCreateNamesSpace - Return namespace descriptor for the given -/// namespace decl. -llvm::DINameSpace +llvm::DINamespace * CGDebugInfo::getOrCreateNameSpace(const NamespaceDecl *NSDecl) { NSDecl = NSDecl->getCanonicalDecl(); auto I = NameSpaceCache.find(NSDecl); if (I != NameSpaceCache.end()) - return llvm::DINameSpace(cast<llvm::MDNode>(I->second)); + return cast<llvm::DINamespace>(I->second); unsigned LineNo = getLineNumber(NSDecl->getLocation()); - llvm::DIFile FileD = getOrCreateFile(NSDecl->getLocation()); - llvm::DIDescriptor Context = - getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext())); - llvm::DINameSpace NS = - DBuilder.createNameSpace(Context, NSDecl->getName(), FileD, LineNo); + llvm::DIFile *FileD = getOrCreateFile(NSDecl->getLocation()); + llvm::DIScope *Context = + getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext())); + llvm::DINamespace *NS = + DBuilder.createNameSpace(Context, NSDecl->getName(), FileD, LineNo); NameSpaceCache[NSDecl].reset(NS); return NS; } @@ -3352,28 +3310,28 @@ void CGDebugInfo::finalize() { // element and the size(), so don't cache/reference them. for (size_t i = 0; i != ObjCInterfaceCache.size(); ++i) { ObjCInterfaceCacheEntry E = ObjCInterfaceCache[i]; - E.Decl.replaceAllUsesWith(CGM.getLLVMContext(), - E.Type->getDecl()->getDefinition() - ? CreateTypeDefinition(E.Type, E.Unit) - : E.Decl); + llvm::DIType *Ty = E.Type->getDecl()->getDefinition() + ? CreateTypeDefinition(E.Type, E.Unit) + : E.Decl; + DBuilder.replaceTemporary(llvm::TempDIType(E.Decl), Ty); } for (auto p : ReplaceMap) { assert(p.second); - llvm::DIType Ty(cast<llvm::MDNode>(p.second)); - assert(Ty.isForwardDecl()); + auto *Ty = cast<llvm::DIType>(p.second); + assert(Ty->isForwardDecl()); auto it = TypeCache.find(p.first); assert(it != TypeCache.end()); assert(it->second); - llvm::DIType RepTy(cast<llvm::MDNode>(it->second)); - Ty.replaceAllUsesWith(CGM.getLLVMContext(), RepTy); + DBuilder.replaceTemporary(llvm::TempDIType(Ty), + cast<llvm::DIType>(it->second)); } for (const auto &p : FwdDeclReplaceMap) { assert(p.second); - llvm::DIDescriptor FwdDecl(cast<llvm::MDNode>(p.second)); + llvm::TempMDNode FwdDecl(cast<llvm::MDNode>(p.second)); llvm::Metadata *Repl; auto it = DeclCache.find(p.first); @@ -3385,15 +3343,14 @@ void CGDebugInfo::finalize() { else Repl = it->second; - FwdDecl.replaceAllUsesWith(CGM.getLLVMContext(), - llvm::DIDescriptor(cast<llvm::MDNode>(Repl))); + DBuilder.replaceTemporary(std::move(FwdDecl), cast<llvm::MDNode>(Repl)); } // We keep our own list of retained types, because we need to look // up the final type in the type cache. for (std::vector<void *>::const_iterator RI = RetainedTypes.begin(), RE = RetainedTypes.end(); RI != RE; ++RI) - DBuilder.retainType(llvm::DIType(cast<llvm::MDNode>(TypeCache[*RI]))); + DBuilder.retainType(cast<llvm::DIType>(TypeCache[*RI])); DBuilder.finalize(); } @@ -3401,7 +3358,8 @@ void CGDebugInfo::finalize() { void CGDebugInfo::EmitExplicitCastType(QualType Ty) { if (CGM.getCodeGenOpts().getDebugInfo() < CodeGenOptions::LimitedDebugInfo) return; - llvm::DIType DieTy = getOrCreateType(Ty, getOrCreateMainFile()); - // Don't ignore in case of explicit cast where it is referenced indirectly. - DBuilder.retainType(DieTy); + + if (auto *DieTy = getOrCreateType(Ty, getOrCreateMainFile())) + // Don't ignore in case of explicit cast where it is referenced indirectly. + DBuilder.retainType(DieTy); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h index 0be032c..82680a8 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDebugInfo.h @@ -26,88 +26,92 @@ #include "llvm/Support/Allocator.h" namespace llvm { - class MDNode; +class MDNode; } namespace clang { - class CXXMethodDecl; - class VarDecl; - class ObjCInterfaceDecl; - class ObjCIvarDecl; - class ClassTemplateSpecializationDecl; - class GlobalDecl; - class UsingDecl; +class CXXMethodDecl; +class VarDecl; +class ObjCInterfaceDecl; +class ObjCIvarDecl; +class ClassTemplateSpecializationDecl; +class GlobalDecl; +class UsingDecl; namespace CodeGen { - class CodeGenModule; - class CodeGenFunction; - class CGBlockInfo; +class CodeGenModule; +class CodeGenFunction; +class CGBlockInfo; -/// CGDebugInfo - This class gathers all debug information during compilation -/// and is responsible for emitting to llvm globals or pass directly to -/// the backend. +/// This class gathers all debug information during compilation and is +/// responsible for emitting to llvm globals or pass directly to the +/// backend. class CGDebugInfo { - friend class ArtificialLocation; + friend class ApplyDebugLocation; friend class SaveAndRestoreLocation; CodeGenModule &CGM; const CodeGenOptions::DebugInfoKind DebugKind; llvm::DIBuilder DBuilder; - llvm::DICompileUnit TheCU; + llvm::DICompileUnit *TheCU = nullptr; SourceLocation CurLoc; - llvm::DIType VTablePtrType; - llvm::DIType ClassTy; - llvm::DICompositeType ObjTy; - llvm::DIType SelTy; - llvm::DIType OCLImage1dDITy, OCLImage1dArrayDITy, OCLImage1dBufferDITy; - llvm::DIType OCLImage2dDITy, OCLImage2dArrayDITy; - llvm::DIType OCLImage3dDITy; - llvm::DIType OCLEventDITy; - llvm::DIType BlockLiteralGeneric; - - /// TypeCache - Cache of previously constructed Types. + llvm::DIType *VTablePtrType = nullptr; + llvm::DIType *ClassTy = nullptr; + llvm::DICompositeType *ObjTy = nullptr; + llvm::DIType *SelTy = nullptr; + llvm::DIType *OCLImage1dDITy = nullptr; + llvm::DIType *OCLImage1dArrayDITy = nullptr; + llvm::DIType *OCLImage1dBufferDITy = nullptr; + llvm::DIType *OCLImage2dDITy = nullptr; + llvm::DIType *OCLImage2dArrayDITy = nullptr; + llvm::DIType *OCLImage3dDITy = nullptr; + llvm::DIType *OCLEventDITy = nullptr; + + /// Cache of previously constructed Types. llvm::DenseMap<const void *, llvm::TrackingMDRef> TypeCache; struct ObjCInterfaceCacheEntry { const ObjCInterfaceType *Type; - llvm::DIType Decl; - llvm::DIFile Unit; - ObjCInterfaceCacheEntry(const ObjCInterfaceType *Type, llvm::DIType Decl, - llvm::DIFile Unit) + llvm::DIType *Decl; + llvm::DIFile *Unit; + ObjCInterfaceCacheEntry(const ObjCInterfaceType *Type, llvm::DIType *Decl, + llvm::DIFile *Unit) : Type(Type), Decl(Decl), Unit(Unit) {} }; - /// ObjCInterfaceCache - Cache of previously constructed interfaces - /// which may change. + /// Cache of previously constructed interfaces which may change. llvm::SmallVector<ObjCInterfaceCacheEntry, 32> ObjCInterfaceCache; - /// RetainedTypes - list of interfaces we want to keep even if orphaned. + /// Cache of references to AST files such as PCHs or modules. + llvm::DenseMap<uint64_t, llvm::DIModule *> ModuleRefCache; + + /// List of interfaces we want to keep even if orphaned. std::vector<void *> RetainedTypes; - /// ReplaceMap - Cache of forward declared types to RAUW at the end of + /// Cache of forward declared types to RAUW at the end of /// compilation. std::vector<std::pair<const TagType *, llvm::TrackingMDRef>> ReplaceMap; - /// \brief Cache of replaceable forward declarartions (functions and + /// Cache of replaceable forward declarartions (functions and /// variables) to RAUW at the end of compilation. std::vector<std::pair<const DeclaratorDecl *, llvm::TrackingMDRef>> FwdDeclReplaceMap; - // LexicalBlockStack - Keep track of our current nested lexical block. - std::vector<llvm::TrackingMDNodeRef> LexicalBlockStack; + /// Keep track of our current nested lexical block. + std::vector<llvm::TypedTrackingMDRef<llvm::DIScope>> LexicalBlockStack; llvm::DenseMap<const Decl *, llvm::TrackingMDRef> RegionMap; - // FnBeginRegionCount - Keep track of LexicalBlockStack counter at the - // beginning of a function. This is used to pop unbalanced regions at - // the end of a function. + /// Keep track of LexicalBlockStack counter at the beginning of a + /// function. This is used to pop unbalanced regions at the end of a + /// function. std::vector<unsigned> FnBeginRegionCount; - /// DebugInfoNames - This is a storage for names that are - /// constructed on demand. For example, C++ destructors, C++ operators etc.. + /// This is a storage for names that are constructed on demand. For + /// example, C++ destructors, C++ operators etc.. llvm::BumpPtrAllocator DebugInfoNames; StringRef CWDName; llvm::DenseMap<const char *, llvm::TrackingMDRef> DIFileCache; llvm::DenseMap<const FunctionDecl *, llvm::TrackingMDRef> SPCache; - /// \brief Cache declarations relevant to DW_TAG_imported_declarations (C++ + /// Cache declarations relevant to DW_TAG_imported_declarations (C++ /// using declarations) that aren't covered by other more specific caches. llvm::DenseMap<const Decl *, llvm::TrackingMDRef> DeclCache; llvm::DenseMap<const NamespaceDecl *, llvm::TrackingMDRef> NameSpaceCache; @@ -116,100 +120,139 @@ class CGDebugInfo { llvm::DenseMap<const Decl *, llvm::TrackingMDRef> StaticDataMemberCache; /// Helper functions for getOrCreateType. + /// @{ + /// Currently the checksum of an interface includes the number of + /// ivars and property accessors. unsigned Checksum(const ObjCInterfaceDecl *InterfaceDecl); - llvm::DIType CreateType(const BuiltinType *Ty); - llvm::DIType CreateType(const ComplexType *Ty); - llvm::DIType CreateQualifiedType(QualType Ty, llvm::DIFile Fg); - llvm::DIType CreateType(const TypedefType *Ty, llvm::DIFile Fg); - llvm::DIType CreateType(const TemplateSpecializationType *Ty, llvm::DIFile Fg); - llvm::DIType CreateType(const ObjCObjectPointerType *Ty, - llvm::DIFile F); - llvm::DIType CreateType(const PointerType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const BlockPointerType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const FunctionType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const RecordType *Tyg); - llvm::DIType CreateTypeDefinition(const RecordType *Ty); - llvm::DICompositeType CreateLimitedType(const RecordType *Ty); - void CollectContainingType(const CXXRecordDecl *RD, llvm::DICompositeType CT); - llvm::DIType CreateType(const ObjCInterfaceType *Ty, llvm::DIFile F); - llvm::DIType CreateTypeDefinition(const ObjCInterfaceType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const ObjCObjectType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const VectorType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const ArrayType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const LValueReferenceType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const RValueReferenceType *Ty, llvm::DIFile Unit); - llvm::DIType CreateType(const MemberPointerType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const AtomicType *Ty, llvm::DIFile F); - llvm::DIType CreateEnumType(const EnumType *Ty); - llvm::DIType CreateTypeDefinition(const EnumType *Ty); - llvm::DIType CreateSelfType(const QualType &QualTy, llvm::DIType Ty); - llvm::DIType getTypeOrNull(const QualType); - llvm::DICompositeType getOrCreateMethodType(const CXXMethodDecl *Method, - llvm::DIFile F); - llvm::DICompositeType getOrCreateInstanceMethodType( - QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile Unit); - llvm::DICompositeType getOrCreateFunctionType(const Decl *D, QualType FnType, - llvm::DIFile F); - llvm::DIType getOrCreateVTablePtrType(llvm::DIFile F); - llvm::DINameSpace getOrCreateNameSpace(const NamespaceDecl *N); - llvm::DIType getOrCreateTypeDeclaration(QualType PointeeTy, llvm::DIFile F); - llvm::DIType CreatePointerLikeType(llvm::dwarf::Tag Tag, - const Type *Ty, QualType PointeeTy, - llvm::DIFile F); + llvm::DIType *CreateType(const BuiltinType *Ty); + llvm::DIType *CreateType(const ComplexType *Ty); + llvm::DIType *CreateQualifiedType(QualType Ty, llvm::DIFile *Fg); + llvm::DIType *CreateType(const TypedefType *Ty, llvm::DIFile *Fg); + llvm::DIType *CreateType(const TemplateSpecializationType *Ty, + llvm::DIFile *Fg); + llvm::DIType *CreateType(const ObjCObjectPointerType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const PointerType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const BlockPointerType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const FunctionType *Ty, llvm::DIFile *F); + /// Get structure or union type. + llvm::DIType *CreateType(const RecordType *Tyg); + llvm::DIType *CreateTypeDefinition(const RecordType *Ty); + llvm::DICompositeType *CreateLimitedType(const RecordType *Ty); + void CollectContainingType(const CXXRecordDecl *RD, + llvm::DICompositeType *CT); + /// Get Objective-C interface type. + llvm::DIType *CreateType(const ObjCInterfaceType *Ty, llvm::DIFile *F); + llvm::DIType *CreateTypeDefinition(const ObjCInterfaceType *Ty, + llvm::DIFile *F); + /// Get Objective-C object type. + llvm::DIType *CreateType(const ObjCObjectType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const VectorType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const ArrayType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const LValueReferenceType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const RValueReferenceType *Ty, llvm::DIFile *Unit); + llvm::DIType *CreateType(const MemberPointerType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const AtomicType *Ty, llvm::DIFile *F); + /// Get enumeration type. + llvm::DIType *CreateEnumType(const EnumType *Ty); + llvm::DIType *CreateTypeDefinition(const EnumType *Ty); + /// Look up the completed type for a self pointer in the TypeCache and + /// create a copy of it with the ObjectPointer and Artificial flags + /// set. If the type is not cached, a new one is created. This should + /// never happen though, since creating a type for the implicit self + /// argument implies that we already parsed the interface definition + /// and the ivar declarations in the implementation. + llvm::DIType *CreateSelfType(const QualType &QualTy, llvm::DIType *Ty); + /// @} + + /// Get the type from the cache or return null type if it doesn't + /// exist. + llvm::DIType *getTypeOrNull(const QualType); + /// Return the debug type for a C++ method. + /// \arg CXXMethodDecl is of FunctionType. This function type is + /// not updated to include implicit \c this pointer. Use this routine + /// to get a method type which includes \c this pointer. + llvm::DISubroutineType *getOrCreateMethodType(const CXXMethodDecl *Method, + llvm::DIFile *F); + llvm::DISubroutineType * + getOrCreateInstanceMethodType(QualType ThisPtr, const FunctionProtoType *Func, + llvm::DIFile *Unit); + llvm::DISubroutineType * + getOrCreateFunctionType(const Decl *D, QualType FnType, llvm::DIFile *F); + /// \return debug info descriptor for vtable. + llvm::DIType *getOrCreateVTablePtrType(llvm::DIFile *F); + /// \return namespace descriptor for the given namespace decl. + llvm::DINamespace *getOrCreateNameSpace(const NamespaceDecl *N); + llvm::DIType *getOrCreateTypeDeclaration(QualType PointeeTy, llvm::DIFile *F); + llvm::DIType *CreatePointerLikeType(llvm::dwarf::Tag Tag, const Type *Ty, + QualType PointeeTy, llvm::DIFile *F); llvm::Value *getCachedInterfaceTypeOrNull(const QualType Ty); - llvm::DIType getOrCreateStructPtrType(StringRef Name, llvm::DIType &Cache); - - llvm::DISubprogram CreateCXXMemberFunction(const CXXMethodDecl *Method, - llvm::DIFile F, - llvm::DIType RecordTy); - - void CollectCXXMemberFunctions(const CXXRecordDecl *Decl, llvm::DIFile F, + llvm::DIType *getOrCreateStructPtrType(StringRef Name, llvm::DIType *&Cache); + + /// A helper function to create a subprogram for a single member + /// function GlobalDecl. + llvm::DISubprogram *CreateCXXMemberFunction(const CXXMethodDecl *Method, + llvm::DIFile *F, + llvm::DIType *RecordTy); + + /// A helper function to collect debug info for C++ member + /// functions. This is used while creating debug info entry for a + /// Record. + void CollectCXXMemberFunctions(const CXXRecordDecl *Decl, llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &E, - llvm::DIType T); + llvm::DIType *T); - void CollectCXXBases(const CXXRecordDecl *Decl, llvm::DIFile F, + /// A helper function to collect debug info for C++ base + /// classes. This is used while creating debug info entry for a + /// Record. + void CollectCXXBases(const CXXRecordDecl *Decl, llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &EltTys, - llvm::DIType RecordTy); - - llvm::DIArray - CollectTemplateParams(const TemplateParameterList *TPList, - ArrayRef<TemplateArgument> TAList, - llvm::DIFile Unit); - llvm::DIArray - CollectFunctionTemplateParams(const FunctionDecl *FD, llvm::DIFile Unit); - llvm::DIArray + llvm::DIType *RecordTy); + + /// A helper function to collect template parameters. + llvm::DINodeArray CollectTemplateParams(const TemplateParameterList *TPList, + ArrayRef<TemplateArgument> TAList, + llvm::DIFile *Unit); + /// A helper function to collect debug info for function template + /// parameters. + llvm::DINodeArray CollectFunctionTemplateParams(const FunctionDecl *FD, + llvm::DIFile *Unit); + + /// A helper function to collect debug info for template + /// parameters. + llvm::DINodeArray CollectCXXTemplateParams(const ClassTemplateSpecializationDecl *TS, - llvm::DIFile F); + llvm::DIFile *F); - llvm::DIType createFieldType(StringRef name, QualType type, - uint64_t sizeInBitsOverride, SourceLocation loc, - AccessSpecifier AS, - uint64_t offsetInBits, - llvm::DIFile tunit, - llvm::DIScope scope, - const RecordDecl* RD = nullptr); + llvm::DIType *createFieldType(StringRef name, QualType type, + uint64_t sizeInBitsOverride, SourceLocation loc, + AccessSpecifier AS, uint64_t offsetInBits, + llvm::DIFile *tunit, llvm::DIScope *scope, + const RecordDecl *RD = nullptr); - // Helpers for collecting fields of a record. + /// Helpers for collecting fields of a record. + /// @{ void CollectRecordLambdaFields(const CXXRecordDecl *CXXDecl, SmallVectorImpl<llvm::Metadata *> &E, - llvm::DIType RecordTy); - llvm::DIDerivedType CreateRecordStaticField(const VarDecl *Var, - llvm::DIType RecordTy, - const RecordDecl* RD); + llvm::DIType *RecordTy); + llvm::DIDerivedType *CreateRecordStaticField(const VarDecl *Var, + llvm::DIType *RecordTy, + const RecordDecl *RD); void CollectRecordNormalField(const FieldDecl *Field, uint64_t OffsetInBits, - llvm::DIFile F, + llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &E, - llvm::DIType RecordTy, const RecordDecl *RD); - void CollectRecordFields(const RecordDecl *Decl, llvm::DIFile F, + llvm::DIType *RecordTy, const RecordDecl *RD); + void CollectRecordFields(const RecordDecl *Decl, llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &E, - llvm::DICompositeType RecordTy); + llvm::DICompositeType *RecordTy); - void CollectVTableInfo(const CXXRecordDecl *Decl, llvm::DIFile F, + /// If the C++ class has vtable info then insert appropriate debug + /// info entry in EltTys vector. + void CollectVTableInfo(const CXXRecordDecl *Decl, llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &EltTys); + /// @} - // CreateLexicalBlock - Create a new lexical block node and push it on - // the stack. + /// Create a new lexical block node and push it on the stack. void CreateLexicalBlock(SourceLocation Loc); public: @@ -218,90 +261,85 @@ public: void finalize(); - /// setLocation - Update the current source location. If \arg loc is - /// invalid it is ignored. + /// Update the current source location. If \arg loc is invalid it is + /// ignored. void setLocation(SourceLocation Loc); - /// getLocation - Return the current source location. - SourceLocation getLocation() const { return CurLoc; } - - /// EmitLocation - Emit metadata to indicate a change in line/column - /// information in the source file. - /// \param ForceColumnInfo Assume DebugColumnInfo option is true. - void EmitLocation(CGBuilderTy &Builder, SourceLocation Loc, - bool ForceColumnInfo = false); + /// Emit metadata to indicate a change in line/column information in + /// the source file. If the location is invalid, the previous + /// location will be reused. + void EmitLocation(CGBuilderTy &Builder, SourceLocation Loc); - /// EmitFunctionStart - Emit a call to llvm.dbg.function.start to indicate + /// Emit a call to llvm.dbg.function.start to indicate /// start of a new function. /// \param Loc The location of the function header. /// \param ScopeLoc The location of the function body. - void EmitFunctionStart(GlobalDecl GD, - SourceLocation Loc, SourceLocation ScopeLoc, - QualType FnType, llvm::Function *Fn, - CGBuilderTy &Builder); + void EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, + SourceLocation ScopeLoc, QualType FnType, + llvm::Function *Fn, CGBuilderTy &Builder); - /// EmitFunctionEnd - Constructs the debug code for exiting a function. + /// Constructs the debug code for exiting a function. void EmitFunctionEnd(CGBuilderTy &Builder); - /// EmitLexicalBlockStart - Emit metadata to indicate the beginning of a - /// new lexical block and push the block onto the stack. + /// Emit metadata to indicate the beginning of a new lexical block + /// and push the block onto the stack. void EmitLexicalBlockStart(CGBuilderTy &Builder, SourceLocation Loc); - /// EmitLexicalBlockEnd - Emit metadata to indicate the end of a new lexical - /// block and pop the current block. + /// Emit metadata to indicate the end of a new lexical block and pop + /// the current block. void EmitLexicalBlockEnd(CGBuilderTy &Builder, SourceLocation Loc); - /// EmitDeclareOfAutoVariable - Emit call to llvm.dbg.declare for an automatic - /// variable declaration. + /// Emit call to \c llvm.dbg.declare for an automatic variable + /// declaration. void EmitDeclareOfAutoVariable(const VarDecl *Decl, llvm::Value *AI, CGBuilderTy &Builder); - /// EmitDeclareOfBlockDeclRefVariable - Emit call to llvm.dbg.declare for an - /// imported variable declaration in a block. + /// Emit call to \c llvm.dbg.declare for an imported variable + /// declaration in a block. void EmitDeclareOfBlockDeclRefVariable(const VarDecl *variable, llvm::Value *storage, CGBuilderTy &Builder, const CGBlockInfo &blockInfo, llvm::Instruction *InsertPoint = 0); - /// EmitDeclareOfArgVariable - Emit call to llvm.dbg.declare for an argument - /// variable declaration. + /// Emit call to \c llvm.dbg.declare for an argument variable + /// declaration. void EmitDeclareOfArgVariable(const VarDecl *Decl, llvm::Value *AI, unsigned ArgNo, CGBuilderTy &Builder); - /// EmitDeclareOfBlockLiteralArgVariable - Emit call to - /// llvm.dbg.declare for the block-literal argument to a block - /// invocation function. + /// Emit call to \c llvm.dbg.declare for the block-literal argument + /// to a block invocation function. void EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, llvm::Value *Arg, unsigned ArgNo, llvm::Value *LocalAddr, CGBuilderTy &Builder); - /// EmitGlobalVariable - Emit information about a global variable. + /// Emit information about a global variable. void EmitGlobalVariable(llvm::GlobalVariable *GV, const VarDecl *Decl); - /// EmitGlobalVariable - Emit global variable's debug info. + /// Emit global variable's debug info. void EmitGlobalVariable(const ValueDecl *VD, llvm::Constant *Init); - /// \brief - Emit C++ using directive. + /// Emit C++ using directive. void EmitUsingDirective(const UsingDirectiveDecl &UD); - /// EmitExplicitCastType - Emit the type explicitly casted to. + /// Emit the type explicitly casted to. void EmitExplicitCastType(QualType Ty); - /// \brief - Emit C++ using declaration. + /// Emit C++ using declaration. void EmitUsingDecl(const UsingDecl &UD); - /// \brief - Emit C++ namespace alias. - llvm::DIImportedEntity EmitNamespaceAlias(const NamespaceAliasDecl &NA); + /// Emit an @import declaration. + void EmitImportDecl(const ImportDecl &ID); + + /// Emit C++ namespace alias. + llvm::DIImportedEntity *EmitNamespaceAlias(const NamespaceAliasDecl &NA); - /// getOrCreateRecordType - Emit record type's standalone debug info. - llvm::DIType getOrCreateRecordType(QualType Ty, SourceLocation L); + /// Emit record type's standalone debug info. + llvm::DIType *getOrCreateRecordType(QualType Ty, SourceLocation L); - /// getOrCreateInterfaceType - Emit an objective c interface type standalone - /// debug info. - llvm::DIType getOrCreateInterfaceType(QualType Ty, - SourceLocation Loc); + /// Emit an Objective-C interface type standalone debug info. + llvm::DIType *getOrCreateInterfaceType(QualType Ty, SourceLocation Loc); void completeType(const EnumDecl *ED); void completeType(const RecordDecl *RD); @@ -311,175 +349,209 @@ public: void completeTemplateDefinition(const ClassTemplateSpecializationDecl &SD); private: - /// EmitDeclare - Emit call to llvm.dbg.declare for a variable declaration. + /// Emit call to llvm.dbg.declare for a variable declaration. /// Tag accepts custom types DW_TAG_arg_variable and DW_TAG_auto_variable, /// otherwise would be of type llvm::dwarf::Tag. - void EmitDeclare(const VarDecl *decl, llvm::dwarf::LLVMConstants Tag, - llvm::Value *AI, unsigned ArgNo, CGBuilderTy &Builder); + void EmitDeclare(const VarDecl *decl, llvm::dwarf::Tag Tag, llvm::Value *AI, + unsigned ArgNo, CGBuilderTy &Builder); - // EmitTypeForVarWithBlocksAttr - Build up structure info for the byref. - // See BuildByRefType. - llvm::DIType EmitTypeForVarWithBlocksAttr(const VarDecl *VD, - uint64_t *OffSet); + /// Build up structure info for the byref. See \a BuildByRefType. + llvm::DIType *EmitTypeForVarWithBlocksAttr(const VarDecl *VD, + uint64_t *OffSet); - /// getContextDescriptor - Get context info for the decl. - llvm::DIScope getContextDescriptor(const Decl *Decl); + /// Get context info for the decl. + llvm::DIScope *getContextDescriptor(const Decl *Decl); - llvm::DIScope getCurrentContextDescriptor(const Decl *Decl); + llvm::DIScope *getCurrentContextDescriptor(const Decl *Decl); - /// \brief Create a forward decl for a RecordType in a given context. - llvm::DICompositeType getOrCreateRecordFwdDecl(const RecordType *, - llvm::DIDescriptor); + /// Create a forward decl for a RecordType in a given context. + llvm::DICompositeType *getOrCreateRecordFwdDecl(const RecordType *, + llvm::DIScope *); - /// createContextChain - Create a set of decls for the context chain. - llvm::DIDescriptor createContextChain(const Decl *Decl); - - /// getCurrentDirname - Return current directory name. + /// Return current directory name. StringRef getCurrentDirname(); - /// CreateCompileUnit - Create new compile unit. + /// Create new compile unit. void CreateCompileUnit(); - /// getOrCreateFile - Get the file debug info descriptor for the input - /// location. - llvm::DIFile getOrCreateFile(SourceLocation Loc); + /// Get the file debug info descriptor for the input location. + llvm::DIFile *getOrCreateFile(SourceLocation Loc); - /// getOrCreateMainFile - Get the file info for main compile unit. - llvm::DIFile getOrCreateMainFile(); + /// Get the file info for main compile unit. + llvm::DIFile *getOrCreateMainFile(); - /// getOrCreateType - Get the type from the cache or create a new type if - /// necessary. - llvm::DIType getOrCreateType(QualType Ty, llvm::DIFile Fg); + /// Get the type from the cache or create a new type if necessary. + llvm::DIType *getOrCreateType(QualType Ty, llvm::DIFile *Fg); - /// getOrCreateLimitedType - Get the type from the cache or create a new - /// partial type if necessary. - llvm::DIType getOrCreateLimitedType(const RecordType *Ty, llvm::DIFile F); + /// Get a reference to a clang module. + llvm::DIModule * + getOrCreateModuleRef(ExternalASTSource::ASTSourceDescriptor Mod); + + /// Get the type from the cache or create a new partial type if + /// necessary. + llvm::DIType *getOrCreateLimitedType(const RecordType *Ty, llvm::DIFile *F); - /// CreateTypeNode - Create type metadata for a source language type. - llvm::DIType CreateTypeNode(QualType Ty, llvm::DIFile Fg); + /// Create type metadata for a source language type. + llvm::DIType *CreateTypeNode(QualType Ty, llvm::DIFile *Fg); - /// getObjCInterfaceDecl - return the underlying ObjCInterfaceDecl - /// if Ty is an ObjCInterface or a pointer to one. - ObjCInterfaceDecl* getObjCInterfaceDecl(QualType Ty); + /// Return the underlying ObjCInterfaceDecl if \arg Ty is an + /// ObjCInterface or a pointer to one. + ObjCInterfaceDecl *getObjCInterfaceDecl(QualType Ty); - /// CreateMemberType - Create new member and increase Offset by FType's size. - llvm::DIType CreateMemberType(llvm::DIFile Unit, QualType FType, - StringRef Name, uint64_t *Offset); + /// Create new member and increase Offset by FType's size. + llvm::DIType *CreateMemberType(llvm::DIFile *Unit, QualType FType, + StringRef Name, uint64_t *Offset); - /// \brief Retrieve the DIDescriptor, if any, for the canonical form of this + /// Retrieve the DIDescriptor, if any, for the canonical form of this /// declaration. - llvm::DIDescriptor getDeclarationOrDefinition(const Decl *D); + llvm::DINode *getDeclarationOrDefinition(const Decl *D); - /// getFunctionDeclaration - Return debug info descriptor to describe method + /// \return debug info descriptor to describe method /// declaration for the given method definition. - llvm::DISubprogram getFunctionDeclaration(const Decl *D); + llvm::DISubprogram *getFunctionDeclaration(const Decl *D); - /// Return debug info descriptor to describe in-class static data member - /// declaration for the given out-of-class definition. - llvm::DIDerivedType + /// \return debug info descriptor to describe in-class static data + /// member declaration for the given out-of-class definition. If D + /// is an out-of-class definition of a static data member of a + /// class, find its corresponding in-class declaration. + llvm::DIDerivedType * getOrCreateStaticDataMemberDeclarationOrNull(const VarDecl *D); - /// \brief Create a DISubprogram describing the forward - /// decalration represented in the given FunctionDecl. - llvm::DISubprogram getFunctionForwardDeclaration(const FunctionDecl *FD); - - /// \brief Create a DIGlobalVariable describing the forward - /// decalration represented in the given VarDecl. - llvm::DIGlobalVariable getGlobalVariableForwardDeclaration(const VarDecl *VD); - - /// Return a global variable that represents one of the collection of - /// global variables created for an anonmyous union. - llvm::DIGlobalVariable - CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile Unit, unsigned LineNo, - StringRef LinkageName, llvm::GlobalVariable *Var, - llvm::DIDescriptor DContext); - - /// getFunctionName - Get function name for the given FunctionDecl. If the - /// name is constructed on demand (e.g. C++ destructor) then the name - /// is stored on the side. + /// Create a subprogram describing the forward declaration + /// represented in the given FunctionDecl. + llvm::DISubprogram *getFunctionForwardDeclaration(const FunctionDecl *FD); + + /// Create a global variable describing the forward decalration + /// represented in the given VarDecl. + llvm::DIGlobalVariable * + getGlobalVariableForwardDeclaration(const VarDecl *VD); + + /// \brief Return a global variable that represents one of the + /// collection of global variables created for an anonmyous union. + /// + /// Recursively collect all of the member fields of a global + /// anonymous decl and create static variables for them. The first + /// time this is called it needs to be on a union and then from + /// there we can have additional unnamed fields. + llvm::DIGlobalVariable * + CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile *Unit, + unsigned LineNo, StringRef LinkageName, + llvm::GlobalVariable *Var, llvm::DIScope *DContext); + + /// Get function name for the given FunctionDecl. If the name is + /// constructed on demand (e.g., C++ destructor) then the name is + /// stored on the side. StringRef getFunctionName(const FunctionDecl *FD); - /// getObjCMethodName - Returns the unmangled name of an Objective-C method. + /// Returns the unmangled name of an Objective-C method. /// This is the display name for the debugging info. StringRef getObjCMethodName(const ObjCMethodDecl *FD); - /// getSelectorName - Return selector name. This is used for debugging + /// Return selector name. This is used for debugging /// info. StringRef getSelectorName(Selector S); - /// getClassName - Get class name including template argument list. + /// Get class name including template argument list. StringRef getClassName(const RecordDecl *RD); - /// getVTableName - Get vtable name for the given Class. + /// Get the vtable name for the given class. StringRef getVTableName(const CXXRecordDecl *Decl); - /// getLineNumber - Get line number for the location. If location is invalid + /// Get line number for the location. If location is invalid /// then use current location. unsigned getLineNumber(SourceLocation Loc); - /// getColumnNumber - Get column number for the location. If location is + /// Get column number for the location. If location is /// invalid then use current location. /// \param Force Assume DebugColumnInfo option is true. - unsigned getColumnNumber(SourceLocation Loc, bool Force=false); + unsigned getColumnNumber(SourceLocation Loc, bool Force = false); - /// \brief Collect various properties of a FunctionDecl. + /// Collect various properties of a FunctionDecl. /// \param GD A GlobalDecl whose getDecl() must return a FunctionDecl. - void collectFunctionDeclProps(GlobalDecl GD, - llvm::DIFile Unit, + void collectFunctionDeclProps(GlobalDecl GD, llvm::DIFile *Unit, StringRef &Name, StringRef &LinkageName, - llvm::DIDescriptor &FDContext, - llvm::DIArray &TParamsArray, + llvm::DIScope *&FDContext, + llvm::DINodeArray &TParamsArray, unsigned &Flags); - /// \brief Collect various properties of a VarDecl. - void collectVarDeclProps(const VarDecl *VD, llvm::DIFile &Unit, - unsigned &LineNo, QualType &T, - StringRef &Name, StringRef &LinkageName, - llvm::DIDescriptor &VDContext); + /// Collect various properties of a VarDecl. + void collectVarDeclProps(const VarDecl *VD, llvm::DIFile *&Unit, + unsigned &LineNo, QualType &T, StringRef &Name, + StringRef &LinkageName, llvm::DIScope *&VDContext); - /// internString - Allocate a copy of \p A using the DebugInfoNames allocator + /// Allocate a copy of \p A using the DebugInfoNames allocator /// and return a reference to it. If multiple arguments are given the strings /// are concatenated. StringRef internString(StringRef A, StringRef B = StringRef()) { char *Data = DebugInfoNames.Allocate<char>(A.size() + B.size()); - std::memcpy(Data, A.data(), A.size()); - std::memcpy(Data + A.size(), B.data(), B.size()); + if (!A.empty()) + std::memcpy(Data, A.data(), A.size()); + if (!B.empty()) + std::memcpy(Data + A.size(), B.data(), B.size()); return StringRef(Data, A.size() + B.size()); } }; +/// A scoped helper to set the current debug location to the specified +/// location or preferred location of the specified Expr. class ApplyDebugLocation { -protected: +private: + void init(SourceLocation TemporaryLocation, bool DefaultToEmpty = false); + ApplyDebugLocation(CodeGenFunction &CGF, bool DefaultToEmpty, + SourceLocation TemporaryLocation); + llvm::DebugLoc OriginalLocation; CodeGenFunction &CGF; public: - ApplyDebugLocation(CodeGenFunction &CGF, - SourceLocation TemporaryLocation = SourceLocation(), - bool ForceColumnInfo = false); + /// Set the location to the (valid) TemporaryLocation. + ApplyDebugLocation(CodeGenFunction &CGF, SourceLocation TemporaryLocation); + ApplyDebugLocation(CodeGenFunction &CGF, const Expr *E); ApplyDebugLocation(CodeGenFunction &CGF, llvm::DebugLoc Loc); + ~ApplyDebugLocation(); -}; -/// ArtificialLocation - An RAII object that temporarily switches to -/// an artificial debug location that has a valid scope, but no line -/// information. This is useful when emitting compiler-generated -/// helper functions that have no source location associated with -/// them. The DWARF specification allows the compiler to use the -/// special line number 0 to indicate code that can not be attributed -/// to any source location. -/// -/// This is necessary because passing an empty SourceLocation to -/// CGDebugInfo::setLocation() will result in the last valid location -/// being reused. -class ArtificialLocation : public ApplyDebugLocation { -public: - ArtificialLocation(CodeGenFunction &CGF); -}; + /// \brief Apply TemporaryLocation if it is valid. Otherwise switch + /// to an artificial debug location that has a valid scope, but no + /// line information. + /// + /// Artificial locations are useful when emitting compiler-generated + /// helper functions that have no source location associated with + /// them. The DWARF specification allows the compiler to use the + /// special line number 0 to indicate code that can not be + /// attributed to any source location. Note that passing an empty + /// SourceLocation to CGDebugInfo::setLocation() will result in the + /// last valid location being reused. + static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF) { + return ApplyDebugLocation(CGF, false, SourceLocation()); + } + /// \brief Apply TemporaryLocation if it is valid. Otherwise switch + /// to an artificial debug location that has a valid scope, but no + /// line information. + static ApplyDebugLocation + CreateDefaultArtificial(CodeGenFunction &CGF, + SourceLocation TemporaryLocation) { + return ApplyDebugLocation(CGF, false, TemporaryLocation); + } + + /// Set the IRBuilder to not attach debug locations. Note that + /// passing an empty SourceLocation to \a CGDebugInfo::setLocation() + /// will result in the last valid location being reused. Note that + /// all instructions that do not have a location at the beginning of + /// a function are counted towards to funciton prologue. + static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF) { + return ApplyDebugLocation(CGF, true, SourceLocation()); + } + /// \brief Apply TemporaryLocation if it is valid. Otherwise set the IRBuilder + /// to not attach debug locations. + static ApplyDebugLocation + CreateDefaultEmpty(CodeGenFunction &CGF, SourceLocation TemporaryLocation) { + return ApplyDebugLocation(CGF, true, TemporaryLocation); + } +}; } // namespace CodeGen } // namespace clang - #endif diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp index 15a1a7f..96aa8c68 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDecl.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "CodeGenFunction.h" +#include "CGCleanup.h" #include "CGDebugInfo.h" #include "CGOpenCLRuntime.h" #include "CodeGenModule.h" @@ -34,6 +35,7 @@ using namespace CodeGen; void CodeGenFunction::EmitDecl(const Decl &D) { switch (D.getKind()) { case Decl::TranslationUnit: + case Decl::ExternCContext: case Decl::Namespace: case Decl::UnresolvedUsingTypename: case Decl::ClassTemplateSpecialization: @@ -77,6 +79,7 @@ void CodeGenFunction::EmitDecl(const Decl &D) { case Decl::Captured: case Decl::ClassScopeFunctionSpecialization: case Decl::UsingShadow: + case Decl::ObjCTypeParam: llvm_unreachable("Declaration should not be in declstmts!"); case Decl::Function: // void X(); case Decl::Record: // struct/union/class X; @@ -154,6 +157,8 @@ static std::string getStaticDeclName(CodeGenModule &CGM, const VarDecl &D) { assert(!D.isExternallyVisible() && "name shouldn't matter"); std::string ContextName; const DeclContext *DC = D.getDeclContext(); + if (auto *CD = dyn_cast<CapturedDecl>(DC)) + DC = cast<DeclContext>(CD->getNonClosureContext()); if (const auto *FD = dyn_cast<FunctionDecl>(DC)) ContextName = CGM.getMangledName(FD); else if (const auto *BD = dyn_cast<BlockDecl>(DC)) @@ -206,6 +211,9 @@ llvm::Constant *CodeGenModule::getOrCreateStaticVarDecl( GV->setAlignment(getContext().getDeclAlign(&D).getQuantity()); setGlobalVisibility(GV, &D); + if (supportsCOMDAT() && GV->isWeakForLinker()) + GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); + if (D.getTLSKind()) setTLSMode(GV, D); @@ -512,10 +520,7 @@ namespace { : Addr(addr), Size(size) {} void Emit(CodeGenFunction &CGF, Flags flags) override { - llvm::Value *castAddr = CGF.Builder.CreateBitCast(Addr, CGF.Int8PtrTy); - CGF.Builder.CreateCall2(CGF.CGM.getLLVMLifetimeEndFn(), - Size, castAddr) - ->setDoesNotThrow(); + CGF.EmitLifetimeEnd(Size, Addr); } }; } @@ -575,9 +580,9 @@ static bool isAccessedBy(const VarDecl &var, const Stmt *s) { } } - for (Stmt::const_child_range children = s->children(); children; ++children) - // children might be null; as in missing decl or conditional of an if-stmt. - if ((*children) && isAccessedBy(var, *children)) + for (const Stmt *SubStmt : s->children()) + // SubStmt might be null; as in missing decl or conditional of an if-stmt. + if (SubStmt && isAccessedBy(var, SubStmt)) return true; return false; @@ -631,8 +636,9 @@ void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D, if (capturedByInit) { // We can use a simple GEP for this because it can't have been // moved yet. - tempLV.setAddress(Builder.CreateStructGEP(tempLV.getAddress(), - getByRefValueLLVMField(cast<VarDecl>(D)))); + tempLV.setAddress(Builder.CreateStructGEP( + nullptr, tempLV.getAddress(), + getByRefValueLLVMField(cast<VarDecl>(D)).second)); } llvm::PointerType *ty @@ -793,8 +799,9 @@ static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc, // If necessary, get a pointer to the element and emit it. if (!Elt->isNullValue() && !isa<llvm::UndefValue>(Elt)) - emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i), - isVolatile, Builder); + emitStoresForInitAfterMemset( + Elt, Builder.CreateConstGEP2_32(Init->getType(), Loc, 0, i), + isVolatile, Builder); } return; } @@ -807,8 +814,9 @@ static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc, // If necessary, get a pointer to the element and emit it. if (!Elt->isNullValue() && !isa<llvm::UndefValue>(Elt)) - emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i), - isVolatile, Builder); + emitStoresForInitAfterMemset( + Elt, Builder.CreateConstGEP2_32(Init->getType(), Loc, 0, i), + isVolatile, Builder); } } @@ -833,21 +841,6 @@ static bool shouldUseMemSetPlusStoresToInitialize(llvm::Constant *Init, canEmitInitWithFewStoresAfterMemset(Init, StoreBudget); } -/// Should we use the LLVM lifetime intrinsics for the given local variable? -static bool shouldUseLifetimeMarkers(CodeGenFunction &CGF, const VarDecl &D, - unsigned Size) { - // For now, only in optimized builds. - if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) - return false; - - // Limit the size of marked objects to 32 bytes. We don't want to increase - // compile time by marking tiny objects. - unsigned SizeThreshold = 32; - - return Size > SizeThreshold; -} - - /// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a /// variable declaration with auto, register, or no storage class specifier. /// These turn into simple stack objects, or GlobalValues depending on target. @@ -857,6 +850,35 @@ void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D) { EmitAutoVarCleanups(emission); } +/// Emit a lifetime.begin marker if some criteria are satisfied. +/// \return a pointer to the temporary size Value if a marker was emitted, null +/// otherwise +llvm::Value *CodeGenFunction::EmitLifetimeStart(uint64_t Size, + llvm::Value *Addr) { + // For now, only in optimized builds. + if (CGM.getCodeGenOpts().OptimizationLevel == 0) + return nullptr; + + // Disable lifetime markers in msan builds. + // FIXME: Remove this when msan works with lifetime markers. + if (getLangOpts().Sanitize.has(SanitizerKind::Memory)) + return nullptr; + + llvm::Value *SizeV = llvm::ConstantInt::get(Int64Ty, Size); + Addr = Builder.CreateBitCast(Addr, Int8PtrTy); + llvm::CallInst *C = + Builder.CreateCall(CGM.getLLVMLifetimeStartFn(), {SizeV, Addr}); + C->setDoesNotThrow(); + return SizeV; +} + +void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) { + Addr = Builder.CreateBitCast(Addr, Int8PtrTy); + llvm::CallInst *C = + Builder.CreateCall(CGM.getLLVMLifetimeEndFn(), {Size, Addr}); + C->setDoesNotThrow(); +} + /// EmitAutoVarAlloca - Emit the alloca and debug information for a /// local variable. Does not emit initialization or destruction. CodeGenFunction::AutoVarEmission @@ -952,13 +974,8 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // Emit a lifetime intrinsic if meaningful. There's no point // in doing this if we don't have a valid insertion point (?). uint64_t size = CGM.getDataLayout().getTypeAllocSize(LTy); - if (HaveInsertPoint() && shouldUseLifetimeMarkers(*this, D, size)) { - llvm::Value *sizeV = llvm::ConstantInt::get(Int64Ty, size); - - emission.SizeForLifetimeMarkers = sizeV; - llvm::Value *castAddr = Builder.CreateBitCast(Alloc, Int8PtrTy); - Builder.CreateCall2(CGM.getLLVMLifetimeStartFn(), sizeV, castAddr) - ->setDoesNotThrow(); + if (HaveInsertPoint()) { + emission.SizeForLifetimeMarkers = EmitLifetimeStart(size, Alloc); } else { assert(!emission.useLifetimeMarkers()); } @@ -1058,8 +1075,8 @@ static bool isCapturedBy(const VarDecl &var, const Expr *e) { return false; } - for (Stmt::const_child_range children = e->children(); children; ++children) - if (isCapturedBy(var, cast<Expr>(*children))) + for (const Stmt *SubStmt : e->children()) + if (isCapturedBy(var, cast<Expr>(SubStmt))) return true; return false; @@ -1087,7 +1104,7 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) { if (emission.wasEmittedAsGlobal()) return; const VarDecl &D = *emission.Variable; - ApplyDebugLocation DL(*this, D.getLocation()); + auto DL = ApplyDebugLocation::CreateDefaultArtificial(*this, D.getLocation()); QualType type = D.getType(); // If this local has an initializer, emit it now. @@ -1304,6 +1321,8 @@ void CodeGenFunction::EmitAutoVarCleanups(const AutoVarEmission &emission) { EHStack.pushCleanup<CallLifetimeEnd>(NormalCleanup, emission.getAllocatedAddress(), emission.getSizeForLifetimeMarkers()); + EHCleanupScope &cleanup = cast<EHCleanupScope>(*EHStack.begin()); + cleanup.setLifetimeMarker(); } // Check the type for a cleanup. diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp index 19e4bdd..00d6d5c 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGDeclCXX.cpp @@ -139,9 +139,32 @@ void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D, const Expr *Init = D.getInit(); QualType T = D.getType(); + // The address space of a static local variable (DeclPtr) may be different + // from the address space of the "this" argument of the constructor. In that + // case, we need an addrspacecast before calling the constructor. + // + // struct StructWithCtor { + // __device__ StructWithCtor() {...} + // }; + // __device__ void foo() { + // __shared__ StructWithCtor s; + // ... + // } + // + // For example, in the above CUDA code, the static local variable s has a + // "shared" address space qualifier, but the constructor of StructWithCtor + // expects "this" in the "generic" address space. + unsigned ExpectedAddrSpace = getContext().getTargetAddressSpace(T); + unsigned ActualAddrSpace = DeclPtr->getType()->getPointerAddressSpace(); + if (ActualAddrSpace != ExpectedAddrSpace) { + llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(T); + llvm::PointerType *PTy = llvm::PointerType::get(LTy, ExpectedAddrSpace); + DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(DeclPtr, PTy); + } + if (!T->isReferenceType()) { if (getLangOpts().OpenMP && D.hasAttr<OMPThreadPrivateDeclAttr>()) - (void)CGM.getOpenMPRuntime().EmitOMPThreadPrivateVarDefinition( + (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition( &D, DeclPtr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(), PerformInit, this); if (PerformInit) @@ -236,18 +259,23 @@ llvm::Function *CodeGenModule::CreateGlobalInitOrDestructFunction( Fn->setSection(Section); } + SetLLVMFunctionAttributes(nullptr, getTypes().arrangeNullaryFunction(), Fn); + Fn->setCallingConv(getRuntimeCC()); if (!getLangOpts().Exceptions) Fn->setDoesNotThrow(); if (!isInSanitizerBlacklist(Fn, Loc)) { - if (getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address | + SanitizerKind::KernelAddress)) Fn->addFnAttr(llvm::Attribute::SanitizeAddress); if (getLangOpts().Sanitize.has(SanitizerKind::Thread)) Fn->addFnAttr(llvm::Attribute::SanitizeThread); if (getLangOpts().Sanitize.has(SanitizerKind::Memory)) Fn->addFnAttr(llvm::Attribute::SanitizeMemory); + if (getLangOpts().Sanitize.has(SanitizerKind::SafeStack)) + Fn->addFnAttr(llvm::Attribute::SafeStack); } return Fn; @@ -267,15 +295,7 @@ void CodeGenModule::EmitPointerToInitFunc(const VarDecl *D, addUsedGlobal(PtrArray); // If the GV is already in a comdat group, then we have to join it. - llvm::Comdat *C = GV->getComdat(); - - // LinkOnce and Weak linkage are lowered down to a single-member comdat group. - // Make an explicit group so we can join it. - if (!C && (GV->hasWeakLinkage() || GV->hasLinkOnceLinkage())) { - C = TheModule.getOrInsertComdat(GV->getName()); - GV->setComdat(C); - } - if (C) + if (llvm::Comdat *C = GV->getComdat()) PtrArray->setComdat(C); } @@ -283,6 +303,11 @@ void CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D, llvm::GlobalVariable *Addr, bool PerformInit) { + // Check if we've already initialized this decl. + auto I = DelayedCXXInitPosition.find(D); + if (I != DelayedCXXInitPosition.end() && I->second == ~0U) + return; + llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false); SmallString<256> FnName; { @@ -312,11 +337,9 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D, CXXThreadLocalInitVars.push_back(Addr); } else if (PerformInit && ISA) { EmitPointerToInitFunc(D, Addr, Fn, ISA); - DelayedCXXInitPosition.erase(D); } else if (auto *IPA = D->getAttr<InitPriorityAttr>()) { OrderGlobalInits Key(IPA->getPriority(), PrioritizedCXXGlobalInits.size()); PrioritizedCXXGlobalInits.push_back(std::make_pair(Key, Fn)); - DelayedCXXInitPosition.erase(D); } else if (isTemplateInstantiation(D->getTemplateSpecializationKind())) { // C++ [basic.start.init]p2: // Definitions of explicitly specialized class template static data @@ -331,24 +354,24 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D, // minor startup time optimization. In the MS C++ ABI, there are no guard // variables, so this COMDAT key is required for correctness. AddGlobalCtor(Fn, 65535, COMDATKey); - DelayedCXXInitPosition.erase(D); } else if (D->hasAttr<SelectAnyAttr>()) { // SelectAny globals will be comdat-folded. Put the initializer into a // COMDAT group associated with the global, so the initializers get folded // too. AddGlobalCtor(Fn, 65535, COMDATKey); - DelayedCXXInitPosition.erase(D); } else { - llvm::DenseMap<const Decl *, unsigned>::iterator I = - DelayedCXXInitPosition.find(D); + I = DelayedCXXInitPosition.find(D); // Re-do lookup in case of re-hash. if (I == DelayedCXXInitPosition.end()) { CXXGlobalInits.push_back(Fn); - } else { - assert(CXXGlobalInits[I->second] == nullptr); + } else if (I->second != ~0U) { + assert(I->second < CXXGlobalInits.size() && + CXXGlobalInits[I->second] == nullptr); CXXGlobalInits[I->second] = Fn; - DelayedCXXInitPosition.erase(I); } } + + // Remember that we already emitted the initializer for this global. + DelayedCXXInitPosition[D] = ~0U; } void CodeGenModule::EmitCXXThreadLocalInitFunc() { @@ -401,6 +424,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() { CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, LocalCXXGlobalInits); AddGlobalCtor(Fn, Priority); } + PrioritizedCXXGlobalInits.clear(); } SmallString<128> FileName; @@ -411,7 +435,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() { // priority emitted above. FileName = llvm::sys::path::filename(MainFile->getName()); } else { - FileName = SmallString<128>("<null>"); + FileName = "<null>"; } for (size_t i = 0; i < FileName.size(); ++i) { @@ -428,7 +452,6 @@ CodeGenModule::EmitCXXGlobalInitFunc() { AddGlobalCtor(Fn); CXXGlobalInits.clear(); - PrioritizedCXXGlobalInits.clear(); } void CodeGenModule::EmitCXXGlobalDtorFunc() { @@ -477,11 +500,11 @@ CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn, ArrayRef<llvm::Function *> Decls, llvm::GlobalVariable *Guard) { { - ApplyDebugLocation NL(*this); + auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(GlobalDecl(), getContext().VoidTy, Fn, getTypes().arrangeNullaryFunction(), FunctionArgList()); // Emit an artificial location for this function. - ArtificialLocation AL(*this); + auto AL = ApplyDebugLocation::CreateArtificial(*this); llvm::BasicBlock *ExitBlock = nullptr; if (Guard) { @@ -528,11 +551,11 @@ void CodeGenFunction::GenerateCXXGlobalDtorsFunc(llvm::Function *Fn, const std::vector<std::pair<llvm::WeakVH, llvm::Constant*> > &DtorsAndObjects) { { - ApplyDebugLocation NL(*this); + auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(GlobalDecl(), getContext().VoidTy, Fn, getTypes().arrangeNullaryFunction(), FunctionArgList()); // Emit an artificial location for this function. - ArtificialLocation AL(*this); + auto AL = ApplyDebugLocation::CreateArtificial(*this); // Emit the dtors, in reverse order from construction. for (unsigned i = 0, e = DtorsAndObjects.size(); i != e; ++i) { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp index cb8eb8f..7b8368e 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGException.cpp @@ -16,23 +16,19 @@ #include "CGCleanup.h" #include "CGObjCRuntime.h" #include "TargetInfo.h" +#include "clang/AST/Mangle.h" #include "clang/AST/StmtCXX.h" #include "clang/AST/StmtObjC.h" +#include "clang/AST/StmtVisitor.h" +#include "clang/Basic/TargetBuiltins.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/Support/SaveAndRestore.h" using namespace clang; using namespace CodeGen; -static llvm::Constant *getAllocateExceptionFn(CodeGenModule &CGM) { - // void *__cxa_allocate_exception(size_t thrown_size); - - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.Int8PtrTy, CGM.SizeTy, /*IsVarArgs=*/false); - - return CGM.CreateRuntimeFunction(FTy, "__cxa_allocate_exception"); -} - static llvm::Constant *getFreeExceptionFn(CodeGenModule &CGM) { // void __cxa_free_exception(void *thrown_exception); @@ -42,44 +38,6 @@ static llvm::Constant *getFreeExceptionFn(CodeGenModule &CGM) { return CGM.CreateRuntimeFunction(FTy, "__cxa_free_exception"); } -static llvm::Constant *getThrowFn(CodeGenModule &CGM) { - // void __cxa_throw(void *thrown_exception, std::type_info *tinfo, - // void (*dest) (void *)); - - llvm::Type *Args[3] = { CGM.Int8PtrTy, CGM.Int8PtrTy, CGM.Int8PtrTy }; - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.VoidTy, Args, /*IsVarArgs=*/false); - - return CGM.CreateRuntimeFunction(FTy, "__cxa_throw"); -} - -static llvm::Constant *getGetExceptionPtrFn(CodeGenModule &CGM) { - // void *__cxa_get_exception_ptr(void*); - - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.Int8PtrTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); - - return CGM.CreateRuntimeFunction(FTy, "__cxa_get_exception_ptr"); -} - -static llvm::Constant *getBeginCatchFn(CodeGenModule &CGM) { - // void *__cxa_begin_catch(void*); - - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.Int8PtrTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); - - return CGM.CreateRuntimeFunction(FTy, "__cxa_begin_catch"); -} - -static llvm::Constant *getEndCatchFn(CodeGenModule &CGM) { - // void __cxa_end_catch(); - - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.VoidTy, /*IsVarArgs=*/false); - - return CGM.CreateRuntimeFunction(FTy, "__cxa_end_catch"); -} - static llvm::Constant *getUnexpectedFn(CodeGenModule &CGM) { // void __cxa_call_unexpected(void *thrown_exception); @@ -89,23 +47,30 @@ static llvm::Constant *getUnexpectedFn(CodeGenModule &CGM) { return CGM.CreateRuntimeFunction(FTy, "__cxa_call_unexpected"); } -static llvm::Constant *getTerminateFn(CodeGenModule &CGM) { +llvm::Constant *CodeGenModule::getTerminateFn() { // void __terminate(); llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.VoidTy, /*IsVarArgs=*/false); + llvm::FunctionType::get(VoidTy, /*IsVarArgs=*/false); StringRef name; // In C++, use std::terminate(). - if (CGM.getLangOpts().CPlusPlus) - name = "_ZSt9terminatev"; // FIXME: mangling! - else if (CGM.getLangOpts().ObjC1 && - CGM.getLangOpts().ObjCRuntime.hasTerminate()) + if (getLangOpts().CPlusPlus && + getTarget().getCXXABI().isItaniumFamily()) { + name = "_ZSt9terminatev"; + } else if (getLangOpts().CPlusPlus && + getTarget().getCXXABI().isMicrosoft()) { + if (getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2015)) + name = "__std_terminate"; + else + name = "\01?terminate@@YAXXZ"; + } else if (getLangOpts().ObjC1 && + getLangOpts().ObjCRuntime.hasTerminate()) name = "objc_terminate"; else name = "abort"; - return CGM.CreateRuntimeFunction(FTy, name); + return CreateRuntimeFunction(FTy, name); } static llvm::Constant *getCatchallRethrowFn(CodeGenModule &CGM, @@ -126,7 +91,12 @@ namespace { // This function must have prototype void(void*). const char *CatchallRethrowFn; - static const EHPersonality &get(CodeGenModule &CGM); + static const EHPersonality &get(CodeGenModule &CGM, + const FunctionDecl *FD); + static const EHPersonality &get(CodeGenFunction &CGF) { + return get(CGF.CGM, dyn_cast_or_null<FunctionDecl>(CGF.CurCodeDecl)); + } + static const EHPersonality GNU_C; static const EHPersonality GNU_C_SJLJ; static const EHPersonality GNU_C_SEH; @@ -137,6 +107,9 @@ namespace { static const EHPersonality GNU_CPlusPlus; static const EHPersonality GNU_CPlusPlus_SJLJ; static const EHPersonality GNU_CPlusPlus_SEH; + static const EHPersonality MSVC_except_handler; + static const EHPersonality MSVC_C_specific_handler; + static const EHPersonality MSVC_CxxFrameHandler3; }; } @@ -159,6 +132,12 @@ const EHPersonality EHPersonality::GNU_ObjCXX = { "__gnustep_objcxx_personality_v0", nullptr }; const EHPersonality EHPersonality::GNUstep_ObjC = { "__gnustep_objc_personality_v0", nullptr }; +const EHPersonality +EHPersonality::MSVC_except_handler = { "_except_handler3", nullptr }; +const EHPersonality +EHPersonality::MSVC_C_specific_handler = { "__C_specific_handler", nullptr }; +const EHPersonality +EHPersonality::MSVC_CxxFrameHandler3 = { "__CxxFrameHandler3", nullptr }; /// On Win64, use libgcc's SEH personality function. We fall back to dwarf on /// other platforms, unless the user asked for SjLj exceptions. @@ -231,9 +210,29 @@ static const EHPersonality &getObjCXXPersonality(const llvm::Triple &T, llvm_unreachable("bad runtime kind"); } -const EHPersonality &EHPersonality::get(CodeGenModule &CGM) { +static const EHPersonality &getSEHPersonalityMSVC(const llvm::Triple &T) { + if (T.getArch() == llvm::Triple::x86) + return EHPersonality::MSVC_except_handler; + return EHPersonality::MSVC_C_specific_handler; +} + +const EHPersonality &EHPersonality::get(CodeGenModule &CGM, + const FunctionDecl *FD) { const llvm::Triple &T = CGM.getTarget().getTriple(); const LangOptions &L = CGM.getLangOpts(); + + // Try to pick a personality function that is compatible with MSVC if we're + // not compiling Obj-C. Obj-C users better have an Obj-C runtime that supports + // the GCC-style personality function. + if (T.isWindowsMSVCEnvironment() && !L.ObjC1) { + if (L.SjLjExceptions) + return EHPersonality::GNU_CPlusPlus_SJLJ; + else if (FD && FD->usesSEHTry()) + return getSEHPersonalityMSVC(T); + else + return EHPersonality::MSVC_CxxFrameHandler3; + } + if (L.CPlusPlus && L.ObjC1) return getObjCXXPersonality(T, L); else if (L.CPlusPlus) @@ -318,7 +317,7 @@ void CodeGenModule::SimplifyPersonality() { if (!LangOpts.ObjCRuntime.isNeXTFamily()) return; - const EHPersonality &ObjCXX = EHPersonality::get(*this); + const EHPersonality &ObjCXX = EHPersonality::get(*this, /*FD=*/nullptr); const EHPersonality &CXX = getCXXPersonality(getTarget().getTriple(), LangOpts); if (&ObjCXX == &CXX) @@ -369,17 +368,16 @@ namespace { // differs from EmitAnyExprToMem only in that, if a final copy-ctor // call is required, an exception within that copy ctor causes // std::terminate to be invoked. -static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *e, - llvm::Value *addr) { +void CodeGenFunction::EmitAnyExprToExn(const Expr *e, llvm::Value *addr) { // Make sure the exception object is cleaned up if there's an // exception during initialization. - CGF.pushFullExprCleanup<FreeException>(EHCleanup, addr); - EHScopeStack::stable_iterator cleanup = CGF.EHStack.stable_begin(); + pushFullExprCleanup<FreeException>(EHCleanup, addr); + EHScopeStack::stable_iterator cleanup = EHStack.stable_begin(); // __cxa_allocate_exception returns a void*; we need to cast this // to the appropriate type for the object. - llvm::Type *ty = CGF.ConvertTypeForMem(e->getType())->getPointerTo(); - llvm::Value *typedAddr = CGF.Builder.CreateBitCast(addr, ty); + llvm::Type *ty = ConvertTypeForMem(e->getType())->getPointerTo(); + llvm::Value *typedAddr = Builder.CreateBitCast(addr, ty); // FIXME: this isn't quite right! If there's a final unelided call // to a copy constructor, then according to [except.terminate]p1 we @@ -388,11 +386,11 @@ static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *e, // evaluated but before the exception is caught. But the best way // to handle that is to teach EmitAggExpr to do the final copy // differently if it can't be elided. - CGF.EmitAnyExprToMem(e, typedAddr, e->getType().getQualifiers(), - /*IsInit*/ true); + EmitAnyExprToMem(e, typedAddr, e->getType().getQualifiers(), + /*IsInit*/ true); // Deactivate the cleanup block. - CGF.DeactivateCleanupBlock(cleanup, cast<llvm::Instruction>(typedAddr)); + DeactivateCleanupBlock(cleanup, cast<llvm::Instruction>(typedAddr)); } llvm::Value *CodeGenFunction::getExceptionSlot() { @@ -417,67 +415,18 @@ llvm::Value *CodeGenFunction::getSelectorFromSlot() { void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint) { - if (!E->getSubExpr()) { - CGM.getCXXABI().emitRethrow(*this, /*isNoReturn*/true); - - // throw is an expression, and the expression emitters expect us - // to leave ourselves at a valid insertion point. - if (KeepInsertionPoint) - EmitBlock(createBasicBlock("throw.cont")); - - return; - } - - if (CGM.getTarget().getTriple().isKnownWindowsMSVCEnvironment()) { - ErrorUnsupported(E, "throw expression"); - return; - } - - QualType ThrowType = E->getSubExpr()->getType(); - - if (ThrowType->isObjCObjectPointerType()) { - const Stmt *ThrowStmt = E->getSubExpr(); - const ObjCAtThrowStmt S(E->getExprLoc(), - const_cast<Stmt *>(ThrowStmt)); - CGM.getObjCRuntime().EmitThrowStmt(*this, S, false); - // This will clear insertion point which was not cleared in - // call to EmitThrowStmt. - if (KeepInsertionPoint) - EmitBlock(createBasicBlock("throw.cont")); - return; - } - - // Now allocate the exception object. - llvm::Type *SizeTy = ConvertType(getContext().getSizeType()); - uint64_t TypeSize = getContext().getTypeSizeInChars(ThrowType).getQuantity(); - - llvm::Constant *AllocExceptionFn = getAllocateExceptionFn(CGM); - llvm::CallInst *ExceptionPtr = - EmitNounwindRuntimeCall(AllocExceptionFn, - llvm::ConstantInt::get(SizeTy, TypeSize), - "exception"); - - EmitAnyExprToExn(*this, E->getSubExpr(), ExceptionPtr); - - // Now throw the exception. - llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType, - /*ForEH=*/true); - - // The address of the destructor. If the exception type has a - // trivial destructor (or isn't a record), we just pass null. - llvm::Constant *Dtor = nullptr; - if (const RecordType *RecordTy = ThrowType->getAs<RecordType>()) { - CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordTy->getDecl()); - if (!Record->hasTrivialDestructor()) { - CXXDestructorDecl *DtorD = Record->getDestructor(); - Dtor = CGM.getAddrOfCXXStructor(DtorD, StructorType::Complete); - Dtor = llvm::ConstantExpr::getBitCast(Dtor, Int8PtrTy); + if (const Expr *SubExpr = E->getSubExpr()) { + QualType ThrowType = SubExpr->getType(); + if (ThrowType->isObjCObjectPointerType()) { + const Stmt *ThrowStmt = E->getSubExpr(); + const ObjCAtThrowStmt S(E->getExprLoc(), const_cast<Stmt *>(ThrowStmt)); + CGM.getObjCRuntime().EmitThrowStmt(*this, S, false); + } else { + CGM.getCXXABI().emitThrow(*this, E); } + } else { + CGM.getCXXABI().emitRethrow(*this, /*isNoReturn=*/true); } - if (!Dtor) Dtor = llvm::Constant::getNullValue(Int8PtrTy); - - llvm::Value *args[] = { ExceptionPtr, TypeInfo, Dtor }; - EmitNoreturnRuntimeCallOrInvoke(getThrowFn(CGM), args); // throw is an expression, and the expression emitters expect us // to leave ourselves at a valid insertion point. @@ -509,6 +458,10 @@ void CodeGenFunction::EmitStartEHSpec(const Decl *D) { EHStack.pushTerminate(); } } else if (EST == EST_Dynamic || EST == EST_DynamicNone) { + // TODO: Revisit exception specifications for the MS ABI. There is a way to + // encode these in an object file but MSVC doesn't do anything with it. + if (getTarget().getCXXABI().isMicrosoft()) + return; unsigned NumExceptions = Proto->getNumExceptions(); EHFilterScope *Filter = EHStack.pushFilter(NumExceptions); @@ -543,8 +496,9 @@ static void emitFilterDispatchBlock(CodeGenFunction &CGF, llvm::Value *zero = CGF.Builder.getInt32(0); llvm::Value *failsFilter = - CGF.Builder.CreateICmpSLT(selector, zero, "ehspec.fails"); - CGF.Builder.CreateCondBr(failsFilter, unexpectedBB, CGF.getEHResumeBlock(false)); + CGF.Builder.CreateICmpSLT(selector, zero, "ehspec.fails"); + CGF.Builder.CreateCondBr(failsFilter, unexpectedBB, + CGF.getEHResumeBlock(false)); CGF.EmitBlock(unexpectedBB); } @@ -582,6 +536,10 @@ void CodeGenFunction::EmitEndEHSpec(const Decl *D) { EHStack.popTerminate(); } } else if (EST == EST_Dynamic || EST == EST_DynamicNone) { + // TODO: Revisit exception specifications for the MS ABI. There is a way to + // encode these in an object file but MSVC doesn't do anything with it. + if (getTarget().getCXXABI().isMicrosoft()) + return; EHFilterScope &filterScope = cast<EHFilterScope>(*EHStack.begin()); emitFilterDispatchBlock(*this, filterScope); EHStack.popFilter(); @@ -589,11 +547,6 @@ void CodeGenFunction::EmitEndEHSpec(const Decl *D) { } void CodeGenFunction::EmitCXXTryStmt(const CXXTryStmt &S) { - if (CGM.getTarget().getTriple().isKnownWindowsMSVCEnvironment()) { - ErrorUnsupported(&S, "try statement"); - return; - } - EnterCXXTryStmt(S); EmitStmt(S.getTryBlock()); ExitCXXTryStmt(S); @@ -622,7 +575,8 @@ void CodeGenFunction::EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) { if (CaughtType->isObjCObjectPointerType()) TypeInfo = CGM.getObjCRuntime().GetEHType(CaughtType); else - TypeInfo = CGM.GetAddrOfRTTIDescriptor(CaughtType, /*ForEH=*/true); + TypeInfo = + CGM.getAddrOfCXXCatchHandlerType(CaughtType, C->getCaughtType()); CatchScope->setHandler(I, TypeInfo, Handler); } else { // No exception decl indicates '...', a catch-all. @@ -695,8 +649,15 @@ llvm::BasicBlock *CodeGenFunction::getInvokeDestImpl() { assert(EHStack.requiresLandingPad()); assert(!EHStack.empty()); - if (!CGM.getLangOpts().Exceptions) - return nullptr; + // If exceptions are disabled, there are usually no landingpads. However, when + // SEH is enabled, functions using SEH still get landingpads. + const LangOptions &LO = CGM.getLangOpts(); + if (!LO.Exceptions) { + if (!LO.Borland && !LO.MicrosoftExt) + return nullptr; + if (!currentFunctionUsesSEHTry()) + return nullptr; + } // Check the innermost scope for a cached landing pad. If this is // a non-EH cleanup, we'll check enclosing scopes in EmitLandingPad. @@ -734,17 +695,19 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() { // Save the current IR generation state. CGBuilderTy::InsertPoint savedIP = Builder.saveAndClearIP(); - ApplyDebugLocation AutoRestoreLocation(*this, CurEHLocation); + auto DL = ApplyDebugLocation::CreateDefaultArtificial(*this, CurEHLocation); - const EHPersonality &personality = EHPersonality::get(CGM); + const EHPersonality &personality = EHPersonality::get(*this); + + if (!CurFn->hasPersonalityFn()) + CurFn->setPersonalityFn(getOpaquePersonalityFn(CGM, personality)); // Create and configure the landing pad. llvm::BasicBlock *lpad = createBasicBlock("lpad"); EmitBlock(lpad); - llvm::LandingPadInst *LPadInst = - Builder.CreateLandingPad(llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), - getOpaquePersonalityFn(CGM, personality), 0); + llvm::LandingPadInst *LPadInst = Builder.CreateLandingPad( + llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), 0); llvm::Value *LPadExn = Builder.CreateExtractValue(LPadInst, 0); Builder.CreateStore(LPadExn, getExceptionSlot()); @@ -762,8 +725,8 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() { bool hasFilter = false; SmallVector<llvm::Value*, 4> filterTypes; llvm::SmallPtrSet<llvm::Value*, 4> catchTypes; - for (EHScopeStack::iterator I = EHStack.begin(), E = EHStack.end(); - I != E; ++I) { + for (EHScopeStack::iterator I = EHStack.begin(), E = EHStack.end(); I != E; + ++I) { switch (I->getKind()) { case EHScope::Cleanup: @@ -857,263 +820,6 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() { return lpad; } -namespace { - /// A cleanup to call __cxa_end_catch. In many cases, the caught - /// exception type lets us state definitively that the thrown exception - /// type does not have a destructor. In particular: - /// - Catch-alls tell us nothing, so we have to conservatively - /// assume that the thrown exception might have a destructor. - /// - Catches by reference behave according to their base types. - /// - Catches of non-record types will only trigger for exceptions - /// of non-record types, which never have destructors. - /// - Catches of record types can trigger for arbitrary subclasses - /// of the caught type, so we have to assume the actual thrown - /// exception type might have a throwing destructor, even if the - /// caught type's destructor is trivial or nothrow. - struct CallEndCatch : EHScopeStack::Cleanup { - CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {} - bool MightThrow; - - void Emit(CodeGenFunction &CGF, Flags flags) override { - if (!MightThrow) { - CGF.EmitNounwindRuntimeCall(getEndCatchFn(CGF.CGM)); - return; - } - - CGF.EmitRuntimeCallOrInvoke(getEndCatchFn(CGF.CGM)); - } - }; -} - -/// Emits a call to __cxa_begin_catch and enters a cleanup to call -/// __cxa_end_catch. -/// -/// \param EndMightThrow - true if __cxa_end_catch might throw -static llvm::Value *CallBeginCatch(CodeGenFunction &CGF, - llvm::Value *Exn, - bool EndMightThrow) { - llvm::CallInst *call = - CGF.EmitNounwindRuntimeCall(getBeginCatchFn(CGF.CGM), Exn); - - CGF.EHStack.pushCleanup<CallEndCatch>(NormalAndEHCleanup, EndMightThrow); - - return call; -} - -/// A "special initializer" callback for initializing a catch -/// parameter during catch initialization. -static void InitCatchParam(CodeGenFunction &CGF, - const VarDecl &CatchParam, - llvm::Value *ParamAddr, - SourceLocation Loc) { - // Load the exception from where the landing pad saved it. - llvm::Value *Exn = CGF.getExceptionFromSlot(); - - CanQualType CatchType = - CGF.CGM.getContext().getCanonicalType(CatchParam.getType()); - llvm::Type *LLVMCatchTy = CGF.ConvertTypeForMem(CatchType); - - // If we're catching by reference, we can just cast the object - // pointer to the appropriate pointer. - if (isa<ReferenceType>(CatchType)) { - QualType CaughtType = cast<ReferenceType>(CatchType)->getPointeeType(); - bool EndCatchMightThrow = CaughtType->isRecordType(); - - // __cxa_begin_catch returns the adjusted object pointer. - llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndCatchMightThrow); - - // We have no way to tell the personality function that we're - // catching by reference, so if we're catching a pointer, - // __cxa_begin_catch will actually return that pointer by value. - if (const PointerType *PT = dyn_cast<PointerType>(CaughtType)) { - QualType PointeeType = PT->getPointeeType(); - - // When catching by reference, generally we should just ignore - // this by-value pointer and use the exception object instead. - if (!PointeeType->isRecordType()) { - - // Exn points to the struct _Unwind_Exception header, which - // we have to skip past in order to reach the exception data. - unsigned HeaderSize = - CGF.CGM.getTargetCodeGenInfo().getSizeOfUnwindException(); - AdjustedExn = CGF.Builder.CreateConstGEP1_32(Exn, HeaderSize); - - // However, if we're catching a pointer-to-record type that won't - // work, because the personality function might have adjusted - // the pointer. There's actually no way for us to fully satisfy - // the language/ABI contract here: we can't use Exn because it - // might have the wrong adjustment, but we can't use the by-value - // pointer because it's off by a level of abstraction. - // - // The current solution is to dump the adjusted pointer into an - // alloca, which breaks language semantics (because changing the - // pointer doesn't change the exception) but at least works. - // The better solution would be to filter out non-exact matches - // and rethrow them, but this is tricky because the rethrow - // really needs to be catchable by other sites at this landing - // pad. The best solution is to fix the personality function. - } else { - // Pull the pointer for the reference type off. - llvm::Type *PtrTy = - cast<llvm::PointerType>(LLVMCatchTy)->getElementType(); - - // Create the temporary and write the adjusted pointer into it. - llvm::Value *ExnPtrTmp = CGF.CreateTempAlloca(PtrTy, "exn.byref.tmp"); - llvm::Value *Casted = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy); - CGF.Builder.CreateStore(Casted, ExnPtrTmp); - - // Bind the reference to the temporary. - AdjustedExn = ExnPtrTmp; - } - } - - llvm::Value *ExnCast = - CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.byref"); - CGF.Builder.CreateStore(ExnCast, ParamAddr); - return; - } - - // Scalars and complexes. - TypeEvaluationKind TEK = CGF.getEvaluationKind(CatchType); - if (TEK != TEK_Aggregate) { - llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, false); - - // If the catch type is a pointer type, __cxa_begin_catch returns - // the pointer by value. - if (CatchType->hasPointerRepresentation()) { - llvm::Value *CastExn = - CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.casted"); - - switch (CatchType.getQualifiers().getObjCLifetime()) { - case Qualifiers::OCL_Strong: - CastExn = CGF.EmitARCRetainNonBlock(CastExn); - // fallthrough - - case Qualifiers::OCL_None: - case Qualifiers::OCL_ExplicitNone: - case Qualifiers::OCL_Autoreleasing: - CGF.Builder.CreateStore(CastExn, ParamAddr); - return; - - case Qualifiers::OCL_Weak: - CGF.EmitARCInitWeak(ParamAddr, CastExn); - return; - } - llvm_unreachable("bad ownership qualifier!"); - } - - // Otherwise, it returns a pointer into the exception object. - - llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok - llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy); - - LValue srcLV = CGF.MakeNaturalAlignAddrLValue(Cast, CatchType); - LValue destLV = CGF.MakeAddrLValue(ParamAddr, CatchType, - CGF.getContext().getDeclAlign(&CatchParam)); - switch (TEK) { - case TEK_Complex: - CGF.EmitStoreOfComplex(CGF.EmitLoadOfComplex(srcLV, Loc), destLV, - /*init*/ true); - return; - case TEK_Scalar: { - llvm::Value *ExnLoad = CGF.EmitLoadOfScalar(srcLV, Loc); - CGF.EmitStoreOfScalar(ExnLoad, destLV, /*init*/ true); - return; - } - case TEK_Aggregate: - llvm_unreachable("evaluation kind filtered out!"); - } - llvm_unreachable("bad evaluation kind"); - } - - assert(isa<RecordType>(CatchType) && "unexpected catch type!"); - - llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok - - // Check for a copy expression. If we don't have a copy expression, - // that means a trivial copy is okay. - const Expr *copyExpr = CatchParam.getInit(); - if (!copyExpr) { - llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, true); - llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy); - CGF.EmitAggregateCopy(ParamAddr, adjustedExn, CatchType); - return; - } - - // We have to call __cxa_get_exception_ptr to get the adjusted - // pointer before copying. - llvm::CallInst *rawAdjustedExn = - CGF.EmitNounwindRuntimeCall(getGetExceptionPtrFn(CGF.CGM), Exn); - - // Cast that to the appropriate type. - llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy); - - // The copy expression is defined in terms of an OpaqueValueExpr. - // Find it and map it to the adjusted expression. - CodeGenFunction::OpaqueValueMapping - opaque(CGF, OpaqueValueExpr::findInCopyConstruct(copyExpr), - CGF.MakeAddrLValue(adjustedExn, CatchParam.getType())); - - // Call the copy ctor in a terminate scope. - CGF.EHStack.pushTerminate(); - - // Perform the copy construction. - CharUnits Alignment = CGF.getContext().getDeclAlign(&CatchParam); - CGF.EmitAggExpr(copyExpr, - AggValueSlot::forAddr(ParamAddr, Alignment, Qualifiers(), - AggValueSlot::IsNotDestructed, - AggValueSlot::DoesNotNeedGCBarriers, - AggValueSlot::IsNotAliased)); - - // Leave the terminate scope. - CGF.EHStack.popTerminate(); - - // Undo the opaque value mapping. - opaque.pop(); - - // Finally we can call __cxa_begin_catch. - CallBeginCatch(CGF, Exn, true); -} - -/// Begins a catch statement by initializing the catch variable and -/// calling __cxa_begin_catch. -static void BeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *S) { - // We have to be very careful with the ordering of cleanups here: - // C++ [except.throw]p4: - // The destruction [of the exception temporary] occurs - // immediately after the destruction of the object declared in - // the exception-declaration in the handler. - // - // So the precise ordering is: - // 1. Construct catch variable. - // 2. __cxa_begin_catch - // 3. Enter __cxa_end_catch cleanup - // 4. Enter dtor cleanup - // - // We do this by using a slightly abnormal initialization process. - // Delegation sequence: - // - ExitCXXTryStmt opens a RunCleanupsScope - // - EmitAutoVarAlloca creates the variable and debug info - // - InitCatchParam initializes the variable from the exception - // - CallBeginCatch calls __cxa_begin_catch - // - CallBeginCatch enters the __cxa_end_catch cleanup - // - EmitAutoVarCleanups enters the variable destructor cleanup - // - EmitCXXTryStmt emits the code for the catch body - // - EmitCXXTryStmt close the RunCleanupsScope - - VarDecl *CatchParam = S->getExceptionDecl(); - if (!CatchParam) { - llvm::Value *Exn = CGF.getExceptionFromSlot(); - CallBeginCatch(CGF, Exn, true); - return; - } - - // Emit the local. - CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam); - InitCatchParam(CGF, *CatchParam, var.getObjectAddress(CGF), S->getLocStart()); - CGF.EmitAutoVarCleanups(var); -} - /// Emit the structure of the dispatch block for the given catch scope. /// It is an invariant that the dispatch block already exists. static void emitCatchDispatchBlock(CodeGenFunction &CGF, @@ -1252,11 +958,10 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) { RunCleanupsScope CatchScope(*this); // Initialize the catch variable and set up the cleanups. - BeginCatch(*this, C); + CGM.getCXXABI().emitBeginCatch(*this, C); // Emit the PGO counter increment. - RegionCounter CatchCnt = getPGORegionCounter(C); - CatchCnt.beginRegion(Builder); + incrementProfileCounter(C); // Perform the body of the catch. EmitStmt(C->getHandlerBlock()); @@ -1284,9 +989,8 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) { Builder.CreateBr(ContBB); } - RegionCounter ContCnt = getPGORegionCounter(&S); EmitBlock(ContBB); - ContCnt.beginRegion(Builder); + incrementProfileCounter(&S); } namespace { @@ -1480,68 +1184,6 @@ void CodeGenFunction::FinallyInfo::exit(CodeGenFunction &CGF) { CGF.PopCleanupBlock(); } -/// In a terminate landing pad, should we use __clang__call_terminate -/// or just a naked call to std::terminate? -/// -/// __clang_call_terminate calls __cxa_begin_catch, which then allows -/// std::terminate to usefully report something about the -/// violating exception. -static bool useClangCallTerminate(CodeGenModule &CGM) { - // Only do this for Itanium-family ABIs in C++ mode. - return (CGM.getLangOpts().CPlusPlus && - CGM.getTarget().getCXXABI().isItaniumFamily()); -} - -/// Get or define the following function: -/// void @__clang_call_terminate(i8* %exn) nounwind noreturn -/// This code is used only in C++. -static llvm::Constant *getClangCallTerminateFn(CodeGenModule &CGM) { - llvm::FunctionType *fnTy = - llvm::FunctionType::get(CGM.VoidTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); - llvm::Constant *fnRef = - CGM.CreateRuntimeFunction(fnTy, "__clang_call_terminate"); - - llvm::Function *fn = dyn_cast<llvm::Function>(fnRef); - if (fn && fn->empty()) { - fn->setDoesNotThrow(); - fn->setDoesNotReturn(); - - // What we really want is to massively penalize inlining without - // forbidding it completely. The difference between that and - // 'noinline' is negligible. - fn->addFnAttr(llvm::Attribute::NoInline); - - // Allow this function to be shared across translation units, but - // we don't want it to turn into an exported symbol. - fn->setLinkage(llvm::Function::LinkOnceODRLinkage); - fn->setVisibility(llvm::Function::HiddenVisibility); - - // Set up the function. - llvm::BasicBlock *entry = - llvm::BasicBlock::Create(CGM.getLLVMContext(), "", fn); - CGBuilderTy builder(entry); - - // Pull the exception pointer out of the parameter list. - llvm::Value *exn = &*fn->arg_begin(); - - // Call __cxa_begin_catch(exn). - llvm::CallInst *catchCall = builder.CreateCall(getBeginCatchFn(CGM), exn); - catchCall->setDoesNotThrow(); - catchCall->setCallingConv(CGM.getRuntimeCC()); - - // Call std::terminate(). - llvm::CallInst *termCall = builder.CreateCall(getTerminateFn(CGM)); - termCall->setDoesNotThrow(); - termCall->setDoesNotReturn(); - termCall->setCallingConv(CGM.getRuntimeCC()); - - // std::terminate cannot return. - builder.CreateUnreachable(); - } - - return fnRef; -} - llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() { if (TerminateLandingPad) return TerminateLandingPad; @@ -1553,20 +1195,20 @@ llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() { Builder.SetInsertPoint(TerminateLandingPad); // Tell the backend that this is a landing pad. - const EHPersonality &Personality = EHPersonality::get(CGM); - llvm::LandingPadInst *LPadInst = - Builder.CreateLandingPad(llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), - getOpaquePersonalityFn(CGM, Personality), 0); + const EHPersonality &Personality = EHPersonality::get(*this); + + if (!CurFn->hasPersonalityFn()) + CurFn->setPersonalityFn(getOpaquePersonalityFn(CGM, Personality)); + + llvm::LandingPadInst *LPadInst = Builder.CreateLandingPad( + llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), 0); LPadInst->addClause(getCatchAllValue(*this)); - llvm::CallInst *terminateCall; - if (useClangCallTerminate(CGM)) { - // Extract out the exception pointer. - llvm::Value *exn = Builder.CreateExtractValue(LPadInst, 0); - terminateCall = EmitNounwindRuntimeCall(getClangCallTerminateFn(CGM), exn); - } else { - terminateCall = EmitNounwindRuntimeCall(getTerminateFn(CGM)); - } + llvm::Value *Exn = 0; + if (getLangOpts().CPlusPlus) + Exn = Builder.CreateExtractValue(LPadInst, 0); + llvm::CallInst *terminateCall = + CGM.getCXXABI().emitTerminateForUnexpectedException(*this, Exn); terminateCall->setDoesNotReturn(); Builder.CreateUnreachable(); @@ -1586,14 +1228,11 @@ llvm::BasicBlock *CodeGenFunction::getTerminateHandler() { // end of the function by FinishFunction. TerminateHandler = createBasicBlock("terminate.handler"); Builder.SetInsertPoint(TerminateHandler); - llvm::CallInst *terminateCall; - if (useClangCallTerminate(CGM)) { - // Load the exception pointer. - llvm::Value *exn = getExceptionFromSlot(); - terminateCall = EmitNounwindRuntimeCall(getClangCallTerminateFn(CGM), exn); - } else { - terminateCall = EmitNounwindRuntimeCall(getTerminateFn(CGM)); - } + llvm::Value *Exn = 0; + if (getLangOpts().CPlusPlus) + Exn = getExceptionFromSlot(); + llvm::CallInst *terminateCall = + CGM.getCXXABI().emitTerminateForUnexpectedException(*this, Exn); terminateCall->setDoesNotReturn(); Builder.CreateUnreachable(); @@ -1612,15 +1251,14 @@ llvm::BasicBlock *CodeGenFunction::getEHResumeBlock(bool isCleanup) { EHResumeBlock = createBasicBlock("eh.resume"); Builder.SetInsertPoint(EHResumeBlock); - const EHPersonality &Personality = EHPersonality::get(CGM); + const EHPersonality &Personality = EHPersonality::get(*this); // This can always be a call because we necessarily didn't find // anything on the EH stack which needs our help. const char *RethrowName = Personality.CatchallRethrowFn; if (RethrowName != nullptr && !isCleanup) { EmitRuntimeCall(getCatchallRethrowFn(CGM, RethrowName), - getExceptionFromSlot()) - ->setDoesNotReturn(); + getExceptionFromSlot())->setDoesNotReturn(); Builder.CreateUnreachable(); Builder.restoreIP(SavedIP); return EHResumeBlock; @@ -1642,9 +1280,510 @@ llvm::BasicBlock *CodeGenFunction::getEHResumeBlock(bool isCleanup) { } void CodeGenFunction::EmitSEHTryStmt(const SEHTryStmt &S) { - CGM.ErrorUnsupported(&S, "SEH __try"); + EnterSEHTryStmt(S); + { + JumpDest TryExit = getJumpDestInCurrentScope("__try.__leave"); + + SEHTryEpilogueStack.push_back(&TryExit); + EmitStmt(S.getTryBlock()); + SEHTryEpilogueStack.pop_back(); + + if (!TryExit.getBlock()->use_empty()) + EmitBlock(TryExit.getBlock(), /*IsFinished=*/true); + else + delete TryExit.getBlock(); + } + ExitSEHTryStmt(S); +} + +namespace { +struct PerformSEHFinally : EHScopeStack::Cleanup { + llvm::Function *OutlinedFinally; + PerformSEHFinally(llvm::Function *OutlinedFinally) + : OutlinedFinally(OutlinedFinally) {} + + void Emit(CodeGenFunction &CGF, Flags F) override { + ASTContext &Context = CGF.getContext(); + CodeGenModule &CGM = CGF.CGM; + + CallArgList Args; + + // Compute the two argument values. + QualType ArgTys[2] = {Context.UnsignedCharTy, Context.VoidPtrTy}; + llvm::Value *LocalAddrFn = CGM.getIntrinsic(llvm::Intrinsic::localaddress); + llvm::Value *FP = CGF.Builder.CreateCall(LocalAddrFn); + llvm::Value *IsForEH = + llvm::ConstantInt::get(CGF.ConvertType(ArgTys[0]), F.isForEHCleanup()); + Args.add(RValue::get(IsForEH), ArgTys[0]); + Args.add(RValue::get(FP), ArgTys[1]); + + // Arrange a two-arg function info and type. + FunctionProtoType::ExtProtoInfo EPI; + const auto *FPT = cast<FunctionProtoType>( + Context.getFunctionType(Context.VoidTy, ArgTys, EPI)); + const CGFunctionInfo &FnInfo = + CGM.getTypes().arrangeFreeFunctionCall(Args, FPT, + /*chainCall=*/false); + + CGF.EmitCall(FnInfo, OutlinedFinally, ReturnValueSlot(), Args); + } +}; +} + +namespace { +/// Find all local variable captures in the statement. +struct CaptureFinder : ConstStmtVisitor<CaptureFinder> { + CodeGenFunction &ParentCGF; + const VarDecl *ParentThis; + SmallVector<const VarDecl *, 4> Captures; + llvm::Value *SEHCodeSlot = nullptr; + CaptureFinder(CodeGenFunction &ParentCGF, const VarDecl *ParentThis) + : ParentCGF(ParentCGF), ParentThis(ParentThis) {} + + // Return true if we need to do any capturing work. + bool foundCaptures() { + return !Captures.empty() || SEHCodeSlot; + } + + void Visit(const Stmt *S) { + // See if this is a capture, then recurse. + ConstStmtVisitor<CaptureFinder>::Visit(S); + for (const Stmt *Child : S->children()) + if (Child) + Visit(Child); + } + + void VisitDeclRefExpr(const DeclRefExpr *E) { + // If this is already a capture, just make sure we capture 'this'. + if (E->refersToEnclosingVariableOrCapture()) { + Captures.push_back(ParentThis); + return; + } + + const auto *D = dyn_cast<VarDecl>(E->getDecl()); + if (D && D->isLocalVarDeclOrParm() && D->hasLocalStorage()) + Captures.push_back(D); + } + + void VisitCXXThisExpr(const CXXThisExpr *E) { + Captures.push_back(ParentThis); + } + + void VisitCallExpr(const CallExpr *E) { + // We only need to add parent frame allocations for these builtins in x86. + if (ParentCGF.getTarget().getTriple().getArch() != llvm::Triple::x86) + return; + + unsigned ID = E->getBuiltinCallee(); + switch (ID) { + case Builtin::BI__exception_code: + case Builtin::BI_exception_code: + // This is the simple case where we are the outermost finally. All we + // have to do here is make sure we escape this and recover it in the + // outlined handler. + if (!SEHCodeSlot) + SEHCodeSlot = ParentCGF.SEHCodeSlotStack.back(); + break; + } + } +}; +} + +llvm::Value *CodeGenFunction::recoverAddrOfEscapedLocal( + CodeGenFunction &ParentCGF, llvm::Value *ParentVar, llvm::Value *ParentFP) { + llvm::CallInst *RecoverCall = nullptr; + CGBuilderTy Builder(AllocaInsertPt); + if (auto *ParentAlloca = dyn_cast<llvm::AllocaInst>(ParentVar)) { + // Mark the variable escaped if nobody else referenced it and compute the + // localescape index. + auto InsertPair = ParentCGF.EscapedLocals.insert( + std::make_pair(ParentAlloca, ParentCGF.EscapedLocals.size())); + int FrameEscapeIdx = InsertPair.first->second; + // call i8* @llvm.localrecover(i8* bitcast(@parentFn), i8* %fp, i32 N) + llvm::Function *FrameRecoverFn = llvm::Intrinsic::getDeclaration( + &CGM.getModule(), llvm::Intrinsic::localrecover); + llvm::Constant *ParentI8Fn = + llvm::ConstantExpr::getBitCast(ParentCGF.CurFn, Int8PtrTy); + RecoverCall = Builder.CreateCall( + FrameRecoverFn, {ParentI8Fn, ParentFP, + llvm::ConstantInt::get(Int32Ty, FrameEscapeIdx)}); + + } else { + // If the parent didn't have an alloca, we're doing some nested outlining. + // Just clone the existing localrecover call, but tweak the FP argument to + // use our FP value. All other arguments are constants. + auto *ParentRecover = + cast<llvm::IntrinsicInst>(ParentVar->stripPointerCasts()); + assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::localrecover && + "expected alloca or localrecover in parent LocalDeclMap"); + RecoverCall = cast<llvm::CallInst>(ParentRecover->clone()); + RecoverCall->setArgOperand(1, ParentFP); + RecoverCall->insertBefore(AllocaInsertPt); + } + + // Bitcast the variable, rename it, and insert it in the local decl map. + llvm::Value *ChildVar = + Builder.CreateBitCast(RecoverCall, ParentVar->getType()); + ChildVar->setName(ParentVar->getName()); + return ChildVar; +} + +void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, + const Stmt *OutlinedStmt, + bool IsFilter) { + // Find all captures in the Stmt. + CaptureFinder Finder(ParentCGF, ParentCGF.CXXABIThisDecl); + Finder.Visit(OutlinedStmt); + + // We can exit early on x86_64 when there are no captures. We just have to + // save the exception code in filters so that __exception_code() works. + if (!Finder.foundCaptures() && + CGM.getTarget().getTriple().getArch() != llvm::Triple::x86) { + if (IsFilter) + EmitSEHExceptionCodeSave(ParentCGF, nullptr, nullptr); + return; + } + + llvm::Value *EntryEBP = nullptr; + llvm::Value *ParentFP; + if (IsFilter && CGM.getTarget().getTriple().getArch() == llvm::Triple::x86) { + // 32-bit SEH filters need to be careful about FP recovery. The end of the + // EH registration is passed in as the EBP physical register. We can + // recover that with llvm.frameaddress(1), and adjust that to recover the + // parent's true frame pointer. + CGBuilderTy Builder(AllocaInsertPt); + EntryEBP = Builder.CreateCall( + CGM.getIntrinsic(llvm::Intrinsic::frameaddress), {Builder.getInt32(1)}); + llvm::Function *RecoverFPIntrin = + CGM.getIntrinsic(llvm::Intrinsic::x86_seh_recoverfp); + llvm::Constant *ParentI8Fn = + llvm::ConstantExpr::getBitCast(ParentCGF.CurFn, Int8PtrTy); + ParentFP = Builder.CreateCall(RecoverFPIntrin, {ParentI8Fn, EntryEBP}); + } else { + // Otherwise, for x64 and 32-bit finally functions, the parent FP is the + // second parameter. + auto AI = CurFn->arg_begin(); + ++AI; + ParentFP = AI; + } + + // Create llvm.localrecover calls for all captures. + for (const VarDecl *VD : Finder.Captures) { + if (isa<ImplicitParamDecl>(VD)) { + CGM.ErrorUnsupported(VD, "'this' captured by SEH"); + CXXThisValue = llvm::UndefValue::get(ConvertTypeForMem(VD->getType())); + continue; + } + if (VD->getType()->isVariablyModifiedType()) { + CGM.ErrorUnsupported(VD, "VLA captured by SEH"); + continue; + } + assert((isa<ImplicitParamDecl>(VD) || VD->isLocalVarDeclOrParm()) && + "captured non-local variable"); + + // If this decl hasn't been declared yet, it will be declared in the + // OutlinedStmt. + auto I = ParentCGF.LocalDeclMap.find(VD); + if (I == ParentCGF.LocalDeclMap.end()) + continue; + llvm::Value *ParentVar = I->second; + + LocalDeclMap[VD] = + recoverAddrOfEscapedLocal(ParentCGF, ParentVar, ParentFP); + } + + if (Finder.SEHCodeSlot) { + SEHCodeSlotStack.push_back( + recoverAddrOfEscapedLocal(ParentCGF, Finder.SEHCodeSlot, ParentFP)); + } + + if (IsFilter) + EmitSEHExceptionCodeSave(ParentCGF, ParentFP, EntryEBP); +} + +/// Arrange a function prototype that can be called by Windows exception +/// handling personalities. On Win64, the prototype looks like: +/// RetTy func(void *EHPtrs, void *ParentFP); +void CodeGenFunction::startOutlinedSEHHelper(CodeGenFunction &ParentCGF, + bool IsFilter, + const Stmt *OutlinedStmt) { + SourceLocation StartLoc = OutlinedStmt->getLocStart(); + + // Get the mangled function name. + SmallString<128> Name; + { + llvm::raw_svector_ostream OS(Name); + const Decl *ParentCodeDecl = ParentCGF.CurCodeDecl; + const NamedDecl *Parent = dyn_cast_or_null<NamedDecl>(ParentCodeDecl); + assert(Parent && "FIXME: handle unnamed decls (lambdas, blocks) with SEH"); + MangleContext &Mangler = CGM.getCXXABI().getMangleContext(); + if (IsFilter) + Mangler.mangleSEHFilterExpression(Parent, OS); + else + Mangler.mangleSEHFinallyBlock(Parent, OS); + } + + FunctionArgList Args; + if (CGM.getTarget().getTriple().getArch() != llvm::Triple::x86 || !IsFilter) { + // All SEH finally functions take two parameters. Win64 filters take two + // parameters. Win32 filters take no parameters. + if (IsFilter) { + Args.push_back(ImplicitParamDecl::Create( + getContext(), nullptr, StartLoc, + &getContext().Idents.get("exception_pointers"), + getContext().VoidPtrTy)); + } else { + Args.push_back(ImplicitParamDecl::Create( + getContext(), nullptr, StartLoc, + &getContext().Idents.get("abnormal_termination"), + getContext().UnsignedCharTy)); + } + Args.push_back(ImplicitParamDecl::Create( + getContext(), nullptr, StartLoc, + &getContext().Idents.get("frame_pointer"), getContext().VoidPtrTy)); + } + + QualType RetTy = IsFilter ? getContext().LongTy : getContext().VoidTy; + + llvm::Function *ParentFn = ParentCGF.CurFn; + const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionDeclaration( + RetTy, Args, FunctionType::ExtInfo(), /*isVariadic=*/false); + + llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo); + llvm::Function *Fn = llvm::Function::Create( + FnTy, llvm::GlobalValue::InternalLinkage, Name.str(), &CGM.getModule()); + // The filter is either in the same comdat as the function, or it's internal. + if (llvm::Comdat *C = ParentFn->getComdat()) { + Fn->setComdat(C); + } else if (ParentFn->hasWeakLinkage() || ParentFn->hasLinkOnceLinkage()) { + llvm::Comdat *C = CGM.getModule().getOrInsertComdat(ParentFn->getName()); + ParentFn->setComdat(C); + Fn->setComdat(C); + } else { + Fn->setLinkage(llvm::GlobalValue::InternalLinkage); + } + + IsOutlinedSEHHelper = true; + + StartFunction(GlobalDecl(), RetTy, Fn, FnInfo, Args, + OutlinedStmt->getLocStart(), OutlinedStmt->getLocStart()); + + CGM.SetLLVMFunctionAttributes(nullptr, FnInfo, CurFn); + EmitCapturedLocals(ParentCGF, OutlinedStmt, IsFilter); +} + +/// Create a stub filter function that will ultimately hold the code of the +/// filter expression. The EH preparation passes in LLVM will outline the code +/// from the main function body into this stub. +llvm::Function * +CodeGenFunction::GenerateSEHFilterFunction(CodeGenFunction &ParentCGF, + const SEHExceptStmt &Except) { + const Expr *FilterExpr = Except.getFilterExpr(); + startOutlinedSEHHelper(ParentCGF, true, FilterExpr); + + // Emit the original filter expression, convert to i32, and return. + llvm::Value *R = EmitScalarExpr(FilterExpr); + R = Builder.CreateIntCast(R, ConvertType(getContext().LongTy), + FilterExpr->getType()->isSignedIntegerType()); + Builder.CreateStore(R, ReturnValue); + + FinishFunction(FilterExpr->getLocEnd()); + + return CurFn; +} + +llvm::Function * +CodeGenFunction::GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF, + const SEHFinallyStmt &Finally) { + const Stmt *FinallyBlock = Finally.getBlock(); + startOutlinedSEHHelper(ParentCGF, false, FinallyBlock); + + // Mark finally block calls as nounwind and noinline to make LLVM's job a + // little easier. + // FIXME: Remove these restrictions in the future. + CurFn->addFnAttr(llvm::Attribute::NoUnwind); + CurFn->addFnAttr(llvm::Attribute::NoInline); + + // Emit the original filter expression, convert to i32, and return. + EmitStmt(FinallyBlock); + + FinishFunction(FinallyBlock->getLocEnd()); + + return CurFn; +} + +void CodeGenFunction::EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF, + llvm::Value *ParentFP, + llvm::Value *EntryEBP) { + // Get the pointer to the EXCEPTION_POINTERS struct. This is returned by the + // __exception_info intrinsic. + if (CGM.getTarget().getTriple().getArch() != llvm::Triple::x86) { + // On Win64, the info is passed as the first parameter to the filter. + auto AI = CurFn->arg_begin(); + SEHInfo = AI; + SEHCodeSlotStack.push_back( + CreateMemTemp(getContext().IntTy, "__exception_code")); + } else { + // On Win32, the EBP on entry to the filter points to the end of an + // exception registration object. It contains 6 32-bit fields, and the info + // pointer is stored in the second field. So, GEP 20 bytes backwards and + // load the pointer. + SEHInfo = Builder.CreateConstInBoundsGEP1_32(Int8Ty, EntryEBP, -20); + SEHInfo = Builder.CreateBitCast(SEHInfo, Int8PtrTy->getPointerTo()); + SEHInfo = Builder.CreateLoad(Int8PtrTy, SEHInfo); + SEHCodeSlotStack.push_back(recoverAddrOfEscapedLocal( + ParentCGF, ParentCGF.SEHCodeSlotStack.back(), ParentFP)); + } + + // Save the exception code in the exception slot to unify exception access in + // the filter function and the landing pad. + // struct EXCEPTION_POINTERS { + // EXCEPTION_RECORD *ExceptionRecord; + // CONTEXT *ContextRecord; + // }; + // int exceptioncode = exception_pointers->ExceptionRecord->ExceptionCode; + llvm::Type *RecordTy = CGM.Int32Ty->getPointerTo(); + llvm::Type *PtrsTy = llvm::StructType::get(RecordTy, CGM.VoidPtrTy, nullptr); + llvm::Value *Ptrs = Builder.CreateBitCast(SEHInfo, PtrsTy->getPointerTo()); + llvm::Value *Rec = Builder.CreateStructGEP(PtrsTy, Ptrs, 0); + Rec = Builder.CreateLoad(Rec); + llvm::Value *Code = Builder.CreateLoad(Rec); + assert(!SEHCodeSlotStack.empty() && "emitting EH code outside of __except"); + Builder.CreateStore(Code, SEHCodeSlotStack.back()); +} + +llvm::Value *CodeGenFunction::EmitSEHExceptionInfo() { + // Sema should diagnose calling this builtin outside of a filter context, but + // don't crash if we screw up. + if (!SEHInfo) + return llvm::UndefValue::get(Int8PtrTy); + assert(SEHInfo->getType() == Int8PtrTy); + return SEHInfo; +} + +llvm::Value *CodeGenFunction::EmitSEHExceptionCode() { + assert(!SEHCodeSlotStack.empty() && "emitting EH code outside of __except"); + return Builder.CreateLoad(Int32Ty, SEHCodeSlotStack.back()); +} + +llvm::Value *CodeGenFunction::EmitSEHAbnormalTermination() { + // Abnormal termination is just the first parameter to the outlined finally + // helper. + auto AI = CurFn->arg_begin(); + return Builder.CreateZExt(&*AI, Int32Ty); +} + +void CodeGenFunction::EnterSEHTryStmt(const SEHTryStmt &S) { + CodeGenFunction HelperCGF(CGM, /*suppressNewContext=*/true); + if (const SEHFinallyStmt *Finally = S.getFinallyHandler()) { + // Outline the finally block. + llvm::Function *FinallyFunc = + HelperCGF.GenerateSEHFinallyFunction(*this, *Finally); + + // Push a cleanup for __finally blocks. + EHStack.pushCleanup<PerformSEHFinally>(NormalAndEHCleanup, FinallyFunc); + return; + } + + // Otherwise, we must have an __except block. + const SEHExceptStmt *Except = S.getExceptHandler(); + assert(Except); + EHCatchScope *CatchScope = EHStack.pushCatch(1); + SEHCodeSlotStack.push_back( + CreateMemTemp(getContext().IntTy, "__exception_code")); + + // If the filter is known to evaluate to 1, then we can use the clause + // "catch i8* null". We can't do this on x86 because the filter has to save + // the exception code. + llvm::Constant *C = + CGM.EmitConstantExpr(Except->getFilterExpr(), getContext().IntTy, this); + if (CGM.getTarget().getTriple().getArch() != llvm::Triple::x86 && C && + C->isOneValue()) { + CatchScope->setCatchAllHandler(0, createBasicBlock("__except")); + return; + } + + // In general, we have to emit an outlined filter function. Use the function + // in place of the RTTI typeinfo global that C++ EH uses. + llvm::Function *FilterFunc = + HelperCGF.GenerateSEHFilterFunction(*this, *Except); + llvm::Constant *OpaqueFunc = + llvm::ConstantExpr::getBitCast(FilterFunc, Int8PtrTy); + CatchScope->setHandler(0, OpaqueFunc, createBasicBlock("__except")); +} + +void CodeGenFunction::ExitSEHTryStmt(const SEHTryStmt &S) { + // Just pop the cleanup if it's a __finally block. + if (S.getFinallyHandler()) { + PopCleanupBlock(); + return; + } + + // Otherwise, we must have an __except block. + const SEHExceptStmt *Except = S.getExceptHandler(); + assert(Except && "__try must have __finally xor __except"); + EHCatchScope &CatchScope = cast<EHCatchScope>(*EHStack.begin()); + + // Don't emit the __except block if the __try block lacked invokes. + // TODO: Model unwind edges from instructions, either with iload / istore or + // a try body function. + if (!CatchScope.hasEHBranches()) { + CatchScope.clearHandlerBlocks(); + EHStack.popCatch(); + SEHCodeSlotStack.pop_back(); + return; + } + + // The fall-through block. + llvm::BasicBlock *ContBB = createBasicBlock("__try.cont"); + + // We just emitted the body of the __try; jump to the continue block. + if (HaveInsertPoint()) + Builder.CreateBr(ContBB); + + // Check if our filter function returned true. + emitCatchDispatchBlock(*this, CatchScope); + + // Grab the block before we pop the handler. + llvm::BasicBlock *ExceptBB = CatchScope.getHandler(0).Block; + EHStack.popCatch(); + + EmitBlockAfterUses(ExceptBB); + + // On Win64, the exception pointer is the exception code. Copy it to the slot. + if (CGM.getTarget().getTriple().getArch() != llvm::Triple::x86) { + llvm::Value *Code = + Builder.CreatePtrToInt(getExceptionFromSlot(), IntPtrTy); + Code = Builder.CreateTrunc(Code, Int32Ty); + Builder.CreateStore(Code, SEHCodeSlotStack.back()); + } + + // Emit the __except body. + EmitStmt(Except->getBlock()); + + // End the lifetime of the exception code. + SEHCodeSlotStack.pop_back(); + + if (HaveInsertPoint()) + Builder.CreateBr(ContBB); + + EmitBlock(ContBB); } void CodeGenFunction::EmitSEHLeaveStmt(const SEHLeaveStmt &S) { - CGM.ErrorUnsupported(&S, "SEH __leave"); + // If this code is reachable then emit a stop point (if generating + // debug info). We have to do this ourselves because we are on the + // "simple" statement path. + if (HaveInsertPoint()) + EmitStopPoint(&S); + + // This must be a __leave from a __finally block, which we warn on and is UB. + // Just emit unreachable. + if (!isSEHTryScope()) { + Builder.CreateUnreachable(); + Builder.ClearInsertionPoint(); + return; + } + + EmitBranchThroughCleanup(*SEHTryEpilogueStack.back()); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp index ce7679c..9c3dfe5 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExpr.cpp @@ -31,6 +31,7 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" #include "llvm/Support/ConvertUTF.h" +#include "llvm/Support/MathExtras.h" using namespace clang; using namespace CodeGen; @@ -300,9 +301,26 @@ createReferenceTemporary(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, const Expr *Inner) { switch (M->getStorageDuration()) { case SD_FullExpression: - case SD_Automatic: - return CGF.CreateMemTemp(Inner->getType(), "ref.tmp"); - + case SD_Automatic: { + // If we have a constant temporary array or record try to promote it into a + // constant global under the same rules a normal constant would've been + // promoted. This is easier on the optimizer and generally emits fewer + // instructions. + QualType Ty = Inner->getType(); + if (CGF.CGM.getCodeGenOpts().MergeAllConstants && + (Ty->isArrayType() || Ty->isRecordType()) && + CGF.CGM.isTypeConstant(Ty, true)) + if (llvm::Constant *Init = CGF.CGM.EmitConstantExpr(Inner, Ty, &CGF)) { + auto *GV = new llvm::GlobalVariable( + CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, + llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp"); + GV->setAlignment( + CGF.getContext().getTypeAlignInChars(Ty).getQuantity()); + // FIXME: Should we put the new global into a COMDAT? + return GV; + } + return CGF.CreateMemTemp(Ty, "ref.tmp"); + } case SD_Thread: case SD_Static: return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner); @@ -324,14 +342,15 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { M->getType().getObjCLifetime() != Qualifiers::OCL_None && M->getType().getObjCLifetime() != Qualifiers::OCL_ExplicitNone) { llvm::Value *Object = createReferenceTemporary(*this, M, E); - LValue RefTempDst = MakeAddrLValue(Object, M->getType()); - if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) { + Object = llvm::ConstantExpr::getBitCast( + Var, ConvertTypeForMem(E->getType())->getPointerTo()); // We should not have emitted the initializer for this temporary as a // constant. assert(!Var->hasInitializer()); Var->setInitializer(CGM.EmitNullConstant(E->getType())); } + LValue RefTempDst = MakeAddrLValue(Object, M->getType()); switch (getEvaluationKind(E->getType())) { default: llvm_unreachable("expected scalar or aggregate expression"); @@ -370,8 +389,11 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { // Create and initialize the reference temporary. llvm::Value *Object = createReferenceTemporary(*this, M, E); if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) { - // If the temporary is a global and has a constant initializer, we may - // have already initialized it. + Object = llvm::ConstantExpr::getBitCast( + Var, ConvertTypeForMem(E->getType())->getPointerTo()); + // If the temporary is a global and has a constant initializer or is a + // constant temporary that we promoted to a global, we may have already + // initialized it. if (!Var->hasInitializer()) { Var->setInitializer(CGM.EmitNullConstant(E->getType())); EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true); @@ -478,7 +500,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, SanitizerScope SanScope(this); - SmallVector<std::pair<llvm::Value *, SanitizerKind>, 3> Checks; + SmallVector<std::pair<llvm::Value *, SanitizerMask>, 3> Checks; llvm::BasicBlock *Done = nullptr; bool AllowNullPointers = TCK == TCK_DowncastPointer || TCK == TCK_Upcast || @@ -513,7 +535,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *Min = Builder.getFalse(); llvm::Value *CastAddr = Builder.CreateBitCast(Address, Int8PtrTy); llvm::Value *LargeEnough = - Builder.CreateICmpUGE(Builder.CreateCall2(F, CastAddr, Min), + Builder.CreateICmpUGE(Builder.CreateCall(F, {CastAddr, Min}), llvm::ConstantInt::get(IntPtrTy, Size)); Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize)); } @@ -807,6 +829,7 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { /// length type, this is not possible. /// LValue CodeGenFunction::EmitLValue(const Expr *E) { + ApplyDebugLocation DL(*this, E); switch (E->getStmtClass()) { default: return EmitUnsupportedLValue(E, "l-value expression"); @@ -819,10 +842,14 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E)); case Expr::BinaryOperatorClass: return EmitBinaryOperatorLValue(cast<BinaryOperator>(E)); - case Expr::CompoundAssignOperatorClass: - if (!E->getType()->isAnyComplexType()) + case Expr::CompoundAssignOperatorClass: { + QualType Ty = E->getType(); + if (const AtomicType *AT = Ty->getAs<AtomicType>()) + Ty = AT->getValueType(); + if (!Ty->isAnyComplexType()) return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E)); return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E)); + } case Expr::CallExprClass: case Expr::CXXMemberCallExprClass: case Expr::CXXOperatorCallExprClass: @@ -1135,7 +1162,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile, } // Atomic operations have to be done on integral types. - if (Ty->isAtomicType()) { + if (Ty->isAtomicType() || typeIsSuitableForInlineAtomic(Ty, Volatile)) { LValue lvalue = LValue::MakeAddr(Addr, Ty, CharUnits::fromQuantity(Alignment), getContext(), TBAAInfo); @@ -1178,7 +1205,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile, EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty) }; - SanitizerKind Kind = NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool; + SanitizerMask Kind = NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool; EmitCheck(std::make_pair(Check, Kind), "load_invalid_value", StaticArgs, EmitCheckValue(Load)); } @@ -1254,7 +1281,8 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr, Value = EmitToMemory(Value, Ty); - if (Ty->isAtomicType()) { + if (Ty->isAtomicType() || + (!isInit && typeIsSuitableForInlineAtomic(Ty, Volatile))) { EmitAtomicStore(RValue::get(Value), LValue::MakeAddr(Addr, Ty, CharUnits::fromQuantity(Alignment), @@ -1328,14 +1356,15 @@ RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) { RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV) { const CGBitFieldInfo &Info = LV.getBitFieldInfo(); + CharUnits Align = LV.getAlignment().alignmentAtOffset(Info.StorageOffset); // Get the output type. llvm::Type *ResLTy = ConvertType(LV.getType()); llvm::Value *Ptr = LV.getBitFieldAddr(); - llvm::Value *Val = Builder.CreateLoad(Ptr, LV.isVolatileQualified(), - "bf.load"); - cast<llvm::LoadInst>(Val)->setAlignment(Info.StorageAlignment); + llvm::Value *Val = Builder.CreateAlignedLoad(Ptr, Align.getQuantity(), + LV.isVolatileQualified(), + "bf.load"); if (Info.IsSigned) { assert(static_cast<unsigned>(Info.Offset + Info.Size) <= Info.StorageSize); @@ -1531,6 +1560,7 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, llvm::Value **Result) { const CGBitFieldInfo &Info = Dst.getBitFieldInfo(); + CharUnits Align = Dst.getAlignment().alignmentAtOffset(Info.StorageOffset); llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType()); llvm::Value *Ptr = Dst.getBitFieldAddr(); @@ -1547,9 +1577,9 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, // and mask together with source before storing. if (Info.StorageSize != Info.Size) { assert(Info.StorageSize > Info.Size && "Invalid bitfield size."); - llvm::Value *Val = Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), - "bf.load"); - cast<llvm::LoadInst>(Val)->setAlignment(Info.StorageAlignment); + llvm::Value *Val = Builder.CreateAlignedLoad(Ptr, Align.getQuantity(), + Dst.isVolatileQualified(), + "bf.load"); // Mask the source value as needed. if (!hasBooleanRepresentation(Dst.getType())) @@ -1575,9 +1605,8 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, } // Write the new value back out. - llvm::StoreInst *Store = Builder.CreateStore(SrcVal, Ptr, - Dst.isVolatileQualified()); - Store->setAlignment(Info.StorageAlignment); + Builder.CreateAlignedStore(SrcVal, Ptr, Align.getQuantity(), + Dst.isVolatileQualified()); // Return the new value of the bit-field, if requested. if (Result) { @@ -1692,8 +1721,8 @@ void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) { llvm::Value *Value = Src.getScalarVal(); if (OrigTy->isPointerTy()) Value = Builder.CreatePtrToInt(Value, Ty); - Builder.CreateCall2(F, llvm::MetadataAsValue::get(Ty->getContext(), RegName), - Value); + Builder.CreateCall( + F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value}); } // setObjCGCLValueClass - sets class of the lvalue for the purpose of @@ -1807,7 +1836,7 @@ EmitBitCastOfLValueToProperType(CodeGenFunction &CGF, static LValue EmitThreadPrivateVarDeclLValue( CodeGenFunction &CGF, const VarDecl *VD, QualType T, llvm::Value *V, llvm::Type *RealVarTy, CharUnits Alignment, SourceLocation Loc) { - V = CGF.CGM.getOpenMPRuntime().getOMPAddrOfThreadPrivate(CGF, VD, V, Loc); + V = CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, V, Loc); V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy); return CGF.MakeAddrLValue(V, T, Alignment); } @@ -2050,9 +2079,8 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) { assert(E->getSubExpr()->getType()->isAnyComplexType()); unsigned Idx = E->getOpcode() == UO_Imag; - return MakeAddrLValue(Builder.CreateStructGEP(LV.getAddress(), - Idx, "idx"), - ExprTy); + return MakeAddrLValue( + Builder.CreateStructGEP(nullptr, LV.getAddress(), Idx, "idx"), ExprTy); } case UO_PreInc: case UO_PreDec: { @@ -2217,7 +2245,8 @@ enum class CheckRecoverableKind { }; } -static CheckRecoverableKind getRecoverableKind(SanitizerKind Kind) { +static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) { + assert(llvm::countPopulation(Kind) == 1); switch (Kind) { case SanitizerKind::Vptr: return CheckRecoverableKind::AlwaysRecoverable; @@ -2264,7 +2293,7 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, } void CodeGenFunction::EmitCheck( - ArrayRef<std::pair<llvm::Value *, SanitizerKind>> Checked, + ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked, StringRef CheckName, ArrayRef<llvm::Constant *> StaticArgs, ArrayRef<llvm::Value *> DynamicArgs) { assert(IsSanitizerScope); @@ -2272,15 +2301,24 @@ void CodeGenFunction::EmitCheck( llvm::Value *FatalCond = nullptr; llvm::Value *RecoverableCond = nullptr; + llvm::Value *TrapCond = nullptr; for (int i = 0, n = Checked.size(); i < n; ++i) { llvm::Value *Check = Checked[i].first; + // -fsanitize-trap= overrides -fsanitize-recover=. llvm::Value *&Cond = - CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second) - ? RecoverableCond - : FatalCond; + CGM.getCodeGenOpts().SanitizeTrap.has(Checked[i].second) + ? TrapCond + : CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second) + ? RecoverableCond + : FatalCond; Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check; } + if (TrapCond) + EmitTrapCheck(TrapCond); + if (!FatalCond && !RecoverableCond) + return; + llvm::Value *JointCond; if (FatalCond && RecoverableCond) JointCond = Builder.CreateAnd(FatalCond, RecoverableCond); @@ -2298,15 +2336,6 @@ void CodeGenFunction::EmitCheck( } #endif - if (CGM.getCodeGenOpts().SanitizeUndefinedTrapOnError) { - assert(RecoverKind != CheckRecoverableKind::AlwaysRecoverable && - "Runtime call required for AlwaysRecoverable kind!"); - // Assume that -fsanitize-undefined-trap-on-error overrides - // -fsanitize-recover= options, as we can only print meaningful error - // message and recover if we have a runtime support. - return EmitTrapCheck(JointCond); - } - llvm::BasicBlock *Cont = createBasicBlock("cont"); llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName); llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers); @@ -2375,8 +2404,7 @@ void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked) { TrapBB = createBasicBlock("trap"); Builder.CreateCondBr(Checked, Cont, TrapBB); EmitBlock(TrapBB); - llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::trap); - llvm::CallInst *TrapCall = Builder.CreateCall(F); + llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap); TrapCall->setDoesNotReturn(); TrapCall->setDoesNotThrow(); Builder.CreateUnreachable(); @@ -2387,6 +2415,17 @@ void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked) { EmitBlock(Cont); } +llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) { + llvm::CallInst *TrapCall = Builder.CreateCall(CGM.getIntrinsic(IntrID)); + + if (!CGM.getCodeGenOpts().TrapFuncName.empty()) + TrapCall->addAttribute(llvm::AttributeSet::FunctionIndex, + "trap-func-name", + CGM.getCodeGenOpts().TrapFuncName); + + return TrapCall; +} + /// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an /// array to pointer, return the array subexpression. static const Expr *isSimpleArrayDecayOperand(const Expr *E) { @@ -2648,7 +2687,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, unsigned Idx = RL.getLLVMFieldNo(field); if (Idx != 0) // For structs, we GEP to the field that the record layout suggests. - Addr = Builder.CreateStructGEP(Addr, Idx, field->getName()); + Addr = Builder.CreateStructGEP(nullptr, Addr, Idx, field->getName()); // Get the access type. llvm::Type *PtrTy = llvm::Type::getIntNPtrTy( getLLVMContext(), Info.StorageSize, @@ -2683,7 +2722,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, } else { // For structs, we GEP to the field that the record layout suggests. unsigned idx = CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field); - addr = Builder.CreateStructGEP(addr, idx, field->getName()); + addr = Builder.CreateStructGEP(nullptr, addr, idx, field->getName()); // If this is a reference field, load the reference right now. if (const ReferenceType *refType = type->getAs<ReferenceType>()) { @@ -2762,7 +2801,7 @@ CodeGenFunction::EmitLValueForFieldInitialization(LValue Base, const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(Field->getParent()); unsigned idx = RL.getLLVMFieldNo(Field); - llvm::Value *V = Builder.CreateStructGEP(Base.getAddress(), idx); + llvm::Value *V = Builder.CreateStructGEP(nullptr, Base.getAddress(), idx); assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs"); // Make sure that the address is pointing to the right type. This is critical @@ -2834,7 +2873,6 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) { } OpaqueValueMapping binding(*this, expr); - RegionCounter Cnt = getPGORegionCounter(expr); const Expr *condExpr = expr->getCond(); bool CondExprBool; @@ -2845,7 +2883,7 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) { if (!ContainsLabel(dead)) { // If the true case is live, we need to track its region. if (CondExprBool) - Cnt.beginRegion(Builder); + incrementProfileCounter(expr); return EmitLValue(live); } } @@ -2855,11 +2893,11 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) { llvm::BasicBlock *contBlock = createBasicBlock("cond.end"); ConditionalEvaluation eval(*this); - EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, Cnt.getCount()); + EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, getProfileCount(expr)); // Any temporaries created here are conditional. EmitBlock(lhsBlock); - Cnt.beginRegion(Builder); + incrementProfileCounter(expr); eval.begin(*this); Optional<LValue> lhs = EmitLValueOrThrowExpression(*this, expr->getTrueExpr()); @@ -3007,6 +3045,10 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), Derived, E->getType()); + if (SanOpts.has(SanitizerKind::CFIDerivedCast)) + EmitVTablePtrCheckForCast(E->getType(), Derived, /*MayBeNull=*/false, + CFITCK_DerivedCast, E->getLocStart()); + return MakeAddrLValue(Derived, E->getType()); } case CK_LValueBitCast: { @@ -3016,6 +3058,11 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { LValue LV = EmitLValue(E->getSubExpr()); llvm::Value *V = Builder.CreateBitCast(LV.getAddress(), ConvertType(CE->getTypeAsWritten())); + + if (SanOpts.has(SanitizerKind::CFIUnrelatedCast)) + EmitVTablePtrCheckForCast(E->getType(), V, /*MayBeNull=*/false, + CFITCK_UnrelatedCast, E->getLocStart()); + return MakeAddrLValue(V, E->getType()); } case CK_ObjCObjectLValueCast: { @@ -3059,16 +3106,6 @@ RValue CodeGenFunction::EmitRValueForField(LValue LV, RValue CodeGenFunction::EmitCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue) { - // Force column info to be generated so we can differentiate - // multiple call sites on the same line in the debug info. - // FIXME: This is insufficient. Two calls coming from the same macro - // expansion will still get the same line/column and break debug info. It's - // possible that LLVM can be fixed to not rely on this uniqueness, at which - // point this workaround can be removed. - ApplyDebugLocation DL(*this, E->getLocStart(), - E->getDirectCallee() && - E->getDirectCallee()->isInlineSpecified()); - // Builtins never have block type. if (E->getCallee()->getType()->isBlockPointerType()) return EmitBlockCallExpr(E, ReturnValue); @@ -3202,7 +3239,7 @@ LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) { if (!RV.isScalar()) return MakeAddrLValue(RV.getAggregateAddr(), E->getType()); - assert(E->getCallReturnType()->isReferenceType() && + assert(E->getCallReturnType(getContext())->isReferenceType() && "Can't have a scalar return unless the return type is a " "reference type!"); @@ -3328,16 +3365,6 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, const auto *FnType = cast<FunctionType>(cast<PointerType>(CalleeType)->getPointeeType()); - // Force column info to differentiate multiple inlined call sites on - // the same line, analoguous to EmitCallExpr. - // FIXME: This is insufficient. Two calls coming from the same macro expansion - // will still get the same line/column and break debug info. It's possible - // that LLVM can be fixed to not rely on this uniqueness, at which point this - // workaround can be removed. - bool ForceColumnInfo = false; - if (const FunctionDecl* FD = dyn_cast_or_null<const FunctionDecl>(TargetDecl)) - ForceColumnInfo = FD->isInlineSpecified(); - if (getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function) && (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) { if (llvm::Constant *PrefixSig = @@ -3355,7 +3382,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, llvm::Value *CalleePrefixStruct = Builder.CreateBitCast( Callee, llvm::PointerType::getUnqual(PrefixStructTy)); llvm::Value *CalleeSigPtr = - Builder.CreateConstGEP2_32(CalleePrefixStruct, 0, 0); + Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 0); llvm::Value *CalleeSig = Builder.CreateLoad(CalleeSigPtr); llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig); @@ -3365,7 +3392,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, EmitBlock(TypeCheck); llvm::Value *CalleeRTTIPtr = - Builder.CreateConstGEP2_32(CalleePrefixStruct, 0, 1); + Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 1); llvm::Value *CalleeRTTI = Builder.CreateLoad(CalleeRTTIPtr); llvm::Value *CalleeRTTIMatch = Builder.CreateICmpEQ(CalleeRTTI, FTRTTIConst); @@ -3386,8 +3413,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)), CGM.getContext().VoidPtrTy); EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arg_begin(), - E->arg_end(), E->getDirectCallee(), /*ParamsToSkip*/ 0, - ForceColumnInfo); + E->arg_end(), E->getDirectCallee(), /*ParamsToSkip*/ 0); const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall( Args, FnType, /*isChainCall=*/Chain); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp index 4cf94c0..883b76b 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprAgg.cpp @@ -34,6 +34,7 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> { CodeGenFunction &CGF; CGBuilderTy &Builder; AggValueSlot Dest; + bool IsResultUnused; /// We want to use 'dest' as the return slot except under two /// conditions: @@ -48,7 +49,7 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> { if (!shouldUseDestForReturnSlot()) return ReturnValueSlot(); - return ReturnValueSlot(Dest.getAddr(), Dest.isVolatile()); + return ReturnValueSlot(Dest.getAddr(), Dest.isVolatile(), IsResultUnused); } AggValueSlot EnsureSlot(QualType T) { @@ -61,9 +62,9 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> { } public: - AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest) - : CGF(cgf), Builder(CGF.Builder), Dest(Dest) { - } + AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused) + : CGF(cgf), Builder(CGF.Builder), Dest(Dest), + IsResultUnused(IsResultUnused) { } //===--------------------------------------------------------------------===// // Utilities @@ -98,6 +99,11 @@ public: // Visitor Methods //===--------------------------------------------------------------------===// + void Visit(Expr *E) { + ApplyDebugLocation DL(CGF, E); + StmtVisitor<AggExprEmitter>::Visit(E); + } + void VisitStmt(Stmt *S) { CGF.ErrorUnsupported(S, "aggregate expression"); } @@ -154,10 +160,12 @@ public: EmitAggLoadOfLValue(E); } + void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E); void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO); void VisitChooseExpr(const ChooseExpr *CE); void VisitInitListExpr(InitListExpr *E); void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E); + void VisitNoInitExpr(NoInitExpr *E) { } // Do nothing. void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) { Visit(DAE->getExpr()); } @@ -207,7 +215,7 @@ void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) { LValue LV = CGF.EmitLValue(E); // If the type of the l-value is atomic, then do an atomic load. - if (LV.getType()->isAtomicType()) { + if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(LV)) { CGF.EmitAtomicLoad(LV, E->getExprLoc(), Dest); return; } @@ -579,7 +587,12 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { } case CK_ToUnion: { - if (Dest.isIgnored()) break; + // Evaluate even if the destination is ignored. + if (Dest.isIgnored()) { + CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(), + /*ignoreResult=*/true); + break; + } // GCC union extension QualType Ty = E->getSubExpr()->getType(); @@ -640,7 +653,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { // Build a GEP to refer to the subobject. llvm::Value *valueAddr = - CGF.Builder.CreateStructGEP(valueDest.getAddr(), 0); + CGF.Builder.CreateStructGEP(nullptr, valueDest.getAddr(), 0); valueDest = AggValueSlot::forAddr(valueAddr, valueDest.getAlignment(), valueDest.getQualifiers(), @@ -661,7 +674,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { CGF.EmitAggExpr(E->getSubExpr(), atomicSlot); llvm::Value *valueAddr = - Builder.CreateStructGEP(atomicSlot.getAddr(), 0); + Builder.CreateStructGEP(nullptr, atomicSlot.getAddr(), 0); RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile()); return EmitFinalDestCopy(valueType, rvalue); } @@ -736,7 +749,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { } void AggExprEmitter::VisitCallExpr(const CallExpr *E) { - if (E->getCallReturnType()->isReferenceType()) { + if (E->getCallReturnType(CGF.getContext())->isReferenceType()) { EmitAggLoadOfLValue(E); return; } @@ -860,7 +873,8 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) { LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store); // That copy is an atomic copy if the LHS is atomic. - if (LHS.getType()->isAtomicType()) { + if (LHS.getType()->isAtomicType() || + CGF.LValueIsSuitableForInlineAtomic(LHS)) { CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false); return; } @@ -877,7 +891,8 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) { // If we have an atomic type, evaluate into the destination and then // do an atomic copy. - if (LHS.getType()->isAtomicType()) { + if (LHS.getType()->isAtomicType() || + CGF.LValueIsSuitableForInlineAtomic(LHS)) { EnsureDest(E->getRHS()->getType()); Visit(E->getRHS()); CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false); @@ -909,16 +924,16 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // Bind the common expression if necessary. CodeGenFunction::OpaqueValueMapping binding(CGF, E); - RegionCounter Cnt = CGF.getPGORegionCounter(E); CodeGenFunction::ConditionalEvaluation eval(CGF); - CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount()); + CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, + CGF.getProfileCount(E)); // Save whether the destination's lifetime is externally managed. bool isExternallyDestructed = Dest.isExternallyDestructed(); eval.begin(CGF); CGF.EmitBlock(LHSBlock); - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Visit(E->getTrueExpr()); eval.end(CGF); @@ -1043,6 +1058,9 @@ AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) { return; } else if (isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) { return EmitNullInitializationToLValue(LV); + } else if (isa<NoInitExpr>(E)) { + // Do nothing. + return; } else if (type->isReferenceType()) { RValue RV = CGF.EmitReferenceBindingToExpr(E); return CGF.EmitStoreThroughLValue(RV, LV); @@ -1263,6 +1281,15 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { cleanupDominator->eraseFromParent(); } +void AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) { + AggValueSlot Dest = EnsureSlot(E->getType()); + + LValue DestLV = CGF.MakeAddrLValue(Dest.getAddr(), E->getType(), + Dest.getAlignment()); + EmitInitializationToLValue(E->getBase(), DestLV); + VisitInitListExpr(E->getUpdater()); +} + //===----------------------------------------------------------------------===// // Entry Points into this File //===----------------------------------------------------------------------===// @@ -1382,7 +1409,7 @@ void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) { // Optimize the slot if possible. CheckAggExprForMemSetUse(Slot, E, *this); - AggExprEmitter(*this, Slot).Visit(const_cast<Expr*>(E)); + AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(const_cast<Expr*>(E)); } LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) { @@ -1408,7 +1435,8 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, assert((Record->hasTrivialCopyConstructor() || Record->hasTrivialCopyAssignment() || Record->hasTrivialMoveConstructor() || - Record->hasTrivialMoveAssignment()) && + Record->hasTrivialMoveAssignment() || + Record->isUnion()) && "Trying to aggregate-copy a type without a trivial copy/move " "constructor or assignment operator"); // Ignore empty classes in C++. @@ -1439,7 +1467,34 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, if (alignment.isZero()) alignment = TypeInfo.second; - // FIXME: Handle variable sized types. + llvm::Value *SizeVal = nullptr; + if (TypeInfo.first.isZero()) { + // But note that getTypeInfo returns 0 for a VLA. + if (auto *VAT = dyn_cast_or_null<VariableArrayType>( + getContext().getAsArrayType(Ty))) { + QualType BaseEltTy; + SizeVal = emitArrayLength(VAT, BaseEltTy, DestPtr); + TypeInfo = getContext().getTypeInfoDataSizeInChars(BaseEltTy); + std::pair<CharUnits, CharUnits> LastElementTypeInfo; + if (!isAssignment) + LastElementTypeInfo = getContext().getTypeInfoInChars(BaseEltTy); + assert(!TypeInfo.first.isZero()); + SizeVal = Builder.CreateNUWMul( + SizeVal, + llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity())); + if (!isAssignment) { + SizeVal = Builder.CreateNUWSub( + SizeVal, + llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity())); + SizeVal = Builder.CreateNUWAdd( + SizeVal, llvm::ConstantInt::get( + SizeTy, LastElementTypeInfo.first.getQuantity())); + } + } + } + if (!SizeVal) { + SizeVal = llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity()); + } // FIXME: If we have a volatile struct, the optimizer can remove what might // appear to be `extra' memory ops: @@ -1470,9 +1525,6 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, } else if (const RecordType *RecordTy = Ty->getAs<RecordType>()) { RecordDecl *Record = RecordTy->getDecl(); if (Record->hasObjectMember()) { - CharUnits size = TypeInfo.first; - llvm::Type *SizeTy = ConvertType(getContext().getSizeType()); - llvm::Value *SizeVal = llvm::ConstantInt::get(SizeTy, size.getQuantity()); CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr, SizeVal); return; @@ -1481,10 +1533,6 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, QualType BaseType = getContext().getBaseElementType(Ty); if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) { if (RecordTy->getDecl()->hasObjectMember()) { - CharUnits size = TypeInfo.first; - llvm::Type *SizeTy = ConvertType(getContext().getSizeType()); - llvm::Value *SizeVal = - llvm::ConstantInt::get(SizeTy, size.getQuantity()); CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr, SizeVal); return; @@ -1497,9 +1545,6 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, // the optimizer wishes to expand it in to scalar memory operations. llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty); - Builder.CreateMemCpy(DestPtr, SrcPtr, - llvm::ConstantInt::get(IntPtrTy, - TypeInfo.first.getQuantity()), - alignment.getQuantity(), isVolatile, - /*TBAATag=*/nullptr, TBAAStructTag); + Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, alignment.getQuantity(), + isVolatile, /*TBAATag=*/nullptr, TBAAStructTag); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp index 6d63b3a..c7adcca 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprCXX.cpp @@ -173,7 +173,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( This = EmitLValue(Base).getAddress(); - if (MD->isTrivial()) { + if (MD->isTrivial() || (MD->isDefaulted() && MD->getParent()->isUnion())) { if (isa<CXXDestructorDecl>(MD)) return RValue::get(nullptr); if (isa<CXXConstructorDecl>(MD) && cast<CXXConstructorDecl>(MD)->isDefaultConstructor()) @@ -254,8 +254,15 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( if (const CXXConstructorDecl *Ctor = dyn_cast<CXXConstructorDecl>(MD)) { Callee = CGM.GetAddrOfFunction(GlobalDecl(Ctor, Ctor_Complete), Ty); } else if (UseVirtualCall) { - Callee = CGM.getCXXABI().getVirtualFunctionPointer(*this, MD, This, Ty); + Callee = CGM.getCXXABI().getVirtualFunctionPointer(*this, MD, This, Ty, + CE->getLocStart()); } else { + if (SanOpts.has(SanitizerKind::CFINVCall) && + MD->getParent()->isDynamicClass()) { + llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy); + EmitVTablePtrCheckForCall(MD, VTable, CFITCK_NVCall, CE->getLocStart()); + } + if (getLangOpts().AppleKext && MD->isVirtual() && HasQualifier) Callee = BuildAppleKextVirtualCall(MD, Qualifier, Ty); else if (!DevirtualizedMethod) @@ -684,7 +691,7 @@ static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF, llvm::Value *tsmV = llvm::ConstantInt::get(CGF.SizeTy, typeSizeMultiplier); llvm::Value *result = - CGF.Builder.CreateCall2(umul_with_overflow, size, tsmV); + CGF.Builder.CreateCall(umul_with_overflow, {size, tsmV}); llvm::Value *overflowed = CGF.Builder.CreateExtractValue(result, 1); if (hasOverflow) @@ -723,7 +730,7 @@ static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF, llvm::Value *cookieSizeV = llvm::ConstantInt::get(CGF.SizeTy, cookieSize); llvm::Value *result = - CGF.Builder.CreateCall2(uadd_with_overflow, size, cookieSizeV); + CGF.Builder.CreateCall(uadd_with_overflow, {size, cookieSizeV}); llvm::Value *overflowed = CGF.Builder.CreateExtractValue(result, 1); if (hasOverflow) @@ -778,12 +785,10 @@ static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const Expr *Init, llvm_unreachable("bad evaluation kind"); } -void -CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, - QualType ElementType, - llvm::Value *BeginPtr, - llvm::Value *NumElements, - llvm::Value *AllocSizeWithoutCookie) { +void CodeGenFunction::EmitNewArrayInitializer( + const CXXNewExpr *E, QualType ElementType, llvm::Type *ElementTy, + llvm::Value *BeginPtr, llvm::Value *NumElements, + llvm::Value *AllocSizeWithoutCookie) { // If we have a type with trivial initialization and no initializer, // there's nothing to do. if (!E->hasInitializer()) @@ -809,7 +814,8 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, if (const ConstantArrayType *CAT = dyn_cast_or_null<ConstantArrayType>( AllocType->getAsArrayTypeUnsafe())) { unsigned AS = CurPtr->getType()->getPointerAddressSpace(); - llvm::Type *AllocPtrTy = ConvertTypeForMem(AllocType)->getPointerTo(AS); + ElementTy = ConvertTypeForMem(AllocType); + llvm::Type *AllocPtrTy = ElementTy->getPointerTo(AS); CurPtr = Builder.CreateBitCast(CurPtr, AllocPtrTy); InitListElements *= getContext().getConstantArrayElementCount(CAT); } @@ -839,7 +845,8 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, // initialization loops. StoreAnyExprIntoOneUnit(*this, ILE->getInit(i), ILE->getInit(i)->getType(), CurPtr); - CurPtr = Builder.CreateConstInBoundsGEP1_32(CurPtr, 1, "array.exp.next"); + CurPtr = Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr, 1, + "array.exp.next"); } // The remaining elements are filled with the array filler expression. @@ -952,6 +959,25 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, if (ILE->getNumInits() == 0 && TryMemsetInitialization()) return; + // If we have a struct whose every field is value-initialized, we can + // usually use memset. + if (auto *ILE = dyn_cast<InitListExpr>(Init)) { + if (const RecordType *RType = ILE->getType()->getAs<RecordType>()) { + if (RType->getDecl()->isStruct()) { + unsigned NumFields = 0; + for (auto *Field : RType->getDecl()->fields()) + if (!Field->isUnnamedBitfield()) + ++NumFields; + if (ILE->getNumInits() == NumFields) + for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i) + if (!isa<ImplicitValueInitExpr>(ILE->getInit(i))) + --NumFields; + if (ILE->getNumInits() == NumFields && TryMemsetInitialization()) + return; + } + } + } + // Create the loop blocks. llvm::BasicBlock *EntryBB = Builder.GetInsertBlock(); llvm::BasicBlock *LoopBB = createBasicBlock("new.loop"); @@ -1000,7 +1026,7 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, // Advance to the next element by adjusting the pointer type as necessary. llvm::Value *NextPtr = - Builder.CreateConstInBoundsGEP1_32(CurPtr, 1, "array.next"); + Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr, 1, "array.next"); // Check whether we've gotten to the end of the array and, if so, // exit the loop. @@ -1012,13 +1038,12 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, } static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E, - QualType ElementType, - llvm::Value *NewPtr, - llvm::Value *NumElements, + QualType ElementType, llvm::Type *ElementTy, + llvm::Value *NewPtr, llvm::Value *NumElements, llvm::Value *AllocSizeWithoutCookie) { - ApplyDebugLocation DL(CGF, E->getStartLoc()); + ApplyDebugLocation DL(CGF, E); if (E->isArray()) - CGF.EmitNewArrayInitializer(E, ElementType, NewPtr, NumElements, + CGF.EmitNewArrayInitializer(E, ElementType, ElementTy, NewPtr, NumElements, AllocSizeWithoutCookie); else if (const Expr *Init = E->getInitializer()) StoreAnyExprIntoOneUnit(CGF, Init, E->getAllocatedType(), NewPtr); @@ -1279,10 +1304,9 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // Emit a null check on the allocation result if the allocation // function is allowed to return null (because it has a non-throwing - // exception spec; for this part, we inline - // CXXNewExpr::shouldNullCheckAllocation()) and we have an + // exception spec or is the reserved placement new) and we have an // interesting initializer. - bool nullCheck = allocatorType->isNothrow(getContext()) && + bool nullCheck = E->shouldNullCheckAllocation(getContext()) && (!allocType.isPODType(getContext()) || E->hasInitializer()); llvm::BasicBlock *nullCheckBB = nullptr; @@ -1327,11 +1351,11 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { E, allocType); } - llvm::Type *elementPtrTy - = ConvertTypeForMem(allocType)->getPointerTo(AS); + llvm::Type *elementTy = ConvertTypeForMem(allocType); + llvm::Type *elementPtrTy = elementTy->getPointerTo(AS); llvm::Value *result = Builder.CreateBitCast(allocation, elementPtrTy); - EmitNewInitializer(*this, E, allocType, result, numElements, + EmitNewInitializer(*this, E, allocType, elementTy, result, numElements, allocSizeWithoutCookie); if (E->isArray()) { // NewPtr is a pointer to the base element type. If we're @@ -1524,7 +1548,8 @@ namespace { // The size of an element, multiplied by the number of elements. llvm::Value *Size = llvm::ConstantInt::get(SizeTy, ElementTypeSize.getQuantity()); - Size = CGF.Builder.CreateMul(Size, NumElements); + if (NumElements) + Size = CGF.Builder.CreateMul(Size, NumElements); // Plus the size of the cookie if applicable. if (!CookieSize.isZero()) { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp index 1580bbe..27d1c68 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprComplex.cpp @@ -95,6 +95,7 @@ public: //===--------------------------------------------------------------------===// ComplexPairTy Visit(Expr *E) { + ApplyDebugLocation DL(CGF, E); return StmtVisitor<ComplexExprEmitter, ComplexPairTy>::Visit(E); } @@ -316,14 +317,14 @@ ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue, llvm::Value *Real=nullptr, *Imag=nullptr; if (!IgnoreReal || isVolatile) { - llvm::Value *RealP = Builder.CreateStructGEP(SrcPtr, 0, + llvm::Value *RealP = Builder.CreateStructGEP(nullptr, SrcPtr, 0, SrcPtr->getName() + ".realp"); Real = Builder.CreateAlignedLoad(RealP, AlignR, isVolatile, SrcPtr->getName() + ".real"); } if (!IgnoreImag || isVolatile) { - llvm::Value *ImagP = Builder.CreateStructGEP(SrcPtr, 1, + llvm::Value *ImagP = Builder.CreateStructGEP(nullptr, SrcPtr, 1, SrcPtr->getName() + ".imagp"); Imag = Builder.CreateAlignedLoad(ImagP, AlignI, isVolatile, SrcPtr->getName() + ".imag"); @@ -335,12 +336,13 @@ ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue, /// specified value pointer. void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, LValue lvalue, bool isInit) { - if (lvalue.getType()->isAtomicType()) + if (lvalue.getType()->isAtomicType() || + (!isInit && CGF.LValueIsSuitableForInlineAtomic(lvalue))) return CGF.EmitAtomicStore(RValue::getComplex(Val), lvalue, isInit); llvm::Value *Ptr = lvalue.getAddress(); - llvm::Value *RealPtr = Builder.CreateStructGEP(Ptr, 0, "real"); - llvm::Value *ImagPtr = Builder.CreateStructGEP(Ptr, 1, "imag"); + llvm::Value *RealPtr = Builder.CreateStructGEP(nullptr, Ptr, 0, "real"); + llvm::Value *ImagPtr = Builder.CreateStructGEP(nullptr, Ptr, 1, "imag"); unsigned AlignR = lvalue.getAlignment().getQuantity(); ASTContext &C = CGF.getContext(); QualType ComplexTy = lvalue.getType(); @@ -375,7 +377,7 @@ VisitImaginaryLiteral(const ImaginaryLiteral *IL) { ComplexPairTy ComplexExprEmitter::VisitCallExpr(const CallExpr *E) { - if (E->getCallReturnType()->isReferenceType()) + if (E->getCallReturnType(CGF.getContext())->isReferenceType()) return EmitLoadOfLValue(E); return CGF.EmitCallExpr(E).getComplexVal(); @@ -818,6 +820,8 @@ EmitCompoundAssignLValue(const CompoundAssignOperator *E, TestAndClearIgnoreReal(); TestAndClearIgnoreImag(); QualType LHSTy = E->getLHS()->getType(); + if (const AtomicType *AT = LHSTy->getAs<AtomicType>()) + LHSTy = AT->getValueType(); BinOpInfo OpInfo; @@ -945,13 +949,14 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // Bind the common expression if necessary. CodeGenFunction::OpaqueValueMapping binding(CGF, E); - RegionCounter Cnt = CGF.getPGORegionCounter(E); + CodeGenFunction::ConditionalEvaluation eval(CGF); - CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount()); + CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, + CGF.getProfileCount(E)); eval.begin(CGF); CGF.EmitBlock(LHSBlock); - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); ComplexPairTy LHS = Visit(E->getTrueExpr()); LHSBlock = Builder.GetInsertBlock(); CGF.EmitBranch(ContBlock); @@ -1033,7 +1038,7 @@ ComplexPairTy CodeGenFunction::EmitComplexExpr(const Expr *E, bool IgnoreReal, "Invalid complex expression to emit"); return ComplexExprEmitter(*this, IgnoreReal, IgnoreImag) - .Visit(const_cast<Expr*>(E)); + .Visit(const_cast<Expr *>(E)); } void CodeGenFunction::EmitComplexExprIntoLValue(const Expr *E, LValue dest, @@ -1085,8 +1090,8 @@ EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E) { } LValue CodeGenFunction:: -EmitScalarCompooundAssignWithComplex(const CompoundAssignOperator *E, - llvm::Value *&Result) { +EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E, + llvm::Value *&Result) { CompoundFunc Op = getComplexOp(E->getOpcode()); RValue Val; LValue Ret = ComplexExprEmitter(*this).EmitCompoundAssignLValue(E, Op, Val); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp index 54f7eee..a15c151 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprConstant.cpp @@ -33,6 +33,7 @@ using namespace CodeGen; //===----------------------------------------------------------------------===// namespace { +class ConstExprEmitter; class ConstStructBuilder { CodeGenModule &CGM; CodeGenFunction *CGF; @@ -42,6 +43,10 @@ class ConstStructBuilder { CharUnits LLVMStructAlignment; SmallVector<llvm::Constant *, 32> Elements; public: + static llvm::Constant *BuildStruct(CodeGenModule &CGM, CodeGenFunction *CFG, + ConstExprEmitter *Emitter, + llvm::ConstantStruct *Base, + InitListExpr *Updater); static llvm::Constant *BuildStruct(CodeGenModule &CGM, CodeGenFunction *CGF, InitListExpr *ILE); static llvm::Constant *BuildStruct(CodeGenModule &CGM, CodeGenFunction *CGF, @@ -68,6 +73,8 @@ private: void ConvertStructToPacked(); bool Build(InitListExpr *ILE); + bool Build(ConstExprEmitter *Emitter, llvm::ConstantStruct *Base, + InitListExpr *Updater); void Build(const APValue &Val, const RecordDecl *RD, bool IsPrimaryBase, const CXXRecordDecl *VTableClass, CharUnits BaseOffset); llvm::Constant *Finalize(QualType Ty); @@ -383,14 +390,19 @@ bool ConstStructBuilder::Build(InitListExpr *ILE) { if (!EltInit) return false; - + if (!Field->isBitField()) { // Handle non-bitfield members. AppendField(*Field, Layout.getFieldOffset(FieldNo), EltInit); } else { // Otherwise we have a bitfield. - AppendBitField(*Field, Layout.getFieldOffset(FieldNo), - cast<llvm::ConstantInt>(EltInit)); + if (auto *CI = dyn_cast<llvm::ConstantInt>(EltInit)) { + AppendBitField(*Field, Layout.getFieldOffset(FieldNo), CI); + } else { + // We are trying to initialize a bitfield with a non-trivial constant, + // this must require run-time code. + return false; + } } } @@ -542,6 +554,17 @@ llvm::Constant *ConstStructBuilder::Finalize(QualType Ty) { llvm::Constant *ConstStructBuilder::BuildStruct(CodeGenModule &CGM, CodeGenFunction *CGF, + ConstExprEmitter *Emitter, + llvm::ConstantStruct *Base, + InitListExpr *Updater) { + ConstStructBuilder Builder(CGM, CGF); + if (!Builder.Build(Emitter, Base, Updater)) + return nullptr; + return Builder.Finalize(Updater->getType()); +} + +llvm::Constant *ConstStructBuilder::BuildStruct(CodeGenModule &CGM, + CodeGenFunction *CGF, InitListExpr *ILE) { ConstStructBuilder Builder(CGM, CGF); @@ -813,6 +836,82 @@ public: return nullptr; } + llvm::Constant *EmitDesignatedInitUpdater(llvm::Constant *Base, + InitListExpr *Updater) { + QualType ExprType = Updater->getType(); + + if (ExprType->isArrayType()) { + llvm::ArrayType *AType = cast<llvm::ArrayType>(ConvertType(ExprType)); + llvm::Type *ElemType = AType->getElementType(); + + unsigned NumInitElements = Updater->getNumInits(); + unsigned NumElements = AType->getNumElements(); + + std::vector<llvm::Constant *> Elts; + Elts.reserve(NumElements); + + if (llvm::ConstantDataArray *DataArray = + dyn_cast<llvm::ConstantDataArray>(Base)) + for (unsigned i = 0; i != NumElements; ++i) + Elts.push_back(DataArray->getElementAsConstant(i)); + else if (llvm::ConstantArray *Array = + dyn_cast<llvm::ConstantArray>(Base)) + for (unsigned i = 0; i != NumElements; ++i) + Elts.push_back(Array->getOperand(i)); + else + return nullptr; // FIXME: other array types not implemented + + llvm::Constant *fillC = nullptr; + if (Expr *filler = Updater->getArrayFiller()) + if (!isa<NoInitExpr>(filler)) + fillC = CGM.EmitConstantExpr(filler, filler->getType(), CGF); + bool RewriteType = (fillC && fillC->getType() != ElemType); + + for (unsigned i = 0; i != NumElements; ++i) { + Expr *Init = nullptr; + if (i < NumInitElements) + Init = Updater->getInit(i); + + if (!Init && fillC) + Elts[i] = fillC; + else if (!Init || isa<NoInitExpr>(Init)) + ; // Do nothing. + else if (InitListExpr *ChildILE = dyn_cast<InitListExpr>(Init)) + Elts[i] = EmitDesignatedInitUpdater(Elts[i], ChildILE); + else + Elts[i] = CGM.EmitConstantExpr(Init, Init->getType(), CGF); + + if (!Elts[i]) + return nullptr; + RewriteType |= (Elts[i]->getType() != ElemType); + } + + if (RewriteType) { + std::vector<llvm::Type *> Types; + Types.reserve(NumElements); + for (unsigned i = 0; i != NumElements; ++i) + Types.push_back(Elts[i]->getType()); + llvm::StructType *SType = llvm::StructType::get(AType->getContext(), + Types, true); + return llvm::ConstantStruct::get(SType, Elts); + } + + return llvm::ConstantArray::get(AType, Elts); + } + + if (ExprType->isRecordType()) + return ConstStructBuilder::BuildStruct(CGM, CGF, this, + dyn_cast<llvm::ConstantStruct>(Base), Updater); + + return nullptr; + } + + llvm::Constant *VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) { + return EmitDesignatedInitUpdater( + CGM.EmitConstantExpr(E->getBase(), E->getType(), CGF), + E->getUpdater()); + } + llvm::Constant *VisitCXXConstructExpr(CXXConstructExpr *E) { if (!E->getConstructor()->isTrivial()) return nullptr; @@ -998,6 +1097,68 @@ public: } // end anonymous namespace. +bool ConstStructBuilder::Build(ConstExprEmitter *Emitter, + llvm::ConstantStruct *Base, + InitListExpr *Updater) { + assert(Base && "base expression should not be empty"); + + QualType ExprType = Updater->getType(); + RecordDecl *RD = ExprType->getAs<RecordType>()->getDecl(); + const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD); + const llvm::StructLayout *BaseLayout = CGM.getDataLayout().getStructLayout( + Base->getType()); + unsigned FieldNo = -1; + unsigned ElementNo = 0; + + for (FieldDecl *Field : RD->fields()) { + ++FieldNo; + + if (RD->isUnion() && Updater->getInitializedFieldInUnion() != Field) + continue; + + // Skip anonymous bitfields. + if (Field->isUnnamedBitfield()) + continue; + + llvm::Constant *EltInit = Base->getOperand(ElementNo); + + // Bail out if the type of the ConstantStruct does not have the same layout + // as the type of the InitListExpr. + if (CGM.getTypes().ConvertType(Field->getType()) != EltInit->getType() || + Layout.getFieldOffset(ElementNo) != + BaseLayout->getElementOffsetInBits(ElementNo)) + return false; + + // Get the initializer. If we encounter an empty field or a NoInitExpr, + // we use values from the base expression. + Expr *Init = nullptr; + if (ElementNo < Updater->getNumInits()) + Init = Updater->getInit(ElementNo); + + if (!Init || isa<NoInitExpr>(Init)) + ; // Do nothing. + else if (InitListExpr *ChildILE = dyn_cast<InitListExpr>(Init)) + EltInit = Emitter->EmitDesignatedInitUpdater(EltInit, ChildILE); + else + EltInit = CGM.EmitConstantExpr(Init, Field->getType(), CGF); + + ++ElementNo; + + if (!EltInit) + return false; + + if (!Field->isBitField()) + AppendField(Field, Layout.getFieldOffset(FieldNo), EltInit); + else if (llvm::ConstantInt *CI = dyn_cast<llvm::ConstantInt>(EltInit)) + AppendBitField(Field, Layout.getFieldOffset(FieldNo), CI); + else + // Initializing a bitfield with a non-trivial constant? + return false; + } + + return true; +} + llvm::Constant *CodeGenModule::EmitConstantInit(const VarDecl &D, CodeGenFunction *CGF) { // Make a quick check if variable can be default NULL initialized @@ -1110,7 +1271,7 @@ llvm::Constant *CodeGenModule::EmitConstantValue(const APValue &Value, unsigned AS = C->getType()->getPointerAddressSpace(); llvm::Type *CharPtrTy = Int8Ty->getPointerTo(AS); llvm::Constant *Casted = llvm::ConstantExpr::getBitCast(C, CharPtrTy); - Casted = llvm::ConstantExpr::getGetElementPtr(Casted, Offset); + Casted = llvm::ConstantExpr::getGetElementPtr(Int8Ty, Casted, Offset); C = llvm::ConstantExpr::getPointerCast(Casted, C->getType()); } @@ -1291,7 +1452,7 @@ CodeGenModule::getMemberPointerConstant(const UnaryOperator *uo) { // A member function pointer. if (const CXXMethodDecl *method = dyn_cast<CXXMethodDecl>(decl)) - return getCXXABI().EmitMemberPointer(method); + return getCXXABI().EmitMemberFunctionPointer(method); // Otherwise, a member data pointer. uint64_t fieldOffset = getContext().getFieldOffset(decl); @@ -1344,8 +1505,14 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM, } // For unions, stop after the first named field. - if (record->isUnion() && Field->getDeclName()) - break; + if (record->isUnion()) { + if (Field->getIdentifier()) + break; + if (const auto *FieldRD = + dyn_cast_or_null<RecordDecl>(Field->getType()->getAsTagDecl())) + if (FieldRD->findFirstNamedDataMember()) + break; + } } // Fill in the virtual bases, if we're working with the complete object. @@ -1403,10 +1570,6 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) { llvm::Constant *Element = EmitNullConstant(ElementTy); unsigned NumElements = CAT->getSize().getZExtValue(); - - if (Element->isNullValue()) - return llvm::ConstantAggregateZero::get(ATy); - SmallVector<llvm::Constant *, 8> Array(NumElements, Element); return llvm::ConstantArray::get(ATy, Array); } @@ -1416,8 +1579,7 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) { return ::EmitNullConstant(*this, RD, /*complete object*/ true); } - assert(T->isMemberPointerType() && "Should only see member pointers here!"); - assert(!T->getAs<MemberPointerType>()->getPointeeType()->isFunctionType() && + assert(T->isMemberDataPointerType() && "Should only see pointers to data members here!"); return getCXXABI().EmitNullMemberPointer(T->castAs<MemberPointerType>()); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp index 3be14c8..74f6019 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGExprScalar.cpp @@ -16,6 +16,7 @@ #include "CGDebugInfo.h" #include "CGObjCRuntime.h" #include "CodeGenModule.h" +#include "TargetInfo.h" #include "clang/AST/ASTContext.h" #include "clang/AST/DeclObjC.h" #include "clang/AST/RecordLayout.h" @@ -85,7 +86,7 @@ public: return CGF.EmitCheckedLValue(E, TCK); } - void EmitBinOpCheck(ArrayRef<std::pair<Value *, SanitizerKind>> Checks, + void EmitBinOpCheck(ArrayRef<std::pair<Value *, SanitizerMask>> Checks, const BinOpInfo &Info); Value *EmitLoadOfLValue(LValue LV, SourceLocation Loc) { @@ -196,7 +197,7 @@ public: //===--------------------------------------------------------------------===// Value *Visit(Expr *E) { - ApplyDebugLocation DL(CGF, E->getLocStart()); + ApplyDebugLocation DL(CGF, E); return StmtVisitor<ScalarExprEmitter, Value*>::Visit(E); } @@ -320,7 +321,7 @@ public: Value *VisitCastExpr(CastExpr *E); Value *VisitCallExpr(const CallExpr *E) { - if (E->getCallReturnType()->isReferenceType()) + if (E->getCallReturnType(CGF.getContext())->isReferenceType()) return EmitLoadOfLValue(E); Value *V = CGF.EmitCallExpr(E).getScalarVal(); @@ -349,10 +350,9 @@ public: return EmitScalarPrePostIncDec(E, LV, true, true); } - llvm::Value *EmitAddConsiderOverflowBehavior(const UnaryOperator *E, - llvm::Value *InVal, - llvm::Value *NextVal, - bool IsInc); + llvm::Value *EmitIncDecConsiderOverflowBehavior(const UnaryOperator *E, + llvm::Value *InVal, + bool IsInc); llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, bool isInc, bool isPre); @@ -745,23 +745,37 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, QualType OrigSrcType = SrcType; llvm::Type *SrcTy = Src->getType(); - // If casting to/from storage-only half FP, use special intrinsics. - if (SrcType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType && - !CGF.getContext().getLangOpts().HalfArgsAndReturns) { - Src = Builder.CreateCall( - CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16, - CGF.CGM.FloatTy), - Src); - SrcType = CGF.getContext().FloatTy; - SrcTy = CGF.FloatTy; - } - // Handle conversions to bool first, they are special: comparisons against 0. if (DstType->isBooleanType()) return EmitConversionToBool(Src, SrcType); llvm::Type *DstTy = ConvertType(DstType); + // Cast from half through float if half isn't a native type. + if (SrcType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) { + // Cast to FP using the intrinsic if the half type itself isn't supported. + if (DstTy->isFloatingPointTy()) { + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) + return Builder.CreateCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16, DstTy), + Src); + } else { + // Cast to other types through float, using either the intrinsic or FPExt, + // depending on whether the half type itself is supported + // (as opposed to operations on half, available with NativeHalfType). + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) { + Src = Builder.CreateCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16, + CGF.CGM.FloatTy), + Src); + } else { + Src = Builder.CreateFPExt(Src, CGF.CGM.FloatTy, "conv"); + } + SrcType = CGF.getContext().FloatTy; + SrcTy = CGF.FloatTy; + } + } + // Ignore conversions like int -> uint. if (SrcTy == DstTy) return Src; @@ -818,10 +832,20 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, EmitFloatConversionCheck(OrigSrc, OrigSrcType, Src, SrcType, DstType, DstTy); - // Cast to half via float - if (DstType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType && - !CGF.getContext().getLangOpts().HalfArgsAndReturns) + // Cast to half through float if half isn't a native type. + if (DstType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) { + // Make sure we cast in a single step if from another FP type. + if (SrcTy->isFloatingPointTy()) { + // Use the intrinsic if the half type itself isn't supported + // (as opposed to operations on half, available with NativeHalfType). + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) + return Builder.CreateCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16, SrcTy), Src); + // If the half type is supported, just use an fptrunc. + return Builder.CreateFPTrunc(Src, DstTy); + } DstTy = CGF.FloatTy; + } if (isa<llvm::IntegerType>(SrcTy)) { bool InputSigned = SrcType->isSignedIntegerOrEnumerationType(); @@ -847,10 +871,14 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, } if (DstTy != ResTy) { - assert(ResTy->isIntegerTy(16) && "Only half FP requires extra conversion"); - Res = Builder.CreateCall( + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) { + assert(ResTy->isIntegerTy(16) && "Only half FP requires extra conversion"); + Res = Builder.CreateCall( CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16, CGF.CGM.FloatTy), Res); + } else { + Res = Builder.CreateFPTrunc(Res, ResTy, "conv"); + } } return Res; @@ -889,7 +917,7 @@ Value *ScalarExprEmitter::EmitNullValue(QualType Ty) { /// operation). The check passes if all values in \p Checks (which are \c i1), /// are \c true. void ScalarExprEmitter::EmitBinOpCheck( - ArrayRef<std::pair<Value *, SanitizerKind>> Checks, const BinOpInfo &Info) { + ArrayRef<std::pair<Value *, SanitizerMask>> Checks, const BinOpInfo &Info) { assert(CGF.IsSanitizerScope); StringRef CheckName; SmallVector<llvm::Constant *, 4> StaticData; @@ -1138,6 +1166,16 @@ static llvm::Constant *getMaskElt(llvm::ShuffleVectorInst *SVI, unsigned Idx, return llvm::ConstantInt::get(I32Ty, Off+MV); } +static llvm::Constant *getAsInt32(llvm::ConstantInt *C, llvm::Type *I32Ty) { + if (C->getBitWidth() != 32) { + assert(llvm::ConstantInt::isValueValidForType(I32Ty, + C->getZExtValue()) && + "Index operand too large for shufflevector mask!"); + return llvm::ConstantInt::get(I32Ty, C->getZExtValue()); + } + return C; +} + Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) { bool Ignore = TestAndClearIgnoreResultAssign(); (void)Ignore; @@ -1188,7 +1226,8 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) { Value *LHS = nullptr, *RHS = nullptr; if (CurIdx == 0) { // insert into undef -> shuffle (src, undef) - Args.push_back(C); + // shufflemask must use an i32 + Args.push_back(getAsInt32(C, CGF.Int32Ty)); Args.resize(ResElts, llvm::UndefValue::get(CGF.Int32Ty)); LHS = EI->getVectorOperand(); @@ -1355,6 +1394,15 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { llvm_unreachable("wrong cast for pointers in different address spaces" "(must be an address space cast)!"); } + + if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast)) { + if (auto PT = DestTy->getAs<PointerType>()) + CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Src, + /*MayBeNull=*/true, + CodeGenFunction::CFITCK_UnrelatedCast, + CE->getLocStart()); + } + return Builder.CreateBitCast(Src, DstTy); } case CK_AddressSpaceConversion: { @@ -1384,6 +1432,12 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { CGF.EmitTypeCheck(CodeGenFunction::TCK_DowncastPointer, CE->getExprLoc(), Derived, DestTy->getPointeeType()); + if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast)) + CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived, + /*MayBeNull=*/true, + CodeGenFunction::CFITCK_DerivedCast, + CE->getLocStart()); + return Derived; } case CK_UncheckedDerivedToBase: @@ -1412,13 +1466,13 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { // anything here. if (!E->getType()->isVariableArrayType()) { assert(isa<llvm::PointerType>(V->getType()) && "Expected pointer"); + llvm::Type *NewTy = ConvertType(E->getType()); V = CGF.Builder.CreatePointerCast( - V, ConvertType(E->getType())->getPointerTo( - V->getType()->getPointerAddressSpace())); + V, NewTy->getPointerTo(V->getType()->getPointerAddressSpace())); assert(isa<llvm::ArrayType>(V->getType()->getPointerElementType()) && "Expected pointer to array"); - V = Builder.CreateStructGEP(V, 0, "arraydecay"); + V = Builder.CreateStructGEP(NewTy, V, 0, "arraydecay"); } // Make sure the array decay ends up being the right type. This matters if @@ -1571,26 +1625,32 @@ Value *ScalarExprEmitter::VisitStmtExpr(const StmtExpr *E) { // Unary Operators //===----------------------------------------------------------------------===// -llvm::Value *ScalarExprEmitter:: -EmitAddConsiderOverflowBehavior(const UnaryOperator *E, - llvm::Value *InVal, - llvm::Value *NextVal, bool IsInc) { +static BinOpInfo createBinOpInfoFromIncDec(const UnaryOperator *E, + llvm::Value *InVal, bool IsInc) { + BinOpInfo BinOp; + BinOp.LHS = InVal; + BinOp.RHS = llvm::ConstantInt::get(InVal->getType(), 1, false); + BinOp.Ty = E->getType(); + BinOp.Opcode = IsInc ? BO_Add : BO_Sub; + BinOp.FPContractable = false; + BinOp.E = E; + return BinOp; +} + +llvm::Value *ScalarExprEmitter::EmitIncDecConsiderOverflowBehavior( + const UnaryOperator *E, llvm::Value *InVal, bool IsInc) { + llvm::Value *Amount = + llvm::ConstantInt::get(InVal->getType(), IsInc ? 1 : -1, true); + StringRef Name = IsInc ? "inc" : "dec"; switch (CGF.getLangOpts().getSignedOverflowBehavior()) { case LangOptions::SOB_Defined: - return Builder.CreateAdd(InVal, NextVal, IsInc ? "inc" : "dec"); + return Builder.CreateAdd(InVal, Amount, Name); case LangOptions::SOB_Undefined: if (!CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) - return Builder.CreateNSWAdd(InVal, NextVal, IsInc ? "inc" : "dec"); + return Builder.CreateNSWAdd(InVal, Amount, Name); // Fall through. case LangOptions::SOB_Trapping: - BinOpInfo BinOp; - BinOp.LHS = InVal; - BinOp.RHS = NextVal; - BinOp.Ty = E->getType(); - BinOp.Opcode = BO_Add; - BinOp.FPContractable = false; - BinOp.E = E; - return EmitOverflowCheckedBinOp(BinOp); + return EmitOverflowCheckedBinOp(createBinOpInfoFromIncDec(E, InVal, IsInc)); } llvm_unreachable("Unknown SignedOverflowBehaviorTy"); } @@ -1668,27 +1728,20 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, // Most common case by far: integer increment. } else if (type->isIntegerType()) { - - llvm::Value *amt = llvm::ConstantInt::get(value->getType(), amount, true); - // Note that signed integer inc/dec with width less than int can't // overflow because of promotion rules; we're just eliding a few steps here. bool CanOverflow = value->getType()->getIntegerBitWidth() >= CGF.IntTy->getIntegerBitWidth(); if (CanOverflow && type->isSignedIntegerOrEnumerationType()) { - value = EmitAddConsiderOverflowBehavior(E, value, amt, isInc); + value = EmitIncDecConsiderOverflowBehavior(E, value, isInc); } else if (CanOverflow && type->isUnsignedIntegerType() && CGF.SanOpts.has(SanitizerKind::UnsignedIntegerOverflow)) { - BinOpInfo BinOp; - BinOp.LHS = value; - BinOp.RHS = llvm::ConstantInt::get(value->getType(), 1, false); - BinOp.Ty = E->getType(); - BinOp.Opcode = isInc ? BO_Add : BO_Sub; - BinOp.FPContractable = false; - BinOp.E = E; - value = EmitOverflowCheckedBinOp(BinOp); - } else + value = + EmitOverflowCheckedBinOp(createBinOpInfoFromIncDec(E, value, isInc)); + } else { + llvm::Value *amt = llvm::ConstantInt::get(value->getType(), amount, true); value = Builder.CreateAdd(value, amt, isInc ? "inc" : "dec"); + } // Next most common: pointer increment. } else if (const PointerType *ptr = type->getAs<PointerType>()) { @@ -1742,13 +1795,16 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, // Add the inc/dec to the real part. llvm::Value *amt; - if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType && - !CGF.getContext().getLangOpts().HalfArgsAndReturns) { + if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) { // Another special case: half FP increment should be done via float - value = Builder.CreateCall( - CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16, - CGF.CGM.FloatTy), - input); + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) { + value = Builder.CreateCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16, + CGF.CGM.FloatTy), + input, "incdec.conv"); + } else { + value = Builder.CreateFPExt(input, CGF.CGM.FloatTy, "incdec.conv"); + } } if (value->getType()->isFloatTy()) @@ -1758,20 +1814,29 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, amt = llvm::ConstantFP::get(VMContext, llvm::APFloat(static_cast<double>(amount))); else { + // Remaining types are either Half or LongDouble. Convert from float. llvm::APFloat F(static_cast<float>(amount)); bool ignored; - F.convert(CGF.getTarget().getLongDoubleFormat(), + // Don't use getFloatTypeSemantics because Half isn't + // necessarily represented using the "half" LLVM type. + F.convert(value->getType()->isHalfTy() + ? CGF.getTarget().getHalfFormat() + : CGF.getTarget().getLongDoubleFormat(), llvm::APFloat::rmTowardZero, &ignored); amt = llvm::ConstantFP::get(VMContext, F); } value = Builder.CreateFAdd(value, amt, isInc ? "inc" : "dec"); - if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType && - !CGF.getContext().getLangOpts().HalfArgsAndReturns) - value = Builder.CreateCall( - CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16, - CGF.CGM.FloatTy), - value); + if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) { + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) { + value = Builder.CreateCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16, + CGF.CGM.FloatTy), + value, "incdec.conv"); + } else { + value = Builder.CreateFPTrunc(value, input->getType(), "incdec.conv"); + } + } // Objective-C pointer types. } else { @@ -1794,10 +1859,9 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, llvm::BasicBlock *opBB = Builder.GetInsertBlock(); llvm::BasicBlock *contBB = CGF.createBasicBlock("atomic_cont", CGF.CurFn); auto Pair = CGF.EmitAtomicCompareExchange( - LV, RValue::get(atomicPHI), RValue::get(CGF.EmitToMemory(value, type)), - E->getExprLoc()); - llvm::Value *old = Pair.first.getScalarVal(); - llvm::Value *success = Pair.second.getScalarVal(); + LV, RValue::get(atomicPHI), RValue::get(value), E->getExprLoc()); + llvm::Value *old = CGF.EmitToMemory(Pair.first.getScalarVal(), type); + llvm::Value *success = Pair.second; atomicPHI->addIncoming(old, opBB); Builder.CreateCondBr(success, contBB, opBB); Builder.SetInsertPoint(contBB); @@ -1985,6 +2049,13 @@ ScalarExprEmitter::VisitUnaryExprOrTypeTraitExpr( return size; } + } else if (E->getKind() == UETT_OpenMPRequiredSimdAlign) { + auto Alignment = + CGF.getContext() + .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( + E->getTypeOfArgument()->getPointeeType())) + .getQuantity(); + return llvm::ConstantInt::get(CGF.SizeTy, Alignment); } // If this isn't sizeof(vla), the result must be constant; use the constant @@ -2056,7 +2127,7 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( BinOpInfo OpInfo; if (E->getComputationResultType()->isAnyComplexType()) - return CGF.EmitScalarCompooundAssignWithComplex(E, Result); + return CGF.EmitScalarCompoundAssignWithComplex(E, Result); // Emit the RHS first. __block variables need to have the rhs evaluated // first, plus this should improve codegen a little. @@ -2138,10 +2209,9 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( llvm::BasicBlock *opBB = Builder.GetInsertBlock(); llvm::BasicBlock *contBB = CGF.createBasicBlock("atomic_cont", CGF.CurFn); auto Pair = CGF.EmitAtomicCompareExchange( - LHSLV, RValue::get(atomicPHI), - RValue::get(CGF.EmitToMemory(Result, LHSTy)), E->getExprLoc()); - llvm::Value *old = Pair.first.getScalarVal(); - llvm::Value *success = Pair.second.getScalarVal(); + LHSLV, RValue::get(atomicPHI), RValue::get(Result), E->getExprLoc()); + llvm::Value *old = CGF.EmitToMemory(Pair.first.getScalarVal(), LHSTy); + llvm::Value *success = Pair.second; atomicPHI->addIncoming(old, opBB); Builder.CreateCondBr(success, contBB, opBB); Builder.SetInsertPoint(contBB); @@ -2184,7 +2254,7 @@ Value *ScalarExprEmitter::EmitCompoundAssign(const CompoundAssignOperator *E, void ScalarExprEmitter::EmitUndefinedBehaviorIntegerDivAndRemCheck( const BinOpInfo &Ops, llvm::Value *Zero, bool isDiv) { - SmallVector<std::pair<llvm::Value *, SanitizerKind>, 2> Checks; + SmallVector<std::pair<llvm::Value *, SanitizerMask>, 2> Checks; if (CGF.SanOpts.has(SanitizerKind::IntegerDivideByZero)) { Checks.push_back(std::make_pair(Builder.CreateICmpNE(Ops.RHS, Zero), @@ -2296,7 +2366,7 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) { llvm::Function *intrinsic = CGF.CGM.getIntrinsic(IID, opTy); - Value *resultAndOverflow = Builder.CreateCall2(intrinsic, Ops.LHS, Ops.RHS); + Value *resultAndOverflow = Builder.CreateCall(intrinsic, {Ops.LHS, Ops.RHS}); Value *result = Builder.CreateExtractValue(resultAndOverflow, 0); Value *overflow = Builder.CreateExtractValue(resultAndOverflow, 1); @@ -2309,7 +2379,7 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) { if (!isSigned || CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) { CodeGenFunction::SanitizerScope SanScope(&CGF); llvm::Value *NotOverflow = Builder.CreateNot(overflow); - SanitizerKind Kind = isSigned ? SanitizerKind::SignedIntegerOverflow + SanitizerMask Kind = isSigned ? SanitizerKind::SignedIntegerOverflow : SanitizerKind::UnsignedIntegerOverflow; EmitBinOpCheck(std::make_pair(NotOverflow, Kind), Ops); } else @@ -2476,10 +2546,9 @@ static Value* buildFMulAdd(llvm::BinaryOperator *MulOp, Value *Addend, "neg"); } - Value *FMulAdd = - Builder.CreateCall3( + Value *FMulAdd = Builder.CreateCall( CGF.CGM.getIntrinsic(llvm::Intrinsic::fmuladd, Addend->getType()), - MulOp0, MulOp1, Addend); + {MulOp0, MulOp1, Addend}); MulOp->eraseFromParent(); return FMulAdd; @@ -2664,21 +2733,34 @@ Value *ScalarExprEmitter::EmitShl(const BinOpInfo &Ops) { if (Ops.LHS->getType() != RHS->getType()) RHS = Builder.CreateIntCast(RHS, Ops.LHS->getType(), false, "sh_prom"); - if (CGF.SanOpts.has(SanitizerKind::Shift) && !CGF.getLangOpts().OpenCL && - isa<llvm::IntegerType>(Ops.LHS->getType())) { + bool SanitizeBase = CGF.SanOpts.has(SanitizerKind::ShiftBase) && + Ops.Ty->hasSignedIntegerRepresentation(); + bool SanitizeExponent = CGF.SanOpts.has(SanitizerKind::ShiftExponent); + // OpenCL 6.3j: shift values are effectively % word size of LHS. + if (CGF.getLangOpts().OpenCL) + RHS = + Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shl.mask"); + else if ((SanitizeBase || SanitizeExponent) && + isa<llvm::IntegerType>(Ops.LHS->getType())) { CodeGenFunction::SanitizerScope SanScope(&CGF); + SmallVector<std::pair<Value *, SanitizerMask>, 2> Checks; llvm::Value *WidthMinusOne = GetWidthMinusOneValue(Ops.LHS, RHS); - llvm::Value *Valid = Builder.CreateICmpULE(RHS, WidthMinusOne); + llvm::Value *ValidExponent = Builder.CreateICmpULE(RHS, WidthMinusOne); - if (Ops.Ty->hasSignedIntegerRepresentation()) { - llvm::BasicBlock *Orig = Builder.GetInsertBlock(); - llvm::BasicBlock *Cont = CGF.createBasicBlock("cont"); - llvm::BasicBlock *CheckBitsShifted = CGF.createBasicBlock("check"); - Builder.CreateCondBr(Valid, CheckBitsShifted, Cont); + if (SanitizeExponent) { + Checks.push_back( + std::make_pair(ValidExponent, SanitizerKind::ShiftExponent)); + } + if (SanitizeBase) { // Check whether we are shifting any non-zero bits off the top of the - // integer. - CGF.EmitBlock(CheckBitsShifted); + // integer. We only emit this check if exponent is valid - otherwise + // instructions below will have undefined behavior themselves. + llvm::BasicBlock *Orig = Builder.GetInsertBlock(); + llvm::BasicBlock *Cont = CGF.createBasicBlock("cont"); + llvm::BasicBlock *CheckShiftBase = CGF.createBasicBlock("check"); + Builder.CreateCondBr(ValidExponent, CheckShiftBase, Cont); + CGF.EmitBlock(CheckShiftBase); llvm::Value *BitsShiftedOff = Builder.CreateLShr(Ops.LHS, Builder.CreateSub(WidthMinusOne, RHS, "shl.zeros", @@ -2693,19 +2775,17 @@ Value *ScalarExprEmitter::EmitShl(const BinOpInfo &Ops) { BitsShiftedOff = Builder.CreateLShr(BitsShiftedOff, One); } llvm::Value *Zero = llvm::ConstantInt::get(BitsShiftedOff->getType(), 0); - llvm::Value *SecondCheck = Builder.CreateICmpEQ(BitsShiftedOff, Zero); + llvm::Value *ValidBase = Builder.CreateICmpEQ(BitsShiftedOff, Zero); CGF.EmitBlock(Cont); - llvm::PHINode *P = Builder.CreatePHI(Valid->getType(), 2); - P->addIncoming(Valid, Orig); - P->addIncoming(SecondCheck, CheckBitsShifted); - Valid = P; + llvm::PHINode *BaseCheck = Builder.CreatePHI(ValidBase->getType(), 2); + BaseCheck->addIncoming(Builder.getTrue(), Orig); + BaseCheck->addIncoming(ValidBase, CheckShiftBase); + Checks.push_back(std::make_pair(BaseCheck, SanitizerKind::ShiftBase)); } - EmitBinOpCheck(std::make_pair(Valid, SanitizerKind::Shift), Ops); + assert(!Checks.empty()); + EmitBinOpCheck(Checks, Ops); } - // OpenCL 6.3j: shift values are effectively % word size of LHS. - if (CGF.getLangOpts().OpenCL) - RHS = Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shl.mask"); return Builder.CreateShl(Ops.LHS, RHS, "shl"); } @@ -2717,18 +2797,18 @@ Value *ScalarExprEmitter::EmitShr(const BinOpInfo &Ops) { if (Ops.LHS->getType() != RHS->getType()) RHS = Builder.CreateIntCast(RHS, Ops.LHS->getType(), false, "sh_prom"); - if (CGF.SanOpts.has(SanitizerKind::Shift) && !CGF.getLangOpts().OpenCL && - isa<llvm::IntegerType>(Ops.LHS->getType())) { + // OpenCL 6.3j: shift values are effectively % word size of LHS. + if (CGF.getLangOpts().OpenCL) + RHS = + Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shr.mask"); + else if (CGF.SanOpts.has(SanitizerKind::ShiftExponent) && + isa<llvm::IntegerType>(Ops.LHS->getType())) { CodeGenFunction::SanitizerScope SanScope(&CGF); llvm::Value *Valid = Builder.CreateICmpULE(RHS, GetWidthMinusOneValue(Ops.LHS, RHS)); - EmitBinOpCheck(std::make_pair(Valid, SanitizerKind::Shift), Ops); + EmitBinOpCheck(std::make_pair(Valid, SanitizerKind::ShiftExponent), Ops); } - // OpenCL 6.3j: shift values are effectively % word size of LHS. - if (CGF.getLangOpts().OpenCL) - RHS = Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shr.mask"); - if (Ops.Ty->hasUnsignedIntegerRepresentation()) return Builder.CreateLShr(Ops.LHS, RHS, "shr"); return Builder.CreateAShr(Ops.LHS, RHS, "shr"); @@ -2846,7 +2926,7 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc, Value *CR6Param = Builder.getInt32(CR6); llvm::Function *F = CGF.CGM.getIntrinsic(ID); - Result = Builder.CreateCall3(F, CR6Param, FirstVecArg, SecondVecArg, ""); + Result = Builder.CreateCall(F, {CR6Param, FirstVecArg, SecondVecArg}); return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType()); } @@ -2975,11 +3055,9 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) { } Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { - RegionCounter Cnt = CGF.getPGORegionCounter(E); - // Perform vector logical and on comparisons with zero vectors. if (E->getType()->isVectorType()) { - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *LHS = Visit(E->getLHS()); Value *RHS = Visit(E->getRHS()); @@ -3002,7 +3080,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { bool LHSCondVal; if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) { if (LHSCondVal) { // If we have 1 && X, just emit X. - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS()); // ZExt result to int or bool. @@ -3020,7 +3098,8 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { CodeGenFunction::ConditionalEvaluation eval(CGF); // Branch on the LHS first. If it is false, go to the failure (cont) block. - CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock, Cnt.getCount()); + CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock, + CGF.getProfileCount(E->getRHS())); // Any edges into the ContBlock are now from an (indeterminate number of) // edges from this first condition. All of these values will be false. Start @@ -3033,7 +3112,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { eval.begin(CGF); CGF.EmitBlock(RHSBlock); - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS()); eval.end(CGF); @@ -3043,7 +3122,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { // Emit an unconditional branch from this block to ContBlock. { // There is no need to emit line number for unconditional branch. - ApplyDebugLocation DL(CGF); + auto NL = ApplyDebugLocation::CreateEmpty(CGF); CGF.EmitBlock(ContBlock); } // Insert an entry into the phi node for the edge with the value of RHSCond. @@ -3054,11 +3133,9 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { } Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { - RegionCounter Cnt = CGF.getPGORegionCounter(E); - // Perform vector logical or on comparisons with zero vectors. if (E->getType()->isVectorType()) { - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *LHS = Visit(E->getLHS()); Value *RHS = Visit(E->getRHS()); @@ -3081,7 +3158,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { bool LHSCondVal; if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) { if (!LHSCondVal) { // If we have 0 || X, just emit X. - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS()); // ZExt result to int or bool. @@ -3100,7 +3177,8 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { // Branch on the LHS first. If it is true, go to the success (cont) block. CGF.EmitBranchOnBoolExpr(E->getLHS(), ContBlock, RHSBlock, - Cnt.getParentCount() - Cnt.getCount()); + CGF.getCurrentProfileCount() - + CGF.getProfileCount(E->getRHS())); // Any edges into the ContBlock are now from an (indeterminate number of) // edges from this first condition. All of these values will be true. Start @@ -3115,7 +3193,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { // Emit the RHS condition as a bool value. CGF.EmitBlock(RHSBlock); - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS()); eval.end(CGF); @@ -3166,7 +3244,6 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // Bind the common expression if necessary. CodeGenFunction::OpaqueValueMapping binding(CGF, E); - RegionCounter Cnt = CGF.getPGORegionCounter(E); Expr *condExpr = E->getCond(); Expr *lhsExpr = E->getTrueExpr(); @@ -3182,7 +3259,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // If the dead side doesn't have labels we need, just emit the Live part. if (!CGF.ContainsLabel(dead)) { if (CondExprBool) - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *Result = Visit(live); // If the live part is a throw expression, it acts like it has a void @@ -3199,7 +3276,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // the select function. if (CGF.getLangOpts().OpenCL && condExpr->getType()->isVectorType()) { - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); llvm::Value *CondV = CGF.EmitScalarExpr(condExpr); llvm::Value *LHS = Visit(lhsExpr); @@ -3244,7 +3321,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // safe to evaluate the LHS and RHS unconditionally. if (isCheapEnoughToEvaluateUnconditionally(lhsExpr, CGF) && isCheapEnoughToEvaluateUnconditionally(rhsExpr, CGF)) { - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); llvm::Value *CondV = CGF.EvaluateExprAsBool(condExpr); llvm::Value *LHS = Visit(lhsExpr); @@ -3262,10 +3339,11 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end"); CodeGenFunction::ConditionalEvaluation eval(CGF); - CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock, Cnt.getCount()); + CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock, + CGF.getProfileCount(lhsExpr)); CGF.EmitBlock(LHSBlock); - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); eval.begin(CGF); Value *LHS = Visit(lhsExpr); eval.end(CGF); @@ -3393,14 +3471,8 @@ Value *CodeGenFunction::EmitScalarExpr(const Expr *E, bool IgnoreResultAssign) { assert(E && hasScalarEvaluationKind(E->getType()) && "Invalid scalar expression to emit"); - bool hasDebugInfo = getDebugInfo(); - if (isa<CXXDefaultArgExpr>(E)) - disableDebugInfo(); - Value *V = ScalarExprEmitter(*this, IgnoreResultAssign) - .Visit(const_cast<Expr*>(E)); - if (isa<CXXDefaultArgExpr>(E) && hasDebugInfo) - enableDebugInfo(); - return V; + return ScalarExprEmitter(*this, IgnoreResultAssign) + .Visit(const_cast<Expr *>(E)); } /// EmitScalarConversion - Emit a conversion from the specified type to the diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp index 89f43c2..1163d63 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp @@ -8,13 +8,14 @@ //===----------------------------------------------------------------------===// #include "CGLoopInfo.h" +#include "clang/AST/Attr.h" +#include "clang/Sema/LoopHint.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constants.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Metadata.h" -using namespace clang; -using namespace CodeGen; +using namespace clang::CodeGen; using namespace llvm; static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) { @@ -26,8 +27,8 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) { SmallVector<Metadata *, 4> Args; // Reserve operand 0 for loop id self reference. - MDNode *TempNode = MDNode::getTemporary(Ctx, None); - Args.push_back(TempNode); + auto TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); // Setting vectorizer.width if (Attrs.VectorizerWidth > 0) { @@ -58,7 +59,6 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) { // Set the first operand to itself. MDNode *LoopID = MDNode::get(Ctx, Args); LoopID->replaceOperandWith(0, LoopID); - MDNode::deleteTemporary(TempNode); return LoopID; } @@ -78,7 +78,34 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs) LoopID = createMetadata(Header->getContext(), Attrs); } -void LoopInfoStack::push(BasicBlock *Header) { +void LoopInfoStack::push(BasicBlock *Header, + ArrayRef<const clang::Attr *> Attrs) { + for (const auto *Attr : Attrs) { + const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); + + // Skip non loop hint attributes + if (!LH) + continue; + + LoopHintAttr::OptionType Option = LH->getOption(); + LoopHintAttr::LoopHintState State = LH->getState(); + switch (Option) { + case LoopHintAttr::Vectorize: + case LoopHintAttr::Interleave: + if (State == LoopHintAttr::AssumeSafety) { + // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. + setParallel(true); + } + break; + case LoopHintAttr::VectorizeWidth: + case LoopHintAttr::InterleaveCount: + case LoopHintAttr::Unroll: + case LoopHintAttr::UnrollCount: + // Nothing to do here for these loop hints. + break; + } + } + Active.push_back(LoopInfo(Header, StagedAttrs)); // Clear the attributes so nested loops do not inherit them. StagedAttrs.clear(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h index b169399..2249937 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGLoopInfo.h @@ -15,6 +15,7 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGLOOPINFO_H #define LLVM_CLANG_LIB_CODEGEN_CGLOOPINFO_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/Value.h" @@ -27,6 +28,7 @@ class MDNode; } // end namespace llvm namespace clang { +class Attr; namespace CodeGen { /// \brief Attributes that may be specified on loops. @@ -78,15 +80,16 @@ private: /// This stack can be used to prepare attributes which are applied when a loop /// is emitted. class LoopInfoStack { - LoopInfoStack(const LoopInfoStack &) LLVM_DELETED_FUNCTION; - void operator=(const LoopInfoStack &) LLVM_DELETED_FUNCTION; + LoopInfoStack(const LoopInfoStack &) = delete; + void operator=(const LoopInfoStack &) = delete; public: LoopInfoStack() {} /// \brief Begin a new structured loop. The set of staged attributes will be /// applied to the loop and then cleared. - void push(llvm::BasicBlock *Header); + void push(llvm::BasicBlock *Header, + llvm::ArrayRef<const Attr *> Attrs = llvm::None); /// \brief End the current loop. void pop(); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp index 34c6d94..eb76ad1 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjC.cpp @@ -31,10 +31,9 @@ using namespace CodeGen; typedef llvm::PointerIntPair<llvm::Value*,1,bool> TryEmitResult; static TryEmitResult tryEmitARCRetainScalarExpr(CodeGenFunction &CGF, const Expr *e); -static RValue AdjustRelatedResultType(CodeGenFunction &CGF, - QualType ET, - const ObjCMethodDecl *Method, - RValue Result); +static RValue AdjustObjCObjectType(CodeGenFunction &CGF, + QualType ET, + RValue Result); /// Given the address of a variable of pointer type, find the correct /// null to store into it. @@ -55,13 +54,15 @@ llvm::Value *CodeGenFunction::EmitObjCStringLiteral(const ObjCStringLiteral *E) /// EmitObjCBoxedExpr - This routine generates code to call /// the appropriate expression boxing method. This will either be -/// one of +[NSNumber numberWith<Type>:], or +[NSString stringWithUTF8String:]. +/// one of +[NSNumber numberWith<Type>:], or +[NSString stringWithUTF8String:], +/// or [NSValue valueWithBytes:objCType:]. /// llvm::Value * CodeGenFunction::EmitObjCBoxedExpr(const ObjCBoxedExpr *E) { // Generate the correct selector for this literal's concrete type. // Get the method. const ObjCMethodDecl *BoxingMethod = E->getBoxingMethod(); + const Expr *SubExpr = E->getSubExpr(); assert(BoxingMethod && "BoxingMethod is null"); assert(BoxingMethod->isClassMethod() && "BoxingMethod must be a class method"); Selector Sel = BoxingMethod->getSelector(); @@ -74,7 +75,35 @@ CodeGenFunction::EmitObjCBoxedExpr(const ObjCBoxedExpr *E) { llvm::Value *Receiver = Runtime.GetClass(*this, ClassDecl); CallArgList Args; - EmitCallArgs(Args, BoxingMethod, E->arg_begin(), E->arg_end()); + const ParmVarDecl *ArgDecl = *BoxingMethod->param_begin(); + QualType ArgQT = ArgDecl->getType().getUnqualifiedType(); + + // ObjCBoxedExpr supports boxing of structs and unions + // via [NSValue valueWithBytes:objCType:] + const QualType ValueType(SubExpr->getType().getCanonicalType()); + if (ValueType->isObjCBoxableRecordType()) { + // Emit CodeGen for first parameter + // and cast value to correct type + llvm::Value *Temporary = CreateMemTemp(SubExpr->getType()); + EmitAnyExprToMem(SubExpr, Temporary, Qualifiers(), /*isInit*/ true); + llvm::Value *BitCast = Builder.CreateBitCast(Temporary, + ConvertType(ArgQT)); + Args.add(RValue::get(BitCast), ArgQT); + + // Create char array to store type encoding + std::string Str; + getContext().getObjCEncodingForType(ValueType, Str); + llvm::GlobalVariable *GV = CGM.GetAddrOfConstantCString(Str); + + // Cast type encoding to correct type + const ParmVarDecl *EncodingDecl = BoxingMethod->parameters()[1]; + QualType EncodingQT = EncodingDecl->getType().getUnqualifiedType(); + llvm::Value *Cast = Builder.CreateBitCast(GV, ConvertType(EncodingQT)); + + Args.add(RValue::get(Cast), EncodingQT); + } else { + Args.add(EmitAnyExpr(SubExpr), ArgQT); + } RValue result = Runtime.GenerateMessageSend( *this, ReturnValueSlot(), BoxingMethod->getReturnType(), Sel, Receiver, @@ -102,8 +131,8 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, ArrayType::Normal, /*IndexTypeQuals=*/0); // Allocate the temporary array(s). - llvm::Value *Objects = CreateMemTemp(ElementArrayType, "objects"); - llvm::Value *Keys = nullptr; + llvm::AllocaInst *Objects = CreateMemTemp(ElementArrayType, "objects"); + llvm::AllocaInst *Keys = nullptr; if (DLE) Keys = CreateMemTemp(ElementArrayType, "keys"); @@ -119,10 +148,9 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, if (ALE) { // Emit the element and store it to the appropriate array slot. const Expr *Rhs = ALE->getElement(i); - LValue LV = LValue::MakeAddr(Builder.CreateStructGEP(Objects, i), - ElementType, - Context.getTypeAlignInChars(Rhs->getType()), - Context); + LValue LV = LValue::MakeAddr( + Builder.CreateStructGEP(Objects->getAllocatedType(), Objects, i), + ElementType, Context.getTypeAlignInChars(Rhs->getType()), Context); llvm::Value *value = EmitScalarExpr(Rhs); EmitStoreThroughLValue(RValue::get(value), LV, true); @@ -132,19 +160,17 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, } else { // Emit the key and store it to the appropriate array slot. const Expr *Key = DLE->getKeyValueElement(i).Key; - LValue KeyLV = LValue::MakeAddr(Builder.CreateStructGEP(Keys, i), - ElementType, - Context.getTypeAlignInChars(Key->getType()), - Context); + LValue KeyLV = LValue::MakeAddr( + Builder.CreateStructGEP(Keys->getAllocatedType(), Keys, i), + ElementType, Context.getTypeAlignInChars(Key->getType()), Context); llvm::Value *keyValue = EmitScalarExpr(Key); EmitStoreThroughLValue(RValue::get(keyValue), KeyLV, /*isInit=*/true); // Emit the value and store it to the appropriate array slot. - const Expr *Value = DLE->getKeyValueElement(i).Value; - LValue ValueLV = LValue::MakeAddr(Builder.CreateStructGEP(Objects, i), - ElementType, - Context.getTypeAlignInChars(Value->getType()), - Context); + const Expr *Value = DLE->getKeyValueElement(i).Value; + LValue ValueLV = LValue::MakeAddr( + Builder.CreateStructGEP(Objects->getAllocatedType(), Objects, i), + ElementType, Context.getTypeAlignInChars(Value->getType()), Context); llvm::Value *valueValue = EmitScalarExpr(Value); EmitStoreThroughLValue(RValue::get(valueValue), ValueLV, /*isInit=*/true); if (TrackNeededObjects) { @@ -221,23 +247,22 @@ llvm::Value *CodeGenFunction::EmitObjCProtocolExpr(const ObjCProtocolExpr *E) { return CGM.getObjCRuntime().GenerateProtocolRef(*this, E->getProtocol()); } -/// \brief Adjust the type of the result of an Objective-C message send -/// expression when the method has a related result type. -static RValue AdjustRelatedResultType(CodeGenFunction &CGF, - QualType ExpT, - const ObjCMethodDecl *Method, - RValue Result) { - if (!Method) +/// \brief Adjust the type of an Objective-C object that doesn't match up due +/// to type erasure at various points, e.g., related result types or the use +/// of parameterized classes. +static RValue AdjustObjCObjectType(CodeGenFunction &CGF, QualType ExpT, + RValue Result) { + if (!ExpT->isObjCRetainableType()) return Result; - if (!Method->hasRelatedResultType() || - CGF.getContext().hasSameType(ExpT, Method->getReturnType()) || - !Result.isScalar()) + // If the converted types are the same, we're done. + llvm::Type *ExpLLVMTy = CGF.ConvertType(ExpT); + if (ExpLLVMTy == Result.getScalarVal()->getType()) return Result; - - // We have applied a related result type. Cast the rvalue appropriately. + + // We have applied a substitution. Cast the rvalue appropriately. return RValue::get(CGF.Builder.CreateBitCast(Result.getScalarVal(), - CGF.ConvertType(ExpT))); + ExpLLVMTy)); } /// Decide whether to extend the lifetime of the receiver of a @@ -422,7 +447,7 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E, Builder.CreateStore(newSelf, selfAddr); } - return AdjustRelatedResultType(*this, E->getType(), method, result); + return AdjustObjCObjectType(*this, E->getType(), result); } namespace { @@ -472,8 +497,7 @@ void CodeGenFunction::StartObjCMethod(const ObjCMethodDecl *OMD, args.push_back(OMD->getSelfDecl()); args.push_back(OMD->getCmdDecl()); - for (const auto *PI : OMD->params()) - args.push_back(PI); + args.append(OMD->param_begin(), OMD->param_end()); CurGD = OMD; CurEHLocation = OMD->getLocEnd(); @@ -501,8 +525,7 @@ void CodeGenFunction::GenerateObjCMethod(const ObjCMethodDecl *OMD) { StartObjCMethod(OMD, OMD->getClassInterface()); PGO.assignRegionCounters(OMD, CurFn); assert(isa<CompoundStmt>(OMD->getBody())); - RegionCounter Cnt = getPGORegionCounter(OMD->getBody()); - Cnt.beginRegion(Builder); + incrementProfileCounter(OMD->getBody()); EmitCompoundStmtWithoutScope(*cast<CompoundStmt>(OMD->getBody())); FinishFunction(OMD->getBodyRBrace()); } @@ -1435,7 +1458,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // Fast enumeration state. QualType StateTy = CGM.getObjCFastEnumerationStateType(); - llvm::Value *StatePtr = CreateMemTemp(StateTy, "state.ptr"); + llvm::AllocaInst *StatePtr = CreateMemTemp(StateTy, "state.ptr"); EmitNullInitialization(StatePtr, StateTy); // Number of elements in the items array. @@ -1507,11 +1530,11 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // If the limit pointer was zero to begin with, the collection is // empty; skip all this. Set the branch weight assuming this has the same // probability of exiting the loop as any other loop exit. - uint64_t EntryCount = PGO.getCurrentRegionCount(); - RegionCounter Cnt = getPGORegionCounter(&S); - Builder.CreateCondBr(Builder.CreateICmpEQ(initialBufferLimit, zero, "iszero"), - EmptyBB, LoopInitBB, - PGO.createBranchWeights(EntryCount, Cnt.getCount())); + uint64_t EntryCount = getCurrentProfileCount(); + Builder.CreateCondBr( + Builder.CreateICmpEQ(initialBufferLimit, zero, "iszero"), EmptyBB, + LoopInitBB, + createProfileWeights(EntryCount, getProfileCount(S.getBody()))); // Otherwise, initialize the loop. EmitBlock(LoopInitBB); @@ -1519,8 +1542,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // Save the initial mutations value. This is the value at an // address that was written into the state object by // countByEnumeratingWithState:objects:count:. - llvm::Value *StateMutationsPtrPtr = - Builder.CreateStructGEP(StatePtr, 2, "mutationsptr.ptr"); + llvm::Value *StateMutationsPtrPtr = Builder.CreateStructGEP( + StatePtr->getAllocatedType(), StatePtr, 2, "mutationsptr.ptr"); llvm::Value *StateMutationsPtr = Builder.CreateLoad(StateMutationsPtrPtr, "mutationsptr"); @@ -1540,7 +1563,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ llvm::PHINode *count = Builder.CreatePHI(UnsignedLongLTy, 3, "forcoll.count"); count->addIncoming(initialBufferLimit, LoopInitBB); - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); // Check whether the mutations value has changed from where it was // at start. StateMutationsPtr should actually be invariant between @@ -1600,8 +1623,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // Fetch the buffer out of the enumeration state. // TODO: this pointer should actually be invariant between // refreshes, which would help us do certain loop optimizations. - llvm::Value *StateItemsPtr = - Builder.CreateStructGEP(StatePtr, 1, "stateitems.ptr"); + llvm::Value *StateItemsPtr = Builder.CreateStructGEP( + StatePtr->getAllocatedType(), StatePtr, 1, "stateitems.ptr"); llvm::Value *EnumStateItems = Builder.CreateLoad(StateItemsPtr, "stateitems"); @@ -1652,9 +1675,9 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // Set the branch weights based on the simplifying assumption that this is // like a while-loop, i.e., ignoring that the false branch fetches more // elements and then returns to the loop. - Builder.CreateCondBr(Builder.CreateICmpULT(indexPlusOne, count), - LoopBodyBB, FetchMoreBB, - PGO.createBranchWeights(Cnt.getCount(), EntryCount)); + Builder.CreateCondBr( + Builder.CreateICmpULT(indexPlusOne, count), LoopBodyBB, FetchMoreBB, + createProfileWeights(getProfileCount(S.getBody()), EntryCount)); index->addIncoming(indexPlusOne, AfterBody.getBlock()); count->addIncoming(count, AfterBody.getBlock()); @@ -1985,7 +2008,8 @@ CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) { } // Call the marker asm if we made one, which we do only at -O0. - if (marker) Builder.CreateCall(marker); + if (marker) + Builder.CreateCall(marker); return emitARCValueOperation(*this, value, CGM.getARCEntrypoints().objc_retainAutoreleasedReturnValue, @@ -3000,13 +3024,9 @@ CodeGenFunction::GenerateObjCAtomicGetterCopyHelperFunction( SmallVector<Expr*, 4> ConstructorArgs; ConstructorArgs.push_back(&SRC); - CXXConstructExpr::arg_iterator A = CXXConstExpr->arg_begin(); - ++A; - - for (CXXConstructExpr::arg_iterator AEnd = CXXConstExpr->arg_end(); - A != AEnd; ++A) - ConstructorArgs.push_back(*A); - + ConstructorArgs.append(std::next(CXXConstExpr->arg_begin()), + CXXConstExpr->arg_end()); + CXXConstructExpr *TheCXXConstructExpr = CXXConstructExpr::Create(C, Ty, SourceLocation(), CXXConstExpr->getConstructor(), diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp index c0dc3b8..b52d623 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCGNU.cpp @@ -46,54 +46,49 @@ namespace { /// avoids constructing the type more than once if it's used more than once. class LazyRuntimeFunction { CodeGenModule *CGM; - std::vector<llvm::Type*> ArgTys; + llvm::FunctionType *FTy; const char *FunctionName; llvm::Constant *Function; - public: - /// Constructor leaves this class uninitialized, because it is intended to - /// be used as a field in another class and not all of the types that are - /// used as arguments will necessarily be available at construction time. - LazyRuntimeFunction() - : CGM(nullptr), FunctionName(nullptr), Function(nullptr) {} - /// Initialises the lazy function with the name, return type, and the types - /// of the arguments. - LLVM_END_WITH_NULL - void init(CodeGenModule *Mod, const char *name, - llvm::Type *RetTy, ...) { - CGM =Mod; - FunctionName = name; - Function = nullptr; - ArgTys.clear(); - va_list Args; - va_start(Args, RetTy); - while (llvm::Type *ArgTy = va_arg(Args, llvm::Type*)) - ArgTys.push_back(ArgTy); - va_end(Args); - // Push the return type on at the end so we can pop it off easily - ArgTys.push_back(RetTy); - } - /// Overloaded cast operator, allows the class to be implicitly cast to an - /// LLVM constant. - operator llvm::Constant*() { - if (!Function) { - if (!FunctionName) return nullptr; - // We put the return type on the end of the vector, so pop it back off - llvm::Type *RetTy = ArgTys.back(); - ArgTys.pop_back(); - llvm::FunctionType *FTy = llvm::FunctionType::get(RetTy, ArgTys, false); - Function = - cast<llvm::Constant>(CGM->CreateRuntimeFunction(FTy, FunctionName)); - // We won't need to use the types again, so we may as well clean up the - // vector now - ArgTys.resize(0); - } - return Function; - } - operator llvm::Function*() { - return cast<llvm::Function>((llvm::Constant*)*this); - } +public: + /// Constructor leaves this class uninitialized, because it is intended to + /// be used as a field in another class and not all of the types that are + /// used as arguments will necessarily be available at construction time. + LazyRuntimeFunction() + : CGM(nullptr), FunctionName(nullptr), Function(nullptr) {} + /// Initialises the lazy function with the name, return type, and the types + /// of the arguments. + LLVM_END_WITH_NULL + void init(CodeGenModule *Mod, const char *name, llvm::Type *RetTy, ...) { + CGM = Mod; + FunctionName = name; + Function = nullptr; + std::vector<llvm::Type *> ArgTys; + va_list Args; + va_start(Args, RetTy); + while (llvm::Type *ArgTy = va_arg(Args, llvm::Type *)) + ArgTys.push_back(ArgTy); + va_end(Args); + FTy = llvm::FunctionType::get(RetTy, ArgTys, false); + } + + llvm::FunctionType *getType() { return FTy; } + + /// Overloaded cast operator, allows the class to be implicitly cast to an + /// LLVM constant. + operator llvm::Constant *() { + if (!Function) { + if (!FunctionName) + return nullptr; + Function = + cast<llvm::Constant>(CGM->CreateRuntimeFunction(FTy, FunctionName)); + } + return Function; + } + operator llvm::Function *() { + return cast<llvm::Function>((llvm::Constant *)*this); + } }; @@ -171,8 +166,9 @@ protected: /// where the C code specifies const char*. llvm::Constant *MakeConstantString(const std::string &Str, const std::string &Name="") { - llvm::Constant *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str()); - return llvm::ConstantExpr::getGetElementPtr(ConstStr, Zeros); + auto *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str()); + return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), + ConstStr, Zeros); } /// Emits a linkonce_odr string, whose name is the prefix followed by the /// string value. This allows the linker to combine the strings between @@ -181,13 +177,14 @@ protected: llvm::Constant *ExportUniqueString(const std::string &Str, const std::string prefix) { std::string name = prefix + Str; - llvm::Constant *ConstStr = TheModule.getGlobalVariable(name); + auto *ConstStr = TheModule.getGlobalVariable(name); if (!ConstStr) { llvm::Constant *value = llvm::ConstantDataArray::getString(VMContext,Str); ConstStr = new llvm::GlobalVariable(TheModule, value->getType(), true, llvm::GlobalValue::LinkOnceODRLinkage, value, prefix + Str); } - return llvm::ConstantExpr::getGetElementPtr(ConstStr, Zeros); + return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), + ConstStr, Zeros); } /// Generates a global structure, initialized by the elements in the vector. /// The element types must match the types of the structure elements in the @@ -237,8 +234,9 @@ protected: NameAndAttributes += TypeStr; NameAndAttributes += '\0'; NameAndAttributes += PD->getNameAsString(); - return llvm::ConstantExpr::getGetElementPtr( - CGM.GetAddrOfConstantCString(NameAndAttributes), Zeros); + auto *ConstStr = CGM.GetAddrOfConstantCString(NameAndAttributes); + return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), + ConstStr, Zeros); } return MakeConstantString(PD->getNameAsString()); } @@ -672,8 +670,8 @@ class CGObjCGNUstep : public CGObjCGNU { slot->setMetadata(msgSendMDKind, node); // Load the imp from the slot - llvm::Value *imp = - Builder.CreateLoad(Builder.CreateStructGEP(slot.getInstruction(), 4)); + llvm::Value *imp = Builder.CreateLoad( + Builder.CreateStructGEP(nullptr, slot.getInstruction(), 4)); // The lookup function may have changed the receiver, so make sure we use // the new one. @@ -690,7 +688,7 @@ class CGObjCGNUstep : public CGObjCGNU { CGF.EmitNounwindRuntimeCall(SlotLookupSuperFn, lookupArgs); slot->setOnlyReadsMemory(); - return Builder.CreateLoad(Builder.CreateStructGEP(slot, 4)); + return Builder.CreateLoad(Builder.CreateStructGEP(nullptr, slot, 4)); } public: CGObjCGNUstep(CodeGenModule &Mod) : CGObjCGNU(Mod, 9, 3) { @@ -1013,7 +1011,7 @@ CGObjCGNU::CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion, llvm::Value *CGObjCGNU::GetClassNamed(CodeGenFunction &CGF, const std::string &Name, bool isWeak) { - llvm::Value *ClassName = CGM.GetAddrOfConstantCString(Name); + llvm::GlobalVariable *ClassNameGV = CGM.GetAddrOfConstantCString(Name); // With the incompatible ABI, this will need to be replaced with a direct // reference to the class symbol. For the compatible nonfragile ABI we are // still performing this lookup at run time but emitting the symbol for the @@ -1023,7 +1021,8 @@ llvm::Value *CGObjCGNU::GetClassNamed(CodeGenFunction &CGF, // with memoized versions or with static references if it's safe to do so. if (!isWeak) EmitClassRef(Name); - ClassName = CGF.Builder.CreateStructGEP(ClassName, 0); + llvm::Value *ClassName = + CGF.Builder.CreateStructGEP(ClassNameGV->getValueType(), ClassNameGV, 0); llvm::Constant *ClassLookupFn = CGM.CreateRuntimeFunction(llvm::FunctionType::get(IdTy, PtrToInt8Ty, true), @@ -1056,9 +1055,9 @@ llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, Selector Sel, } if (!SelValue) { SelValue = llvm::GlobalAlias::create( - SelectorTy->getElementType(), 0, llvm::GlobalValue::PrivateLinkage, + SelectorTy, llvm::GlobalValue::PrivateLinkage, ".objc_selector_" + Sel.getAsString(), &TheModule); - Types.push_back(TypedSelector(TypeEncoding, SelValue)); + Types.emplace_back(TypeEncoding, SelValue); } if (lval) { @@ -1143,21 +1142,22 @@ llvm::Constant *CGObjCGNUstep::GetEHType(QualType T) { // It's quite ugly hard-coding this. Ideally we'd generate it using the host // platform's name mangling. const char *vtableName = "_ZTVN7gnustep7libobjc22__objc_class_type_infoE"; - llvm::Constant *Vtable = TheModule.getGlobalVariable(vtableName); + auto *Vtable = TheModule.getGlobalVariable(vtableName); if (!Vtable) { Vtable = new llvm::GlobalVariable(TheModule, PtrToInt8Ty, true, llvm::GlobalValue::ExternalLinkage, nullptr, vtableName); } llvm::Constant *Two = llvm::ConstantInt::get(IntTy, 2); - Vtable = llvm::ConstantExpr::getGetElementPtr(Vtable, Two); - Vtable = llvm::ConstantExpr::getBitCast(Vtable, PtrToInt8Ty); + auto *BVtable = llvm::ConstantExpr::getBitCast( + llvm::ConstantExpr::getGetElementPtr(Vtable->getValueType(), Vtable, Two), + PtrToInt8Ty); llvm::Constant *typeName = ExportUniqueString(className, "__objc_eh_typename_"); std::vector<llvm::Constant*> fields; - fields.push_back(Vtable); + fields.push_back(BVtable); fields.push_back(typeName); llvm::Constant *TI = MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty, @@ -1261,25 +1261,25 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF, if (IsClassMessage) { if (!MetaClassPtrAlias) { MetaClassPtrAlias = llvm::GlobalAlias::create( - IdTy->getElementType(), 0, llvm::GlobalValue::InternalLinkage, + IdTy, llvm::GlobalValue::InternalLinkage, ".objc_metaclass_ref" + Class->getNameAsString(), &TheModule); } ReceiverClass = MetaClassPtrAlias; } else { if (!ClassPtrAlias) { ClassPtrAlias = llvm::GlobalAlias::create( - IdTy->getElementType(), 0, llvm::GlobalValue::InternalLinkage, + IdTy, llvm::GlobalValue::InternalLinkage, ".objc_class_ref" + Class->getNameAsString(), &TheModule); } ReceiverClass = ClassPtrAlias; } } // Cast the pointer to a simplified version of the class structure + llvm::Type *CastTy = llvm::StructType::get(IdTy, IdTy, nullptr); ReceiverClass = Builder.CreateBitCast(ReceiverClass, - llvm::PointerType::getUnqual( - llvm::StructType::get(IdTy, IdTy, nullptr))); + llvm::PointerType::getUnqual(CastTy)); // Get the superclass pointer - ReceiverClass = Builder.CreateStructGEP(ReceiverClass, 1); + ReceiverClass = Builder.CreateStructGEP(CastTy, ReceiverClass, 1); // Load the superclass pointer ReceiverClass = Builder.CreateLoad(ReceiverClass); // Construct the structure used to look up the IMP @@ -1287,8 +1287,10 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF, Receiver->getType(), IdTy, nullptr); llvm::Value *ObjCSuper = Builder.CreateAlloca(ObjCSuperTy); - Builder.CreateStore(Receiver, Builder.CreateStructGEP(ObjCSuper, 0)); - Builder.CreateStore(ReceiverClass, Builder.CreateStructGEP(ObjCSuper, 1)); + Builder.CreateStore(Receiver, + Builder.CreateStructGEP(ObjCSuperTy, ObjCSuper, 0)); + Builder.CreateStore(ReceiverClass, + Builder.CreateStructGEP(ObjCSuperTy, ObjCSuper, 1)); ObjCSuper = EnforceType(Builder, ObjCSuper, PtrToObjCSuperTy); @@ -2119,9 +2121,8 @@ void CGObjCGNU::RegisterAlias(const ObjCCompatibleAliasDecl *OAD) { // Get the class declaration for which the alias is specified. ObjCInterfaceDecl *ClassDecl = const_cast<ObjCInterfaceDecl *>(OAD->getClassInterface()); - std::string ClassName = ClassDecl->getNameAsString(); - std::string AliasName = OAD->getNameAsString(); - ClassAliases.push_back(ClassAliasPair(ClassName,AliasName)); + ClassAliases.emplace_back(ClassDecl->getNameAsString(), + OAD->getNameAsString()); } void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) { @@ -2294,7 +2295,8 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) { offsetPointerIndexes[2] = llvm::ConstantInt::get(IndexTy, ivarIndex); // Get the correct ivar field llvm::Constant *offsetValue = llvm::ConstantExpr::getGetElementPtr( - IvarList, offsetPointerIndexes); + cast<llvm::GlobalVariable>(IvarList)->getValueType(), IvarList, + offsetPointerIndexes); // Get the existing variable, if one exists. llvm::GlobalVariable *offset = TheModule.getNamedGlobal(Name); if (offset) { @@ -2366,7 +2368,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { std::vector<llvm::Constant*> Elements; llvm::Constant *Statics = NULLPtr; // Generate statics list: - if (ConstantStrings.size()) { + if (!ConstantStrings.empty()) { llvm::ArrayType *StaticsArrayTy = llvm::ArrayType::get(PtrToInt8Ty, ConstantStrings.size() + 1); ConstantStrings.push_back(NULLPtr); @@ -2439,8 +2441,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { // Number of static selectors Elements.push_back(llvm::ConstantInt::get(LongTy, SelectorCount)); - llvm::Constant *SelectorList = MakeGlobalArray(SelStructTy, Selectors, - ".objc_selector_list"); + llvm::GlobalVariable *SelectorList = + MakeGlobalArray(SelStructTy, Selectors, ".objc_selector_list"); Elements.push_back(llvm::ConstantExpr::getBitCast(SelectorList, SelStructPtrTy)); @@ -2450,8 +2452,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { llvm::Constant *Idxs[] = {Zeros[0], llvm::ConstantInt::get(Int32Ty, i), Zeros[0]}; // FIXME: We're generating redundant loads and stores here! - llvm::Constant *SelPtr = llvm::ConstantExpr::getGetElementPtr(SelectorList, - makeArrayRef(Idxs, 2)); + llvm::Constant *SelPtr = llvm::ConstantExpr::getGetElementPtr( + SelectorList->getValueType(), SelectorList, makeArrayRef(Idxs, 2)); // If selectors are defined as an opaque type, cast the pointer to this // type. SelPtr = llvm::ConstantExpr::getBitCast(SelPtr, SelectorTy); @@ -2562,8 +2564,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { true); if (TheClass) { TheClass = llvm::ConstantExpr::getBitCast(TheClass, PtrTy); - Builder.CreateCall2(RegisterAlias, TheClass, - MakeConstantString(iter->second)); + Builder.CreateCall(RegisterAlias, + {TheClass, MakeConstantString(iter->second)}); } } // Jump to end: @@ -2679,7 +2681,7 @@ llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF, llvm::Value *AddrWeakObj) { CGBuilderTy &B = CGF.Builder; AddrWeakObj = EnforceType(B, AddrWeakObj, PtrToIdTy); - return B.CreateCall(WeakReadFn, AddrWeakObj); + return B.CreateCall(WeakReadFn.getType(), WeakReadFn, AddrWeakObj); } void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF, @@ -2687,7 +2689,7 @@ void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF, CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, PtrToIdTy); - B.CreateCall2(WeakAssignFn, src, dst); + B.CreateCall(WeakAssignFn.getType(), WeakAssignFn, {src, dst}); } void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF, @@ -2696,11 +2698,9 @@ void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF, CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, PtrToIdTy); - if (!threadlocal) - B.CreateCall2(GlobalAssignFn, src, dst); - else - // FIXME. Add threadloca assign API - llvm_unreachable("EmitObjCGlobalAssign - Threal Local API NYI"); + // FIXME. Add threadloca assign API + assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI"); + B.CreateCall(GlobalAssignFn.getType(), GlobalAssignFn, {src, dst}); } void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF, @@ -2709,7 +2709,7 @@ void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF, CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, IdTy); - B.CreateCall3(IvarAssignFn, src, dst, ivarOffset); + B.CreateCall(IvarAssignFn.getType(), IvarAssignFn, {src, dst, ivarOffset}); } void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF, @@ -2717,7 +2717,7 @@ void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF, CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, PtrToIdTy); - B.CreateCall2(StrongCastAssignFn, src, dst); + B.CreateCall(StrongCastAssignFn.getType(), StrongCastAssignFn, {src, dst}); } void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF, @@ -2728,7 +2728,7 @@ void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF, DestPtr = EnforceType(B, DestPtr, PtrTy); SrcPtr = EnforceType(B, SrcPtr, PtrTy); - B.CreateCall3(MemMoveFn, DestPtr, SrcPtr, Size); + B.CreateCall(MemMoveFn.getType(), MemMoveFn, {DestPtr, SrcPtr, Size}); } llvm::GlobalVariable *CGObjCGNU::ObjCIvarOffsetVariable( diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp index f91e8e1..a45446a 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCMac.cpp @@ -486,7 +486,6 @@ public: } ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm); - ~ObjCCommonTypesHelper(){} }; /// ObjCTypesHelper - Helper class that encapsulates lazy @@ -595,7 +594,6 @@ public: public: ObjCTypesHelper(CodeGen::CodeGenModule &cgm); - ~ObjCTypesHelper() {} }; /// ObjCNonFragileABITypesHelper - will have all types needed by objective-c's @@ -733,7 +731,6 @@ public: llvm::Type *EHTypePtrTy; ObjCNonFragileABITypesHelper(CodeGen::CodeGenModule &cgm); - ~ObjCNonFragileABITypesHelper(){} }; class CGObjCCommonMac : public CodeGen::CGObjCRuntime { @@ -1678,14 +1675,13 @@ struct NullReturnState { /// getConstantGEP() - Help routine to construct simple GEPs. static llvm::Constant *getConstantGEP(llvm::LLVMContext &VMContext, - llvm::Constant *C, - unsigned idx0, + llvm::GlobalVariable *C, unsigned idx0, unsigned idx1) { llvm::Value *Idxs[] = { llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), idx0), llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), idx1) }; - return llvm::ConstantExpr::getGetElementPtr(C, Idxs); + return llvm::ConstantExpr::getGetElementPtr(C->getValueType(), C, Idxs); } /// hasObjCExceptionAttribute - Return true if this class or any super @@ -1791,8 +1787,9 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, CGF.CreateTempAlloca(ObjCTypes.SuperTy, "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); - CGF.Builder.CreateStore(ReceiverAsObject, - CGF.Builder.CreateStructGEP(ObjCSuper, 0)); + CGF.Builder.CreateStore( + ReceiverAsObject, + CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 0)); // If this is a class message the metaclass is passed as the target. llvm::Value *Target; @@ -1805,20 +1802,20 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, // the class's "isa" pointer. The following assumes that // isa" is the first ivar in a class (which it must be). Target = EmitClassRef(CGF, Class->getSuperClass()); - Target = CGF.Builder.CreateStructGEP(Target, 0); + Target = CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, Target, 0); Target = CGF.Builder.CreateLoad(Target); } else { - llvm::Value *MetaClassPtr = EmitMetaClassRef(Class); - llvm::Value *SuperPtr = CGF.Builder.CreateStructGEP(MetaClassPtr, 1); + llvm::Constant *MetaClassPtr = EmitMetaClassRef(Class); + llvm::Value *SuperPtr = + CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, MetaClassPtr, 1); llvm::Value *Super = CGF.Builder.CreateLoad(SuperPtr); Target = Super; } - } - else if (isCategoryImpl) + } else if (isCategoryImpl) Target = EmitClassRef(CGF, Class->getSuperClass()); else { llvm::Value *ClassPtr = EmitSuperClassRef(Class); - ClassPtr = CGF.Builder.CreateStructGEP(ClassPtr, 1); + ClassPtr = CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, ClassPtr, 1); Target = CGF.Builder.CreateLoad(ClassPtr); } // FIXME: We shouldn't need to do this cast, rectify the ASTContext and @@ -1826,8 +1823,8 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, llvm::Type *ClassTy = CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType()); Target = CGF.Builder.CreateBitCast(Target, ClassTy); - CGF.Builder.CreateStore(Target, - CGF.Builder.CreateStructGEP(ObjCSuper, 1)); + CGF.Builder.CreateStore( + Target, CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 1)); return EmitMessageSend(CGF, Return, ResultType, EmitSelector(CGF, Sel), ObjCSuper, ObjCTypes.SuperPtrCTy, @@ -3810,15 +3807,16 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Enter a try block: // - Call objc_exception_try_enter to push ExceptionData on top of // the EH stack. - CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), ExceptionData); + CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), + ExceptionData); // - Call setjmp on the exception data buffer. llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0); llvm::Value *GEPIndexes[] = { Zero, Zero, Zero }; - llvm::Value *SetJmpBuffer = - CGF.Builder.CreateGEP(ExceptionData, GEPIndexes, "setjmp_buffer"); - llvm::CallInst *SetJmpResult = - CGF.EmitNounwindRuntimeCall(ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result"); + llvm::Value *SetJmpBuffer = CGF.Builder.CreateGEP( + ObjCTypes.ExceptionDataTy, ExceptionData, GEPIndexes, "setjmp_buffer"); + llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall( + ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result"); SetJmpResult->setCanReturnTwice(); // If setjmp returned 0, enter the protected block; otherwise, @@ -5263,6 +5261,7 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul // const uint32_t size; // sizeof(struct _protocol_t) // const uint32_t flags; // = 0 // const char ** extendedMethodTypes; + // const char *demangledName; // } // Holder for struct _protocol_list_t * @@ -5275,6 +5274,7 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul MethodListnfABIPtrTy, MethodListnfABIPtrTy, MethodListnfABIPtrTy, MethodListnfABIPtrTy, PropertyListPtrTy, IntTy, IntTy, Int8PtrPtrTy, + Int8PtrTy, nullptr); // struct _protocol_t* @@ -6207,6 +6207,7 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocolRef( /// const uint32_t size; // sizeof(struct _protocol_t) /// const uint32_t flags; // = 0 /// const char ** extendedMethodTypes; +/// const char *demangledName; /// } /// @endcode /// @@ -6258,7 +6259,7 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol( MethodTypesExt.insert(MethodTypesExt.end(), OptMethodTypesExt.begin(), OptMethodTypesExt.end()); - llvm::Constant *Values[11]; + llvm::Constant *Values[12]; // isa is NULL Values[0] = llvm::Constant::getNullValue(ObjCTypes.ObjectPtrTy); Values[1] = GetClassName(PD->getObjCRuntimeNameAsString()); @@ -6291,6 +6292,9 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol( Values[10] = EmitProtocolMethodTypes("\01l_OBJC_$_PROTOCOL_METHOD_TYPES_" + PD->getObjCRuntimeNameAsString(), MethodTypesExt, ObjCTypes); + // const char *demangledName; + Values[11] = llvm::Constant::getNullValue(ObjCTypes.Int8PtrTy); + llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ProtocolnfABITy, Values); @@ -6562,7 +6566,8 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF, args[1].RV = RValue::get(mref); // Load the function to call from the message ref table. - llvm::Value *callee = CGF.Builder.CreateStructGEP(mref, 0); + llvm::Value *callee = + CGF.Builder.CreateStructGEP(ObjCTypes.MessageRefTy, mref, 0); callee = CGF.Builder.CreateLoad(callee, "msgSend_fn"); callee = CGF.Builder.CreateBitCast(callee, MSI.MessengerType); @@ -6727,8 +6732,9 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); - CGF.Builder.CreateStore(ReceiverAsObject, - CGF.Builder.CreateStructGEP(ObjCSuper, 0)); + CGF.Builder.CreateStore( + ReceiverAsObject, + CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 0)); // If this is a class message the metaclass is passed as the target. llvm::Value *Target; @@ -6742,8 +6748,8 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, llvm::Type *ClassTy = CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType()); Target = CGF.Builder.CreateBitCast(Target, ClassTy); - CGF.Builder.CreateStore(Target, - CGF.Builder.CreateStructGEP(ObjCSuper, 1)); + CGF.Builder.CreateStore( + Target, CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 1)); return (isVTableDispatchedSelector(Sel)) ? EmitVTableMessageSend(CGF, Return, ResultType, Sel, @@ -6992,10 +6998,10 @@ CGObjCNonFragileABIMac::GetInterfaceEHType(const ObjCInterfaceDecl *ID, llvm::Value *VTableIdx = llvm::ConstantInt::get(CGM.Int32Ty, 2); llvm::Constant *Values[] = { - llvm::ConstantExpr::getGetElementPtr(VTableGV, VTableIdx), - GetClassName(ID->getObjCRuntimeNameAsString()), - GetClassGlobal(ClassName.str()) - }; + llvm::ConstantExpr::getGetElementPtr(VTableGV->getValueType(), VTableGV, + VTableIdx), + GetClassName(ID->getObjCRuntimeNameAsString()), + GetClassGlobal(ClassName.str())}; llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.EHTypeTy, Values); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp index 3d013da..2ac6bb2 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGObjCRuntime.cpp @@ -134,7 +134,7 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF, CGBitFieldInfo *Info = new (CGF.CGM.getContext()) CGBitFieldInfo( CGBitFieldInfo::MakeInfo(CGF.CGM.getTypes(), Ivar, BitOffset, BitFieldSize, CGF.CGM.getContext().toBits(StorageSize), - Alignment.getQuantity())); + CharUnits::fromQuantity(0))); V = CGF.Builder.CreateBitCast(V, llvm::Type::getIntNPtrTy(CGF.getLLVMContext(), diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 22ee00f..8148839 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -13,6 +13,7 @@ #include "CGOpenMPRuntime.h" #include "CodeGenFunction.h" +#include "CGCleanup.h" #include "clang/AST/Decl.h" #include "clang/AST/StmtOpenMP.h" #include "llvm/ADT/ArrayRef.h" @@ -27,61 +28,243 @@ using namespace clang; using namespace CodeGen; namespace { -/// \brief API for captured statement code generation in OpenMP constructs. +/// \brief Base class for handling code generation inside OpenMP regions. class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo { public: - CGOpenMPRegionInfo(const OMPExecutableDirective &D, const CapturedStmt &CS, - const VarDecl *ThreadIDVar) - : CGCapturedStmtInfo(CS, CR_OpenMP), ThreadIDVar(ThreadIDVar), - Directive(D) { - assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); - } + /// \brief Kinds of OpenMP regions used in codegen. + enum CGOpenMPRegionKind { + /// \brief Region with outlined function for standalone 'parallel' + /// directive. + ParallelOutlinedRegion, + /// \brief Region with outlined function for standalone 'task' directive. + TaskOutlinedRegion, + /// \brief Region for constructs that do not require function outlining, + /// like 'for', 'sections', 'atomic' etc. directives. + InlinedRegion, + }; - /// \brief Gets a variable or parameter for storing global thread id + CGOpenMPRegionInfo(const CapturedStmt &CS, + const CGOpenMPRegionKind RegionKind, + const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind) + : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind), + CodeGen(CodeGen), Kind(Kind) {} + + CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind, + const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind) + : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen), + Kind(Kind) {} + + /// \brief Get a variable or parameter for storing global thread id /// inside OpenMP construct. - const VarDecl *getThreadIDVariable() const { return ThreadIDVar; } + virtual const VarDecl *getThreadIDVariable() const = 0; + + /// \brief Emit the captured statement body. + virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) override; + + /// \brief Get an LValue for the current ThreadID variable. + /// \return LValue for thread id variable. This LValue always has type int32*. + virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF); - /// \brief Gets an LValue for the current ThreadID variable. - LValue getThreadIDVariableLValue(CodeGenFunction &CGF); + CGOpenMPRegionKind getRegionKind() const { return RegionKind; } + + OpenMPDirectiveKind getDirectiveKind() const { return Kind; } static bool classof(const CGCapturedStmtInfo *Info) { return Info->getKind() == CR_OpenMP; } - /// \brief Emit the captured statement body. - void EmitBody(CodeGenFunction &CGF, Stmt *S) override; +protected: + CGOpenMPRegionKind RegionKind; + const RegionCodeGenTy &CodeGen; + OpenMPDirectiveKind Kind; +}; + +/// \brief API for captured statement code generation in OpenMP constructs. +class CGOpenMPOutlinedRegionInfo : public CGOpenMPRegionInfo { +public: + CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar, + const RegionCodeGenTy &CodeGen, + OpenMPDirectiveKind Kind) + : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind), + ThreadIDVar(ThreadIDVar) { + assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); + } + /// \brief Get a variable or parameter for storing global thread id + /// inside OpenMP construct. + const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; } + + /// \brief Get the name of the capture helper. + StringRef getHelperName() const override { return ".omp_outlined."; } + + static bool classof(const CGCapturedStmtInfo *Info) { + return CGOpenMPRegionInfo::classof(Info) && + cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == + ParallelOutlinedRegion; + } + +private: + /// \brief A variable or parameter storing global thread id for OpenMP + /// constructs. + const VarDecl *ThreadIDVar; +}; + +/// \brief API for captured statement code generation in OpenMP constructs. +class CGOpenMPTaskOutlinedRegionInfo : public CGOpenMPRegionInfo { +public: + CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS, + const VarDecl *ThreadIDVar, + const RegionCodeGenTy &CodeGen, + OpenMPDirectiveKind Kind) + : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind), + ThreadIDVar(ThreadIDVar) { + assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); + } + /// \brief Get a variable or parameter for storing global thread id + /// inside OpenMP construct. + const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; } + + /// \brief Get an LValue for the current ThreadID variable. + LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override; /// \brief Get the name of the capture helper. StringRef getHelperName() const override { return ".omp_outlined."; } + static bool classof(const CGCapturedStmtInfo *Info) { + return CGOpenMPRegionInfo::classof(Info) && + cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == + TaskOutlinedRegion; + } + private: /// \brief A variable or parameter storing global thread id for OpenMP /// constructs. const VarDecl *ThreadIDVar; - /// \brief OpenMP executable directive associated with the region. - const OMPExecutableDirective &Directive; }; + +/// \brief API for inlined captured statement code generation in OpenMP +/// constructs. +class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo { +public: + CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI, + const RegionCodeGenTy &CodeGen, + OpenMPDirectiveKind Kind) + : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind), OldCSI(OldCSI), + OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {} + // \brief Retrieve the value of the context parameter. + llvm::Value *getContextValue() const override { + if (OuterRegionInfo) + return OuterRegionInfo->getContextValue(); + llvm_unreachable("No context value for inlined OpenMP region"); + } + virtual void setContextValue(llvm::Value *V) override { + if (OuterRegionInfo) { + OuterRegionInfo->setContextValue(V); + return; + } + llvm_unreachable("No context value for inlined OpenMP region"); + } + /// \brief Lookup the captured field decl for a variable. + const FieldDecl *lookup(const VarDecl *VD) const override { + if (OuterRegionInfo) + return OuterRegionInfo->lookup(VD); + // If there is no outer outlined region,no need to lookup in a list of + // captured variables, we can use the original one. + return nullptr; + } + FieldDecl *getThisFieldDecl() const override { + if (OuterRegionInfo) + return OuterRegionInfo->getThisFieldDecl(); + return nullptr; + } + /// \brief Get a variable or parameter for storing global thread id + /// inside OpenMP construct. + const VarDecl *getThreadIDVariable() const override { + if (OuterRegionInfo) + return OuterRegionInfo->getThreadIDVariable(); + return nullptr; + } + + /// \brief Get the name of the capture helper. + StringRef getHelperName() const override { + if (auto *OuterRegionInfo = getOldCSI()) + return OuterRegionInfo->getHelperName(); + llvm_unreachable("No helper name for inlined OpenMP construct"); + } + + CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; } + + static bool classof(const CGCapturedStmtInfo *Info) { + return CGOpenMPRegionInfo::classof(Info) && + cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion; + } + +private: + /// \brief CodeGen info about outer OpenMP region. + CodeGenFunction::CGCapturedStmtInfo *OldCSI; + CGOpenMPRegionInfo *OuterRegionInfo; +}; + +/// \brief RAII for emitting code of OpenMP constructs. +class InlinedOpenMPRegionRAII { + CodeGenFunction &CGF; + +public: + /// \brief Constructs region for combined constructs. + /// \param CodeGen Code generation sequence for combined directives. Includes + /// a list of functions used for code generation of implicitly inlined + /// regions. + InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, + OpenMPDirectiveKind Kind) + : CGF(CGF) { + // Start emission for the construct. + CGF.CapturedStmtInfo = + new CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, CodeGen, Kind); + } + ~InlinedOpenMPRegionRAII() { + // Restore original CapturedStmtInfo only if we're done with code emission. + auto *OldCSI = + cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); + delete CGF.CapturedStmtInfo; + CGF.CapturedStmtInfo = OldCSI; + } +}; + } // namespace LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { return CGF.MakeNaturalAlignAddrLValue( - CGF.GetAddrOfLocalVar(ThreadIDVar), - CGF.getContext().getPointerType(ThreadIDVar->getType())); + CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(getThreadIDVariable()), + CGF.PointerAlignInBytes), + getThreadIDVariable() + ->getType() + ->castAs<PointerType>() + ->getPointeeType()); } -void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, Stmt *S) { - CodeGenFunction::OMPPrivateScope PrivateScope(CGF); - CGF.EmitOMPPrivateClause(Directive, PrivateScope); - CGF.EmitOMPFirstprivateClause(Directive, PrivateScope); - if (PrivateScope.Privatize()) - // Emit implicit barrier to synchronize threads and avoid data races. - CGF.CGM.getOpenMPRuntime().EmitOMPBarrierCall(CGF, Directive.getLocStart(), - /*IsExplicit=*/false); - CGCapturedStmtInfo::EmitBody(CGF, S); +void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) { + // 1.2.2 OpenMP Language Terminology + // Structured block - An executable statement with a single entry at the + // top and a single exit at the bottom. + // The point of exit cannot be a branch out of the structured block. + // longjmp() and throw() must not violate the entry/exit criteria. + CGF.EHStack.pushTerminate(); + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + CodeGen(CGF); + } + CGF.EHStack.popTerminate(); +} + +LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue( + CodeGenFunction &CGF) { + return CGF.MakeNaturalAlignAddrLValue( + CGF.GetAddrOfLocalVar(getThreadIDVariable()), + getThreadIDVariable()->getType()); } CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM) - : CGM(CGM), DefaultOpenMPPSource(nullptr) { + : CGM(CGM), DefaultOpenMPPSource(nullptr), KmpRoutineEntryPtrTy(nullptr) { IdentTy = llvm::StructType::create( "ident_t", CGM.Int32Ty /* reserved_1 */, CGM.Int32Ty /* flags */, CGM.Int32Ty /* reserved_2 */, CGM.Int32Ty /* reserved_3 */, @@ -93,18 +276,37 @@ CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM) KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8); } -llvm::Value * -CGOpenMPRuntime::EmitOpenMPOutlinedFunction(const OMPExecutableDirective &D, - const VarDecl *ThreadIDVar) { +void CGOpenMPRuntime::clear() { + InternalVars.clear(); +} + +llvm::Value *CGOpenMPRuntime::emitParallelOutlinedFunction( + const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, + OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { + assert(ThreadIDVar->getType()->isPointerType() && + "thread id variable must be of type kmp_int32 *"); const CapturedStmt *CS = cast<CapturedStmt>(D.getAssociatedStmt()); CodeGenFunction CGF(CGM, true); - CGOpenMPRegionInfo CGInfo(D, *CS, ThreadIDVar); - CGF.CapturedStmtInfo = &CGInfo; + CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind); + CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); + return CGF.GenerateCapturedStmtFunction(*CS); +} + +llvm::Value *CGOpenMPRuntime::emitTaskOutlinedFunction( + const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, + OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { + assert(!ThreadIDVar->getType()->isPointerType() && + "thread id variable must be of type kmp_int32 for tasks"); + auto *CS = cast<CapturedStmt>(D.getAssociatedStmt()); + CodeGenFunction CGF(CGM, true); + CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, + InnermostKind); + CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); return CGF.GenerateCapturedStmtFunction(*CS); } llvm::Value * -CGOpenMPRuntime::GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags) { +CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) { llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags); if (!Entry) { if (!DefaultOpenMPPSource) { @@ -134,12 +336,13 @@ CGOpenMPRuntime::GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags) { return Entry; } -llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation( - CodeGenFunction &CGF, SourceLocation Loc, OpenMPLocationFlags Flags) { +llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, + SourceLocation Loc, + OpenMPLocationFlags Flags) { // If no debug info is generated - return global default location. if (CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::NoDebugInfo || Loc.isInvalid()) - return GetOrCreateDefaultOpenMPLocation(Flags); + return getOrCreateDefaultLocation(Flags); assert(CGF.CurFn && "No function in current CodeGenFunction."); @@ -159,14 +362,14 @@ llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation( CGBuilderTy::InsertPointGuard IPG(CGF.Builder); CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt); - CGF.Builder.CreateMemCpy(LocValue, GetOrCreateDefaultOpenMPLocation(Flags), + CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags), llvm::ConstantExpr::getSizeOf(IdentTy), CGM.PointerAlignInBytes); } // char **psource = &.kmpc_loc_<flags>.addr.psource; - auto *PSource = - CGF.Builder.CreateConstInBoundsGEP2_32(LocValue, 0, IdentField_PSource); + auto *PSource = CGF.Builder.CreateConstInBoundsGEP2_32(IdentTy, LocValue, 0, + IdentField_PSource); auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding()); if (OMPDebugLoc == nullptr) { @@ -189,8 +392,8 @@ llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation( return LocValue; } -llvm::Value *CGOpenMPRuntime::GetOpenMPThreadID(CodeGenFunction &CGF, - SourceLocation Loc) { +llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, + SourceLocation Loc) { assert(CGF.CurFn && "No function in current CodeGenFunction."); llvm::Value *ThreadID = nullptr; @@ -204,36 +407,35 @@ llvm::Value *CGOpenMPRuntime::GetOpenMPThreadID(CodeGenFunction &CGF, } if (auto OMPRegionInfo = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { - // Check if this an outlined function with thread id passed as argument. - auto ThreadIDVar = OMPRegionInfo->getThreadIDVariable(); - auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); - auto RVal = CGF.EmitLoadOfLValue(LVal, Loc); - LVal = CGF.MakeNaturalAlignAddrLValue(RVal.getScalarVal(), - ThreadIDVar->getType()); - ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal(); - // If value loaded in entry block, cache it and use it everywhere in - // function. - if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) { - auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); - Elem.second.ThreadID = ThreadID; + if (OMPRegionInfo->getThreadIDVariable()) { + // Check if this an outlined function with thread id passed as argument. + auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); + ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal(); + // If value loaded in entry block, cache it and use it everywhere in + // function. + if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) { + auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); + Elem.second.ThreadID = ThreadID; + } + return ThreadID; } - } else { - // This is not an outlined function region - need to call __kmpc_int32 - // kmpc_global_thread_num(ident_t *loc). - // Generate thread id value and cache this value for use across the - // function. - CGBuilderTy::InsertPointGuard IPG(CGF.Builder); - CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt); - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc)}; - ThreadID = CGF.EmitRuntimeCall( - CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num), Args); - auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); - Elem.second.ThreadID = ThreadID; } + + // This is not an outlined function region - need to call __kmpc_int32 + // kmpc_global_thread_num(ident_t *loc). + // Generate thread id value and cache this value for use across the + // function. + CGBuilderTy::InsertPointGuard IPG(CGF.Builder); + CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt); + ThreadID = + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num), + emitUpdateLocation(CGF, Loc)); + auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); + Elem.second.ThreadID = ThreadID; return ThreadID; } -void CGOpenMPRuntime::FunctionFinished(CodeGenFunction &CGF) { +void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { assert(CGF.CurFn && "No function in current CodeGenFunction."); if (OpenMPLocThreadIDMap.count(CGF.CurFn)) OpenMPLocThreadIDMap.erase(CGF.CurFn); @@ -248,7 +450,7 @@ llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() { } llvm::Constant * -CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) { +CGOpenMPRuntime::createRuntimeFunction(OpenMPRTLFunction Function) { llvm::Constant *RTLFn = nullptr; switch (Function) { case OMPRTL__kmpc_fork_call: { @@ -334,85 +536,12 @@ CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) { RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier"); break; } - // Build __kmpc_for_static_init*( - // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype, - // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower, - // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride, - // kmp_int[32|64] incr, kmp_int[32|64] chunk); - case OMPRTL__kmpc_for_static_init_4: { - auto ITy = CGM.Int32Ty; - auto PtrTy = llvm::PointerType::getUnqual(ITy); - llvm::Type *TypeParams[] = { - getIdentTyPointerTy(), // loc - CGM.Int32Ty, // tid - CGM.Int32Ty, // schedtype - llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter - PtrTy, // p_lower - PtrTy, // p_upper - PtrTy, // p_stride - ITy, // incr - ITy // chunk - }; - llvm::FunctionType *FnTy = - llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); - RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_4"); - break; - } - case OMPRTL__kmpc_for_static_init_4u: { - auto ITy = CGM.Int32Ty; - auto PtrTy = llvm::PointerType::getUnqual(ITy); - llvm::Type *TypeParams[] = { - getIdentTyPointerTy(), // loc - CGM.Int32Ty, // tid - CGM.Int32Ty, // schedtype - llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter - PtrTy, // p_lower - PtrTy, // p_upper - PtrTy, // p_stride - ITy, // incr - ITy // chunk - }; - llvm::FunctionType *FnTy = - llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); - RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_4u"); - break; - } - case OMPRTL__kmpc_for_static_init_8: { - auto ITy = CGM.Int64Ty; - auto PtrTy = llvm::PointerType::getUnqual(ITy); - llvm::Type *TypeParams[] = { - getIdentTyPointerTy(), // loc - CGM.Int32Ty, // tid - CGM.Int32Ty, // schedtype - llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter - PtrTy, // p_lower - PtrTy, // p_upper - PtrTy, // p_stride - ITy, // incr - ITy // chunk - }; - llvm::FunctionType *FnTy = - llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); - RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_8"); - break; - } - case OMPRTL__kmpc_for_static_init_8u: { - auto ITy = CGM.Int64Ty; - auto PtrTy = llvm::PointerType::getUnqual(ITy); - llvm::Type *TypeParams[] = { - getIdentTyPointerTy(), // loc - CGM.Int32Ty, // tid - CGM.Int32Ty, // schedtype - llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter - PtrTy, // p_lower - PtrTy, // p_upper - PtrTy, // p_stride - ITy, // incr - ITy // chunk - }; + case OMPRTL__kmpc_barrier: { + // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; llvm::FunctionType *FnTy = llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); - RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_8u"); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier"); break; } case OMPRTL__kmpc_for_static_fini: { @@ -452,10 +581,10 @@ CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) { break; } case OMPRTL__kmpc_flush: { - // Build void __kmpc_flush(ident_t *loc, ...); + // Build void __kmpc_flush(ident_t *loc); llvm::Type *TypeParams[] = {getIdentTyPointerTy()}; llvm::FunctionType *FnTy = - llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true); + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush"); break; } @@ -475,38 +604,365 @@ CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) { RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master"); break; } + case OMPRTL__kmpc_omp_taskyield: { + // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid, + // int end_part); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield"); + break; + } + case OMPRTL__kmpc_single: { + // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single"); + break; + } + case OMPRTL__kmpc_end_single: { + // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single"); + break; + } + case OMPRTL__kmpc_omp_task_alloc: { + // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid, + // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, + // kmp_routine_entry_t *task_entry); + assert(KmpRoutineEntryPtrTy != nullptr && + "Type kmp_routine_entry_t must be created."); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, + CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy}; + // Return void * and then cast to particular kmp_task_t type. + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc"); + break; + } + case OMPRTL__kmpc_omp_task: { + // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t + // *new_task); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, + CGM.VoidPtrTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task"); + break; + } + case OMPRTL__kmpc_copyprivate: { + // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid, + // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), + // kmp_int32 didit); + llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; + auto *CpyFnTy = + llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy, + CGM.VoidPtrTy, CpyFnTy->getPointerTo(), + CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate"); + break; + } + case OMPRTL__kmpc_reduce: { + // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, + // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void + // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck); + llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; + auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams, + /*isVarArg=*/false); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy, + CGM.VoidPtrTy, ReduceFnTy->getPointerTo(), + llvm::PointerType::getUnqual(KmpCriticalNameTy)}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce"); + break; + } + case OMPRTL__kmpc_reduce_nowait: { + // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 + // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, + // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name + // *lck); + llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; + auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams, + /*isVarArg=*/false); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy, + CGM.VoidPtrTy, ReduceFnTy->getPointerTo(), + llvm::PointerType::getUnqual(KmpCriticalNameTy)}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait"); + break; + } + case OMPRTL__kmpc_end_reduce: { + // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, + // kmp_critical_name *lck); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), CGM.Int32Ty, + llvm::PointerType::getUnqual(KmpCriticalNameTy)}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce"); + break; + } + case OMPRTL__kmpc_end_reduce_nowait: { + // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, + // kmp_critical_name *lck); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), CGM.Int32Ty, + llvm::PointerType::getUnqual(KmpCriticalNameTy)}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = + CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait"); + break; + } + case OMPRTL__kmpc_omp_task_begin_if0: { + // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t + // *new_task); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, + CGM.VoidPtrTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = + CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0"); + break; + } + case OMPRTL__kmpc_omp_task_complete_if0: { + // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t + // *new_task); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, + CGM.VoidPtrTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, + /*Name=*/"__kmpc_omp_task_complete_if0"); + break; + } + case OMPRTL__kmpc_ordered: { + // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered"); + break; + } + case OMPRTL__kmpc_end_ordered: { + // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered"); + break; + } + case OMPRTL__kmpc_omp_taskwait: { + // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait"); + break; + } + case OMPRTL__kmpc_taskgroup: { + // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup"); + break; + } + case OMPRTL__kmpc_end_taskgroup: { + // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup"); + break; + } + case OMPRTL__kmpc_push_proc_bind: { + // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, + // int proc_bind) + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind"); + break; + } + case OMPRTL__kmpc_omp_task_with_deps: { + // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid, + // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list, + // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty, + CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = + CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps"); + break; + } + case OMPRTL__kmpc_omp_wait_deps: { + // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid, + // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias, + // kmp_depend_info_t *noalias_dep_list); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, + CGM.Int32Ty, CGM.VoidPtrTy, + CGM.Int32Ty, CGM.VoidPtrTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps"); + break; + } + case OMPRTL__kmpc_cancellationpoint: { + // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32 + // global_tid, kmp_int32 cncl_kind) + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint"); + break; + } + case OMPRTL__kmpc_cancel: { + // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid, + // kmp_int32 cncl_kind) + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel"); + break; + } } return RTLFn; } +llvm::Constant *CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, + bool IVSigned) { + assert((IVSize == 32 || IVSize == 64) && + "IV size is not compatible with the omp runtime"); + auto Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4" + : "__kmpc_for_static_init_4u") + : (IVSigned ? "__kmpc_for_static_init_8" + : "__kmpc_for_static_init_8u"); + auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty; + auto PtrTy = llvm::PointerType::getUnqual(ITy); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), // loc + CGM.Int32Ty, // tid + CGM.Int32Ty, // schedtype + llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter + PtrTy, // p_lower + PtrTy, // p_upper + PtrTy, // p_stride + ITy, // incr + ITy // chunk + }; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); + return CGM.CreateRuntimeFunction(FnTy, Name); +} + +llvm::Constant *CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, + bool IVSigned) { + assert((IVSize == 32 || IVSize == 64) && + "IV size is not compatible with the omp runtime"); + auto Name = + IVSize == 32 + ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u") + : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u"); + auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty; + llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc + CGM.Int32Ty, // tid + CGM.Int32Ty, // schedtype + ITy, // lower + ITy, // upper + ITy, // stride + ITy // chunk + }; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); + return CGM.CreateRuntimeFunction(FnTy, Name); +} + +llvm::Constant *CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, + bool IVSigned) { + assert((IVSize == 32 || IVSize == 64) && + "IV size is not compatible with the omp runtime"); + auto Name = + IVSize == 32 + ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u") + : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u"); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), // loc + CGM.Int32Ty, // tid + }; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + return CGM.CreateRuntimeFunction(FnTy, Name); +} + +llvm::Constant *CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, + bool IVSigned) { + assert((IVSize == 32 || IVSize == 64) && + "IV size is not compatible with the omp runtime"); + auto Name = + IVSize == 32 + ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u") + : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u"); + auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty; + auto PtrTy = llvm::PointerType::getUnqual(ITy); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), // loc + CGM.Int32Ty, // tid + llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter + PtrTy, // p_lower + PtrTy, // p_upper + PtrTy // p_stride + }; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); + return CGM.CreateRuntimeFunction(FnTy, Name); +} + llvm::Constant * CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) { + assert(!CGM.getLangOpts().OpenMPUseTLS || + !CGM.getContext().getTargetInfo().isTLSSupported()); // Lookup the entry, lazily creating it if necessary. - return GetOrCreateInternalVariable(CGM.Int8PtrPtrTy, + return getOrCreateInternalVariable(CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)) + ".cache."); } -llvm::Value *CGOpenMPRuntime::getOMPAddrOfThreadPrivate(CodeGenFunction &CGF, - const VarDecl *VD, - llvm::Value *VDAddr, - SourceLocation Loc) { +llvm::Value *CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, + const VarDecl *VD, + llvm::Value *VDAddr, + SourceLocation Loc) { + if (CGM.getLangOpts().OpenMPUseTLS && + CGM.getContext().getTargetInfo().isTLSSupported()) + return VDAddr; + auto VarTy = VDAddr->getType()->getPointerElementType(); - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), - GetOpenMPThreadID(CGF, Loc), + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), CGF.Builder.CreatePointerCast(VDAddr, CGM.Int8PtrTy), CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)), getOrCreateThreadPrivateCache(VD)}; return CGF.EmitRuntimeCall( - CreateRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args); + createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args); } -void CGOpenMPRuntime::EmitOMPThreadPrivateVarInit( +void CGOpenMPRuntime::emitThreadPrivateVarInit( CodeGenFunction &CGF, llvm::Value *VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) { // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime // library. - auto OMPLoc = EmitOpenMPUpdateLocation(CGF, Loc); - CGF.EmitRuntimeCall(CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num), + auto OMPLoc = emitUpdateLocation(CGF, Loc); + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num), OMPLoc); // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor) // to register constructor/destructor for variable. @@ -514,12 +970,16 @@ void CGOpenMPRuntime::EmitOMPThreadPrivateVarInit( CGF.Builder.CreatePointerCast(VDAddr, CGM.VoidPtrTy), Ctor, CopyCtor, Dtor}; CGF.EmitRuntimeCall( - CreateRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args); + createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args); } -llvm::Function *CGOpenMPRuntime::EmitOMPThreadPrivateVarDefinition( +llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition( const VarDecl *VD, llvm::Value *VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF) { + if (CGM.getLangOpts().OpenMPUseTLS && + CGM.getContext().getTargetInfo().isTLSSupported()) + return nullptr; + VD = VD->getDefinition(CGM.getContext()); if (VD && ThreadPrivateWithDefinition.count(VD) == 0) { ThreadPrivateWithDefinition.insert(VD); @@ -620,54 +1080,121 @@ llvm::Function *CGOpenMPRuntime::EmitOMPThreadPrivateVarDefinition( InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction, CGM.getTypes().arrangeNullaryFunction(), ArgList, Loc); - EmitOMPThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); + emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); InitCGF.FinishFunction(); return InitFunction; } - EmitOMPThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); + emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); } return nullptr; } -void CGOpenMPRuntime::EmitOMPParallelCall(CodeGenFunction &CGF, - SourceLocation Loc, - llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct) { - // Build call __kmpc_fork_call(loc, 1, microtask, captured_struct/*context*/) - llvm::Value *Args[] = { - EmitOpenMPUpdateLocation(CGF, Loc), - CGF.Builder.getInt32(1), // Number of arguments after 'microtask' argument - // (there is only one additional argument - 'context') - CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()), - CGF.EmitCastToVoidPtr(CapturedStruct)}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_fork_call); - CGF.EmitRuntimeCall(RTLFn, Args); -} - -void CGOpenMPRuntime::EmitOMPSerialCall(CodeGenFunction &CGF, - SourceLocation Loc, - llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct) { - auto ThreadID = GetOpenMPThreadID(CGF, Loc); - // Build calls: - // __kmpc_serialized_parallel(&Loc, GTid); - llvm::Value *SerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_serialized_parallel); - CGF.EmitRuntimeCall(RTLFn, SerArgs); - - // OutlinedFn(>id, &zero, CapturedStruct); - auto ThreadIDAddr = EmitThreadIDAddress(CGF, Loc); - auto Int32Ty = - CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); - auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr"); - CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); - llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct}; - CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs); +/// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen +/// function. Here is the logic: +/// if (Cond) { +/// ThenGen(); +/// } else { +/// ElseGen(); +/// } +static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, + const RegionCodeGenTy &ThenGen, + const RegionCodeGenTy &ElseGen) { + CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); + + // If the condition constant folds and can be elided, try to avoid emitting + // the condition and the dead arm of the if/else. + bool CondConstant; + if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { + CodeGenFunction::RunCleanupsScope Scope(CGF); + if (CondConstant) { + ThenGen(CGF); + } else { + ElseGen(CGF); + } + return; + } + + // Otherwise, the condition did not fold, or we couldn't elide it. Just + // emit the conditional branch. + auto ThenBlock = CGF.createBasicBlock("omp_if.then"); + auto ElseBlock = CGF.createBasicBlock("omp_if.else"); + auto ContBlock = CGF.createBasicBlock("omp_if.end"); + CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0); + + // Emit the 'then' code. + CGF.EmitBlock(ThenBlock); + { + CodeGenFunction::RunCleanupsScope ThenScope(CGF); + ThenGen(CGF); + } + CGF.EmitBranch(ContBlock); + // Emit the 'else' code if present. + { + // There is no need to emit line number for unconditional branch. + auto NL = ApplyDebugLocation::CreateEmpty(CGF); + CGF.EmitBlock(ElseBlock); + } + { + CodeGenFunction::RunCleanupsScope ThenScope(CGF); + ElseGen(CGF); + } + { + // There is no need to emit line number for unconditional branch. + auto NL = ApplyDebugLocation::CreateEmpty(CGF); + CGF.EmitBranch(ContBlock); + } + // Emit the continuation block for code after the if. + CGF.EmitBlock(ContBlock, /*IsFinished=*/true); +} - // __kmpc_end_serialized_parallel(&Loc, GTid); - llvm::Value *EndSerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID}; - RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel); - CGF.EmitRuntimeCall(RTLFn, EndSerArgs); +void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, + llvm::Value *OutlinedFn, + llvm::Value *CapturedStruct, + const Expr *IfCond) { + auto *RTLoc = emitUpdateLocation(CGF, Loc); + auto &&ThenGen = + [this, OutlinedFn, CapturedStruct, RTLoc](CodeGenFunction &CGF) { + // Build call __kmpc_fork_call(loc, 1, microtask, + // captured_struct/*context*/) + llvm::Value *Args[] = { + RTLoc, + CGF.Builder.getInt32( + 1), // Number of arguments after 'microtask' argument + // (there is only one additional argument - 'context') + CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()), + CGF.EmitCastToVoidPtr(CapturedStruct)}; + auto RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_call); + CGF.EmitRuntimeCall(RTLFn, Args); + }; + auto &&ElseGen = [this, OutlinedFn, CapturedStruct, RTLoc, Loc]( + CodeGenFunction &CGF) { + auto ThreadID = getThreadID(CGF, Loc); + // Build calls: + // __kmpc_serialized_parallel(&Loc, GTid); + llvm::Value *Args[] = {RTLoc, ThreadID}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), + Args); + + // OutlinedFn(>id, &zero, CapturedStruct); + auto ThreadIDAddr = emitThreadIDAddress(CGF, Loc); + auto Int32Ty = CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, + /*Signed*/ true); + auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr"); + CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); + llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct}; + CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs); + + // __kmpc_end_serialized_parallel(&Loc, GTid); + llvm::Value *EndArgs[] = {emitUpdateLocation(CGF, Loc), ThreadID}; + CGF.EmitRuntimeCall( + createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel), EndArgs); + }; + if (IfCond) { + emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen); + } else { + CodeGenFunction::RunCleanupsScope Scope(CGF); + ThenGen(CGF); + } } // If we're inside an (outlined) parallel region, use the region info's @@ -676,13 +1203,14 @@ void CGOpenMPRuntime::EmitOMPSerialCall(CodeGenFunction &CGF, // regular serial code region, get thread ID by calling kmp_int32 // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and // return the address of that temp. -llvm::Value *CGOpenMPRuntime::EmitThreadIDAddress(CodeGenFunction &CGF, +llvm::Value *CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc) { if (auto OMPRegionInfo = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) - return CGF.EmitLoadOfLValue(OMPRegionInfo->getThreadIDVariableLValue(CGF), - SourceLocation()).getScalarVal(); - auto ThreadID = GetOpenMPThreadID(CGF, Loc); + if (OMPRegionInfo->getThreadIDVariable()) + return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(); + + auto ThreadID = getThreadID(CGF, Loc); auto Int32Ty = CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); @@ -693,7 +1221,7 @@ llvm::Value *CGOpenMPRuntime::EmitThreadIDAddress(CodeGenFunction &CGF, } llvm::Constant * -CGOpenMPRuntime::GetOrCreateInternalVariable(llvm::Type *Ty, +CGOpenMPRuntime::getOrCreateInternalVariable(llvm::Type *Ty, const llvm::Twine &Name) { SmallString<256> Buffer; llvm::raw_svector_ostream Out(Buffer); @@ -712,31 +1240,52 @@ CGOpenMPRuntime::GetOrCreateInternalVariable(llvm::Type *Ty, Elem.first()); } -llvm::Value *CGOpenMPRuntime::GetCriticalRegionLock(StringRef CriticalName) { +llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) { llvm::Twine Name(".gomp_critical_user_", CriticalName); - return GetOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var")); + return getOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var")); } -void CGOpenMPRuntime::EmitOMPCriticalRegion( - CodeGenFunction &CGF, StringRef CriticalName, - const std::function<void()> &CriticalOpGen, SourceLocation Loc) { - auto RegionLock = GetCriticalRegionLock(CriticalName); +namespace { +template <size_t N> class CallEndCleanup : public EHScopeStack::Cleanup { + llvm::Value *Callee; + llvm::Value *Args[N]; + +public: + CallEndCleanup(llvm::Value *Callee, ArrayRef<llvm::Value *> CleanupArgs) + : Callee(Callee) { + assert(CleanupArgs.size() == N); + std::copy(CleanupArgs.begin(), CleanupArgs.end(), std::begin(Args)); + } + void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { + CGF.EmitRuntimeCall(Callee, Args); + } +}; +} // namespace + +void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, + StringRef CriticalName, + const RegionCodeGenTy &CriticalOpGen, + SourceLocation Loc) { // __kmpc_critical(ident_t *, gtid, Lock); // CriticalOpGen(); // __kmpc_end_critical(ident_t *, gtid, Lock); // Prepare arguments and build a call to __kmpc_critical - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), - GetOpenMPThreadID(CGF, Loc), RegionLock}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_critical); - CGF.EmitRuntimeCall(RTLFn, Args); - CriticalOpGen(); - // Build a call to __kmpc_end_critical - RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_critical); - CGF.EmitRuntimeCall(RTLFn, Args); -} - -static void EmitOMPIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond, - const std::function<void()> &BodyOpGen) { + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + getCriticalRegionLock(CriticalName)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical), Args); + // Build a call to __kmpc_end_critical + CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_critical), + llvm::makeArrayRef(Args)); + emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); + } +} + +static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond, + OpenMPDirectiveKind Kind, + const RegionCodeGenTy &BodyOpGen) { llvm::Value *CallBool = CGF.EmitScalarConversion( IfCond, CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true), @@ -747,61 +1296,274 @@ static void EmitOMPIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond, // Generate the branch (If-stmt) CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); CGF.EmitBlock(ThenBlock); - BodyOpGen(); + CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, Kind, BodyOpGen); // Emit the rest of bblocks/branches CGF.EmitBranch(ContBlock); CGF.EmitBlock(ContBlock, true); } -void CGOpenMPRuntime::EmitOMPMasterRegion( - CodeGenFunction &CGF, const std::function<void()> &MasterOpGen, - SourceLocation Loc) { +void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &MasterOpGen, + SourceLocation Loc) { // if(__kmpc_master(ident_t *, gtid)) { // MasterOpGen(); // __kmpc_end_master(ident_t *, gtid); // } // Prepare arguments and build a call to __kmpc_master - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), - GetOpenMPThreadID(CGF, Loc)}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_master); - auto *IsMaster = CGF.EmitRuntimeCall(RTLFn, Args); - EmitOMPIfStmt(CGF, IsMaster, [&]() -> void { - MasterOpGen(); - // Build a call to __kmpc_end_master. - // OpenMP [1.2.2 OpenMP Language Terminology] - // For C/C++, an executable statement, possibly compound, with a single - // entry at the top and a single exit at the bottom, or an OpenMP construct. - // * Access to the structured block must not be the result of a branch. - // * The point of exit cannot be a branch out of the structured block. - // * The point of entry must not be a call to setjmp(). - // * longjmp() and throw() must not violate the entry/exit criteria. - // * An expression statement, iteration statement, selection statement, or - // try block is considered to be a structured block if the corresponding - // compound statement obtained by enclosing it in { and } would be a - // structured block. - // It is analyzed in Sema, so we can just call __kmpc_end_master() on - // fallthrough rather than pushing a normal cleanup for it. - RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_master); - CGF.EmitRuntimeCall(RTLFn, Args); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + auto *IsMaster = + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_master), Args); + typedef CallEndCleanup<std::extent<decltype(Args)>::value> + MasterCallEndCleanup; + emitIfStmt(CGF, IsMaster, OMPD_master, [&](CodeGenFunction &CGF) -> void { + CodeGenFunction::RunCleanupsScope Scope(CGF); + CGF.EHStack.pushCleanup<MasterCallEndCleanup>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_master), + llvm::makeArrayRef(Args)); + MasterOpGen(CGF); }); } -void CGOpenMPRuntime::EmitOMPBarrierCall(CodeGenFunction &CGF, - SourceLocation Loc, bool IsExplicit) { - // Build call __kmpc_cancel_barrier(loc, thread_id); - auto Flags = static_cast<OpenMPLocationFlags>( - OMP_IDENT_KMPC | - (IsExplicit ? OMP_IDENT_BARRIER_EXPL : OMP_IDENT_BARRIER_IMPL)); +void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, + SourceLocation Loc) { + // Build call __kmpc_omp_taskyield(loc, thread_id, 0); + llvm::Value *Args[] = { + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args); +} + +void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &TaskgroupOpGen, + SourceLocation Loc) { + // __kmpc_taskgroup(ident_t *, gtid); + // TaskgroupOpGen(); + // __kmpc_end_taskgroup(ident_t *, gtid); + // Prepare arguments and build a call to __kmpc_taskgroup + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args); + // Build a call to __kmpc_end_taskgroup + CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_taskgroup), + llvm::makeArrayRef(Args)); + emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); + } +} + +static llvm::Value *emitCopyprivateCopyFunction( + CodeGenModule &CGM, llvm::Type *ArgsType, + ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs, + ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps) { + auto &C = CGM.getContext(); + // void copy_func(void *LHSArg, void *RHSArg); + FunctionArgList Args; + ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr, + C.VoidPtrTy); + ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr, + C.VoidPtrTy); + Args.push_back(&LHSArg); + Args.push_back(&RHSArg); + FunctionType::ExtInfo EI; + auto &CGFI = CGM.getTypes().arrangeFreeFunctionDeclaration( + C.VoidTy, Args, EI, /*isVariadic=*/false); + auto *Fn = llvm::Function::Create( + CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage, + ".omp.copyprivate.copy_func", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn); + CodeGenFunction CGF(CGM); + CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args); + // Dest = (void*[n])(LHSArg); + // Src = (void*[n])(RHSArg); + auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg), + CGF.PointerAlignInBytes), + ArgsType); + auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg), + CGF.PointerAlignInBytes), + ArgsType); + // *(Type0*)Dst[0] = *(Type0*)Src[0]; + // *(Type1*)Dst[1] = *(Type1*)Src[1]; + // ... + // *(Typen*)Dst[n] = *(Typen*)Src[n]; + for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) { + auto *DestAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad( + CGF.Builder.CreateStructGEP(nullptr, LHS, I), + CGM.PointerAlignInBytes), + CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType()))); + auto *SrcAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad( + CGF.Builder.CreateStructGEP(nullptr, RHS, I), + CGM.PointerAlignInBytes), + CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType()))); + auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl(); + QualType Type = VD->getType(); + CGF.EmitOMPCopy(CGF, Type, DestAddr, SrcAddr, + cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl()), + cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl()), + AssignmentOps[I]); + } + CGF.FinishFunction(); + return Fn; +} + +void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &SingleOpGen, + SourceLocation Loc, + ArrayRef<const Expr *> CopyprivateVars, + ArrayRef<const Expr *> SrcExprs, + ArrayRef<const Expr *> DstExprs, + ArrayRef<const Expr *> AssignmentOps) { + assert(CopyprivateVars.size() == SrcExprs.size() && + CopyprivateVars.size() == DstExprs.size() && + CopyprivateVars.size() == AssignmentOps.size()); + auto &C = CGM.getContext(); + // int32 did_it = 0; + // if(__kmpc_single(ident_t *, gtid)) { + // SingleOpGen(); + // __kmpc_end_single(ident_t *, gtid); + // did_it = 1; + // } + // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>, + // <copy_func>, did_it); + + llvm::AllocaInst *DidIt = nullptr; + if (!CopyprivateVars.empty()) { + // int32 did_it = 0; + auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); + DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it"); + CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(0), DidIt, + DidIt->getAlignment()); + } + // Prepare arguments and build a call to __kmpc_single + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + auto *IsSingle = + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_single), Args); + typedef CallEndCleanup<std::extent<decltype(Args)>::value> + SingleCallEndCleanup; + emitIfStmt(CGF, IsSingle, OMPD_single, [&](CodeGenFunction &CGF) -> void { + CodeGenFunction::RunCleanupsScope Scope(CGF); + CGF.EHStack.pushCleanup<SingleCallEndCleanup>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_single), + llvm::makeArrayRef(Args)); + SingleOpGen(CGF); + if (DidIt) { + // did_it = 1; + CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(1), DidIt, + DidIt->getAlignment()); + } + }); + // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>, + // <copy_func>, did_it); + if (DidIt) { + llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size()); + auto CopyprivateArrayTy = + C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal, + /*IndexTypeQuals=*/0); + // Create a list of all private variables for copyprivate. + auto *CopyprivateList = + CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); + for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) { + auto *Elem = CGF.Builder.CreateStructGEP( + CopyprivateList->getAllocatedType(), CopyprivateList, I); + CGF.Builder.CreateAlignedStore( + CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.EmitLValue(CopyprivateVars[I]).getAddress(), CGF.VoidPtrTy), + Elem, CGM.PointerAlignInBytes); + } + // Build function that copies private values from single region to all other + // threads in the corresponding parallel region. + auto *CpyFn = emitCopyprivateCopyFunction( + CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(), + CopyprivateVars, SrcExprs, DstExprs, AssignmentOps); + auto *BufSize = llvm::ConstantInt::get( + CGM.SizeTy, C.getTypeSizeInChars(CopyprivateArrayTy).getQuantity()); + auto *CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList, + CGF.VoidPtrTy); + auto *DidItVal = + CGF.Builder.CreateAlignedLoad(DidIt, CGF.PointerAlignInBytes); + llvm::Value *Args[] = { + emitUpdateLocation(CGF, Loc), // ident_t *<loc> + getThreadID(CGF, Loc), // i32 <gtid> + BufSize, // size_t <buf_size> + CL, // void *<copyprivate list> + CpyFn, // void (*) (void *, void *) <copy_func> + DidItVal // i32 did_it + }; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args); + } +} + +void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &OrderedOpGen, + SourceLocation Loc) { + // __kmpc_ordered(ident_t *, gtid); + // OrderedOpGen(); + // __kmpc_end_ordered(ident_t *, gtid); + // Prepare arguments and build a call to __kmpc_ordered + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_ordered), Args); + // Build a call to __kmpc_end_ordered + CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_ordered), + llvm::makeArrayRef(Args)); + emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); + } +} + +void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPDirectiveKind Kind, + bool CheckForCancel) { // Build call __kmpc_cancel_barrier(loc, thread_id); - // Replace __kmpc_barrier() function by __kmpc_cancel_barrier() because this - // one provides the same functionality and adds initial support for - // cancellation constructs introduced in OpenMP 4.0. __kmpc_cancel_barrier() - // is provided default by the runtime library so it safe to make such - // replacement. - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc, Flags), - GetOpenMPThreadID(CGF, Loc)}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_cancel_barrier); - CGF.EmitRuntimeCall(RTLFn, Args); + // Build call __kmpc_barrier(loc, thread_id); + OpenMPLocationFlags Flags = OMP_IDENT_KMPC; + if (Kind == OMPD_for) { + Flags = + static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL_FOR); + } else if (Kind == OMPD_sections) { + Flags = static_cast<OpenMPLocationFlags>(Flags | + OMP_IDENT_BARRIER_IMPL_SECTIONS); + } else if (Kind == OMPD_single) { + Flags = + static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL_SINGLE); + } else if (Kind == OMPD_barrier) { + Flags = static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_EXPL); + } else { + Flags = static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL); + } + // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc, + // thread_id); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), + getThreadID(CGF, Loc)}; + if (auto *OMPRegionInfo = + dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { + auto CancelDestination = + CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); + if (CancelDestination.isValid()) { + auto *Result = CGF.EmitRuntimeCall( + createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args); + if (CheckForCancel) { + // if (__kmpc_cancel_barrier()) { + // exit from construct; + // } + auto *ExitBB = CGF.createBasicBlock(".cancel.exit"); + auto *ContBB = CGF.createBasicBlock(".cancel.continue"); + auto *Cmp = CGF.Builder.CreateIsNotNull(Result); + CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); + CGF.EmitBlock(ExitBB); + // exit from construct; + CGF.EmitBranchThroughCleanup(CancelDestination); + CGF.EmitBlock(ContBB, /*IsFinished=*/true); + } + return; + } + } + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args); } /// \brief Schedule types for 'omp for' loops (these enumerators are taken from @@ -817,106 +1579,1258 @@ enum OpenMPSchedType { OMP_sch_auto = 38, /// \brief Lower bound for 'ordered' versions. OMP_ord_lower = 64, - /// \brief Lower bound for 'nomerge' versions. - OMP_nm_lower = 160, + OMP_ord_static_chunked = 65, + OMP_ord_static = 66, + OMP_ord_dynamic_chunked = 67, + OMP_ord_guided_chunked = 68, + OMP_ord_runtime = 69, + OMP_ord_auto = 70, + OMP_sch_default = OMP_sch_static, }; /// \brief Map the OpenMP loop schedule to the runtime enumeration. static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, - bool Chunked) { + bool Chunked, bool Ordered) { switch (ScheduleKind) { case OMPC_SCHEDULE_static: - return Chunked ? OMP_sch_static_chunked : OMP_sch_static; + return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked) + : (Ordered ? OMP_ord_static : OMP_sch_static); case OMPC_SCHEDULE_dynamic: - return OMP_sch_dynamic_chunked; + return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked; case OMPC_SCHEDULE_guided: - return OMP_sch_guided_chunked; - case OMPC_SCHEDULE_auto: - return OMP_sch_auto; + return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked; case OMPC_SCHEDULE_runtime: - return OMP_sch_runtime; + return Ordered ? OMP_ord_runtime : OMP_sch_runtime; + case OMPC_SCHEDULE_auto: + return Ordered ? OMP_ord_auto : OMP_sch_auto; case OMPC_SCHEDULE_unknown: assert(!Chunked && "chunk was specified but schedule kind not known"); - return OMP_sch_static; + return Ordered ? OMP_ord_static : OMP_sch_static; } llvm_unreachable("Unexpected runtime schedule"); } bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const { - auto Schedule = getRuntimeSchedule(ScheduleKind, Chunked); + auto Schedule = getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false); return Schedule == OMP_sch_static; } -void CGOpenMPRuntime::EmitOMPForInit(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPScheduleClauseKind ScheduleKind, - unsigned IVSize, bool IVSigned, - llvm::Value *IL, llvm::Value *LB, - llvm::Value *UB, llvm::Value *ST, - llvm::Value *Chunk) { - OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunk != nullptr); - // Call __kmpc_for_static_init( - // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype, - // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower, - // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride, - // kmp_int[32|64] incr, kmp_int[32|64] chunk); - // TODO: Implement dynamic schedule. - - // If the Chunk was not specified in the clause - use default value 1. - if (Chunk == nullptr) - Chunk = CGF.Builder.getIntN(IVSize, /*C*/ 1); +bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const { + auto Schedule = + getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false); + assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here"); + return Schedule != OMP_sch_static; +} + +void CGOpenMPRuntime::emitForInit(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPScheduleClauseKind ScheduleKind, + unsigned IVSize, bool IVSigned, bool Ordered, + llvm::Value *IL, llvm::Value *LB, + llvm::Value *UB, llvm::Value *ST, + llvm::Value *Chunk) { + OpenMPSchedType Schedule = + getRuntimeSchedule(ScheduleKind, Chunk != nullptr, Ordered); + if (Ordered || + (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && + Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked)) { + // Call __kmpc_dispatch_init( + // ident_t *loc, kmp_int32 tid, kmp_int32 schedule, + // kmp_int[32|64] lower, kmp_int[32|64] upper, + // kmp_int[32|64] stride, kmp_int[32|64] chunk); + // If the Chunk was not specified in the clause - use default value 1. + if (Chunk == nullptr) + Chunk = CGF.Builder.getIntN(IVSize, 1); + llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), + getThreadID(CGF, Loc), + CGF.Builder.getInt32(Schedule), // Schedule type + CGF.Builder.getIntN(IVSize, 0), // Lower + UB, // Upper + CGF.Builder.getIntN(IVSize, 1), // Stride + Chunk // Chunk + }; + CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args); + } else { + // Call __kmpc_for_static_init( + // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype, + // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower, + // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride, + // kmp_int[32|64] incr, kmp_int[32|64] chunk); + if (Chunk == nullptr) { + assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static) && + "expected static non-chunked schedule"); + // If the Chunk was not specified in the clause - use default value 1. + Chunk = CGF.Builder.getIntN(IVSize, 1); + } else + assert((Schedule == OMP_sch_static_chunked || + Schedule == OMP_ord_static_chunked) && + "expected static chunked schedule"); + llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), + getThreadID(CGF, Loc), + CGF.Builder.getInt32(Schedule), // Schedule type + IL, // &isLastIter + LB, // &LB + UB, // &UB + ST, // &Stride + CGF.Builder.getIntN(IVSize, 1), // Incr + Chunk // Chunk + }; + CGF.EmitRuntimeCall(createForStaticInitFunction(IVSize, IVSigned), Args); + } +} + +void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, + SourceLocation Loc) { + // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), + getThreadID(CGF, Loc)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini), + Args); +} + +void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, + SourceLocation Loc, + unsigned IVSize, + bool IVSigned) { + // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), + getThreadID(CGF, Loc)}; + CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args); +} + +llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, + SourceLocation Loc, unsigned IVSize, + bool IVSigned, llvm::Value *IL, + llvm::Value *LB, llvm::Value *UB, + llvm::Value *ST) { + // Call __kmpc_dispatch_next( + // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, + // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, + // kmp_int[32|64] *p_stride); llvm::Value *Args[] = { - EmitOpenMPUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), - GetOpenMPThreadID(CGF, Loc), - CGF.Builder.getInt32(Schedule), // Schedule type - IL, // &isLastIter - LB, // &LB - UB, // &UB - ST, // &Stride - CGF.Builder.getIntN(IVSize, 1), // Incr - Chunk // Chunk + emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), getThreadID(CGF, Loc), + IL, // &isLastIter + LB, // &Lower + UB, // &Upper + ST // &Stride }; - assert((IVSize == 32 || IVSize == 64) && - "Index size is not compatible with the omp runtime"); - auto F = IVSize == 32 ? (IVSigned ? OMPRTL__kmpc_for_static_init_4 - : OMPRTL__kmpc_for_static_init_4u) - : (IVSigned ? OMPRTL__kmpc_for_static_init_8 - : OMPRTL__kmpc_for_static_init_8u); - auto RTLFn = CreateRuntimeFunction(F); - CGF.EmitRuntimeCall(RTLFn, Args); -} - -void CGOpenMPRuntime::EmitOMPForFinish(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPScheduleClauseKind ScheduleKind) { - assert((ScheduleKind == OMPC_SCHEDULE_static || - ScheduleKind == OMPC_SCHEDULE_unknown) && - "Non-static schedule kinds are not yet implemented"); - // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid); - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), - GetOpenMPThreadID(CGF, Loc)}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_for_static_fini); - CGF.EmitRuntimeCall(RTLFn, Args); + llvm::Value *Call = + CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args); + return CGF.EmitScalarConversion( + Call, CGF.getContext().getIntTypeForBitwidth(32, /* Signed */ true), + CGF.getContext().BoolTy); } -void CGOpenMPRuntime::EmitOMPNumThreadsClause(CodeGenFunction &CGF, - llvm::Value *NumThreads, - SourceLocation Loc) { +void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, + llvm::Value *NumThreads, + SourceLocation Loc) { // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads) llvm::Value *Args[] = { - EmitOpenMPUpdateLocation(CGF, Loc), GetOpenMPThreadID(CGF, Loc), + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}; - llvm::Constant *RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_push_num_threads); - CGF.EmitRuntimeCall(RTLFn, Args); + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads), + Args); +} + +void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, + OpenMPProcBindClauseKind ProcBind, + SourceLocation Loc) { + // Constants for proc bind value accepted by the runtime. + enum ProcBindTy { + ProcBindFalse = 0, + ProcBindTrue, + ProcBindMaster, + ProcBindClose, + ProcBindSpread, + ProcBindIntel, + ProcBindDefault + } RuntimeProcBind; + switch (ProcBind) { + case OMPC_PROC_BIND_master: + RuntimeProcBind = ProcBindMaster; + break; + case OMPC_PROC_BIND_close: + RuntimeProcBind = ProcBindClose; + break; + case OMPC_PROC_BIND_spread: + RuntimeProcBind = ProcBindSpread; + break; + case OMPC_PROC_BIND_unknown: + llvm_unreachable("Unsupported proc_bind value."); + } + // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind) + llvm::Value *Args[] = { + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args); +} + +void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, + SourceLocation Loc) { + // Build call void __kmpc_flush(ident_t *loc) + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush), + emitUpdateLocation(CGF, Loc)); +} + +namespace { +/// \brief Indexes of fields for type kmp_task_t. +enum KmpTaskTFields { + /// \brief List of shared variables. + KmpTaskTShareds, + /// \brief Task routine. + KmpTaskTRoutine, + /// \brief Partition id for the untied tasks. + KmpTaskTPartId, + /// \brief Function with call of destructors for private variables. + KmpTaskTDestructors, +}; +} // namespace + +void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) { + if (!KmpRoutineEntryPtrTy) { + // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type. + auto &C = CGM.getContext(); + QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy}; + FunctionProtoType::ExtProtoInfo EPI; + KmpRoutineEntryPtrQTy = C.getPointerType( + C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI)); + KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy); + } +} + +static void addFieldToRecordDecl(ASTContext &C, DeclContext *DC, + QualType FieldTy) { + auto *Field = FieldDecl::Create( + C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy, + C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()), + /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit); + Field->setAccess(AS_public); + DC->addDecl(Field); +} + +namespace { +struct PrivateHelpersTy { + PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy, + const VarDecl *PrivateElemInit) + : Original(Original), PrivateCopy(PrivateCopy), + PrivateElemInit(PrivateElemInit) {} + const VarDecl *Original; + const VarDecl *PrivateCopy; + const VarDecl *PrivateElemInit; +}; +typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy; +} // namespace + +static RecordDecl * +createPrivatesRecordDecl(CodeGenModule &CGM, + const ArrayRef<PrivateDataTy> Privates) { + if (!Privates.empty()) { + auto &C = CGM.getContext(); + // Build struct .kmp_privates_t. { + // /* private vars */ + // }; + auto *RD = C.buildImplicitRecord(".kmp_privates.t"); + RD->startDefinition(); + for (auto &&Pair : Privates) { + auto Type = Pair.second.Original->getType(); + Type = Type.getNonReferenceType(); + addFieldToRecordDecl(C, RD, Type); + } + RD->completeDefinition(); + return RD; + } + return nullptr; +} + +static RecordDecl * +createKmpTaskTRecordDecl(CodeGenModule &CGM, QualType KmpInt32Ty, + QualType KmpRoutineEntryPointerQTy) { + auto &C = CGM.getContext(); + // Build struct kmp_task_t { + // void * shareds; + // kmp_routine_entry_t routine; + // kmp_int32 part_id; + // kmp_routine_entry_t destructors; + // }; + auto *RD = C.buildImplicitRecord("kmp_task_t"); + RD->startDefinition(); + addFieldToRecordDecl(C, RD, C.VoidPtrTy); + addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy); + addFieldToRecordDecl(C, RD, KmpInt32Ty); + addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy); + RD->completeDefinition(); + return RD; +} + +static RecordDecl * +createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, + const ArrayRef<PrivateDataTy> Privates) { + auto &C = CGM.getContext(); + // Build struct kmp_task_t_with_privates { + // kmp_task_t task_data; + // .kmp_privates_t. privates; + // }; + auto *RD = C.buildImplicitRecord("kmp_task_t_with_privates"); + RD->startDefinition(); + addFieldToRecordDecl(C, RD, KmpTaskTQTy); + if (auto *PrivateRD = createPrivatesRecordDecl(CGM, Privates)) { + addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD)); + } + RD->completeDefinition(); + return RD; +} + +/// \brief Emit a proxy function which accepts kmp_task_t as the second +/// argument. +/// \code +/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { +/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, +/// tt->shareds); +/// return 0; +/// } +/// \endcode +static llvm::Value * +emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, + QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, + QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, + QualType SharedsPtrTy, llvm::Value *TaskFunction, + llvm::Value *TaskPrivatesMap) { + auto &C = CGM.getContext(); + FunctionArgList Args; + ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty); + ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, + /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy); + Args.push_back(&GtidArg); + Args.push_back(&TaskTypeArg); + FunctionType::ExtInfo Info; + auto &TaskEntryFnInfo = + CGM.getTypes().arrangeFreeFunctionDeclaration(KmpInt32Ty, Args, Info, + /*isVariadic=*/false); + auto *TaskEntryTy = CGM.getTypes().GetFunctionType(TaskEntryFnInfo); + auto *TaskEntry = + llvm::Function::Create(TaskEntryTy, llvm::GlobalValue::InternalLinkage, + ".omp_task_entry.", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskEntryFnInfo, TaskEntry); + CodeGenFunction CGF(CGM); + CGF.disableDebugInfo(); + CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args); + + // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map, + // tt->task_data.shareds); + auto *GtidParam = CGF.EmitLoadOfScalar( + CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, + C.getTypeAlignInChars(KmpInt32Ty).getQuantity(), KmpInt32Ty, Loc); + auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes); + LValue TDBase = + CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy); + auto *KmpTaskTWithPrivatesQTyRD = + cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl()); + LValue Base = + CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); + auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl()); + auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId); + auto PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI); + auto *PartidParam = CGF.EmitLoadOfLValue(PartIdLVal, Loc).getScalarVal(); + + auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds); + auto SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI); + auto *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.EmitLoadOfLValue(SharedsLVal, Loc).getScalarVal(), + CGF.ConvertTypeForMem(SharedsPtrTy)); + + auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1); + llvm::Value *PrivatesParam; + if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) { + auto PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); + PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + PrivatesLVal.getAddress(), CGF.VoidPtrTy); + } else { + PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); + } + + llvm::Value *CallArgs[] = {GtidParam, PartidParam, PrivatesParam, + TaskPrivatesMap, SharedsParam}; + CGF.EmitCallOrInvoke(TaskFunction, CallArgs); + CGF.EmitStoreThroughLValue( + RValue::get(CGF.Builder.getInt32(/*C=*/0)), + CGF.MakeNaturalAlignAddrLValue(CGF.ReturnValue, KmpInt32Ty)); + CGF.FinishFunction(); + return TaskEntry; +} + +static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM, + SourceLocation Loc, + QualType KmpInt32Ty, + QualType KmpTaskTWithPrivatesPtrQTy, + QualType KmpTaskTWithPrivatesQTy) { + auto &C = CGM.getContext(); + FunctionArgList Args; + ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty); + ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, + /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy); + Args.push_back(&GtidArg); + Args.push_back(&TaskTypeArg); + FunctionType::ExtInfo Info; + auto &DestructorFnInfo = + CGM.getTypes().arrangeFreeFunctionDeclaration(KmpInt32Ty, Args, Info, + /*isVariadic=*/false); + auto *DestructorFnTy = CGM.getTypes().GetFunctionType(DestructorFnInfo); + auto *DestructorFn = + llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage, + ".omp_task_destructor.", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, DestructorFnInfo, DestructorFn); + CodeGenFunction CGF(CGM); + CGF.disableDebugInfo(); + CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, + Args); + + auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes); + LValue Base = + CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy); + auto *KmpTaskTWithPrivatesQTyRD = + cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl()); + auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); + Base = CGF.EmitLValueForField(Base, *FI); + for (auto *Field : + cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) { + if (auto DtorKind = Field->getType().isDestructedType()) { + auto FieldLValue = CGF.EmitLValueForField(Base, Field); + CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType()); + } + } + CGF.FinishFunction(); + return DestructorFn; +} + +/// \brief Emit a privates mapping function for correct handling of private and +/// firstprivate variables. +/// \code +/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1> +/// **noalias priv1,..., <tyn> **noalias privn) { +/// *priv1 = &.privates.priv1; +/// ...; +/// *privn = &.privates.privn; +/// } +/// \endcode +static llvm::Value * +emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, + const ArrayRef<const Expr *> PrivateVars, + const ArrayRef<const Expr *> FirstprivateVars, + QualType PrivatesQTy, + const ArrayRef<PrivateDataTy> Privates) { + auto &C = CGM.getContext(); + FunctionArgList Args; + ImplicitParamDecl TaskPrivatesArg( + C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, + C.getPointerType(PrivatesQTy).withConst().withRestrict()); + Args.push_back(&TaskPrivatesArg); + llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos; + unsigned Counter = 1; + for (auto *E: PrivateVars) { + Args.push_back(ImplicitParamDecl::Create( + C, /*DC=*/nullptr, Loc, + /*Id=*/nullptr, C.getPointerType(C.getPointerType(E->getType())) + .withConst() + .withRestrict())); + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + PrivateVarsPos[VD] = Counter; + ++Counter; + } + for (auto *E : FirstprivateVars) { + Args.push_back(ImplicitParamDecl::Create( + C, /*DC=*/nullptr, Loc, + /*Id=*/nullptr, C.getPointerType(C.getPointerType(E->getType())) + .withConst() + .withRestrict())); + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + PrivateVarsPos[VD] = Counter; + ++Counter; + } + FunctionType::ExtInfo Info; + auto &TaskPrivatesMapFnInfo = + CGM.getTypes().arrangeFreeFunctionDeclaration(C.VoidTy, Args, Info, + /*isVariadic=*/false); + auto *TaskPrivatesMapTy = + CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo); + auto *TaskPrivatesMap = llvm::Function::Create( + TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, + ".omp_task_privates_map.", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskPrivatesMapFnInfo, + TaskPrivatesMap); + TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline); + CodeGenFunction CGF(CGM); + CGF.disableDebugInfo(); + CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap, + TaskPrivatesMapFnInfo, Args); + + // *privi = &.privates.privi; + auto *TaskPrivatesArgAddr = CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(&TaskPrivatesArg), CGM.PointerAlignInBytes); + LValue Base = + CGF.MakeNaturalAlignAddrLValue(TaskPrivatesArgAddr, PrivatesQTy); + auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl()); + Counter = 0; + for (auto *Field : PrivatesQTyRD->fields()) { + auto FieldLVal = CGF.EmitLValueForField(Base, Field); + auto *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]]; + auto RefLVal = CGF.MakeNaturalAlignAddrLValue(CGF.GetAddrOfLocalVar(VD), + VD->getType()); + auto RefLoadRVal = CGF.EmitLoadOfLValue(RefLVal, Loc); + CGF.EmitStoreOfScalar( + FieldLVal.getAddress(), + CGF.MakeNaturalAlignAddrLValue(RefLoadRVal.getScalarVal(), + RefLVal.getType()->getPointeeType())); + ++Counter; + } + CGF.FinishFunction(); + return TaskPrivatesMap; } -void CGOpenMPRuntime::EmitOMPFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, - SourceLocation Loc) { - // Build call void __kmpc_flush(ident_t *loc, ...) - // FIXME: List of variables is ignored by libiomp5 runtime, no need to - // generate it, just request full memory fence. - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), - llvm::ConstantInt::get(CGM.Int32Ty, 0)}; - auto *RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_flush); - CGF.EmitRuntimeCall(RTLFn, Args); +static int array_pod_sort_comparator(const PrivateDataTy *P1, + const PrivateDataTy *P2) { + return P1->first < P2->first ? 1 : (P2->first < P1->first ? -1 : 0); +} + +void CGOpenMPRuntime::emitTaskCall( + CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, + bool Tied, llvm::PointerIntPair<llvm::Value *, 1, bool> Final, + llvm::Value *TaskFunction, QualType SharedsTy, llvm::Value *Shareds, + const Expr *IfCond, ArrayRef<const Expr *> PrivateVars, + ArrayRef<const Expr *> PrivateCopies, + ArrayRef<const Expr *> FirstprivateVars, + ArrayRef<const Expr *> FirstprivateCopies, + ArrayRef<const Expr *> FirstprivateInits, + ArrayRef<std::pair<OpenMPDependClauseKind, const Expr *>> Dependences) { + auto &C = CGM.getContext(); + llvm::SmallVector<PrivateDataTy, 8> Privates; + // Aggregate privates and sort them by the alignment. + auto I = PrivateCopies.begin(); + for (auto *E : PrivateVars) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + Privates.push_back(std::make_pair( + C.getTypeAlignInChars(VD->getType()), + PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()), + /*PrivateElemInit=*/nullptr))); + ++I; + } + I = FirstprivateCopies.begin(); + auto IElemInitRef = FirstprivateInits.begin(); + for (auto *E : FirstprivateVars) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + Privates.push_back(std::make_pair( + C.getTypeAlignInChars(VD->getType()), + PrivateHelpersTy( + VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()), + cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())))); + ++I, ++IElemInitRef; + } + llvm::array_pod_sort(Privates.begin(), Privates.end(), + array_pod_sort_comparator); + auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); + // Build type kmp_routine_entry_t (if not built yet). + emitKmpRoutineEntryT(KmpInt32Ty); + // Build type kmp_task_t (if not built yet). + if (KmpTaskTQTy.isNull()) { + KmpTaskTQTy = C.getRecordType( + createKmpTaskTRecordDecl(CGM, KmpInt32Ty, KmpRoutineEntryPtrQTy)); + } + auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl()); + // Build particular struct kmp_task_t for the given task. + auto *KmpTaskTWithPrivatesQTyRD = + createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates); + auto KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD); + QualType KmpTaskTWithPrivatesPtrQTy = + C.getPointerType(KmpTaskTWithPrivatesQTy); + auto *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy); + auto *KmpTaskTWithPrivatesPtrTy = KmpTaskTWithPrivatesTy->getPointerTo(); + auto KmpTaskTWithPrivatesTySize = + CGM.getSize(C.getTypeSizeInChars(KmpTaskTWithPrivatesQTy)); + QualType SharedsPtrTy = C.getPointerType(SharedsTy); + + // Emit initial values for private copies (if any). + llvm::Value *TaskPrivatesMap = nullptr; + auto *TaskPrivatesMapTy = + std::next(cast<llvm::Function>(TaskFunction)->getArgumentList().begin(), + 3) + ->getType(); + if (!Privates.empty()) { + auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); + TaskPrivatesMap = emitTaskPrivateMappingFunction( + CGM, Loc, PrivateVars, FirstprivateVars, FI->getType(), Privates); + TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + TaskPrivatesMap, TaskPrivatesMapTy); + } else { + TaskPrivatesMap = llvm::ConstantPointerNull::get( + cast<llvm::PointerType>(TaskPrivatesMapTy)); + } + // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid, + // kmp_task_t *tt); + auto *TaskEntry = emitProxyTaskFunction( + CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTy, + KmpTaskTQTy, SharedsPtrTy, TaskFunction, TaskPrivatesMap); + + // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid, + // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, + // kmp_routine_entry_t *task_entry); + // Task flags. Format is taken from + // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h, + // description of kmp_tasking_flags struct. + const unsigned TiedFlag = 0x1; + const unsigned FinalFlag = 0x2; + unsigned Flags = Tied ? TiedFlag : 0; + auto *TaskFlags = + Final.getPointer() + ? CGF.Builder.CreateSelect(Final.getPointer(), + CGF.Builder.getInt32(FinalFlag), + CGF.Builder.getInt32(/*C=*/0)) + : CGF.Builder.getInt32(Final.getInt() ? FinalFlag : 0); + TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); + auto SharedsSize = C.getTypeSizeInChars(SharedsTy); + llvm::Value *AllocArgs[] = { + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), TaskFlags, + KmpTaskTWithPrivatesTySize, CGM.getSize(SharedsSize), + CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskEntry, + KmpRoutineEntryPtrTy)}; + auto *NewTask = CGF.EmitRuntimeCall( + createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs); + auto *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + NewTask, KmpTaskTWithPrivatesPtrTy); + LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, + KmpTaskTWithPrivatesQTy); + LValue TDBase = + CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); + // Fill the data in the resulting kmp_task_t record. + // Copy shareds if there are any. + llvm::Value *KmpTaskSharedsPtr = nullptr; + if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) { + KmpTaskSharedsPtr = CGF.EmitLoadOfScalar( + CGF.EmitLValueForField( + TDBase, *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)), + Loc); + CGF.EmitAggregateCopy(KmpTaskSharedsPtr, Shareds, SharedsTy); + } + // Emit initial values for private copies (if any). + bool NeedsCleanup = false; + if (!Privates.empty()) { + auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); + auto PrivatesBase = CGF.EmitLValueForField(Base, *FI); + FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin(); + LValue SharedsBase; + if (!FirstprivateVars.empty()) { + SharedsBase = CGF.MakeNaturalAlignAddrLValue( + CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)), + SharedsTy); + } + CodeGenFunction::CGCapturedStmtInfo CapturesInfo( + cast<CapturedStmt>(*D.getAssociatedStmt())); + for (auto &&Pair : Privates) { + auto *VD = Pair.second.PrivateCopy; + auto *Init = VD->getAnyInitializer(); + LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); + if (Init) { + if (auto *Elem = Pair.second.PrivateElemInit) { + auto *OriginalVD = Pair.second.Original; + auto *SharedField = CapturesInfo.lookup(OriginalVD); + auto SharedRefLValue = + CGF.EmitLValueForField(SharedsBase, SharedField); + QualType Type = OriginalVD->getType(); + if (Type->isArrayType()) { + // Initialize firstprivate array. + if (!isa<CXXConstructExpr>(Init) || + CGF.isTrivialInitializer(Init)) { + // Perform simple memcpy. + CGF.EmitAggregateAssign(PrivateLValue.getAddress(), + SharedRefLValue.getAddress(), Type); + } else { + // Initialize firstprivate array using element-by-element + // intialization. + CGF.EmitOMPAggregateAssign( + PrivateLValue.getAddress(), SharedRefLValue.getAddress(), + Type, [&CGF, Elem, Init, &CapturesInfo]( + llvm::Value *DestElement, llvm::Value *SrcElement) { + // Clean up any temporaries needed by the initialization. + CodeGenFunction::OMPPrivateScope InitScope(CGF); + InitScope.addPrivate(Elem, [SrcElement]() -> llvm::Value *{ + return SrcElement; + }); + (void)InitScope.Privatize(); + // Emit initialization for single element. + CodeGenFunction::CGCapturedStmtRAII CapInfoRAII( + CGF, &CapturesInfo); + CGF.EmitAnyExprToMem(Init, DestElement, + Init->getType().getQualifiers(), + /*IsInitializer=*/false); + }); + } + } else { + CodeGenFunction::OMPPrivateScope InitScope(CGF); + InitScope.addPrivate(Elem, [SharedRefLValue]() -> llvm::Value *{ + return SharedRefLValue.getAddress(); + }); + (void)InitScope.Privatize(); + CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo); + CGF.EmitExprAsInit(Init, VD, PrivateLValue, + /*capturedByInit=*/false); + } + } else { + CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false); + } + } + NeedsCleanup = NeedsCleanup || FI->getType().isDestructedType(); + ++FI; + } + } + // Provide pointer to function with destructors for privates. + llvm::Value *DestructorFn = + NeedsCleanup ? emitDestructorsFunction(CGM, Loc, KmpInt32Ty, + KmpTaskTWithPrivatesPtrQTy, + KmpTaskTWithPrivatesQTy) + : llvm::ConstantPointerNull::get( + cast<llvm::PointerType>(KmpRoutineEntryPtrTy)); + LValue Destructor = CGF.EmitLValueForField( + TDBase, *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTDestructors)); + CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + DestructorFn, KmpRoutineEntryPtrTy), + Destructor); + + // Process list of dependences. + llvm::Value *DependInfo = nullptr; + unsigned DependencesNumber = Dependences.size(); + if (!Dependences.empty()) { + // Dependence kind for RTL. + enum RTLDependenceKindTy { DepIn = 1, DepOut = 2, DepInOut = 3 }; + enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags }; + RecordDecl *KmpDependInfoRD; + QualType FlagsTy = C.getIntTypeForBitwidth( + C.toBits(C.getTypeSizeInChars(C.BoolTy)), /*Signed=*/false); + llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); + if (KmpDependInfoTy.isNull()) { + KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info"); + KmpDependInfoRD->startDefinition(); + addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType()); + addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType()); + addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy); + KmpDependInfoRD->completeDefinition(); + KmpDependInfoTy = C.getRecordType(KmpDependInfoRD); + } else { + KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl()); + } + // Define type kmp_depend_info[<Dependences.size()>]; + QualType KmpDependInfoArrayTy = C.getConstantArrayType( + KmpDependInfoTy, llvm::APInt(/*numBits=*/64, Dependences.size()), + ArrayType::Normal, /*IndexTypeQuals=*/0); + // kmp_depend_info[<Dependences.size()>] deps; + DependInfo = CGF.CreateMemTemp(KmpDependInfoArrayTy); + for (unsigned i = 0; i < DependencesNumber; ++i) { + auto Addr = CGF.EmitLValue(Dependences[i].second); + auto *Size = llvm::ConstantInt::get( + CGF.SizeTy, + C.getTypeSizeInChars(Dependences[i].second->getType()).getQuantity()); + auto Base = CGF.MakeNaturalAlignAddrLValue( + CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, DependInfo, i), + KmpDependInfoTy); + // deps[i].base_addr = &<Dependences[i].second>; + auto BaseAddrLVal = CGF.EmitLValueForField( + Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr)); + CGF.EmitStoreOfScalar( + CGF.Builder.CreatePtrToInt(Addr.getAddress(), CGF.IntPtrTy), + BaseAddrLVal); + // deps[i].len = sizeof(<Dependences[i].second>); + auto LenLVal = CGF.EmitLValueForField( + Base, *std::next(KmpDependInfoRD->field_begin(), Len)); + CGF.EmitStoreOfScalar(Size, LenLVal); + // deps[i].flags = <Dependences[i].first>; + RTLDependenceKindTy DepKind; + switch (Dependences[i].first) { + case OMPC_DEPEND_in: + DepKind = DepIn; + break; + case OMPC_DEPEND_out: + DepKind = DepOut; + break; + case OMPC_DEPEND_inout: + DepKind = DepInOut; + break; + case OMPC_DEPEND_unknown: + llvm_unreachable("Unknown task dependence type"); + } + auto FlagsLVal = CGF.EmitLValueForField( + Base, *std::next(KmpDependInfoRD->field_begin(), Flags)); + CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind), + FlagsLVal); + } + DependInfo = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, DependInfo, 0), + CGF.VoidPtrTy); + } + + // NOTE: routine and part_id fields are intialized by __kmpc_omp_task_alloc() + // libcall. + // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t + // *new_task); + // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid, + // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list, + // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence + // list is not empty + auto *ThreadID = getThreadID(CGF, Loc); + auto *UpLoc = emitUpdateLocation(CGF, Loc); + llvm::Value *TaskArgs[] = {UpLoc, ThreadID, NewTask}; + llvm::Value *DepTaskArgs[] = { + UpLoc, + ThreadID, + NewTask, + DependInfo ? CGF.Builder.getInt32(DependencesNumber) : nullptr, + DependInfo, + DependInfo ? CGF.Builder.getInt32(0) : nullptr, + DependInfo ? llvm::ConstantPointerNull::get(CGF.VoidPtrTy) : nullptr}; + auto &&ThenCodeGen = [this, DependInfo, &TaskArgs, + &DepTaskArgs](CodeGenFunction &CGF) { + // TODO: add check for untied tasks. + CGF.EmitRuntimeCall( + createRuntimeFunction(DependInfo ? OMPRTL__kmpc_omp_task_with_deps + : OMPRTL__kmpc_omp_task), + DependInfo ? makeArrayRef(DepTaskArgs) : makeArrayRef(TaskArgs)); + }; + typedef CallEndCleanup<std::extent<decltype(TaskArgs)>::value> + IfCallEndCleanup; + llvm::Value *DepWaitTaskArgs[] = { + UpLoc, + ThreadID, + DependInfo ? CGF.Builder.getInt32(DependencesNumber) : nullptr, + DependInfo, + DependInfo ? CGF.Builder.getInt32(0) : nullptr, + DependInfo ? llvm::ConstantPointerNull::get(CGF.VoidPtrTy) : nullptr}; + auto &&ElseCodeGen = [this, &TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry, + DependInfo, &DepWaitTaskArgs](CodeGenFunction &CGF) { + CodeGenFunction::RunCleanupsScope LocalScope(CGF); + // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid, + // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 + // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info + // is specified. + if (DependInfo) + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps), + DepWaitTaskArgs); + // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid, + // kmp_task_t *new_task); + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), + TaskArgs); + // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid, + // kmp_task_t *new_task); + CGF.EHStack.pushCleanup<IfCallEndCleanup>( + NormalAndEHCleanup, + createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), + llvm::makeArrayRef(TaskArgs)); + + // Call proxy_task_entry(gtid, new_task); + llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy}; + CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs); + }; + if (IfCond) { + emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); + } else { + CodeGenFunction::RunCleanupsScope Scope(CGF); + ThenCodeGen(CGF); + } } + +static llvm::Value *emitReductionFunction(CodeGenModule &CGM, + llvm::Type *ArgsType, + ArrayRef<const Expr *> LHSExprs, + ArrayRef<const Expr *> RHSExprs, + ArrayRef<const Expr *> ReductionOps) { + auto &C = CGM.getContext(); + + // void reduction_func(void *LHSArg, void *RHSArg); + FunctionArgList Args; + ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr, + C.VoidPtrTy); + ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr, + C.VoidPtrTy); + Args.push_back(&LHSArg); + Args.push_back(&RHSArg); + FunctionType::ExtInfo EI; + auto &CGFI = CGM.getTypes().arrangeFreeFunctionDeclaration( + C.VoidTy, Args, EI, /*isVariadic=*/false); + auto *Fn = llvm::Function::Create( + CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage, + ".omp.reduction.reduction_func", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn); + CodeGenFunction CGF(CGM); + CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args); + + // Dst = (void*[n])(LHSArg); + // Src = (void*[n])(RHSArg); + auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg), + CGF.PointerAlignInBytes), + ArgsType); + auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg), + CGF.PointerAlignInBytes), + ArgsType); + + // ... + // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]); + // ... + CodeGenFunction::OMPPrivateScope Scope(CGF); + for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I) { + Scope.addPrivate( + cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl()), + [&]() -> llvm::Value *{ + return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad( + CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, RHS, I), + CGM.PointerAlignInBytes), + CGF.ConvertTypeForMem(C.getPointerType(RHSExprs[I]->getType()))); + }); + Scope.addPrivate( + cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl()), + [&]() -> llvm::Value *{ + return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad( + CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, LHS, I), + CGM.PointerAlignInBytes), + CGF.ConvertTypeForMem(C.getPointerType(LHSExprs[I]->getType()))); + }); + } + Scope.Privatize(); + for (auto *E : ReductionOps) { + CGF.EmitIgnoredExpr(E); + } + Scope.ForceCleanup(); + CGF.FinishFunction(); + return Fn; +} + +void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, + ArrayRef<const Expr *> LHSExprs, + ArrayRef<const Expr *> RHSExprs, + ArrayRef<const Expr *> ReductionOps, + bool WithNowait, bool SimpleReduction) { + // Next code should be emitted for reduction: + // + // static kmp_critical_name lock = { 0 }; + // + // void reduce_func(void *lhs[<n>], void *rhs[<n>]) { + // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]); + // ... + // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1], + // *(Type<n>-1*)rhs[<n>-1]); + // } + // + // ... + // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]}; + // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), + // RedList, reduce_func, &<lock>)) { + // case 1: + // ... + // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); + // ... + // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); + // break; + // case 2: + // ... + // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); + // ... + // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);] + // break; + // default:; + // } + // + // if SimpleReduction is true, only the next code is generated: + // ... + // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); + // ... + + auto &C = CGM.getContext(); + + if (SimpleReduction) { + CodeGenFunction::RunCleanupsScope Scope(CGF); + for (auto *E : ReductionOps) { + CGF.EmitIgnoredExpr(E); + } + return; + } + + // 1. Build a list of reduction variables. + // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; + llvm::APInt ArraySize(/*unsigned int numBits=*/32, RHSExprs.size()); + QualType ReductionArrayTy = + C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal, + /*IndexTypeQuals=*/0); + auto *ReductionList = + CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); + for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I) { + auto *Elem = CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, ReductionList, I); + CGF.Builder.CreateAlignedStore( + CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.EmitLValue(RHSExprs[I]).getAddress(), CGF.VoidPtrTy), + Elem, CGM.PointerAlignInBytes); + } + + // 2. Emit reduce_func(). + auto *ReductionFn = emitReductionFunction( + CGM, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), LHSExprs, + RHSExprs, ReductionOps); + + // 3. Create static kmp_critical_name lock = { 0 }; + auto *Lock = getCriticalRegionLock(".reduction"); + + // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), + // RedList, reduce_func, &<lock>); + auto *IdentTLoc = emitUpdateLocation( + CGF, Loc, + static_cast<OpenMPLocationFlags>(OMP_IDENT_KMPC | OMP_ATOMIC_REDUCE)); + auto *ThreadId = getThreadID(CGF, Loc); + auto *ReductionArrayTySize = llvm::ConstantInt::get( + CGM.SizeTy, C.getTypeSizeInChars(ReductionArrayTy).getQuantity()); + auto *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, + CGF.VoidPtrTy); + llvm::Value *Args[] = { + IdentTLoc, // ident_t *<loc> + ThreadId, // i32 <gtid> + CGF.Builder.getInt32(RHSExprs.size()), // i32 <n> + ReductionArrayTySize, // size_type sizeof(RedList) + RL, // void *RedList + ReductionFn, // void (*) (void *, void *) <reduce_func> + Lock // kmp_critical_name *&<lock> + }; + auto Res = CGF.EmitRuntimeCall( + createRuntimeFunction(WithNowait ? OMPRTL__kmpc_reduce_nowait + : OMPRTL__kmpc_reduce), + Args); + + // 5. Build switch(res) + auto *DefaultBB = CGF.createBasicBlock(".omp.reduction.default"); + auto *SwInst = CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2); + + // 6. Build case 1: + // ... + // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); + // ... + // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); + // break; + auto *Case1BB = CGF.createBasicBlock(".omp.reduction.case1"); + SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); + CGF.EmitBlock(Case1BB); + + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); + llvm::Value *EndArgs[] = { + IdentTLoc, // ident_t *<loc> + ThreadId, // i32 <gtid> + Lock // kmp_critical_name *&<lock> + }; + CGF.EHStack + .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>( + NormalAndEHCleanup, + createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait + : OMPRTL__kmpc_end_reduce), + llvm::makeArrayRef(EndArgs)); + for (auto *E : ReductionOps) { + CGF.EmitIgnoredExpr(E); + } + } + + CGF.EmitBranch(DefaultBB); + + // 7. Build case 2: + // ... + // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); + // ... + // break; + auto *Case2BB = CGF.createBasicBlock(".omp.reduction.case2"); + SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); + CGF.EmitBlock(Case2BB); + + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + if (!WithNowait) { + // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>); + llvm::Value *EndArgs[] = { + IdentTLoc, // ident_t *<loc> + ThreadId, // i32 <gtid> + Lock // kmp_critical_name *&<lock> + }; + CGF.EHStack + .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>( + NormalAndEHCleanup, + createRuntimeFunction(OMPRTL__kmpc_end_reduce), + llvm::makeArrayRef(EndArgs)); + } + auto I = LHSExprs.begin(); + for (auto *E : ReductionOps) { + const Expr *XExpr = nullptr; + const Expr *EExpr = nullptr; + const Expr *UpExpr = nullptr; + BinaryOperatorKind BO = BO_Comma; + if (auto *BO = dyn_cast<BinaryOperator>(E)) { + if (BO->getOpcode() == BO_Assign) { + XExpr = BO->getLHS(); + UpExpr = BO->getRHS(); + } + } + // Try to emit update expression as a simple atomic. + auto *RHSExpr = UpExpr; + if (RHSExpr) { + // Analyze RHS part of the whole expression. + if (auto *ACO = dyn_cast<AbstractConditionalOperator>( + RHSExpr->IgnoreParenImpCasts())) { + // If this is a conditional operator, analyze its condition for + // min/max reduction operator. + RHSExpr = ACO->getCond(); + } + if (auto *BORHS = + dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) { + EExpr = BORHS->getRHS(); + BO = BORHS->getOpcode(); + } + } + if (XExpr) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()); + LValue X = CGF.EmitLValue(XExpr); + RValue E; + if (EExpr) + E = CGF.EmitAnyExpr(EExpr); + CGF.EmitOMPAtomicSimpleUpdateExpr( + X, E, BO, /*IsXLHSInRHSPart=*/true, llvm::Monotonic, Loc, + [&CGF, UpExpr, VD](RValue XRValue) { + CodeGenFunction::OMPPrivateScope PrivateScope(CGF); + PrivateScope.addPrivate( + VD, [&CGF, VD, XRValue]() -> llvm::Value *{ + auto *LHSTemp = CGF.CreateMemTemp(VD->getType()); + CGF.EmitStoreThroughLValue( + XRValue, + CGF.MakeNaturalAlignAddrLValue(LHSTemp, VD->getType())); + return LHSTemp; + }); + (void)PrivateScope.Privatize(); + return CGF.EmitAnyExpr(UpExpr); + }); + } else { + // Emit as a critical region. + emitCriticalRegion(CGF, ".atomic_reduction", [E](CodeGenFunction &CGF) { + CGF.EmitIgnoredExpr(E); + }, Loc); + } + ++I; + } + } + + CGF.EmitBranch(DefaultBB); + CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); +} + +void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, + SourceLocation Loc) { + // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 + // global_tid); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + // Ignore return result until untied tasks are supported. + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args); +} + +void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, + OpenMPDirectiveKind InnerKind, + const RegionCodeGenTy &CodeGen) { + InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind); + CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); +} + +namespace { +enum RTCancelKind { + CancelNoreq = 0, + CancelParallel = 1, + CancelLoop = 2, + CancelSections = 3, + CancelTaskgroup = 4 +}; +} + +static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) { + RTCancelKind CancelKind = CancelNoreq; + if (CancelRegion == OMPD_parallel) + CancelKind = CancelParallel; + else if (CancelRegion == OMPD_for) + CancelKind = CancelLoop; + else if (CancelRegion == OMPD_sections) + CancelKind = CancelSections; + else { + assert(CancelRegion == OMPD_taskgroup); + CancelKind = CancelTaskgroup; + } + return CancelKind; +} + +void CGOpenMPRuntime::emitCancellationPointCall( + CodeGenFunction &CGF, SourceLocation Loc, + OpenMPDirectiveKind CancelRegion) { + // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32 + // global_tid, kmp_int32 cncl_kind); + if (auto *OMPRegionInfo = + dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { + auto CancelDest = + CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); + if (CancelDest.isValid()) { + llvm::Value *Args[] = { + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; + // Ignore return result until untied tasks are supported. + auto *Result = CGF.EmitRuntimeCall( + createRuntimeFunction(OMPRTL__kmpc_cancellationpoint), Args); + // if (__kmpc_cancellationpoint()) { + // __kmpc_cancel_barrier(); + // exit from construct; + // } + auto *ExitBB = CGF.createBasicBlock(".cancel.exit"); + auto *ContBB = CGF.createBasicBlock(".cancel.continue"); + auto *Cmp = CGF.Builder.CreateIsNotNull(Result); + CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); + CGF.EmitBlock(ExitBB); + // __kmpc_cancel_barrier(); + emitBarrierCall(CGF, Loc, OMPD_unknown, /*CheckForCancel=*/false); + // exit from construct; + CGF.EmitBranchThroughCleanup(CancelDest); + CGF.EmitBlock(ContBB, /*IsFinished=*/true); + } + } +} + +void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPDirectiveKind CancelRegion) { + // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid, + // kmp_int32 cncl_kind); + if (auto *OMPRegionInfo = + dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { + auto CancelDest = + CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); + if (CancelDest.isValid()) { + llvm::Value *Args[] = { + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; + // Ignore return result until untied tasks are supported. + auto *Result = + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_cancel), Args); + // if (__kmpc_cancel()) { + // __kmpc_cancel_barrier(); + // exit from construct; + // } + auto *ExitBB = CGF.createBasicBlock(".cancel.exit"); + auto *ContBB = CGF.createBasicBlock(".cancel.continue"); + auto *Cmp = CGF.Builder.CreateIsNotNull(Result); + CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); + CGF.EmitBlock(ExitBB); + // __kmpc_cancel_barrier(); + emitBarrierCall(CGF, Loc, OMPD_unknown, /*CheckForCancel=*/false); + // exit from construct; + CGF.EmitBranchThroughCleanup(CancelDest); + CGF.EmitBlock(ContBB, /*IsFinished=*/true); + } + } +} + diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h index 6daf817..44bc8a1 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGOpenMPRuntime.h @@ -14,6 +14,7 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H #define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H +#include "clang/AST/Type.h" #include "clang/Basic/OpenMPKinds.h" #include "clang/Basic/SourceLocation.h" #include "llvm/ADT/DenseMap.h" @@ -42,9 +43,9 @@ namespace CodeGen { class CodeGenFunction; class CodeGenModule; -class CGOpenMPRuntime { -public: +typedef llvm::function_ref<void(CodeGenFunction &)> RegionCodeGenTy; +class CGOpenMPRuntime { private: enum OpenMPRTLFunction { /// \brief Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, @@ -67,11 +68,9 @@ private: // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32 // global_tid); OMPRTL__kmpc_cancel_barrier, - // Calls for static scheduling 'omp for' loops. - OMPRTL__kmpc_for_static_init_4, - OMPRTL__kmpc_for_static_init_4u, - OMPRTL__kmpc_for_static_init_8, - OMPRTL__kmpc_for_static_init_8u, + // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid); + OMPRTL__kmpc_barrier, + // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid); OMPRTL__kmpc_for_static_fini, // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 // global_tid); @@ -82,12 +81,79 @@ private: // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, // kmp_int32 num_threads); OMPRTL__kmpc_push_num_threads, - // Call to void __kmpc_flush(ident_t *loc, ...); + // Call to void __kmpc_flush(ident_t *loc); OMPRTL__kmpc_flush, // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid); OMPRTL__kmpc_master, // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid); OMPRTL__kmpc_end_master, + // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid, + // int end_part); + OMPRTL__kmpc_omp_taskyield, + // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid); + OMPRTL__kmpc_single, + // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid); + OMPRTL__kmpc_end_single, + // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid, + // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, + // kmp_routine_entry_t *task_entry); + OMPRTL__kmpc_omp_task_alloc, + // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t * + // new_task); + OMPRTL__kmpc_omp_task, + // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid, + // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), + // kmp_int32 didit); + OMPRTL__kmpc_copyprivate, + // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, + // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void + // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck); + OMPRTL__kmpc_reduce, + // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 + // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, + // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name + // *lck); + OMPRTL__kmpc_reduce_nowait, + // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, + // kmp_critical_name *lck); + OMPRTL__kmpc_end_reduce, + // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, + // kmp_critical_name *lck); + OMPRTL__kmpc_end_reduce_nowait, + // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid, + // kmp_task_t * new_task); + OMPRTL__kmpc_omp_task_begin_if0, + // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid, + // kmp_task_t * new_task); + OMPRTL__kmpc_omp_task_complete_if0, + // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid); + OMPRTL__kmpc_ordered, + // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid); + OMPRTL__kmpc_end_ordered, + // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 + // global_tid); + OMPRTL__kmpc_omp_taskwait, + // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid); + OMPRTL__kmpc_taskgroup, + // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid); + OMPRTL__kmpc_end_taskgroup, + // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, + // int proc_bind); + OMPRTL__kmpc_push_proc_bind, + // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32 + // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t + // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list); + OMPRTL__kmpc_omp_task_with_deps, + // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 + // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 + // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); + OMPRTL__kmpc_omp_wait_deps, + // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32 + // global_tid, kmp_int32 cncl_kind); + OMPRTL__kmpc_cancellationpoint, + // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid, + // kmp_int32 cncl_kind); + OMPRTL__kmpc_cancel, }; /// \brief Values for bit flags used in the ident_t to describe the fields. @@ -118,7 +184,7 @@ private: /// \brief Map of flags and corresponding default locations. typedef llvm::DenseMap<unsigned, llvm::Value *> OpenMPDefaultLocMapTy; OpenMPDefaultLocMapTy OpenMPDefaultLocMap; - llvm::Value *GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags); + llvm::Value *getOrCreateDefaultLocation(OpenMPLocationFlags Flags); /// \brief Describes ident structure that describes a source location. /// All descriptions are taken from /// http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h @@ -186,13 +252,38 @@ private: /// variables. llvm::StringMap<llvm::AssertingVH<llvm::Constant>, llvm::BumpPtrAllocator> InternalVars; + /// \brief Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); + llvm::Type *KmpRoutineEntryPtrTy; + QualType KmpRoutineEntryPtrQTy; + /// \brief Type typedef struct kmp_task { + /// void * shareds; /**< pointer to block of pointers to + /// shared vars */ + /// kmp_routine_entry_t routine; /**< pointer to routine to call for + /// executing task */ + /// kmp_int32 part_id; /**< part id for the task */ + /// kmp_routine_entry_t destructors; /* pointer to function to invoke + /// deconstructors of firstprivate C++ objects */ + /// } kmp_task_t; + QualType KmpTaskTQTy; + /// \brief Type typedef struct kmp_depend_info { + /// kmp_intptr_t base_addr; + /// size_t len; + /// struct { + /// bool in:1; + /// bool out:1; + /// } flags; + /// } kmp_depend_info_t; + QualType KmpDependInfoTy; + + + /// \brief Build type kmp_routine_entry_t (if not built yet). + void emitKmpRoutineEntryT(QualType KmpInt32Ty); /// \brief Emits object of ident_t type with info for source location. /// \param Flags Flags for OpenMP location. /// - llvm::Value * - EmitOpenMPUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPLocationFlags Flags = OMP_IDENT_KMPC); + llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPLocationFlags Flags = OMP_IDENT_KMPC); /// \brief Returns pointer to ident_t type. llvm::Type *getIdentTyPointerTy(); @@ -203,7 +294,23 @@ private: /// \brief Returns specified OpenMP runtime function. /// \param Function OpenMP runtime function. /// \return Specified function. - llvm::Constant *CreateRuntimeFunction(OpenMPRTLFunction Function); + llvm::Constant *createRuntimeFunction(OpenMPRTLFunction Function); + + /// \brief Returns __kmpc_for_static_init_* runtime function for the specified + /// size \a IVSize and sign \a IVSigned. + llvm::Constant *createForStaticInitFunction(unsigned IVSize, bool IVSigned); + + /// \brief Returns __kmpc_dispatch_init_* runtime function for the specified + /// size \a IVSize and sign \a IVSigned. + llvm::Constant *createDispatchInitFunction(unsigned IVSize, bool IVSigned); + + /// \brief Returns __kmpc_dispatch_next_* runtime function for the specified + /// size \a IVSize and sign \a IVSigned. + llvm::Constant *createDispatchNextFunction(unsigned IVSize, bool IVSigned); + + /// \brief Returns __kmpc_dispatch_fini_* runtime function for the specified + /// size \a IVSize and sign \a IVSigned. + llvm::Constant *createDispatchFiniFunction(unsigned IVSize, bool IVSigned); /// \brief If the specified mangled name is not in the module, create and /// return threadprivate cache object. This object is a pointer's worth of @@ -214,12 +321,12 @@ private: /// \brief Emits address of the word in a memory where current thread id is /// stored. - virtual llvm::Value *EmitThreadIDAddress(CodeGenFunction &CGF, + virtual llvm::Value *emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc); /// \brief Gets thread id value for the current thread. /// - llvm::Value *GetOpenMPThreadID(CodeGenFunction &CGF, SourceLocation Loc); + llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc); /// \brief Gets (if variable with the given name already exist) or creates /// internal global variable with the specified Name. The created variable has @@ -227,7 +334,7 @@ private: /// \param Ty Type of the global variable. If it is exist already the type /// must be the same. /// \param Name Name of the variable. - llvm::Constant *GetOrCreateInternalVariable(llvm::Type *Ty, + llvm::Constant *getOrCreateInternalVariable(llvm::Type *Ty, const llvm::Twine &Name); /// \brief Set of threadprivate variables with the generated initializer. @@ -239,78 +346,118 @@ private: /// \param CopyCtor Pointer to a global copy function for \a VD. /// \param Dtor Pointer to a global destructor function for \a VD. /// \param Loc Location of threadprivate declaration. - void EmitOMPThreadPrivateVarInit(CodeGenFunction &CGF, llvm::Value *VDAddr, - llvm::Value *Ctor, llvm::Value *CopyCtor, - llvm::Value *Dtor, SourceLocation Loc); + void emitThreadPrivateVarInit(CodeGenFunction &CGF, llvm::Value *VDAddr, + llvm::Value *Ctor, llvm::Value *CopyCtor, + llvm::Value *Dtor, SourceLocation Loc); /// \brief Returns corresponding lock object for the specified critical region /// name. If the lock object does not exist it is created, otherwise the /// reference to the existing copy is returned. /// \param CriticalName Name of the critical region. /// - llvm::Value *GetCriticalRegionLock(StringRef CriticalName); + llvm::Value *getCriticalRegionLock(StringRef CriticalName); public: explicit CGOpenMPRuntime(CodeGenModule &CGM); virtual ~CGOpenMPRuntime() {} + virtual void clear(); - /// \brief Emits outlined function for the specified OpenMP directive \a D - /// (required for parallel and task directives). This outlined function has - /// type void(*)(kmp_int32 /*ThreadID*/, kmp_int32 /*BoundID*/, struct - /// context_vars*). + /// \brief Emits outlined function for the specified OpenMP parallel directive + /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, + /// kmp_int32 BoundID, struct context_vars*). + /// \param D OpenMP directive. + /// \param ThreadIDVar Variable for thread id in the current OpenMP region. + /// \param InnermostKind Kind of innermost directive (for simple directives it + /// is a directive itself, for combined - its innermost directive). + /// \param CodeGen Code generation sequence for the \a D directive. + virtual llvm::Value *emitParallelOutlinedFunction( + const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, + OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen); + + /// \brief Emits outlined function for the OpenMP task directive \a D. This + /// outlined function has type void(*)(kmp_int32 ThreadID, kmp_int32 + /// PartID, struct context_vars*). /// \param D OpenMP directive. /// \param ThreadIDVar Variable for thread id in the current OpenMP region. + /// \param InnermostKind Kind of innermost directive (for simple directives it + /// is a directive itself, for combined - its innermost directive). + /// \param CodeGen Code generation sequence for the \a D directive. /// - virtual llvm::Value * - EmitOpenMPOutlinedFunction(const OMPExecutableDirective &D, - const VarDecl *ThreadIDVar); + virtual llvm::Value *emitTaskOutlinedFunction( + const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, + OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen); /// \brief Cleans up references to the objects in finished function. /// - void FunctionFinished(CodeGenFunction &CGF); + void functionFinished(CodeGenFunction &CGF); - /// \brief Emits code for parallel call of the \a OutlinedFn with variables - /// captured in a record which address is stored in \a CapturedStruct. + /// \brief Emits code for parallel or serial call of the \a OutlinedFn with + /// variables captured in a record which address is stored in \a + /// CapturedStruct. /// \param OutlinedFn Outlined function to be run in parallel threads. Type of - /// this function is void(*)(kmp_int32, kmp_int32, struct context_vars*). + /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). /// \param CapturedStruct A pointer to the record with the references to /// variables used in \a OutlinedFn function. + /// \param IfCond Condition in the associated 'if' clause, if it was + /// specified, nullptr otherwise. /// - virtual void EmitOMPParallelCall(CodeGenFunction &CGF, SourceLocation Loc, - llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct); - - /// \brief Emits code for serial call of the \a OutlinedFn with variables - /// captured in a record which address is stored in \a CapturedStruct. - /// \param OutlinedFn Outlined function to be run in serial mode. - /// \param CapturedStruct A pointer to the record with the references to - /// variables used in \a OutlinedFn function. - /// - virtual void EmitOMPSerialCall(CodeGenFunction &CGF, SourceLocation Loc, - llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct); + virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, + llvm::Value *OutlinedFn, + llvm::Value *CapturedStruct, + const Expr *IfCond); /// \brief Emits a critical region. /// \param CriticalName Name of the critical region. /// \param CriticalOpGen Generator for the statement associated with the given /// critical region. - virtual void EmitOMPCriticalRegion(CodeGenFunction &CGF, - StringRef CriticalName, - const std::function<void()> &CriticalOpGen, - SourceLocation Loc); + virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, + const RegionCodeGenTy &CriticalOpGen, + SourceLocation Loc); /// \brief Emits a master region. /// \param MasterOpGen Generator for the statement associated with the given /// master region. - virtual void EmitOMPMasterRegion(CodeGenFunction &CGF, - const std::function<void()> &MasterOpGen, + virtual void emitMasterRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &MasterOpGen, + SourceLocation Loc); + + /// \brief Emits code for a taskyield directive. + virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc); + + /// \brief Emit a taskgroup region. + /// \param TaskgroupOpGen Generator for the statement associated with the + /// given taskgroup region. + virtual void emitTaskgroupRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc); - /// \brief Emits explicit barrier for OpenMP threads. - /// \param IsExplicit true, if it is explicitly specified barrier. + /// \brief Emits a single region. + /// \param SingleOpGen Generator for the statement associated with the given + /// single region. + virtual void emitSingleRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &SingleOpGen, + SourceLocation Loc, + ArrayRef<const Expr *> CopyprivateVars, + ArrayRef<const Expr *> DestExprs, + ArrayRef<const Expr *> SrcExprs, + ArrayRef<const Expr *> AssignmentOps); + + /// \brief Emit an ordered region. + /// \param OrderedOpGen Generator for the statement associated with the given + /// ordered region. + virtual void emitOrderedRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &OrderedOpGen, + SourceLocation Loc); + + /// \brief Emit an implicit/explicit barrier for OpenMP threads. + /// \param Kind Directive for which this implicit barrier call must be + /// generated. Must be OMPD_barrier for explicit barrier generation. + /// \param CheckForCancel true if check for possible cancellation must be + /// performed, false otherwise. /// - virtual void EmitOMPBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, - bool IsExplicit = true); + virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPDirectiveKind Kind, + bool CheckForCancel = true); /// \brief Check if the specified \a ScheduleKind is static non-chunked. /// This kind of worksharing directive is emitted without outer loop. @@ -320,6 +467,12 @@ public: virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const; + /// \brief Check if the specified \a ScheduleKind is dynamic. + /// This kind of worksharing directive is emitted without outer loop. + /// \param ScheduleKind Schedule Kind specified in the 'schedule' clause. + /// + virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const; + /// \brief Call the appropriate runtime routine to initialize it before start /// of loop. /// @@ -332,6 +485,7 @@ public: /// \param SchedKind Schedule kind, specified by the 'schedule' clause. /// \param IVSize Size of the iteration variable in bits. /// \param IVSigned Sign of the interation variable. + /// \param Ordered true if loop is ordered, false otherwise. /// \param IL Address of the output variable in which the flag of the /// last iteration is returned. /// \param LB Address of the output variable in which the lower iteration @@ -343,29 +497,64 @@ public: /// \param Chunk Value of the chunk for the static_chunked scheduled loop. /// For the default (nullptr) value, the chunk 1 will be used. /// - virtual void EmitOMPForInit(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPScheduleClauseKind SchedKind, - unsigned IVSize, bool IVSigned, llvm::Value *IL, - llvm::Value *LB, llvm::Value *UB, llvm::Value *ST, - llvm::Value *Chunk = nullptr); + virtual void emitForInit(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPScheduleClauseKind SchedKind, unsigned IVSize, + bool IVSigned, bool Ordered, llvm::Value *IL, + llvm::Value *LB, llvm::Value *UB, llvm::Value *ST, + llvm::Value *Chunk = nullptr); + + /// \brief Call the appropriate runtime routine to notify that we finished + /// iteration of the ordered loop with the dynamic scheduling. + /// + /// \param CGF Reference to current CodeGenFunction. + /// \param Loc Clang source location. + /// \param IVSize Size of the iteration variable in bits. + /// \param IVSigned Sign of the interation variable. + /// + virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, + SourceLocation Loc, unsigned IVSize, + bool IVSigned); /// \brief Call the appropriate runtime routine to notify that we finished /// all the work with current loop. /// /// \param CGF Reference to current CodeGenFunction. /// \param Loc Clang source location. - /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause. /// - virtual void EmitOMPForFinish(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPScheduleClauseKind ScheduleKind); + virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc); + + /// Call __kmpc_dispatch_next( + /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, + /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, + /// kmp_int[32|64] *p_stride); + /// \param IVSize Size of the iteration variable in bits. + /// \param IVSigned Sign of the interation variable. + /// \param IL Address of the output variable in which the flag of the + /// last iteration is returned. + /// \param LB Address of the output variable in which the lower iteration + /// number is returned. + /// \param UB Address of the output variable in which the upper iteration + /// number is returned. + /// \param ST Address of the output variable in which the stride value is + /// returned. + virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc, + unsigned IVSize, bool IVSigned, + llvm::Value *IL, llvm::Value *LB, + llvm::Value *UB, llvm::Value *ST); /// \brief Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads' /// clause. /// \param NumThreads An integer value of threads. - virtual void EmitOMPNumThreadsClause(CodeGenFunction &CGF, - llvm::Value *NumThreads, - SourceLocation Loc); + virtual void emitNumThreadsClause(CodeGenFunction &CGF, + llvm::Value *NumThreads, + SourceLocation Loc); + + /// \brief Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 + /// global_tid, int proc_bind) to generate code for 'proc_bind' clause. + virtual void emitProcBindClause(CodeGenFunction &CGF, + OpenMPProcBindClauseKind ProcBind, + SourceLocation Loc); /// \brief Returns address of the threadprivate variable for the current /// thread. @@ -373,10 +562,10 @@ public: /// \param VDAddr Address of the global variable \a VD. /// \param Loc Location of the reference to threadprivate var. /// \return Address of the threadprivate variable for the current thread. - virtual llvm::Value *getOMPAddrOfThreadPrivate(CodeGenFunction &CGF, - const VarDecl *VD, - llvm::Value *VDAddr, - SourceLocation Loc); + virtual llvm::Value *getAddrOfThreadPrivate(CodeGenFunction &CGF, + const VarDecl *VD, + llvm::Value *VDAddr, + SourceLocation Loc); /// \brief Emit a code for initialization of threadprivate variable. It emits /// a call to runtime library which adds initial value to the newly created @@ -387,15 +576,139 @@ public: /// \param Loc Location of threadprivate declaration. /// \param PerformInit true if initialization expression is not constant. virtual llvm::Function * - EmitOMPThreadPrivateVarDefinition(const VarDecl *VD, llvm::Value *VDAddr, - SourceLocation Loc, bool PerformInit, - CodeGenFunction *CGF = nullptr); + emitThreadPrivateVarDefinition(const VarDecl *VD, llvm::Value *VDAddr, + SourceLocation Loc, bool PerformInit, + CodeGenFunction *CGF = nullptr); /// \brief Emit flush of the variables specified in 'omp flush' directive. /// \param Vars List of variables to flush. - virtual void EmitOMPFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars, - SourceLocation Loc); + virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars, + SourceLocation Loc); + + /// \brief Emit task region for the task directive. The task region is + /// emitted in several steps: + /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 + /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, + /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the + /// function: + /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { + /// TaskFunction(gtid, tt->part_id, tt->shareds); + /// return 0; + /// } + /// 2. Copy a list of shared variables to field shareds of the resulting + /// structure kmp_task_t returned by the previous call (if any). + /// 3. Copy a pointer to destructions function to field destructions of the + /// resulting structure kmp_task_t. + /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, + /// kmp_task_t *new_task), where new_task is a resulting structure from + /// previous items. + /// \param D Current task directive. + /// \param Tied true if the task is tied (the task is tied to the thread that + /// can suspend its task region), false - untied (the task is not tied to any + /// thread). + /// \param Final Contains either constant bool value, or llvm::Value * of i1 + /// type for final clause. If the value is true, the task forces all of its + /// child tasks to become final and included tasks. + /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 + /// /*part_id*/, captured_struct */*__context*/); + /// \param SharedsTy A type which contains references the shared variables. + /// \param Shareds Context with the list of shared variables from the \p + /// TaskFunction. + /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr + /// otherwise. + /// \param PrivateVars List of references to private variables for the task + /// directive. + /// \param PrivateCopies List of private copies for each private variable in + /// \p PrivateVars. + /// \param FirstprivateVars List of references to private variables for the + /// task directive. + /// \param FirstprivateCopies List of private copies for each private variable + /// in \p FirstprivateVars. + /// \param FirstprivateInits List of references to auto generated variables + /// used for initialization of a single array element. Used if firstprivate + /// variable is of array type. + /// \param Dependences List of dependences for the 'task' construct, including + /// original expression and dependency type. + virtual void emitTaskCall( + CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, + bool Tied, llvm::PointerIntPair<llvm::Value *, 1, bool> Final, + llvm::Value *TaskFunction, QualType SharedsTy, llvm::Value *Shareds, + const Expr *IfCond, ArrayRef<const Expr *> PrivateVars, + ArrayRef<const Expr *> PrivateCopies, + ArrayRef<const Expr *> FirstprivateVars, + ArrayRef<const Expr *> FirstprivateCopies, + ArrayRef<const Expr *> FirstprivateInits, + ArrayRef<std::pair<OpenMPDependClauseKind, const Expr *>> Dependences); + + /// \brief Emit code for the directive that does not require outlining. + /// + /// \param InnermostKind Kind of innermost directive (for simple directives it + /// is a directive itself, for combined - its innermost directive). + /// \param CodeGen Code generation sequence for the \a D directive. + virtual void emitInlinedDirective(CodeGenFunction &CGF, + OpenMPDirectiveKind InnermostKind, + const RegionCodeGenTy &CodeGen); + /// \brief Emit a code for reduction clause. Next code should be emitted for + /// reduction: + /// \code + /// + /// static kmp_critical_name lock = { 0 }; + /// + /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) { + /// ... + /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]); + /// ... + /// } + /// + /// ... + /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]}; + /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), + /// RedList, reduce_func, &<lock>)) { + /// case 1: + /// ... + /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); + /// ... + /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); + /// break; + /// case 2: + /// ... + /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); + /// ... + /// break; + /// default:; + /// } + /// \endcode + /// + /// \param LHSExprs List of LHS in \a ReductionOps reduction operations. + /// \param RHSExprs List of RHS in \a ReductionOps reduction operations. + /// \param ReductionOps List of reduction operations in form 'LHS binop RHS' + /// or 'operator binop(LHS, RHS)'. + /// \param WithNowait true if parent directive has also nowait clause, false + /// otherwise. + virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, + ArrayRef<const Expr *> LHSExprs, + ArrayRef<const Expr *> RHSExprs, + ArrayRef<const Expr *> ReductionOps, + bool WithNowait, bool SimpleReduction); + + /// \brief Emit code for 'taskwait' directive. + virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc); + + /// \brief Emit code for 'cancellation point' construct. + /// \param CancelRegion Region kind for which the cancellation point must be + /// emitted. + /// + virtual void emitCancellationPointCall(CodeGenFunction &CGF, + SourceLocation Loc, + OpenMPDirectiveKind CancelRegion); + + /// \brief Emit code for 'cancel' construct. + /// \param CancelRegion Region kind for which the cancel must be emitted. + /// + virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPDirectiveKind CancelRegion); }; + } // namespace CodeGen } // namespace clang diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayout.h b/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayout.h index 2de0b2f..d4ad33e 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayout.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayout.h @@ -78,16 +78,16 @@ struct CGBitFieldInfo { /// bitfield. unsigned StorageSize; - /// The alignment which should be used when accessing the bitfield. - unsigned StorageAlignment; + /// The offset of the bitfield storage from the start of the struct. + CharUnits StorageOffset; CGBitFieldInfo() - : Offset(), Size(), IsSigned(), StorageSize(), StorageAlignment() {} + : Offset(), Size(), IsSigned(), StorageSize(), StorageOffset() {} CGBitFieldInfo(unsigned Offset, unsigned Size, bool IsSigned, - unsigned StorageSize, unsigned StorageAlignment) + unsigned StorageSize, CharUnits StorageOffset) : Offset(Offset), Size(Size), IsSigned(IsSigned), - StorageSize(StorageSize), StorageAlignment(StorageAlignment) {} + StorageSize(StorageSize), StorageOffset(StorageOffset) {} void print(raw_ostream &OS) const; void dump() const; @@ -99,7 +99,7 @@ struct CGBitFieldInfo { const FieldDecl *FD, uint64_t Offset, uint64_t Size, uint64_t StorageSize, - uint64_t StorageAlignment); + CharUnits StorageOffset); }; /// CGRecordLayout - This class handles struct and union layout info while @@ -109,8 +109,8 @@ struct CGBitFieldInfo { class CGRecordLayout { friend class CodeGenTypes; - CGRecordLayout(const CGRecordLayout &) LLVM_DELETED_FUNCTION; - void operator=(const CGRecordLayout &) LLVM_DELETED_FUNCTION; + CGRecordLayout(const CGRecordLayout &) = delete; + void operator=(const CGRecordLayout &) = delete; private: /// The LLVM type corresponding to this record layout; used when diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp index 7ad394b..f91eceb 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp @@ -99,10 +99,25 @@ struct CGRecordLowering { MemberInfo StorageInfo(CharUnits Offset, llvm::Type *Data) { return MemberInfo(Offset, MemberInfo::Field, Data); } - bool useMSABI() { + + /// The Microsoft bitfield layout rule allocates discrete storage + /// units of the field's formal type and only combines adjacent + /// fields of the same formal type. We want to emit a layout with + /// these discrete storage units instead of combining them into a + /// continuous run. + bool isDiscreteBitFieldABI() { return Context.getTargetInfo().getCXXABI().isMicrosoft() || D->isMsStruct(Context); } + + /// The Itanium base layout rule allows virtual bases to overlap + /// other bases, which complicates layout in specific ways. + /// + /// Note specifically that the ms_struct attribute doesn't change this. + bool isOverlappingVBaseABI() { + return !Context.getTargetInfo().getCXXABI().isMicrosoft(); + } + /// \brief Wraps llvm::Type::getIntNTy with some implicit arguments. llvm::Type *getIntNType(uint64_t NumBits) { return llvm::Type::getIntNTy(Types.getLLVMContext(), @@ -119,8 +134,9 @@ struct CGRecordLowering { /// for itanium bitfields that are smaller than their declared type. llvm::Type *getStorageType(const FieldDecl *FD) { llvm::Type *Type = Types.ConvertTypeForMem(FD->getType()); - return useMSABI() || !FD->isBitField() ? Type : - getIntNType(std::min(FD->getBitWidthValue(Context), + if (!FD->isBitField()) return Type; + if (isDiscreteBitFieldABI()) return Type; + return getIntNType(std::min(FD->getBitWidthValue(Context), (unsigned)Context.toBits(getSize(Type)))); } /// \brief Gets the llvm Basesubobject type from a CXXRecordDecl. @@ -137,15 +153,10 @@ struct CGRecordLowering { return CharUnits::fromQuantity(DataLayout.getABITypeAlignment(Type)); } bool isZeroInitializable(const FieldDecl *FD) { - const Type *Type = FD->getType()->getBaseElementTypeUnsafe(); - if (const MemberPointerType *MPT = Type->getAs<MemberPointerType>()) - return Types.getCXXABI().isZeroInitializable(MPT); - if (const RecordType *RT = Type->getAs<RecordType>()) - return isZeroInitializable(RT->getDecl()); - return true; + return Types.isZeroInitializable(FD->getType()); } bool isZeroInitializable(const RecordDecl *RD) { - return Types.getCGRecordLayout(RD).isZeroInitializable(); + return Types.isZeroInitializable(RD); } void appendPaddingBytes(CharUnits Size) { if (!Size.isZero()) @@ -198,8 +209,8 @@ struct CGRecordLowering { bool IsZeroInitializableAsBase : 1; bool Packed : 1; private: - CGRecordLowering(const CGRecordLowering &) LLVM_DELETED_FUNCTION; - void operator =(const CGRecordLowering &) LLVM_DELETED_FUNCTION; + CGRecordLowering(const CGRecordLowering &) = delete; + void operator =(const CGRecordLowering &) = delete; }; } // namespace { @@ -217,11 +228,7 @@ void CGRecordLowering::setBitFieldInfo( Info.Offset = (unsigned)(getFieldBitOffset(FD) - Context.toBits(StartOffset)); Info.Size = FD->getBitWidthValue(Context); Info.StorageSize = (unsigned)DataLayout.getTypeAllocSizeInBits(StorageType); - // Here we calculate the actual storage alignment of the bits. E.g if we've - // got an alignment >= 2 and the bitfield starts at offset 6 we've got an - // alignment of 2. - Info.StorageAlignment = - Layout.getAlignment().alignmentAtOffset(StartOffset).getQuantity(); + Info.StorageOffset = StartOffset; if (Info.Size > Info.StorageSize) Info.Size = Info.StorageSize; // Reverse the bit offsets for big endian machines. Because we represent @@ -303,9 +310,13 @@ void CGRecordLowering::lowerUnion() { // If this is the case, then we aught not to try and come up with a "better" // type, it might not be very easy to come up with a Constant which // correctly initializes it. - if (!SeenNamedMember && Field->getDeclName()) { - SeenNamedMember = true; - if (!isZeroInitializable(Field)) { + if (!SeenNamedMember) { + SeenNamedMember = Field->getIdentifier(); + if (!SeenNamedMember) + if (const auto *FieldRD = + dyn_cast_or_null<RecordDecl>(Field->getType()->getAsTagDecl())) + SeenNamedMember = FieldRD->findFirstNamedDataMember(); + if (SeenNamedMember && !isZeroInitializable(Field)) { IsZeroInitializable = IsZeroInitializableAsBase = false; StorageType = FieldType; } @@ -365,7 +376,7 @@ CGRecordLowering::accumulateBitFields(RecordDecl::field_iterator Field, // used to determine if the ASTRecordLayout is treating these two bitfields as // contiguous. StartBitOffset is offset of the beginning of the Run. uint64_t StartBitOffset, Tail = 0; - if (useMSABI()) { + if (isDiscreteBitFieldABI()) { for (; Field != FieldEnd; ++Field) { uint64_t BitOffset = getFieldBitOffset(*Field); // Zero-width bitfields end runs. @@ -438,8 +449,12 @@ void CGRecordLowering::accumulateBases() { for (const auto &Base : RD->bases()) { if (Base.isVirtual()) continue; + + // Bases can be zero-sized even if not technically empty if they + // contain only a trailing array member. const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); - if (!BaseDecl->isEmpty()) + if (!BaseDecl->isEmpty() && + !Context.getASTRecordLayout(BaseDecl).getSize().isZero()) Members.push_back(MemberInfo(Layout.getBaseClassOffset(BaseDecl), MemberInfo::Base, getStorageType(BaseDecl), BaseDecl)); } @@ -461,7 +476,7 @@ void CGRecordLowering::accumulateVBases() { // smaller than the nvsize. Here we check to see if such a base is placed // before the nvsize and set the scissor offset to that, instead of the // nvsize. - if (!useMSABI()) + if (isOverlappingVBaseABI()) for (const auto &Base : RD->vbases()) { const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); if (BaseDecl->isEmpty()) @@ -482,7 +497,8 @@ void CGRecordLowering::accumulateVBases() { CharUnits Offset = Layout.getVBaseClassOffset(BaseDecl); // If the vbase is a primary virtual base of some base, then it doesn't // get its own storage location but instead lives inside of that base. - if (!useMSABI() && Context.isNearlyEmpty(BaseDecl) && + if (isOverlappingVBaseABI() && + Context.isNearlyEmpty(BaseDecl) && !hasOwnStorage(RD, BaseDecl)) { Members.push_back(MemberInfo(Offset, MemberInfo::VBase, nullptr, BaseDecl)); @@ -631,7 +647,7 @@ CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types, const FieldDecl *FD, uint64_t Offset, uint64_t Size, uint64_t StorageSize, - uint64_t StorageAlignment) { + CharUnits StorageOffset) { // This function is vestigial from CGRecordLayoutBuilder days but is still // used in GCObjCRuntime.cpp. That usage has a "fixme" attached to it that // when addressed will allow for the removal of this function. @@ -663,7 +679,7 @@ CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types, Offset = StorageSize - (Offset + Size); } - return CGBitFieldInfo(Offset, Size, IsSigned, StorageSize, StorageAlignment); + return CGBitFieldInfo(Offset, Size, IsSigned, StorageSize, StorageOffset); } CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D, @@ -836,7 +852,7 @@ void CGBitFieldInfo::print(raw_ostream &OS) const { << " Size:" << Size << " IsSigned:" << IsSigned << " StorageSize:" << StorageSize - << " StorageAlignment:" << StorageAlignment << ">"; + << " StorageOffset:" << StorageOffset.getQuantity() << ">"; } void CGBitFieldInfo::dump() const { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp index e3bdf86..7a0b8a3 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGStmt.cpp @@ -88,6 +88,7 @@ void CodeGenFunction::EmitStmt(const Stmt *S) { case Stmt::ContinueStmtClass: case Stmt::DefaultStmtClass: case Stmt::CaseStmtClass: + case Stmt::SEHLeaveStmtClass: llvm_unreachable("should have emitted these statements as simple"); #define STMT(Type, Base) @@ -173,9 +174,6 @@ void CodeGenFunction::EmitStmt(const Stmt *S) { case Stmt::SEHTryStmtClass: EmitSEHTryStmt(cast<SEHTryStmt>(*S)); break; - case Stmt::SEHLeaveStmtClass: - EmitSEHLeaveStmt(cast<SEHLeaveStmt>(*S)); - break; case Stmt::OMPParallelDirectiveClass: EmitOMPParallelDirective(cast<OMPParallelDirective>(*S)); break; @@ -224,6 +222,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S) { case Stmt::OMPTaskwaitDirectiveClass: EmitOMPTaskwaitDirective(cast<OMPTaskwaitDirective>(*S)); break; + case Stmt::OMPTaskgroupDirectiveClass: + EmitOMPTaskgroupDirective(cast<OMPTaskgroupDirective>(*S)); + break; case Stmt::OMPFlushDirectiveClass: EmitOMPFlushDirective(cast<OMPFlushDirective>(*S)); break; @@ -239,6 +240,12 @@ void CodeGenFunction::EmitStmt(const Stmt *S) { case Stmt::OMPTeamsDirectiveClass: EmitOMPTeamsDirective(cast<OMPTeamsDirective>(*S)); break; + case Stmt::OMPCancellationPointDirectiveClass: + EmitOMPCancellationPointDirective(cast<OMPCancellationPointDirective>(*S)); + break; + case Stmt::OMPCancelDirectiveClass: + EmitOMPCancelDirective(cast<OMPCancelDirective>(*S)); + break; } } @@ -256,6 +263,7 @@ bool CodeGenFunction::EmitSimpleStmt(const Stmt *S) { case Stmt::ContinueStmtClass: EmitContinueStmt(cast<ContinueStmt>(*S)); break; case Stmt::DefaultStmtClass: EmitDefaultStmt(cast<DefaultStmt>(*S)); break; case Stmt::CaseStmtClass: EmitCaseStmt(cast<CaseStmt>(*S)); break; + case Stmt::SEHLeaveStmtClass: EmitSEHLeaveStmt(cast<SEHLeaveStmt>(*S)); break; } return true; @@ -424,9 +432,8 @@ void CodeGenFunction::EmitLabel(const LabelDecl *D) { ResolveBranchFixups(Dest.getBlock()); } - RegionCounter Cnt = getPGORegionCounter(D->getStmt()); EmitBlock(Dest.getBlock()); - Cnt.beginRegion(Builder); + incrementProfileCounter(D->getStmt()); } /// Change the cleanup scope of the labels in this lexical scope to @@ -514,7 +521,6 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { // C99 6.8.4.1: The first substatement is executed if the expression compares // unequal to 0. The condition must be a scalar type. LexicalScope ConditionScope(*this, S.getCond()->getSourceRange()); - RegionCounter Cnt = getPGORegionCounter(&S); if (S.getConditionVariable()) EmitAutoVarDecl(*S.getConditionVariable()); @@ -533,7 +539,7 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { // This avoids emitting dead code and simplifies the CFG substantially. if (!ContainsLabel(Skipped)) { if (CondConstant) - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); if (Executed) { RunCleanupsScope ExecutedScope(*this); EmitStmt(Executed); @@ -550,11 +556,12 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { if (S.getElse()) ElseBlock = createBasicBlock("if.else"); - EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, Cnt.getCount()); + EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, + getProfileCount(S.getThen())); // Emit the 'then' code. EmitBlock(ThenBlock); - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); { RunCleanupsScope ThenScope(*this); EmitStmt(S.getThen()); @@ -564,8 +571,8 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { // Emit the 'else' code if present. if (const Stmt *Else = S.getElse()) { { - // There is no need to emit line number for unconditional branch. - ApplyDebugLocation DL(*this); + // There is no need to emit line number for an unconditional branch. + auto NL = ApplyDebugLocation::CreateEmpty(*this); EmitBlock(ElseBlock); } { @@ -573,8 +580,8 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { EmitStmt(Else); } { - // There is no need to emit line number for unconditional branch. - ApplyDebugLocation DL(*this); + // There is no need to emit line number for an unconditional branch. + auto NL = ApplyDebugLocation::CreateEmpty(*this); EmitBranch(ContBlock); } } @@ -679,14 +686,12 @@ void CodeGenFunction::EmitCondBrHints(llvm::LLVMContext &Context, void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> WhileAttrs) { - RegionCounter Cnt = getPGORegionCounter(&S); - // Emit the header for the loop, which will also become // the continue target. JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond"); EmitBlock(LoopHeader.getBlock()); - LoopStack.push(LoopHeader.getBlock()); + LoopStack.push(LoopHeader.getBlock(), WhileAttrs); // Create an exit block for when the condition fails, which will // also become the break target. @@ -725,9 +730,9 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); if (ConditionScope.requiresCleanups()) ExitBlock = createBasicBlock("while.exit"); - llvm::BranchInst *CondBr = - Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, - PGO.createLoopWeights(S.getCond(), Cnt)); + llvm::BranchInst *CondBr = Builder.CreateCondBr( + BoolCondVal, LoopBody, ExitBlock, + createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); @@ -743,7 +748,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, { RunCleanupsScope BodyScope(*this); EmitBlock(LoopBody); - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); EmitStmt(S.getBody()); } @@ -772,7 +777,7 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, JumpDest LoopExit = getJumpDestInCurrentScope("do.end"); JumpDest LoopCond = getJumpDestInCurrentScope("do.cond"); - RegionCounter Cnt = getPGORegionCounter(&S); + uint64_t ParentCount = getCurrentProfileCount(); // Store the blocks to use for break and continue. BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond)); @@ -780,9 +785,9 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, // Emit the body of the loop. llvm::BasicBlock *LoopBody = createBasicBlock("do.body"); - LoopStack.push(LoopBody); + LoopStack.push(LoopBody, DoAttrs); - EmitBlockWithFallThrough(LoopBody, Cnt); + EmitBlockWithFallThrough(LoopBody, &S); { RunCleanupsScope BodyScope(*this); EmitStmt(S.getBody()); @@ -809,9 +814,10 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, // As long as the condition is true, iterate the loop. if (EmitBoolCondBranch) { - llvm::BranchInst *CondBr = - Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.getBlock(), - PGO.createLoopWeights(S.getCond(), Cnt)); + uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount; + llvm::BranchInst *CondBr = Builder.CreateCondBr( + BoolCondVal, LoopBody, LoopExit.getBlock(), + createProfileWeightsForLoop(S.getCond(), BackedgeCount)); // Attach metadata to loop body conditional branch. EmitCondBrHints(LoopBody->getContext(), CondBr, DoAttrs); @@ -838,8 +844,6 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, if (S.getInit()) EmitStmt(S.getInit()); - RegionCounter Cnt = getPGORegionCounter(&S); - // Start the loop with a block that tests the condition. // If there's an increment, the continue scope will be overwritten // later. @@ -847,7 +851,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, llvm::BasicBlock *CondBlock = Continue.getBlock(); EmitBlock(CondBlock); - LoopStack.push(CondBlock); + LoopStack.push(CondBlock, ForAttrs); // If the for loop doesn't have an increment we can just use the // condition as the continue block. Otherwise we'll need to create @@ -881,9 +885,9 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, // C99 6.8.5p2/p4: The first substatement is executed if the expression // compares unequal to 0. The condition must be a scalar type. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - llvm::BranchInst *CondBr = - Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, - PGO.createLoopWeights(S.getCond(), Cnt)); + llvm::BranchInst *CondBr = Builder.CreateCondBr( + BoolCondVal, ForBody, ExitBlock, + createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); // Attach metadata to loop body conditional branch. EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs); @@ -898,7 +902,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, // Treat it as a non-zero constant. Don't even create a new block for the // body, just fall into it. } - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); { // Create a separate cleanup scope for the body, in case it is not @@ -939,15 +943,13 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, EmitStmt(S.getRangeStmt()); EmitStmt(S.getBeginEndStmt()); - RegionCounter Cnt = getPGORegionCounter(&S); - // Start the loop with a block that tests the condition. // If there's an increment, the continue scope will be overwritten // later. llvm::BasicBlock *CondBlock = createBasicBlock("for.cond"); EmitBlock(CondBlock); - LoopStack.push(CondBlock); + LoopStack.push(CondBlock, ForAttrs); // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. @@ -962,7 +964,8 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, // to bool, is true. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); llvm::BranchInst *CondBr = Builder.CreateCondBr( - BoolCondVal, ForBody, ExitBlock, PGO.createLoopWeights(S.getCond(), Cnt)); + BoolCondVal, ForBody, ExitBlock, + createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); // Attach metadata to loop body conditional branch. EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs); @@ -973,7 +976,7 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, } EmitBlock(ForBody); - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); // Create a block for the increment. In case of a 'continue', we jump there. JumpDest Continue = getJumpDestInCurrentScope("for.inc"); @@ -1022,6 +1025,12 @@ void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) { /// if the function returns void, or may be missing one if the function returns /// non-void. Fun stuff :). void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) { + // Returning from an outlined SEH helper is UB, and we already warn on it. + if (IsOutlinedSEHHelper) { + Builder.CreateUnreachable(); + Builder.ClearInsertionPoint(); + } + // Emit the result value, even if unused, to evalute the side effects. const Expr *RV = S.getRetValue(); @@ -1133,13 +1142,11 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) { llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(getContext()); llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(getContext()); - RegionCounter CaseCnt = getPGORegionCounter(&S); - // Emit the code for this case. We do this first to make sure it is // properly chained from our predecessor before generating the // switch machinery to enter this block. llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb"); - EmitBlockWithFallThrough(CaseDest, CaseCnt); + EmitBlockWithFallThrough(CaseDest, &S); EmitStmt(S.getSubStmt()); // If range is empty, do nothing. @@ -1150,7 +1157,7 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) { // FIXME: parameters such as this should not be hardcoded. if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) { // Range is small enough to add multiple switch instruction cases. - uint64_t Total = CaseCnt.getCount(); + uint64_t Total = getProfileCount(&S); unsigned NCases = Range.getZExtValue() + 1; // We only have one region counter for the entire set of cases here, so we // need to divide the weights evenly between the generated cases, ensuring @@ -1189,9 +1196,9 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) { llvm::MDNode *Weights = nullptr; if (SwitchWeights) { - uint64_t ThisCount = CaseCnt.getCount(); + uint64_t ThisCount = getProfileCount(&S); uint64_t DefaultCount = (*SwitchWeights)[0]; - Weights = PGO.createBranchWeights(ThisCount, DefaultCount); + Weights = createProfileWeights(ThisCount, DefaultCount); // Since we're chaining the switch default through each large case range, we // need to update the weight for the default, ie, the first case, to include @@ -1224,7 +1231,6 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) { return; } - RegionCounter CaseCnt = getPGORegionCounter(&S); llvm::ConstantInt *CaseVal = Builder.getInt(S.getLHS()->EvaluateKnownConstInt(getContext())); @@ -1239,7 +1245,7 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) { // Only do this optimization if there are no cleanups that need emitting. if (isObviouslyBranchWithoutCleanups(Block)) { if (SwitchWeights) - SwitchWeights->push_back(CaseCnt.getCount()); + SwitchWeights->push_back(getProfileCount(&S)); SwitchInsn->addCase(CaseVal, Block.getBlock()); // If there was a fallthrough into this case, make sure to redirect it to @@ -1253,9 +1259,9 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) { } llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb"); - EmitBlockWithFallThrough(CaseDest, CaseCnt); + EmitBlockWithFallThrough(CaseDest, &S); if (SwitchWeights) - SwitchWeights->push_back(CaseCnt.getCount()); + SwitchWeights->push_back(getProfileCount(&S)); SwitchInsn->addCase(CaseVal, CaseDest); // Recursively emitting the statement is acceptable, but is not wonderful for @@ -1276,12 +1282,11 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) { llvm::ConstantInt *CaseVal = Builder.getInt(CurCase->getLHS()->EvaluateKnownConstInt(getContext())); - CaseCnt = getPGORegionCounter(NextCase); if (SwitchWeights) - SwitchWeights->push_back(CaseCnt.getCount()); + SwitchWeights->push_back(getProfileCount(NextCase)); if (CGM.getCodeGenOpts().ProfileInstrGenerate) { CaseDest = createBasicBlock("sw.bb"); - EmitBlockWithFallThrough(CaseDest, CaseCnt); + EmitBlockWithFallThrough(CaseDest, &S); } SwitchInsn->addCase(CaseVal, CaseDest); @@ -1297,8 +1302,7 @@ void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S) { assert(DefaultBlock->empty() && "EmitDefaultStmt: Default block already defined?"); - RegionCounter Cnt = getPGORegionCounter(&S); - EmitBlockWithFallThrough(DefaultBlock, Cnt); + EmitBlockWithFallThrough(DefaultBlock, &S); EmitStmt(S.getSubStmt()); } @@ -1520,10 +1524,8 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { const SwitchCase *Case = nullptr; if (FindCaseStatementsForValue(S, ConstantCondValue, CaseStmts, getContext(), Case)) { - if (Case) { - RegionCounter CaseCnt = getPGORegionCounter(Case); - CaseCnt.beginRegion(Builder); - } + if (Case) + incrementProfileCounter(Case); RunCleanupsScope ExecutedScope(*this); // Emit the condition variable if needed inside the entire cleanup scope @@ -1540,8 +1542,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { // specified series of statements and we're good. for (unsigned i = 0, e = CaseStmts.size(); i != e; ++i) EmitStmt(CaseStmts[i]); - RegionCounter ExitCnt = getPGORegionCounter(&S); - ExitCnt.beginRegion(Builder); + incrementProfileCounter(&S); // Now we want to restore the saved switch instance so that nested // switches continue to function properly @@ -1572,7 +1573,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { Case; Case = Case->getNextSwitchCase()) { if (isa<DefaultStmt>(Case)) - DefaultCount = getPGORegionCounter(Case).getCount(); + DefaultCount = getProfileCount(Case); NumCases += 1; } SwitchWeights = new SmallVector<uint64_t, 16>(); @@ -1621,8 +1622,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { // Emit continuation. EmitBlock(SwitchExit.getBlock(), true); - RegionCounter ExitCnt = getPGORegionCounter(&S); - ExitCnt.beginRegion(Builder); + incrementProfileCounter(&S); if (SwitchWeights) { assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() && @@ -1630,7 +1630,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { // If there's only one jump destination there's no sense weighting it. if (SwitchWeights->size() > 1) SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof, - PGO.createBranchWeights(*SwitchWeights)); + createProfileWeights(*SwitchWeights)); delete SwitchWeights; } SwitchInsn = SavedSwitchInsn; @@ -1696,7 +1696,7 @@ SimplifyConstraint(const char *Constraint, const TargetInfo &Target, static std::string AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr, const TargetInfo &Target, CodeGenModule &CGM, - const AsmStmt &Stmt) { + const AsmStmt &Stmt, const bool EarlyClobber) { const DeclRefExpr *AsmDeclRef = dyn_cast<DeclRefExpr>(&AsmExpr); if (!AsmDeclRef) return Constraint; @@ -1721,7 +1721,7 @@ AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr, } // Canonicalize the register here before returning it. Register = Target.getNormalizedGCCRegisterName(Register); - return "{" + Register.str() + "}"; + return (EarlyClobber ? "&{" : "{") + Register.str() + "}"; } llvm::Value* @@ -1759,6 +1759,16 @@ llvm::Value* CodeGenFunction::EmitAsmInput( const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr, std::string &ConstraintStr) { + // If this can't be a register or memory, i.e., has to be a constant + // (immediate or symbolic), try to emit it as such. + if (!Info.allowsRegister() && !Info.allowsMemory()) { + llvm::APSInt Result; + if (InputExpr->EvaluateAsInt(Result, getContext())) + return llvm::ConstantInt::get(getLLVMContext(), Result); + assert(!Info.requiresImmediateConstant() && + "Required-immediate inlineasm arg isn't constant?"); + } + if (Info.allowsRegister() || !Info.allowsMemory()) if (CodeGenFunction::hasScalarEvaluationKind(InputExpr->getType())) return EmitScalarExpr(InputExpr); @@ -1842,6 +1852,14 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { std::vector<llvm::Value*> InOutArgs; std::vector<llvm::Type*> InOutArgTypes; + // An inline asm can be marked readonly if it meets the following conditions: + // - it doesn't have any sideeffects + // - it doesn't clobber memory + // - it doesn't return a value by-reference + // It can be marked readnone if it doesn't have any input memory constraints + // in addition to meeting the conditions listed above. + bool ReadOnly = true, ReadNone = true; + for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) { TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[i]; @@ -1854,7 +1872,8 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { OutExpr = OutExpr->IgnoreParenNoopCasts(getContext()); OutputConstraint = AddVariableConstraints(OutputConstraint, *OutExpr, - getTarget(), CGM, S); + getTarget(), CGM, S, + Info.earlyClobber()); LValue Dest = EmitLValue(OutExpr); if (!Constraints.empty()) @@ -1904,6 +1923,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { Args.push_back(Dest.getAddress()); Constraints += "=*"; Constraints += OutputConstraint; + ReadOnly = ReadNone = false; } if (Info.isReadWrite()) { @@ -1948,6 +1968,9 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i]; + if (Info.allowsMemory()) + ReadNone = false; + if (!Constraints.empty()) Constraints += ','; @@ -1956,10 +1979,9 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { InputConstraint = SimplifyConstraint(InputConstraint.c_str(), getTarget(), &OutputConstraintInfos); - InputConstraint = - AddVariableConstraints(InputConstraint, - *InputExpr->IgnoreParenNoopCasts(getContext()), - getTarget(), CGM, S); + InputConstraint = AddVariableConstraints( + InputConstraint, *InputExpr->IgnoreParenNoopCasts(getContext()), + getTarget(), CGM, S, false /* No EarlyClobber */); llvm::Value *Arg = EmitAsmInput(Info, InputExpr, Constraints); @@ -2013,7 +2035,9 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { for (unsigned i = 0, e = S.getNumClobbers(); i != e; i++) { StringRef Clobber = S.getClobber(i); - if (Clobber != "memory" && Clobber != "cc") + if (Clobber == "memory") + ReadOnly = ReadNone = false; + else if (Clobber != "cc") Clobber = getTarget().getNormalizedGCCRegisterName(Clobber); if (!Constraints.empty()) @@ -2053,6 +2077,16 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { Result->addAttribute(llvm::AttributeSet::FunctionIndex, llvm::Attribute::NoUnwind); + // Attach readnone and readonly attributes. + if (!HasSideEffect) { + if (ReadNone) + Result->addAttribute(llvm::AttributeSet::FunctionIndex, + llvm::Attribute::ReadNone); + else if (ReadOnly) + Result->addAttribute(llvm::AttributeSet::FunctionIndex, + llvm::Attribute::ReadOnly); + } + // Slap the source location of the inline asm into a !srcloc metadata on the // call. if (const GCCAsmStmt *gccAsmStmt = dyn_cast<GCCAsmStmt>(&S)) { @@ -2145,7 +2179,7 @@ CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) { // Emit the CapturedDecl CodeGenFunction CGF(CGM, true); - CGF.CapturedStmtInfo = new CGCapturedStmtInfo(S, K); + CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K)); llvm::Function *F = CGF.GenerateCapturedStmtFunction(S); delete CGF.CapturedStmtInfo; @@ -2187,6 +2221,8 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) { llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage, CapturedStmtInfo->getHelperName(), &CGM.getModule()); CGM.SetInternalFunctionAttributes(CD, F, FuncInfo); + if (CD->isNothrow()) + F->addFnAttr(llvm::Attribute::NoUnwind); // Generate the function. StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp index 78fd37c..e5f507a 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -20,288 +20,539 @@ using namespace clang; using namespace CodeGen; -namespace { -/// \brief RAII for emitting code of CapturedStmt without function outlining. -class InlinedOpenMPRegion { - CodeGenFunction &CGF; - CodeGenFunction::CGCapturedStmtInfo *PrevCapturedStmtInfo; - const Decl *StoredCurCodeDecl; - - /// \brief A class to emit CapturedStmt construct as inlined statement without - /// generating a function for outlined code. - class CGInlinedOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo { - public: - CGInlinedOpenMPRegionInfo() : CGCapturedStmtInfo() {} - }; - -public: - InlinedOpenMPRegion(CodeGenFunction &CGF, const Stmt *S) - : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo), - StoredCurCodeDecl(CGF.CurCodeDecl) { - CGF.CurCodeDecl = cast<CapturedStmt>(S)->getCapturedDecl(); - CGF.CapturedStmtInfo = new CGInlinedOpenMPRegionInfo(); - } - ~InlinedOpenMPRegion() { - delete CGF.CapturedStmtInfo; - CGF.CapturedStmtInfo = PrevCapturedStmtInfo; - CGF.CurCodeDecl = StoredCurCodeDecl; - } -}; -} // namespace - //===----------------------------------------------------------------------===// // OpenMP Directive Emission //===----------------------------------------------------------------------===// +void CodeGenFunction::EmitOMPAggregateAssign( + llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType, + const llvm::function_ref<void(llvm::Value *, llvm::Value *)> &CopyGen) { + // Perform element-by-element initialization. + QualType ElementTy; + auto SrcBegin = SrcAddr; + auto DestBegin = DestAddr; + auto ArrayTy = OriginalType->getAsArrayTypeUnsafe(); + auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin); + // Cast from pointer to array type to pointer to single element. + SrcBegin = Builder.CreatePointerBitCastOrAddrSpaceCast(SrcBegin, + DestBegin->getType()); + auto DestEnd = Builder.CreateGEP(DestBegin, NumElements); + // The basic structure here is a while-do loop. + auto BodyBB = createBasicBlock("omp.arraycpy.body"); + auto DoneBB = createBasicBlock("omp.arraycpy.done"); + auto IsEmpty = + Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty"); + Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); + + // Enter the loop body, making that address the current address. + auto EntryBB = Builder.GetInsertBlock(); + EmitBlock(BodyBB); + auto SrcElementCurrent = + Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); + SrcElementCurrent->addIncoming(SrcBegin, EntryBB); + auto DestElementCurrent = Builder.CreatePHI(DestBegin->getType(), 2, + "omp.arraycpy.destElementPast"); + DestElementCurrent->addIncoming(DestBegin, EntryBB); + + // Emit copy. + CopyGen(DestElementCurrent, SrcElementCurrent); + + // Shift the address forward by one element. + auto DestElementNext = Builder.CreateConstGEP1_32( + DestElementCurrent, /*Idx0=*/1, "omp.arraycpy.dest.element"); + auto SrcElementNext = Builder.CreateConstGEP1_32( + SrcElementCurrent, /*Idx0=*/1, "omp.arraycpy.src.element"); + // Check whether we've reached the end. + auto Done = + Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); + Builder.CreateCondBr(Done, DoneBB, BodyBB); + DestElementCurrent->addIncoming(DestElementNext, Builder.GetInsertBlock()); + SrcElementCurrent->addIncoming(SrcElementNext, Builder.GetInsertBlock()); + + // Done. + EmitBlock(DoneBB, /*IsFinished=*/true); +} -/// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen -/// function. Here is the logic: -/// if (Cond) { -/// CodeGen(true); -/// } else { -/// CodeGen(false); -/// } -static void EmitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, - const std::function<void(bool)> &CodeGen) { - CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); - - // If the condition constant folds and can be elided, try to avoid emitting - // the condition and the dead arm of the if/else. - bool CondConstant; - if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { - CodeGen(CondConstant); - return; - } - - // Otherwise, the condition did not fold, or we couldn't elide it. Just - // emit the conditional branch. - auto ThenBlock = CGF.createBasicBlock(/*name*/ "omp_if.then"); - auto ElseBlock = CGF.createBasicBlock(/*name*/ "omp_if.else"); - auto ContBlock = CGF.createBasicBlock(/*name*/ "omp_if.end"); - CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount*/ 0); - - // Emit the 'then' code. - CGF.EmitBlock(ThenBlock); - CodeGen(/*ThenBlock*/ true); - CGF.EmitBranch(ContBlock); - // Emit the 'else' code if present. - { - // There is no need to emit line number for unconditional branch. - ApplyDebugLocation DL(CGF); - CGF.EmitBlock(ElseBlock); - } - CodeGen(/*ThenBlock*/ false); - { - // There is no need to emit line number for unconditional branch. - ApplyDebugLocation DL(CGF); - CGF.EmitBranch(ContBlock); - } - // Emit the continuation block for code after the if. - CGF.EmitBlock(ContBlock, /*IsFinished*/ true); -} - -void CodeGenFunction::EmitOMPAggregateAssign(LValue OriginalAddr, - llvm::Value *PrivateAddr, - const Expr *AssignExpr, - QualType OriginalType, - const VarDecl *VDInit) { - EmitBlock(createBasicBlock(".omp.assign.begin.")); - if (!isa<CXXConstructExpr>(AssignExpr) || isTrivialInitializer(AssignExpr)) { - // Perform simple memcpy. - EmitAggregateAssign(PrivateAddr, OriginalAddr.getAddress(), - AssignExpr->getType()); - } else { - // Perform element-by-element initialization. - QualType ElementTy; - auto SrcBegin = OriginalAddr.getAddress(); - auto DestBegin = PrivateAddr; - auto ArrayTy = OriginalType->getAsArrayTypeUnsafe(); - auto SrcNumElements = emitArrayLength(ArrayTy, ElementTy, SrcBegin); - auto DestNumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin); - auto SrcEnd = Builder.CreateGEP(SrcBegin, SrcNumElements); - auto DestEnd = Builder.CreateGEP(DestBegin, DestNumElements); - // The basic structure here is a do-while loop, because we don't - // need to check for the zero-element case. - auto BodyBB = createBasicBlock("omp.arraycpy.body"); - auto DoneBB = createBasicBlock("omp.arraycpy.done"); - auto IsEmpty = - Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty"); - Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); - - // Enter the loop body, making that address the current address. - auto EntryBB = Builder.GetInsertBlock(); - EmitBlock(BodyBB); - auto SrcElementPast = Builder.CreatePHI(SrcBegin->getType(), 2, - "omp.arraycpy.srcElementPast"); - SrcElementPast->addIncoming(SrcEnd, EntryBB); - auto DestElementPast = Builder.CreatePHI(DestBegin->getType(), 2, - "omp.arraycpy.destElementPast"); - DestElementPast->addIncoming(DestEnd, EntryBB); - - // Shift the address back by one element. - auto NegativeOne = llvm::ConstantInt::get(SizeTy, -1, true); - auto DestElement = Builder.CreateGEP(DestElementPast, NegativeOne, - "omp.arraycpy.dest.element"); - auto SrcElement = Builder.CreateGEP(SrcElementPast, NegativeOne, - "omp.arraycpy.src.element"); - { - // Create RunCleanScope to cleanup possible temps. - CodeGenFunction::RunCleanupsScope Init(*this); - // Emit initialization for single element. - LocalDeclMap[VDInit] = SrcElement; - EmitAnyExprToMem(AssignExpr, DestElement, - AssignExpr->getType().getQualifiers(), - /*IsInitializer*/ false); - LocalDeclMap.erase(VDInit); +void CodeGenFunction::EmitOMPCopy(CodeGenFunction &CGF, + QualType OriginalType, llvm::Value *DestAddr, + llvm::Value *SrcAddr, const VarDecl *DestVD, + const VarDecl *SrcVD, const Expr *Copy) { + if (OriginalType->isArrayType()) { + auto *BO = dyn_cast<BinaryOperator>(Copy); + if (BO && BO->getOpcode() == BO_Assign) { + // Perform simple memcpy for simple copying. + CGF.EmitAggregateAssign(DestAddr, SrcAddr, OriginalType); + } else { + // For arrays with complex element types perform element by element + // copying. + CGF.EmitOMPAggregateAssign( + DestAddr, SrcAddr, OriginalType, + [&CGF, Copy, SrcVD, DestVD](llvm::Value *DestElement, + llvm::Value *SrcElement) { + // Working with the single array element, so have to remap + // destination and source variables to corresponding array + // elements. + CodeGenFunction::OMPPrivateScope Remap(CGF); + Remap.addPrivate(DestVD, [DestElement]() -> llvm::Value *{ + return DestElement; + }); + Remap.addPrivate( + SrcVD, [SrcElement]() -> llvm::Value *{ return SrcElement; }); + (void)Remap.Privatize(); + CGF.EmitIgnoredExpr(Copy); + }); } - - // Check whether we've reached the end. - auto Done = - Builder.CreateICmpEQ(DestElement, DestBegin, "omp.arraycpy.done"); - Builder.CreateCondBr(Done, DoneBB, BodyBB); - DestElementPast->addIncoming(DestElement, Builder.GetInsertBlock()); - SrcElementPast->addIncoming(SrcElement, Builder.GetInsertBlock()); - - // Done. - EmitBlock(DoneBB, true); + } else { + // Remap pseudo source variable to private copy. + CodeGenFunction::OMPPrivateScope Remap(CGF); + Remap.addPrivate(SrcVD, [SrcAddr]() -> llvm::Value *{ return SrcAddr; }); + Remap.addPrivate(DestVD, [DestAddr]() -> llvm::Value *{ return DestAddr; }); + (void)Remap.Privatize(); + // Emit copying of the whole variable. + CGF.EmitIgnoredExpr(Copy); } - EmitBlock(createBasicBlock(".omp.assign.end.")); } -void CodeGenFunction::EmitOMPFirstprivateClause( - const OMPExecutableDirective &D, - CodeGenFunction::OMPPrivateScope &PrivateScope) { - auto PrivateFilter = [](const OMPClause *C) -> bool { - return C->getClauseKind() == OMPC_firstprivate; - }; - for (OMPExecutableDirective::filtered_clause_iterator<decltype(PrivateFilter)> - I(D.clauses(), PrivateFilter); I; ++I) { +bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, + OMPPrivateScope &PrivateScope) { + llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate; + for (auto &&I = D.getClausesOfKind(OMPC_firstprivate); I; ++I) { auto *C = cast<OMPFirstprivateClause>(*I); auto IRef = C->varlist_begin(); auto InitsRef = C->inits().begin(); for (auto IInit : C->private_copies()) { auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); - auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); - bool IsRegistered; - if (*InitsRef != nullptr) { - // Emit VarDecl with copy init for arrays. - auto *FD = CapturedStmtInfo->lookup(OrigVD); - LValue Base = MakeNaturalAlignAddrLValue( - CapturedStmtInfo->getContextValue(), - getContext().getTagDeclType(FD->getParent())); - auto OriginalAddr = EmitLValueForField(Base, FD); - auto VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl()); - IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { - auto Emission = EmitAutoVarAlloca(*VD); - // Emit initialization of aggregate firstprivate vars. - EmitOMPAggregateAssign(OriginalAddr, Emission.getAllocatedAddress(), - VD->getInit(), (*IRef)->getType(), VDInit); - EmitAutoVarCleanups(Emission); - return Emission.getAllocatedAddress(); - }); - } else - IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { - // Emit private VarDecl with copy init. - EmitDecl(*VD); - return GetAddrOfLocalVar(VD); - }); - assert(IsRegistered && "counter already registered as private"); - // Silence the warning about unused variable. - (void)IsRegistered; + if (EmittedAsFirstprivate.count(OrigVD) == 0) { + EmittedAsFirstprivate.insert(OrigVD); + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); + auto *VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl()); + bool IsRegistered; + DeclRefExpr DRE( + const_cast<VarDecl *>(OrigVD), + /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup( + OrigVD) != nullptr, + (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); + auto *OriginalAddr = EmitLValue(&DRE).getAddress(); + QualType Type = OrigVD->getType(); + if (Type->isArrayType()) { + // Emit VarDecl with copy init for arrays. + // Get the address of the original variable captured in current + // captured region. + IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + auto Emission = EmitAutoVarAlloca(*VD); + auto *Init = VD->getInit(); + if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) { + // Perform simple memcpy. + EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr, + Type); + } else { + EmitOMPAggregateAssign( + Emission.getAllocatedAddress(), OriginalAddr, Type, + [this, VDInit, Init](llvm::Value *DestElement, + llvm::Value *SrcElement) { + // Clean up any temporaries needed by the initialization. + RunCleanupsScope InitScope(*this); + // Emit initialization for single element. + LocalDeclMap[VDInit] = SrcElement; + EmitAnyExprToMem(Init, DestElement, + Init->getType().getQualifiers(), + /*IsInitializer*/ false); + LocalDeclMap.erase(VDInit); + }); + } + EmitAutoVarCleanups(Emission); + return Emission.getAllocatedAddress(); + }); + } else { + IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + // Emit private VarDecl with copy init. + // Remap temp VDInit variable to the address of the original + // variable + // (for proper handling of captured global variables). + LocalDeclMap[VDInit] = OriginalAddr; + EmitDecl(*VD); + LocalDeclMap.erase(VDInit); + return GetAddrOfLocalVar(VD); + }); + } + assert(IsRegistered && + "firstprivate var already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + } ++IRef, ++InitsRef; } } + return !EmittedAsFirstprivate.empty(); } void CodeGenFunction::EmitOMPPrivateClause( const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { - auto PrivateFilter = [](const OMPClause *C) -> bool { - return C->getClauseKind() == OMPC_private; - }; - for (OMPExecutableDirective::filtered_clause_iterator<decltype(PrivateFilter)> - I(D.clauses(), PrivateFilter); I; ++I) { + llvm::DenseSet<const VarDecl *> EmittedAsPrivate; + for (auto &&I = D.getClausesOfKind(OMPC_private); I; ++I) { auto *C = cast<OMPPrivateClause>(*I); auto IRef = C->varlist_begin(); for (auto IInit : C->private_copies()) { auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); - auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); + if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { + auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); + bool IsRegistered = + PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + // Emit private VarDecl with copy init. + EmitDecl(*VD); + return GetAddrOfLocalVar(VD); + }); + assert(IsRegistered && "private var already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + } + ++IRef; + } + } +} + +bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { + // threadprivate_var1 = master_threadprivate_var1; + // operator=(threadprivate_var2, master_threadprivate_var2); + // ... + // __kmpc_barrier(&loc, global_tid); + llvm::DenseSet<const VarDecl *> CopiedVars; + llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr; + for (auto &&I = D.getClausesOfKind(OMPC_copyin); I; ++I) { + auto *C = cast<OMPCopyinClause>(*I); + auto IRef = C->varlist_begin(); + auto ISrcRef = C->source_exprs().begin(); + auto IDestRef = C->destination_exprs().begin(); + for (auto *AssignOp : C->assignment_ops()) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); + QualType Type = VD->getType(); + if (CopiedVars.insert(VD->getCanonicalDecl()).second) { + + // Get the address of the master variable. If we are emitting code with + // TLS support, the address is passed from the master as field in the + // captured declaration. + llvm::Value *MasterAddr; + if (getLangOpts().OpenMPUseTLS && + getContext().getTargetInfo().isTLSSupported()) { + assert(CapturedStmtInfo->lookup(VD) && + "Copyin threadprivates should have been captured!"); + DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(), + VK_LValue, (*IRef)->getExprLoc()); + MasterAddr = EmitLValue(&DRE).getAddress(); + } else { + MasterAddr = VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD) + : CGM.GetAddrOfGlobal(VD); + } + // Get the address of the threadprivate variable. + auto *PrivateAddr = EmitLValue(*IRef).getAddress(); + if (CopiedVars.size() == 1) { + // At first check if current thread is a master thread. If it is, no + // need to copy data. + CopyBegin = createBasicBlock("copyin.not.master"); + CopyEnd = createBasicBlock("copyin.not.master.end"); + Builder.CreateCondBr( + Builder.CreateICmpNE( + Builder.CreatePtrToInt(MasterAddr, CGM.IntPtrTy), + Builder.CreatePtrToInt(PrivateAddr, CGM.IntPtrTy)), + CopyBegin, CopyEnd); + EmitBlock(CopyBegin); + } + auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); + auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); + EmitOMPCopy(*this, Type, PrivateAddr, MasterAddr, DestVD, SrcVD, + AssignOp); + } + ++IRef; + ++ISrcRef; + ++IDestRef; + } + } + if (CopyEnd) { + // Exit out of copying procedure for non-master thread. + EmitBlock(CopyEnd, /*IsFinished=*/true); + return true; + } + return false; +} + +bool CodeGenFunction::EmitOMPLastprivateClauseInit( + const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) { + bool HasAtLeastOneLastprivate = false; + llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; + for (auto &&I = D.getClausesOfKind(OMPC_lastprivate); I; ++I) { + HasAtLeastOneLastprivate = true; + auto *C = cast<OMPLastprivateClause>(*I); + auto IRef = C->varlist_begin(); + auto IDestRef = C->destination_exprs().begin(); + for (auto *IInit : C->private_copies()) { + // Keep the address of the original variable for future update at the end + // of the loop. + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); + if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) { + auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); + PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> llvm::Value *{ + DeclRefExpr DRE( + const_cast<VarDecl *>(OrigVD), + /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup( + OrigVD) != nullptr, + (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); + return EmitLValue(&DRE).getAddress(); + }); + // Check if the variable is also a firstprivate: in this case IInit is + // not generated. Initialization of this variable will happen in codegen + // for 'firstprivate' clause. + if (IInit) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); + bool IsRegistered = + PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + // Emit private VarDecl with copy init. + EmitDecl(*VD); + return GetAddrOfLocalVar(VD); + }); + assert(IsRegistered && + "lastprivate var already registered as private"); + (void)IsRegistered; + } + } + ++IRef, ++IDestRef; + } + } + return HasAtLeastOneLastprivate; +} + +void CodeGenFunction::EmitOMPLastprivateClauseFinal( + const OMPExecutableDirective &D, llvm::Value *IsLastIterCond) { + // Emit following code: + // if (<IsLastIterCond>) { + // orig_var1 = private_orig_var1; + // ... + // orig_varn = private_orig_varn; + // } + llvm::BasicBlock *ThenBB = nullptr; + llvm::BasicBlock *DoneBB = nullptr; + if (IsLastIterCond) { + ThenBB = createBasicBlock(".omp.lastprivate.then"); + DoneBB = createBasicBlock(".omp.lastprivate.done"); + Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB); + EmitBlock(ThenBB); + } + llvm::DenseMap<const Decl *, const Expr *> LoopCountersAndUpdates; + const Expr *LastIterVal = nullptr; + const Expr *IVExpr = nullptr; + const Expr *IncExpr = nullptr; + if (auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) { + if (isOpenMPWorksharingDirective(D.getDirectiveKind())) { + LastIterVal = cast<VarDecl>(cast<DeclRefExpr>( + LoopDirective->getUpperBoundVariable()) + ->getDecl()) + ->getAnyInitializer(); + IVExpr = LoopDirective->getIterationVariable(); + IncExpr = LoopDirective->getInc(); + auto IUpdate = LoopDirective->updates().begin(); + for (auto *E : LoopDirective->counters()) { + auto *D = cast<DeclRefExpr>(E)->getDecl()->getCanonicalDecl(); + LoopCountersAndUpdates[D] = *IUpdate; + ++IUpdate; + } + } + } + { + llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; + bool FirstLCV = true; + for (auto &&I = D.getClausesOfKind(OMPC_lastprivate); I; ++I) { + auto *C = cast<OMPLastprivateClause>(*I); + auto IRef = C->varlist_begin(); + auto ISrcRef = C->source_exprs().begin(); + auto IDestRef = C->destination_exprs().begin(); + for (auto *AssignOp : C->assignment_ops()) { + auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); + QualType Type = PrivateVD->getType(); + auto *CanonicalVD = PrivateVD->getCanonicalDecl(); + if (AlreadyEmittedVars.insert(CanonicalVD).second) { + // If lastprivate variable is a loop control variable for loop-based + // directive, update its value before copyin back to original + // variable. + if (auto *UpExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) { + if (FirstLCV && LastIterVal) { + EmitAnyExprToMem(LastIterVal, EmitLValue(IVExpr).getAddress(), + IVExpr->getType().getQualifiers(), + /*IsInitializer=*/false); + EmitIgnoredExpr(IncExpr); + FirstLCV = false; + } + EmitIgnoredExpr(UpExpr); + } + auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); + auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); + // Get the address of the original variable. + auto *OriginalAddr = GetAddrOfLocalVar(DestVD); + // Get the address of the private variable. + auto *PrivateAddr = GetAddrOfLocalVar(PrivateVD); + EmitOMPCopy(*this, Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, + AssignOp); + } + ++IRef; + ++ISrcRef; + ++IDestRef; + } + } + } + if (IsLastIterCond) { + EmitBlock(DoneBB, /*IsFinished=*/true); + } +} + +void CodeGenFunction::EmitOMPReductionClauseInit( + const OMPExecutableDirective &D, + CodeGenFunction::OMPPrivateScope &PrivateScope) { + for (auto &&I = D.getClausesOfKind(OMPC_reduction); I; ++I) { + auto *C = cast<OMPReductionClause>(*I); + auto ILHS = C->lhs_exprs().begin(); + auto IRHS = C->rhs_exprs().begin(); + for (auto IRef : C->varlists()) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl()); + auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); + auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + // Store the address of the original variable associated with the LHS + // implicit variable. + PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef]() -> llvm::Value *{ + DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), + CapturedStmtInfo->lookup(OrigVD) != nullptr, + IRef->getType(), VK_LValue, IRef->getExprLoc()); + return EmitLValue(&DRE).getAddress(); + }); + // Emit reduction copy. bool IsRegistered = - PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { - // Emit private VarDecl with copy init. - EmitDecl(*VD); - return GetAddrOfLocalVar(VD); + PrivateScope.addPrivate(OrigVD, [this, PrivateVD]() -> llvm::Value *{ + // Emit private VarDecl with reduction init. + EmitDecl(*PrivateVD); + return GetAddrOfLocalVar(PrivateVD); }); - assert(IsRegistered && "counter already registered as private"); + assert(IsRegistered && "private var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; - ++IRef; + ++ILHS, ++IRHS; } } } -/// \brief Emits code for OpenMP parallel directive in the parallel region. -static void EmitOMPParallelCall(CodeGenFunction &CGF, - const OMPParallelDirective &S, - llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct) { - if (auto C = S.getSingleClause(/*K*/ OMPC_num_threads)) { +void CodeGenFunction::EmitOMPReductionClauseFinal( + const OMPExecutableDirective &D) { + llvm::SmallVector<const Expr *, 8> LHSExprs; + llvm::SmallVector<const Expr *, 8> RHSExprs; + llvm::SmallVector<const Expr *, 8> ReductionOps; + bool HasAtLeastOneReduction = false; + for (auto &&I = D.getClausesOfKind(OMPC_reduction); I; ++I) { + HasAtLeastOneReduction = true; + auto *C = cast<OMPReductionClause>(*I); + LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end()); + RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end()); + ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end()); + } + if (HasAtLeastOneReduction) { + // Emit nowait reduction if nowait clause is present or directive is a + // parallel directive (it always has implicit barrier). + CGM.getOpenMPRuntime().emitReduction( + *this, D.getLocEnd(), LHSExprs, RHSExprs, ReductionOps, + D.getSingleClause(OMPC_nowait) || + isOpenMPParallelDirective(D.getDirectiveKind()) || + D.getDirectiveKind() == OMPD_simd, + D.getDirectiveKind() == OMPD_simd); + } +} + +static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, + const OMPExecutableDirective &S, + OpenMPDirectiveKind InnermostKind, + const RegionCodeGenTy &CodeGen) { + auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); + auto CapturedStruct = CGF.GenerateCapturedStmtArgument(*CS); + auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( + S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); + if (auto C = S.getSingleClause(OMPC_num_threads)) { CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); auto NumThreadsClause = cast<OMPNumThreadsClause>(C); auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), /*IgnoreResultAssign*/ true); - CGF.CGM.getOpenMPRuntime().EmitOMPNumThreadsClause( + CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( CGF, NumThreads, NumThreadsClause->getLocStart()); } - CGF.CGM.getOpenMPRuntime().EmitOMPParallelCall(CGF, S.getLocStart(), - OutlinedFn, CapturedStruct); + if (auto *C = S.getSingleClause(OMPC_proc_bind)) { + CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); + auto *ProcBindClause = cast<OMPProcBindClause>(C); + CGF.CGM.getOpenMPRuntime().emitProcBindClause( + CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart()); + } + const Expr *IfCond = nullptr; + if (auto C = S.getSingleClause(OMPC_if)) { + IfCond = cast<OMPIfClause>(C)->getCondition(); + } + CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn, + CapturedStruct, IfCond); } void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { - auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); - auto CapturedStruct = GenerateCapturedStmtArgument(*CS); - auto OutlinedFn = CGM.getOpenMPRuntime().EmitOpenMPOutlinedFunction( - S, *CS->getCapturedDecl()->param_begin()); - if (auto C = S.getSingleClause(/*K*/ OMPC_if)) { - auto Cond = cast<OMPIfClause>(C)->getCondition(); - EmitOMPIfClause(*this, Cond, [&](bool ThenBlock) { - if (ThenBlock) - EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct); - else - CGM.getOpenMPRuntime().EmitOMPSerialCall(*this, S.getLocStart(), - OutlinedFn, CapturedStruct); - }); - } else - EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct); + LexicalScope Scope(*this, S.getSourceRange()); + // Emit parallel region as a standalone region. + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + OMPPrivateScope PrivateScope(CGF); + bool Copyins = CGF.EmitOMPCopyinClause(S); + bool Firstprivates = CGF.EmitOMPFirstprivateClause(S, PrivateScope); + if (Copyins || Firstprivates) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables or propagation master's thread + // values of threadprivate variables to local instances of that variables + // of all other implicit threads. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_unknown); + } + CGF.EmitOMPPrivateClause(S, PrivateScope); + CGF.EmitOMPReductionClauseInit(S, PrivateScope); + (void)PrivateScope.Privatize(); + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EmitOMPReductionClauseFinal(S); + // Emit implicit barrier at the end of the 'parallel' directive. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_unknown); + }; + emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen); } -void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S, - bool SeparateIter) { +void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D, + JumpDest LoopExit) { RunCleanupsScope BodyScope(*this); // Update counters values on current iteration. - for (auto I : S.updates()) { + for (auto I : D.updates()) { EmitIgnoredExpr(I); } + // Update the linear variables. + for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { + auto *C = cast<OMPLinearClause>(*I); + for (auto U : C->updates()) { + EmitIgnoredExpr(U); + } + } + // On a continue in the body, jump to the end. auto Continue = getJumpDestInCurrentScope("omp.body.continue"); - BreakContinueStack.push_back(BreakContinue(JumpDest(), Continue)); + BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); // Emit loop body. - EmitStmt(S.getBody()); + EmitStmt(D.getBody()); // The end (updates/cleanups). EmitBlock(Continue.getBlock()); BreakContinueStack.pop_back(); - if (SeparateIter) { // TODO: Update lastprivates if the SeparateIter flag is true. // This will be implemented in a follow-up OMPLastprivateClause patch, but // result should be still correct without it, as we do not make these // variables private yet. - } } -void CodeGenFunction::EmitOMPInnerLoop(const OMPLoopDirective &S, - OMPPrivateScope &LoopScope, - bool SeparateIter) { +void CodeGenFunction::EmitOMPInnerLoop( + const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, + const Expr *IncExpr, + const llvm::function_ref<void(CodeGenFunction &)> &BodyGen, + const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen) { auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end"); - auto Cnt = getPGORegionCounter(&S); // Start the loop with a block that tests the condition. auto CondBlock = createBasicBlock("omp.inner.for.cond"); @@ -311,35 +562,31 @@ void CodeGenFunction::EmitOMPInnerLoop(const OMPLoopDirective &S, // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. auto ExitBlock = LoopExit.getBlock(); - if (LoopScope.requiresCleanups()) + if (RequiresCleanup) ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup"); auto LoopBody = createBasicBlock("omp.inner.for.body"); - // Emit condition: "IV < LastIteration + 1 [ - 1]" - // ("- 1" when lastprivate clause is present - separate one iteration). - llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond(SeparateIter)); - Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, - PGO.createLoopWeights(S.getCond(SeparateIter), Cnt)); - + // Emit condition. + EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S)); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); EmitBranchThroughCleanup(LoopExit); } EmitBlock(LoopBody); - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); // Create a block for the increment. auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc"); BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); - EmitOMPLoopBody(S); - EmitStopPoint(&S); + BodyGen(*this); // Emit "IV = IV + 1" and a back-edge to the condition block. EmitBlock(Continue.getBlock()); - EmitIgnoredExpr(S.getInc()); + EmitIgnoredExpr(IncExpr); + PostIncGen(*this); BreakContinueStack.pop_back(); EmitBranch(CondBlock); LoopStack.pop(); @@ -347,157 +594,415 @@ void CodeGenFunction::EmitOMPInnerLoop(const OMPLoopDirective &S, EmitBlock(LoopExit.getBlock()); } -void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &S) { - auto IC = S.counters().begin(); - for (auto F : S.finals()) { - if (LocalDeclMap.lookup(cast<DeclRefExpr>((*IC))->getDecl())) { - EmitIgnoredExpr(F); +void CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) { + // Emit inits for the linear variables. + for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { + auto *C = cast<OMPLinearClause>(*I); + for (auto Init : C->inits()) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl()); + auto *OrigVD = cast<VarDecl>( + cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())->getDecl()); + DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), + CapturedStmtInfo->lookup(OrigVD) != nullptr, + VD->getInit()->getType(), VK_LValue, + VD->getInit()->getExprLoc()); + AutoVarEmission Emission = EmitAutoVarAlloca(*VD); + EmitExprAsInit(&DRE, VD, + MakeAddrLValue(Emission.getAllocatedAddress(), + VD->getType(), Emission.Alignment), + /*capturedByInit=*/false); + EmitAutoVarCleanups(Emission); } - ++IC; + // Emit the linear steps for the linear clauses. + // If a step is not constant, it is pre-calculated before the loop. + if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep())) + if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) { + EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl())); + // Emit calculation of the linear step. + EmitIgnoredExpr(CS); + } } } -static void EmitOMPAlignedClause(CodeGenFunction &CGF, CodeGenModule &CGM, - const OMPAlignedClause &Clause) { - unsigned ClauseAlignment = 0; - if (auto AlignmentExpr = Clause.getAlignment()) { - auto AlignmentCI = - cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); - ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue()); +static void emitLinearClauseFinal(CodeGenFunction &CGF, + const OMPLoopDirective &D) { + // Emit the final values of the linear variables. + for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { + auto *C = cast<OMPLinearClause>(*I); + auto IC = C->varlist_begin(); + for (auto F : C->finals()) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl()); + DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), + CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, + (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); + auto *OrigAddr = CGF.EmitLValue(&DRE).getAddress(); + CodeGenFunction::OMPPrivateScope VarScope(CGF); + VarScope.addPrivate(OrigVD, + [OrigAddr]() -> llvm::Value *{ return OrigAddr; }); + (void)VarScope.Privatize(); + CGF.EmitIgnoredExpr(F); + ++IC; + } } - for (auto E : Clause.varlists()) { - unsigned Alignment = ClauseAlignment; - if (Alignment == 0) { - // OpenMP [2.8.1, Description] - // If no optional parameter is specified, implementation-defined default - // alignments for SIMD instructions on the target platforms are assumed. - Alignment = CGM.getTargetCodeGenInfo().getOpenMPSimdDefaultAlignment( - E->getType()); +} + +static void emitAlignedClause(CodeGenFunction &CGF, + const OMPExecutableDirective &D) { + for (auto &&I = D.getClausesOfKind(OMPC_aligned); I; ++I) { + auto *Clause = cast<OMPAlignedClause>(*I); + unsigned ClauseAlignment = 0; + if (auto AlignmentExpr = Clause->getAlignment()) { + auto AlignmentCI = + cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); + ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue()); } - assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) && - "alignment is not power of 2"); - if (Alignment != 0) { - llvm::Value *PtrValue = CGF.EmitScalarExpr(E); - CGF.EmitAlignmentAssumption(PtrValue, Alignment); + for (auto E : Clause->varlists()) { + unsigned Alignment = ClauseAlignment; + if (Alignment == 0) { + // OpenMP [2.8.1, Description] + // If no optional parameter is specified, implementation-defined default + // alignments for SIMD instructions on the target platforms are assumed. + Alignment = + CGF.getContext() + .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( + E->getType()->getPointeeType())) + .getQuantity(); + } + assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) && + "alignment is not power of 2"); + if (Alignment != 0) { + llvm::Value *PtrValue = CGF.EmitScalarExpr(E); + CGF.EmitAlignmentAssumption(PtrValue, Alignment); + } } } } -static void EmitPrivateLoopCounters(CodeGenFunction &CGF, +static void emitPrivateLoopCounters(CodeGenFunction &CGF, CodeGenFunction::OMPPrivateScope &LoopScope, ArrayRef<Expr *> Counters) { for (auto *E : Counters) { auto VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - bool IsRegistered = LoopScope.addPrivate(VD, [&]() -> llvm::Value * { + (void)LoopScope.addPrivate(VD, [&]() -> llvm::Value *{ // Emit var without initialization. auto VarEmission = CGF.EmitAutoVarAlloca(*VD); CGF.EmitAutoVarCleanups(VarEmission); return VarEmission.getAllocatedAddress(); }); - assert(IsRegistered && "counter already registered as private"); - // Silence the warning about unused variable. - (void)IsRegistered; } - (void)LoopScope.Privatize(); } -void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { - // Pragma 'simd' code depends on presence of 'lastprivate'. - // If present, we have to separate last iteration of the loop: - // - // if (LastIteration != 0) { - // for (IV in 0..LastIteration-1) BODY; - // BODY with updates of lastprivate vars; - // <Final counter/linear vars updates>; - // } - // - // otherwise (when there's no lastprivate): - // - // for (IV in 0..LastIteration) BODY; - // <Final counter/linear vars updates>; - // +static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, + const Expr *Cond, llvm::BasicBlock *TrueBlock, + llvm::BasicBlock *FalseBlock, uint64_t TrueCount) { + { + CodeGenFunction::OMPPrivateScope PreCondScope(CGF); + emitPrivateLoopCounters(CGF, PreCondScope, S.counters()); + (void)PreCondScope.Privatize(); + // Get initial values of real counters. + for (auto I : S.inits()) { + CGF.EmitIgnoredExpr(I); + } + } + // Check that loop is executed at least one time. + CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); +} +static void +emitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D, + CodeGenFunction::OMPPrivateScope &PrivateScope) { + for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { + auto *C = cast<OMPLinearClause>(*I); + for (auto *E : C->varlists()) { + auto VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + bool IsRegistered = PrivateScope.addPrivate(VD, [&]()->llvm::Value * { + // Emit var without initialization. + auto VarEmission = CGF.EmitAutoVarAlloca(*VD); + CGF.EmitAutoVarCleanups(VarEmission); + return VarEmission.getAllocatedAddress(); + }); + assert(IsRegistered && "linear var already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + } + } +} + +static void emitSafelenClause(CodeGenFunction &CGF, + const OMPExecutableDirective &D) { + if (auto *C = + cast_or_null<OMPSafelenClause>(D.getSingleClause(OMPC_safelen))) { + RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), + /*ignoreResult=*/true); + llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); + CGF.LoopStack.setVectorizerWidth(Val->getZExtValue()); + // In presence of finite 'safelen', it may be unsafe to mark all + // the memory instructions parallel, because loop-carried + // dependences of 'safelen' iterations are possible. + CGF.LoopStack.setParallel(false); + } +} + +void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D) { // Walk clauses and process safelen/lastprivate. - bool SeparateIter = false; LoopStack.setParallel(); LoopStack.setVectorizerEnable(true); - for (auto C : S.clauses()) { - switch (C->getClauseKind()) { - case OMPC_safelen: { - RValue Len = EmitAnyExpr(cast<OMPSafelenClause>(C)->getSafelen(), - AggValueSlot::ignored(), true); - llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); - LoopStack.setVectorizerWidth(Val->getZExtValue()); - // In presence of finite 'safelen', it may be unsafe to mark all - // the memory instructions parallel, because loop-carried - // dependences of 'safelen' iterations are possible. - LoopStack.setParallel(false); - break; - } - case OMPC_aligned: - EmitOMPAlignedClause(*this, CGM, cast<OMPAlignedClause>(*C)); - break; - case OMPC_lastprivate: - SeparateIter = true; - break; - default: - // Not handled yet - ; + emitSafelenClause(*this, D); +} + +void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &D) { + auto IC = D.counters().begin(); + for (auto F : D.finals()) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl()); + if (LocalDeclMap.lookup(OrigVD) || CapturedStmtInfo->lookup(OrigVD)) { + DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), + CapturedStmtInfo->lookup(OrigVD) != nullptr, + (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); + auto *OrigAddr = EmitLValue(&DRE).getAddress(); + OMPPrivateScope VarScope(*this); + VarScope.addPrivate(OrigVD, + [OrigAddr]() -> llvm::Value *{ return OrigAddr; }); + (void)VarScope.Privatize(); + EmitIgnoredExpr(F); } + ++IC; } + emitLinearClauseFinal(*this, D); +} - InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); - RunCleanupsScope DirectiveScope(*this); +void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + // if (PreCond) { + // for (IV in 0..LastIteration) BODY; + // <Final counter/linear vars updates>; + // } + // + + // Emit: if (PreCond) - begin. + // If the condition constant folds and can be elided, avoid emitting the + // whole loop. + bool CondConstant; + llvm::BasicBlock *ContBlock = nullptr; + if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { + if (!CondConstant) + return; + } else { + auto *ThenBlock = CGF.createBasicBlock("simd.if.then"); + ContBlock = CGF.createBasicBlock("simd.if.end"); + emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, + CGF.getProfileCount(&S)); + CGF.EmitBlock(ThenBlock); + CGF.incrementProfileCounter(&S); + } - CGDebugInfo *DI = getDebugInfo(); - if (DI) - DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin()); + // Emit the loop iteration variable. + const Expr *IVExpr = S.getIterationVariable(); + const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl()); + CGF.EmitVarDecl(*IVDecl); + CGF.EmitIgnoredExpr(S.getInit()); + + // Emit the iterations count variable. + // If it is not a variable, Sema decided to calculate iterations count on + // each iteration (e.g., it is foldable into a constant). + if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { + CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); + // Emit calculation of the iterations count. + CGF.EmitIgnoredExpr(S.getCalcLastIteration()); + } + + CGF.EmitOMPSimdInit(S); + + emitAlignedClause(CGF, S); + CGF.EmitOMPLinearClauseInit(S); + bool HasLastprivateClause; + { + OMPPrivateScope LoopScope(CGF); + emitPrivateLoopCounters(CGF, LoopScope, S.counters()); + emitPrivateLinearVars(CGF, S, LoopScope); + CGF.EmitOMPPrivateClause(S, LoopScope); + CGF.EmitOMPReductionClauseInit(S, LoopScope); + HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); + (void)LoopScope.Privatize(); + CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), + S.getInc(), + [&S](CodeGenFunction &CGF) { + CGF.EmitOMPLoopBody(S, JumpDest()); + CGF.EmitStopPoint(&S); + }, + [](CodeGenFunction &) {}); + // Emit final copy of the lastprivate variables at the end of loops. + if (HasLastprivateClause) { + CGF.EmitOMPLastprivateClauseFinal(S); + } + CGF.EmitOMPReductionClauseFinal(S); + } + CGF.EmitOMPSimdFinal(S); + // Emit: if (PreCond) - end. + if (ContBlock) { + CGF.EmitBranch(ContBlock); + CGF.EmitBlock(ContBlock, true); + } + }; + CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); +} + +void CodeGenFunction::EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind, + const OMPLoopDirective &S, + OMPPrivateScope &LoopScope, + bool Ordered, llvm::Value *LB, + llvm::Value *UB, llvm::Value *ST, + llvm::Value *IL, llvm::Value *Chunk) { + auto &RT = CGM.getOpenMPRuntime(); + + // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime). + const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind); + + assert((Ordered || + !RT.isStaticNonchunked(ScheduleKind, /*Chunked=*/Chunk != nullptr)) && + "static non-chunked schedule does not need outer loop"); + + // Emit outer loop. + // + // OpenMP [2.7.1, Loop Construct, Description, table 2-1] + // When schedule(dynamic,chunk_size) is specified, the iterations are + // distributed to threads in the team in chunks as the threads request them. + // Each thread executes a chunk of iterations, then requests another chunk, + // until no chunks remain to be distributed. Each chunk contains chunk_size + // iterations, except for the last chunk to be distributed, which may have + // fewer iterations. When no chunk_size is specified, it defaults to 1. + // + // When schedule(guided,chunk_size) is specified, the iterations are assigned + // to threads in the team in chunks as the executing threads request them. + // Each thread executes a chunk of iterations, then requests another chunk, + // until no chunks remain to be assigned. For a chunk_size of 1, the size of + // each chunk is proportional to the number of unassigned iterations divided + // by the number of threads in the team, decreasing to 1. For a chunk_size + // with value k (greater than 1), the size of each chunk is determined in the + // same way, with the restriction that the chunks do not contain fewer than k + // iterations (except for the last chunk to be assigned, which may have fewer + // than k iterations). + // + // When schedule(auto) is specified, the decision regarding scheduling is + // delegated to the compiler and/or runtime system. The programmer gives the + // implementation the freedom to choose any possible mapping of iterations to + // threads in the team. + // + // When schedule(runtime) is specified, the decision regarding scheduling is + // deferred until run time, and the schedule and chunk size are taken from the + // run-sched-var ICV. If the ICV is set to auto, the schedule is + // implementation defined + // + // while(__kmpc_dispatch_next(&LB, &UB)) { + // idx = LB; + // while (idx <= UB) { BODY; ++idx; + // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only. + // } // inner loop + // } + // + // OpenMP [2.7.1, Loop Construct, Description, table 2-1] + // When schedule(static, chunk_size) is specified, iterations are divided into + // chunks of size chunk_size, and the chunks are assigned to the threads in + // the team in a round-robin fashion in the order of the thread number. + // + // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) { + // while (idx <= UB) { BODY; ++idx; } // inner loop + // LB = LB + ST; + // UB = UB + ST; + // } + // - // Emit the loop iteration variable. const Expr *IVExpr = S.getIterationVariable(); - const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl()); - EmitVarDecl(*IVDecl); - EmitIgnoredExpr(S.getInit()); + const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); + const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); - // Emit the iterations count variable. - // If it is not a variable, Sema decided to calculate iterations count on each - // iteration (e.g., it is foldable into a constant). - if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { - EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); - // Emit calculation of the iterations count. - EmitIgnoredExpr(S.getCalcLastIteration()); + RT.emitForInit( + *this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, Ordered, IL, LB, + (DynamicOrOrdered ? EmitAnyExpr(S.getLastIteration()).getScalarVal() + : UB), + ST, Chunk); + + auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end"); + + // Start the loop with a block that tests the condition. + auto CondBlock = createBasicBlock("omp.dispatch.cond"); + EmitBlock(CondBlock); + LoopStack.push(CondBlock); + + llvm::Value *BoolCondVal = nullptr; + if (!DynamicOrOrdered) { + // UB = min(UB, GlobalUB) + EmitIgnoredExpr(S.getEnsureUpperBound()); + // IV = LB + EmitIgnoredExpr(S.getInit()); + // IV < UB + BoolCondVal = EvaluateExprAsBool(S.getCond()); + } else { + BoolCondVal = RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, + IL, LB, UB, ST); } - if (SeparateIter) { - // Emit: if (LastIteration > 0) - begin. - RegionCounter Cnt = getPGORegionCounter(&S); - auto ThenBlock = createBasicBlock("simd.if.then"); - auto ContBlock = createBasicBlock("simd.if.end"); - EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount()); - EmitBlock(ThenBlock); - Cnt.beginRegion(Builder); - // Emit 'then' code. - { - OMPPrivateScope LoopScope(*this); - EmitPrivateLoopCounters(*this, LoopScope, S.counters()); - EmitOMPInnerLoop(S, LoopScope, /* SeparateIter */ true); - EmitOMPLoopBody(S, /* SeparateIter */ true); - } - EmitOMPSimdFinal(S); - // Emit: if (LastIteration != 0) - end. - EmitBranch(ContBlock); - EmitBlock(ContBlock, true); + // If there are any cleanups between here and the loop-exit scope, + // create a block to stage a loop exit along. + auto ExitBlock = LoopExit.getBlock(); + if (LoopScope.requiresCleanups()) + ExitBlock = createBasicBlock("omp.dispatch.cleanup"); + + auto LoopBody = createBasicBlock("omp.dispatch.body"); + Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock); + if (ExitBlock != LoopExit.getBlock()) { + EmitBlock(ExitBlock); + EmitBranchThroughCleanup(LoopExit); + } + EmitBlock(LoopBody); + + // Emit "IV = LB" (in case of static schedule, we have already calculated new + // LB for loop condition and emitted it above). + if (DynamicOrOrdered) + EmitIgnoredExpr(S.getInit()); + + // Create a block for the increment. + auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc"); + BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); + + // Generate !llvm.loop.parallel metadata for loads and stores for loops + // with dynamic/guided scheduling and without ordered clause. + if (!isOpenMPSimdDirective(S.getDirectiveKind())) { + LoopStack.setParallel((ScheduleKind == OMPC_SCHEDULE_dynamic || + ScheduleKind == OMPC_SCHEDULE_guided) && + !Ordered); } else { - { - OMPPrivateScope LoopScope(*this); - EmitPrivateLoopCounters(*this, LoopScope, S.counters()); - EmitOMPInnerLoop(S, LoopScope); - } - EmitOMPSimdFinal(S); + EmitOMPSimdInit(S); } - if (DI) - DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd()); + SourceLocation Loc = S.getLocStart(); + EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), + [&S, LoopExit](CodeGenFunction &CGF) { + CGF.EmitOMPLoopBody(S, LoopExit); + CGF.EmitStopPoint(&S); + }, + [Ordered, IVSize, IVSigned, Loc](CodeGenFunction &CGF) { + if (Ordered) { + CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd( + CGF, Loc, IVSize, IVSigned); + } + }); + + EmitBlock(Continue.getBlock()); + BreakContinueStack.pop_back(); + if (!DynamicOrOrdered) { + // Emit "LB = LB + Stride", "UB = UB + Stride". + EmitIgnoredExpr(S.getNextLowerBound()); + EmitIgnoredExpr(S.getNextUpperBound()); + } + + EmitBranch(CondBlock); + LoopStack.pop(); + // Emit the fall-through block. + EmitBlock(LoopExit.getBlock()); + + // Tell the runtime we are done. + if (!DynamicOrOrdered) + RT.emitForStaticFinish(*this, S.getLocEnd()); } /// \brief Emit a helper variable and return corresponding lvalue. @@ -508,7 +1013,39 @@ static LValue EmitOMPHelperVar(CodeGenFunction &CGF, return CGF.EmitLValue(Helper); } -void CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { +static std::pair<llvm::Value * /*Chunk*/, OpenMPScheduleClauseKind> +emitScheduleClause(CodeGenFunction &CGF, const OMPLoopDirective &S, + bool OuterRegion) { + // Detect the loop schedule kind and chunk. + auto ScheduleKind = OMPC_SCHEDULE_unknown; + llvm::Value *Chunk = nullptr; + if (auto *C = + cast_or_null<OMPScheduleClause>(S.getSingleClause(OMPC_schedule))) { + ScheduleKind = C->getScheduleKind(); + if (const auto *Ch = C->getChunkSize()) { + if (auto *ImpRef = cast_or_null<DeclRefExpr>(C->getHelperChunkSize())) { + if (OuterRegion) { + const VarDecl *ImpVar = cast<VarDecl>(ImpRef->getDecl()); + CGF.EmitVarDecl(*ImpVar); + CGF.EmitStoreThroughLValue( + CGF.EmitAnyExpr(Ch), + CGF.MakeNaturalAlignAddrLValue(CGF.GetAddrOfLocalVar(ImpVar), + ImpVar->getType())); + } else { + Ch = ImpRef; + } + } + if (!C->getHelperChunkSize() || !OuterRegion) { + Chunk = CGF.EmitScalarExpr(Ch); + Chunk = CGF.EmitScalarConversion(Chunk, Ch->getType(), + S.getIterationVariable()->getType()); + } + } + } + return std::make_pair(Chunk, ScheduleKind); +} + +bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { // Emit the loop iteration variable. auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable()); auto IVDecl = cast<VarDecl>(IVExpr->getDecl()); @@ -525,15 +1062,28 @@ void CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { auto &RT = CGM.getOpenMPRuntime(); + bool HasLastprivateClause; // Check pre-condition. { // Skip the entire loop if we don't meet the precondition. - RegionCounter Cnt = getPGORegionCounter(&S); - auto ThenBlock = createBasicBlock("omp.precond.then"); - auto ContBlock = createBasicBlock("omp.precond.end"); - EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount()); - EmitBlock(ThenBlock); - Cnt.beginRegion(Builder); + // If the condition constant folds and can be elided, avoid emitting the + // whole loop. + bool CondConstant; + llvm::BasicBlock *ContBlock = nullptr; + if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { + if (!CondConstant) + return false; + } else { + auto *ThenBlock = createBasicBlock("omp.precond.then"); + ContBlock = createBasicBlock("omp.precond.end"); + emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock, + getProfileCount(&S)); + EmitBlock(ThenBlock); + incrementProfileCounter(&S); + } + + emitAlignedClause(*this, S); + EmitOMPLinearClauseInit(S); // Emit 'then' code. { // Emit helper vars inits. @@ -547,152 +1097,994 @@ void CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable())); OMPPrivateScope LoopScope(*this); - EmitPrivateLoopCounters(*this, LoopScope, S.counters()); + if (EmitOMPFirstprivateClause(S, LoopScope)) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables. + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), + OMPD_unknown); + } + EmitOMPPrivateClause(S, LoopScope); + HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); + EmitOMPReductionClauseInit(S, LoopScope); + emitPrivateLoopCounters(*this, LoopScope, S.counters()); + emitPrivateLinearVars(*this, S, LoopScope); + (void)LoopScope.Privatize(); // Detect the loop schedule kind and chunk. - auto ScheduleKind = OMPC_SCHEDULE_unknown; - llvm::Value *Chunk = nullptr; - if (auto C = cast_or_null<OMPScheduleClause>( - S.getSingleClause(OMPC_schedule))) { - ScheduleKind = C->getScheduleKind(); - if (auto Ch = C->getChunkSize()) { - Chunk = EmitScalarExpr(Ch); - Chunk = EmitScalarConversion(Chunk, Ch->getType(), - S.getIterationVariable()->getType()); - } - } + llvm::Value *Chunk; + OpenMPScheduleClauseKind ScheduleKind; + auto ScheduleInfo = + emitScheduleClause(*this, S, /*OuterRegion=*/false); + Chunk = ScheduleInfo.first; + ScheduleKind = ScheduleInfo.second; const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); + const bool Ordered = S.getSingleClause(OMPC_ordered) != nullptr; if (RT.isStaticNonchunked(ScheduleKind, - /* Chunked */ Chunk != nullptr)) { + /* Chunked */ Chunk != nullptr) && + !Ordered) { + if (isOpenMPSimdDirective(S.getDirectiveKind())) { + EmitOMPSimdInit(S); + } // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When no chunk_size is specified, the iteration space is divided into // chunks that are approximately equal in size, and at most one chunk is // distributed to each thread. Note that the size of the chunks is // unspecified in this case. - RT.EmitOMPForInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, - IL.getAddress(), LB.getAddress(), UB.getAddress(), - ST.getAddress()); + RT.emitForInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, + Ordered, IL.getAddress(), LB.getAddress(), + UB.getAddress(), ST.getAddress()); + auto LoopExit = getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit")); // UB = min(UB, GlobalUB); EmitIgnoredExpr(S.getEnsureUpperBound()); // IV = LB; EmitIgnoredExpr(S.getInit()); // while (idx <= UB) { BODY; ++idx; } - EmitOMPInnerLoop(S, LoopScope); + EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), + S.getInc(), + [&S, LoopExit](CodeGenFunction &CGF) { + CGF.EmitOMPLoopBody(S, LoopExit); + CGF.EmitStopPoint(&S); + }, + [](CodeGenFunction &) {}); + EmitBlock(LoopExit.getBlock()); // Tell the runtime we are done. - RT.EmitOMPForFinish(*this, S.getLocStart(), ScheduleKind); - } else - ErrorUnsupported(&S, "OpenMP loop with requested schedule"); + RT.emitForStaticFinish(*this, S.getLocStart()); + } else { + // Emit the outer loop, which requests its work chunk [LB..UB] from + // runtime and runs the inner loop to process it. + EmitOMPForOuterLoop(ScheduleKind, S, LoopScope, Ordered, + LB.getAddress(), UB.getAddress(), ST.getAddress(), + IL.getAddress(), Chunk); + } + EmitOMPReductionClauseFinal(S); + // Emit final copy of the lastprivate variables if IsLastIter != 0. + if (HasLastprivateClause) + EmitOMPLastprivateClauseFinal( + S, Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart()))); + } + if (isOpenMPSimdDirective(S.getDirectiveKind())) { + EmitOMPSimdFinal(S); } // We're now done with the loop, so jump to the continuation block. - EmitBranch(ContBlock); - EmitBlock(ContBlock, true); + if (ContBlock) { + EmitBranch(ContBlock); + EmitBlock(ContBlock, true); + } } + return HasLastprivateClause; } void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) { - InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); - RunCleanupsScope DirectiveScope(*this); + LexicalScope Scope(*this, S.getSourceRange()); + bool HasLastprivates = false; + auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) { + HasLastprivates = CGF.EmitOMPWorksharingLoop(S); + }; + CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen); - CGDebugInfo *DI = getDebugInfo(); - if (DI) - DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin()); + // Emit an implicit barrier at the end. + if (!S.getSingleClause(OMPC_nowait) || HasLastprivates) { + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for); + } +} - EmitOMPWorksharingLoop(S); +void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) { + LexicalScope Scope(*this, S.getSourceRange()); + bool HasLastprivates = false; + auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) { + HasLastprivates = CGF.EmitOMPWorksharingLoop(S); + }; + CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); // Emit an implicit barrier at the end. - CGM.getOpenMPRuntime().EmitOMPBarrierCall(*this, S.getLocStart(), - /*IsExplicit*/ false); - if (DI) - DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd()); + if (!S.getSingleClause(OMPC_nowait) || HasLastprivates) { + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for); + } } -void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &) { - llvm_unreachable("CodeGen for 'omp for simd' is not supported yet."); +static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, + const Twine &Name, + llvm::Value *Init = nullptr) { + auto LVal = CGF.MakeNaturalAlignAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); + if (Init) + CGF.EmitScalarInit(Init, LVal); + return LVal; } -void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &) { - llvm_unreachable("CodeGen for 'omp sections' is not supported yet."); +OpenMPDirectiveKind +CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { + auto *Stmt = cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt(); + auto *CS = dyn_cast<CompoundStmt>(Stmt); + if (CS && CS->size() > 1) { + bool HasLastprivates = false; + auto &&CodeGen = [&S, CS, &HasLastprivates](CodeGenFunction &CGF) { + auto &C = CGF.CGM.getContext(); + auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); + // Emit helper vars inits. + LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", + CGF.Builder.getInt32(0)); + auto *GlobalUBVal = CGF.Builder.getInt32(CS->size() - 1); + LValue UB = + createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); + LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", + CGF.Builder.getInt32(1)); + LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", + CGF.Builder.getInt32(0)); + // Loop counter. + LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); + OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue); + CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); + OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue); + CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); + // Generate condition for loop. + BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue, + OK_Ordinary, S.getLocStart(), + /*fpContractable=*/false); + // Increment for loop counter. + UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, + OK_Ordinary, S.getLocStart()); + auto BodyGen = [CS, &S, &IV](CodeGenFunction &CGF) { + // Iterate through all sections and emit a switch construct: + // switch (IV) { + // case 0: + // <SectionStmt[0]>; + // break; + // ... + // case <NumSection> - 1: + // <SectionStmt[<NumSection> - 1]>; + // break; + // } + // .omp.sections.exit: + auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); + auto *SwitchStmt = CGF.Builder.CreateSwitch( + CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB, + CS->size()); + unsigned CaseNumber = 0; + for (auto *SubStmt : CS->children()) { + auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); + CGF.EmitBlock(CaseBB); + SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); + CGF.EmitStmt(SubStmt); + CGF.EmitBranch(ExitBB); + ++CaseNumber; + } + CGF.EmitBlock(ExitBB, /*IsFinished=*/true); + }; + + CodeGenFunction::OMPPrivateScope LoopScope(CGF); + if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_unknown); + } + CGF.EmitOMPPrivateClause(S, LoopScope); + HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); + CGF.EmitOMPReductionClauseInit(S, LoopScope); + (void)LoopScope.Privatize(); + + // Emit static non-chunked loop. + CGF.CGM.getOpenMPRuntime().emitForInit( + CGF, S.getLocStart(), OMPC_SCHEDULE_static, /*IVSize=*/32, + /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(), + LB.getAddress(), UB.getAddress(), ST.getAddress()); + // UB = min(UB, GlobalUB); + auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart()); + auto *MinUBGlobalUB = CGF.Builder.CreateSelect( + CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); + CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); + // IV = LB; + CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV); + // while (idx <= UB) { BODY; ++idx; } + CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen, + [](CodeGenFunction &) {}); + // Tell the runtime we are done. + CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocStart()); + CGF.EmitOMPReductionClauseFinal(S); + + // Emit final copy of the lastprivate variables if IsLastIter != 0. + if (HasLastprivates) + CGF.EmitOMPLastprivateClauseFinal( + S, CGF.Builder.CreateIsNotNull( + CGF.EmitLoadOfScalar(IL, S.getLocStart()))); + }; + + CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen); + // Emit barrier for lastprivates only if 'sections' directive has 'nowait' + // clause. Otherwise the barrier will be generated by the codegen for the + // directive. + if (HasLastprivates && S.getSingleClause(OMPC_nowait)) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables. + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), + OMPD_unknown); + } + return OMPD_sections; + } + // If only one section is found - no need to generate loop, emit as a single + // region. + bool HasFirstprivates; + // No need to generate reductions for sections with single section region, we + // can use original shared variables for all operations. + bool HasReductions = !S.getClausesOfKind(OMPC_reduction).empty(); + // No need to generate lastprivates for sections with single section region, + // we can use original shared variable for all calculations with barrier at + // the end of the sections. + bool HasLastprivates = !S.getClausesOfKind(OMPC_lastprivate).empty(); + auto &&CodeGen = [Stmt, &S, &HasFirstprivates](CodeGenFunction &CGF) { + CodeGenFunction::OMPPrivateScope SingleScope(CGF); + HasFirstprivates = CGF.EmitOMPFirstprivateClause(S, SingleScope); + CGF.EmitOMPPrivateClause(S, SingleScope); + (void)SingleScope.Privatize(); + + CGF.EmitStmt(Stmt); + }; + CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(), + llvm::None, llvm::None, llvm::None, + llvm::None); + // Emit barrier for firstprivates, lastprivates or reductions only if + // 'sections' directive has 'nowait' clause. Otherwise the barrier will be + // generated by the codegen for the directive. + if ((HasFirstprivates || HasLastprivates || HasReductions) && + S.getSingleClause(OMPC_nowait)) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables. + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_unknown); + } + return OMPD_single; +} + +void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) { + LexicalScope Scope(*this, S.getSourceRange()); + OpenMPDirectiveKind EmittedAs = EmitSections(S); + // Emit an implicit barrier at the end. + if (!S.getSingleClause(OMPC_nowait)) { + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), EmittedAs); + } } -void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &) { - llvm_unreachable("CodeGen for 'omp section' is not supported yet."); +void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) { + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen); } -void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &) { - llvm_unreachable("CodeGen for 'omp single' is not supported yet."); +void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) { + llvm::SmallVector<const Expr *, 8> CopyprivateVars; + llvm::SmallVector<const Expr *, 8> DestExprs; + llvm::SmallVector<const Expr *, 8> SrcExprs; + llvm::SmallVector<const Expr *, 8> AssignmentOps; + // Check if there are any 'copyprivate' clauses associated with this + // 'single' + // construct. + // Build a list of copyprivate variables along with helper expressions + // (<source>, <destination>, <destination>=<source> expressions) + for (auto &&I = S.getClausesOfKind(OMPC_copyprivate); I; ++I) { + auto *C = cast<OMPCopyprivateClause>(*I); + CopyprivateVars.append(C->varlists().begin(), C->varlists().end()); + DestExprs.append(C->destination_exprs().begin(), + C->destination_exprs().end()); + SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end()); + AssignmentOps.append(C->assignment_ops().begin(), + C->assignment_ops().end()); + } + LexicalScope Scope(*this, S.getSourceRange()); + // Emit code for 'single' region along with 'copyprivate' clauses + bool HasFirstprivates; + auto &&CodeGen = [&S, &HasFirstprivates](CodeGenFunction &CGF) { + CodeGenFunction::OMPPrivateScope SingleScope(CGF); + HasFirstprivates = CGF.EmitOMPFirstprivateClause(S, SingleScope); + CGF.EmitOMPPrivateClause(S, SingleScope); + (void)SingleScope.Privatize(); + + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(), + CopyprivateVars, DestExprs, SrcExprs, + AssignmentOps); + // Emit an implicit barrier at the end (to avoid data race on firstprivate + // init or if no 'nowait' clause was specified and no 'copyprivate' clause). + if ((!S.getSingleClause(OMPC_nowait) || HasFirstprivates) && + CopyprivateVars.empty()) { + CGM.getOpenMPRuntime().emitBarrierCall( + *this, S.getLocStart(), + S.getSingleClause(OMPC_nowait) ? OMPD_unknown : OMPD_single); + } } void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) { - CGM.getOpenMPRuntime().EmitOMPMasterRegion(*this, [&]() -> void { - InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); - RunCleanupsScope Scope(*this); - EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); - EnsureInsertPoint(); - }, S.getLocStart()); + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart()); } void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) { - CGM.getOpenMPRuntime().EmitOMPCriticalRegion( - *this, S.getDirectiveName().getAsString(), [&]() -> void { - InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); - RunCleanupsScope Scope(*this); - EmitStmt( - cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); - EnsureInsertPoint(); - }, S.getLocStart()); + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitCriticalRegion( + *this, S.getDirectiveName().getAsString(), CodeGen, S.getLocStart()); } -void -CodeGenFunction::EmitOMPParallelForDirective(const OMPParallelForDirective &) { - llvm_unreachable("CodeGen for 'omp parallel for' is not supported yet."); +void CodeGenFunction::EmitOMPParallelForDirective( + const OMPParallelForDirective &S) { + // Emit directive as a combined directive that consists of two implicit + // directives: 'parallel' with 'for' directive. + LexicalScope Scope(*this, S.getSourceRange()); + (void)emitScheduleClause(*this, S, /*OuterRegion=*/true); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitOMPWorksharingLoop(S); + // Emit implicit barrier at the end of parallel region, but this barrier + // is at the end of 'for' directive, so emit it as the implicit barrier for + // this 'for' directive. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_parallel); + }; + emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen); } void CodeGenFunction::EmitOMPParallelForSimdDirective( - const OMPParallelForSimdDirective &) { - llvm_unreachable("CodeGen for 'omp parallel for simd' is not supported yet."); + const OMPParallelForSimdDirective &S) { + // Emit directive as a combined directive that consists of two implicit + // directives: 'parallel' with 'for' directive. + LexicalScope Scope(*this, S.getSourceRange()); + (void)emitScheduleClause(*this, S, /*OuterRegion=*/true); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitOMPWorksharingLoop(S); + // Emit implicit barrier at the end of parallel region, but this barrier + // is at the end of 'for' directive, so emit it as the implicit barrier for + // this 'for' directive. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_parallel); + }; + emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen); } void CodeGenFunction::EmitOMPParallelSectionsDirective( - const OMPParallelSectionsDirective &) { - llvm_unreachable("CodeGen for 'omp parallel sections' is not supported yet."); + const OMPParallelSectionsDirective &S) { + // Emit directive as a combined directive that consists of two implicit + // directives: 'parallel' with 'sections' directive. + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + (void)CGF.EmitSections(S); + // Emit implicit barrier at the end of parallel region. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_parallel); + }; + emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen); } -void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &) { - llvm_unreachable("CodeGen for 'omp task' is not supported yet."); +void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { + // Emit outlined function for task construct. + LexicalScope Scope(*this, S.getSourceRange()); + auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); + auto CapturedStruct = GenerateCapturedStmtArgument(*CS); + auto *I = CS->getCapturedDecl()->param_begin(); + auto *PartId = std::next(I); + // The first function argument for tasks is a thread id, the second one is a + // part id (0 for tied tasks, >=0 for untied task). + llvm::DenseSet<const VarDecl *> EmittedAsPrivate; + // Get list of private variables. + llvm::SmallVector<const Expr *, 8> PrivateVars; + llvm::SmallVector<const Expr *, 8> PrivateCopies; + for (auto &&I = S.getClausesOfKind(OMPC_private); I; ++I) { + auto *C = cast<OMPPrivateClause>(*I); + auto IRef = C->varlist_begin(); + for (auto *IInit : C->private_copies()) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); + if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { + PrivateVars.push_back(*IRef); + PrivateCopies.push_back(IInit); + } + ++IRef; + } + } + EmittedAsPrivate.clear(); + // Get list of firstprivate variables. + llvm::SmallVector<const Expr *, 8> FirstprivateVars; + llvm::SmallVector<const Expr *, 8> FirstprivateCopies; + llvm::SmallVector<const Expr *, 8> FirstprivateInits; + for (auto &&I = S.getClausesOfKind(OMPC_firstprivate); I; ++I) { + auto *C = cast<OMPFirstprivateClause>(*I); + auto IRef = C->varlist_begin(); + auto IElemInitRef = C->inits().begin(); + for (auto *IInit : C->private_copies()) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); + if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { + FirstprivateVars.push_back(*IRef); + FirstprivateCopies.push_back(IInit); + FirstprivateInits.push_back(*IElemInitRef); + } + ++IRef, ++IElemInitRef; + } + } + // Build list of dependences. + llvm::SmallVector<std::pair<OpenMPDependClauseKind, const Expr *>, 8> + Dependences; + for (auto &&I = S.getClausesOfKind(OMPC_depend); I; ++I) { + auto *C = cast<OMPDependClause>(*I); + for (auto *IRef : C->varlists()) { + Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef)); + } + } + auto &&CodeGen = [PartId, &S, &PrivateVars, &FirstprivateVars]( + CodeGenFunction &CGF) { + // Set proper addresses for generated private copies. + auto *CS = cast<CapturedStmt>(S.getAssociatedStmt()); + OMPPrivateScope Scope(CGF); + if (!PrivateVars.empty() || !FirstprivateVars.empty()) { + auto *CopyFn = CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)), + CGF.PointerAlignInBytes); + auto *PrivatesPtr = CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)), + CGF.PointerAlignInBytes); + // Map privates. + llvm::SmallVector<std::pair<const VarDecl *, llvm::Value *>, 16> + PrivatePtrs; + llvm::SmallVector<llvm::Value *, 16> CallArgs; + CallArgs.push_back(PrivatesPtr); + for (auto *E : PrivateVars) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + auto *PrivatePtr = + CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType())); + PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr)); + CallArgs.push_back(PrivatePtr); + } + for (auto *E : FirstprivateVars) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + auto *PrivatePtr = + CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType())); + PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr)); + CallArgs.push_back(PrivatePtr); + } + CGF.EmitRuntimeCall(CopyFn, CallArgs); + for (auto &&Pair : PrivatePtrs) { + auto *Replacement = + CGF.Builder.CreateAlignedLoad(Pair.second, CGF.PointerAlignInBytes); + Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; }); + } + } + (void)Scope.Privatize(); + if (*PartId) { + // TODO: emit code for untied tasks. + } + CGF.EmitStmt(CS->getCapturedStmt()); + }; + auto OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( + S, *I, OMPD_task, CodeGen); + // Check if we should emit tied or untied task. + bool Tied = !S.getSingleClause(OMPC_untied); + // Check if the task is final + llvm::PointerIntPair<llvm::Value *, 1, bool> Final; + if (auto *Clause = S.getSingleClause(OMPC_final)) { + // If the condition constant folds and can be elided, try to avoid emitting + // the condition and the dead arm of the if/else. + auto *Cond = cast<OMPFinalClause>(Clause)->getCondition(); + bool CondConstant; + if (ConstantFoldsToSimpleInteger(Cond, CondConstant)) + Final.setInt(CondConstant); + else + Final.setPointer(EvaluateExprAsBool(Cond)); + } else { + // By default the task is not final. + Final.setInt(/*IntVal=*/false); + } + auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); + const Expr *IfCond = nullptr; + if (auto C = S.getSingleClause(OMPC_if)) { + IfCond = cast<OMPIfClause>(C)->getCondition(); + } + CGM.getOpenMPRuntime().emitTaskCall( + *this, S.getLocStart(), S, Tied, Final, OutlinedFn, SharedsTy, + CapturedStruct, IfCond, PrivateVars, PrivateCopies, FirstprivateVars, + FirstprivateCopies, FirstprivateInits, Dependences); } -void CodeGenFunction::EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &) { - llvm_unreachable("CodeGen for 'omp taskyield' is not supported yet."); +void CodeGenFunction::EmitOMPTaskyieldDirective( + const OMPTaskyieldDirective &S) { + CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart()); } void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) { - CGM.getOpenMPRuntime().EmitOMPBarrierCall(*this, S.getLocStart()); + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier); +} + +void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) { + CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart()); } -void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &) { - llvm_unreachable("CodeGen for 'omp taskwait' is not supported yet."); +void CodeGenFunction::EmitOMPTaskgroupDirective( + const OMPTaskgroupDirective &S) { + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart()); } void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { - CGM.getOpenMPRuntime().EmitOMPFlush( - *this, [&]() -> ArrayRef<const Expr *> { - if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) { - auto FlushClause = cast<OMPFlushClause>(C); - return llvm::makeArrayRef(FlushClause->varlist_begin(), - FlushClause->varlist_end()); - } - return llvm::None; - }(), - S.getLocStart()); + CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> { + if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) { + auto FlushClause = cast<OMPFlushClause>(C); + return llvm::makeArrayRef(FlushClause->varlist_begin(), + FlushClause->varlist_end()); + } + return llvm::None; + }(), S.getLocStart()); +} + +void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) { + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart()); +} + +static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, + QualType SrcType, QualType DestType) { + assert(CGF.hasScalarEvaluationKind(DestType) && + "DestType must have scalar evaluation kind."); + assert(!Val.isAggregate() && "Must be a scalar or complex."); + return Val.isScalar() + ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType) + : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType, + DestType); +} + +static CodeGenFunction::ComplexPairTy +convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, + QualType DestType) { + assert(CGF.getEvaluationKind(DestType) == TEK_Complex && + "DestType must have complex evaluation kind."); + CodeGenFunction::ComplexPairTy ComplexVal; + if (Val.isScalar()) { + // Convert the input element to the element type of the complex. + auto DestElementType = DestType->castAs<ComplexType>()->getElementType(); + auto ScalarVal = + CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestElementType); + ComplexVal = CodeGenFunction::ComplexPairTy( + ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType())); + } else { + assert(Val.isComplex() && "Must be a scalar or complex."); + auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType(); + auto DestElementType = DestType->castAs<ComplexType>()->getElementType(); + ComplexVal.first = CGF.EmitScalarConversion( + Val.getComplexVal().first, SrcElementType, DestElementType); + ComplexVal.second = CGF.EmitScalarConversion( + Val.getComplexVal().second, SrcElementType, DestElementType); + } + return ComplexVal; +} + +static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, + LValue LVal, RValue RVal) { + if (LVal.isGlobalReg()) { + CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); + } else { + CGF.EmitAtomicStore(RVal, LVal, IsSeqCst ? llvm::SequentiallyConsistent + : llvm::Monotonic, + LVal.isVolatile(), /*IsInit=*/false); + } +} + +static void emitSimpleStore(CodeGenFunction &CGF, LValue LVal, RValue RVal, + QualType RValTy) { + switch (CGF.getEvaluationKind(LVal.getType())) { + case TEK_Scalar: + CGF.EmitStoreThroughLValue( + RValue::get(convertToScalarValue(CGF, RVal, RValTy, LVal.getType())), + LVal); + break; + case TEK_Complex: + CGF.EmitStoreOfComplex( + convertToComplexValue(CGF, RVal, RValTy, LVal.getType()), LVal, + /*isInit=*/false); + break; + case TEK_Aggregate: + llvm_unreachable("Must be a scalar or complex."); + } } -void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &) { - llvm_unreachable("CodeGen for 'omp ordered' is not supported yet."); +static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, + const Expr *X, const Expr *V, + SourceLocation Loc) { + // v = x; + assert(V->isLValue() && "V of 'omp atomic read' is not lvalue"); + assert(X->isLValue() && "X of 'omp atomic read' is not lvalue"); + LValue XLValue = CGF.EmitLValue(X); + LValue VLValue = CGF.EmitLValue(V); + RValue Res = XLValue.isGlobalReg() + ? CGF.EmitLoadOfLValue(XLValue, Loc) + : CGF.EmitAtomicLoad(XLValue, Loc, + IsSeqCst ? llvm::SequentiallyConsistent + : llvm::Monotonic, + XLValue.isVolatile()); + // OpenMP, 2.12.6, atomic Construct + // Any atomic construct with a seq_cst clause forces the atomically + // performed operation to include an implicit flush operation without a + // list. + if (IsSeqCst) + CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); + emitSimpleStore(CGF,VLValue, Res, X->getType().getNonReferenceType()); } -void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &) { - llvm_unreachable("CodeGen for 'omp atomic' is not supported yet."); +static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, + const Expr *X, const Expr *E, + SourceLocation Loc) { + // x = expr; + assert(X->isLValue() && "X of 'omp atomic write' is not lvalue"); + emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); + // OpenMP, 2.12.6, atomic Construct + // Any atomic construct with a seq_cst clause forces the atomically + // performed operation to include an implicit flush operation without a + // list. + if (IsSeqCst) + CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); +} + +static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, + RValue Update, + BinaryOperatorKind BO, + llvm::AtomicOrdering AO, + bool IsXLHSInRHSPart) { + auto &Context = CGF.CGM.getContext(); + // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x' + // expression is simple and atomic is allowed for the given type for the + // target platform. + if (BO == BO_Comma || !Update.isScalar() || + !Update.getScalarVal()->getType()->isIntegerTy() || + !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) && + (Update.getScalarVal()->getType() != + X.getAddress()->getType()->getPointerElementType())) || + !X.getAddress()->getType()->getPointerElementType()->isIntegerTy() || + !Context.getTargetInfo().hasBuiltinAtomic( + Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment()))) + return std::make_pair(false, RValue::get(nullptr)); + + llvm::AtomicRMWInst::BinOp RMWOp; + switch (BO) { + case BO_Add: + RMWOp = llvm::AtomicRMWInst::Add; + break; + case BO_Sub: + if (!IsXLHSInRHSPart) + return std::make_pair(false, RValue::get(nullptr)); + RMWOp = llvm::AtomicRMWInst::Sub; + break; + case BO_And: + RMWOp = llvm::AtomicRMWInst::And; + break; + case BO_Or: + RMWOp = llvm::AtomicRMWInst::Or; + break; + case BO_Xor: + RMWOp = llvm::AtomicRMWInst::Xor; + break; + case BO_LT: + RMWOp = X.getType()->hasSignedIntegerRepresentation() + ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min + : llvm::AtomicRMWInst::Max) + : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin + : llvm::AtomicRMWInst::UMax); + break; + case BO_GT: + RMWOp = X.getType()->hasSignedIntegerRepresentation() + ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max + : llvm::AtomicRMWInst::Min) + : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax + : llvm::AtomicRMWInst::UMin); + break; + case BO_Assign: + RMWOp = llvm::AtomicRMWInst::Xchg; + break; + case BO_Mul: + case BO_Div: + case BO_Rem: + case BO_Shl: + case BO_Shr: + case BO_LAnd: + case BO_LOr: + return std::make_pair(false, RValue::get(nullptr)); + case BO_PtrMemD: + case BO_PtrMemI: + case BO_LE: + case BO_GE: + case BO_EQ: + case BO_NE: + case BO_AddAssign: + case BO_SubAssign: + case BO_AndAssign: + case BO_OrAssign: + case BO_XorAssign: + case BO_MulAssign: + case BO_DivAssign: + case BO_RemAssign: + case BO_ShlAssign: + case BO_ShrAssign: + case BO_Comma: + llvm_unreachable("Unsupported atomic update operation"); + } + auto *UpdateVal = Update.getScalarVal(); + if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) { + UpdateVal = CGF.Builder.CreateIntCast( + IC, X.getAddress()->getType()->getPointerElementType(), + X.getType()->hasSignedIntegerRepresentation()); + } + auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getAddress(), UpdateVal, AO); + return std::make_pair(true, RValue::get(Res)); +} + +std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr( + LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, + llvm::AtomicOrdering AO, SourceLocation Loc, + const llvm::function_ref<RValue(RValue)> &CommonGen) { + // Update expressions are allowed to have the following forms: + // x binop= expr; -> xrval + expr; + // x++, ++x -> xrval + 1; + // x--, --x -> xrval - 1; + // x = x binop expr; -> xrval binop expr + // x = expr Op x; - > expr binop xrval; + auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart); + if (!Res.first) { + if (X.isGlobalReg()) { + // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop + // 'xrval'. + EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X); + } else { + // Perform compare-and-swap procedure. + EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified()); + } + } + return Res; +} + +static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, + const Expr *X, const Expr *E, + const Expr *UE, bool IsXLHSInRHSPart, + SourceLocation Loc) { + assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) && + "Update expr in 'atomic update' must be a binary operator."); + auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts()); + // Update expressions are allowed to have the following forms: + // x binop= expr; -> xrval + expr; + // x++, ++x -> xrval + 1; + // x--, --x -> xrval - 1; + // x = x binop expr; -> xrval binop expr + // x = expr Op x; - > expr binop xrval; + assert(X->isLValue() && "X of 'omp atomic update' is not lvalue"); + LValue XLValue = CGF.EmitLValue(X); + RValue ExprRValue = CGF.EmitAnyExpr(E); + auto AO = IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic; + auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts()); + auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts()); + auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; + auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; + auto Gen = + [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) -> RValue { + CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); + CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); + return CGF.EmitAnyExpr(UE); + }; + (void)CGF.EmitOMPAtomicSimpleUpdateExpr( + XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen); + // OpenMP, 2.12.6, atomic Construct + // Any atomic construct with a seq_cst clause forces the atomically + // performed operation to include an implicit flush operation without a + // list. + if (IsSeqCst) + CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); +} + +static RValue convertToType(CodeGenFunction &CGF, RValue Value, + QualType SourceType, QualType ResType) { + switch (CGF.getEvaluationKind(ResType)) { + case TEK_Scalar: + return RValue::get(convertToScalarValue(CGF, Value, SourceType, ResType)); + case TEK_Complex: { + auto Res = convertToComplexValue(CGF, Value, SourceType, ResType); + return RValue::getComplex(Res.first, Res.second); + } + case TEK_Aggregate: + break; + } + llvm_unreachable("Must be a scalar or complex."); +} + +static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, + bool IsPostfixUpdate, const Expr *V, + const Expr *X, const Expr *E, + const Expr *UE, bool IsXLHSInRHSPart, + SourceLocation Loc) { + assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue"); + assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue"); + RValue NewVVal; + LValue VLValue = CGF.EmitLValue(V); + LValue XLValue = CGF.EmitLValue(X); + RValue ExprRValue = CGF.EmitAnyExpr(E); + auto AO = IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic; + QualType NewVValType; + if (UE) { + // 'x' is updated with some additional value. + assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) && + "Update expr in 'atomic capture' must be a binary operator."); + auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts()); + // Update expressions are allowed to have the following forms: + // x binop= expr; -> xrval + expr; + // x++, ++x -> xrval + 1; + // x--, --x -> xrval - 1; + // x = x binop expr; -> xrval binop expr + // x = expr Op x; - > expr binop xrval; + auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts()); + auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts()); + auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; + NewVValType = XRValExpr->getType(); + auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; + auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, + IsSeqCst, IsPostfixUpdate](RValue XRValue) -> RValue { + CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); + CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); + RValue Res = CGF.EmitAnyExpr(UE); + NewVVal = IsPostfixUpdate ? XRValue : Res; + return Res; + }; + auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( + XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen); + if (Res.first) { + // 'atomicrmw' instruction was generated. + if (IsPostfixUpdate) { + // Use old value from 'atomicrmw'. + NewVVal = Res.second; + } else { + // 'atomicrmw' does not provide new value, so evaluate it using old + // value of 'x'. + CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); + CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); + NewVVal = CGF.EmitAnyExpr(UE); + } + } + } else { + // 'x' is simply rewritten with some 'expr'. + NewVValType = X->getType().getNonReferenceType(); + ExprRValue = convertToType(CGF, ExprRValue, E->getType(), + X->getType().getNonReferenceType()); + auto &&Gen = [&CGF, &NewVVal, ExprRValue](RValue XRValue) -> RValue { + NewVVal = XRValue; + return ExprRValue; + }; + // Try to perform atomicrmw xchg, otherwise simple exchange. + auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( + XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO, + Loc, Gen); + if (Res.first) { + // 'atomicrmw' instruction was generated. + NewVVal = IsPostfixUpdate ? Res.second : ExprRValue; + } + } + // Emit post-update store to 'v' of old/new 'x' value. + emitSimpleStore(CGF, VLValue, NewVVal, NewVValType); + // OpenMP, 2.12.6, atomic Construct + // Any atomic construct with a seq_cst clause forces the atomically + // performed operation to include an implicit flush operation without a + // list. + if (IsSeqCst) + CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); +} + +static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, + bool IsSeqCst, bool IsPostfixUpdate, + const Expr *X, const Expr *V, const Expr *E, + const Expr *UE, bool IsXLHSInRHSPart, + SourceLocation Loc) { + switch (Kind) { + case OMPC_read: + EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc); + break; + case OMPC_write: + EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc); + break; + case OMPC_unknown: + case OMPC_update: + EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc); + break; + case OMPC_capture: + EmitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE, + IsXLHSInRHSPart, Loc); + break; + case OMPC_if: + case OMPC_final: + case OMPC_num_threads: + case OMPC_private: + case OMPC_firstprivate: + case OMPC_lastprivate: + case OMPC_reduction: + case OMPC_safelen: + case OMPC_collapse: + case OMPC_default: + case OMPC_seq_cst: + case OMPC_shared: + case OMPC_linear: + case OMPC_aligned: + case OMPC_copyin: + case OMPC_copyprivate: + case OMPC_flush: + case OMPC_proc_bind: + case OMPC_schedule: + case OMPC_ordered: + case OMPC_nowait: + case OMPC_untied: + case OMPC_threadprivate: + case OMPC_depend: + case OMPC_mergeable: + llvm_unreachable("Clause is not allowed in 'omp atomic'."); + } +} + +void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) { + bool IsSeqCst = S.getSingleClause(/*K=*/OMPC_seq_cst); + OpenMPClauseKind Kind = OMPC_unknown; + for (auto *C : S.clauses()) { + // Find first clause (skip seq_cst clause, if it is first). + if (C->getClauseKind() != OMPC_seq_cst) { + Kind = C->getClauseKind(); + break; + } + } + + const auto *CS = + S.getAssociatedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true); + if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS)) { + enterFullExpression(EWC); + } + // Processing for statements under 'atomic capture'. + if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) { + for (const auto *C : Compound->body()) { + if (const auto *EWC = dyn_cast<ExprWithCleanups>(C)) { + enterFullExpression(EWC); + } + } + } + + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S, Kind, IsSeqCst](CodeGenFunction &CGF) { + EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(), + S.getV(), S.getExpr(), S.getUpdateExpr(), + S.isXLHSInRHSPart(), S.getLocStart()); + }; + CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen); } void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) { @@ -703,3 +2095,23 @@ void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &) { llvm_unreachable("CodeGen for 'omp teams' is not supported yet."); } +void CodeGenFunction::EmitOMPCancellationPointDirective( + const OMPCancellationPointDirective &S) { + CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getLocStart(), + S.getCancelRegion()); +} + +void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) { + CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), + S.getCancelRegion()); +} + +CodeGenFunction::JumpDest +CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) { + if (Kind == OMPD_parallel || Kind == OMPD_task) + return ReturnBlock; + else if (Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections) + return BreakContinueStack.empty() ? JumpDest() + : BreakContinueStack.back().BreakBlock; + return JumpDest(); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp index bd280ea..e3df5a4 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTT.cpp @@ -18,7 +18,7 @@ using namespace clang; using namespace CodeGen; -static llvm::Constant * +static llvm::GlobalVariable * GetAddrOfVTTVTable(CodeGenVTables &CGVT, CodeGenModule &CGM, const CXXRecordDecl *MostDerivedClass, const VTTVTable &VTable, @@ -47,8 +47,8 @@ CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT, llvm::Type *Int8PtrTy = CGM.Int8PtrTy, *Int64Ty = CGM.Int64Ty; llvm::ArrayType *ArrayType = llvm::ArrayType::get(Int8PtrTy, Builder.getVTTComponents().size()); - - SmallVector<llvm::Constant *, 8> VTables; + + SmallVector<llvm::GlobalVariable *, 8> VTables; SmallVector<VTableAddressPointsMapTy, 8> VTableAddressPoints; for (const VTTVTable *i = Builder.getVTTVTables().begin(), *e = Builder.getVTTVTables().end(); i != e; ++i) { @@ -61,7 +61,7 @@ CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT, for (const VTTComponent *i = Builder.getVTTComponents().begin(), *e = Builder.getVTTComponents().end(); i != e; ++i) { const VTTVTable &VTTVT = Builder.getVTTVTables()[i->VTableIndex]; - llvm::Constant *VTable = VTables[i->VTableIndex]; + llvm::GlobalVariable *VTable = VTables[i->VTableIndex]; uint64_t AddressPoint; if (VTTVT.getBase() == RD) { // Just get the address point for the regular vtable. @@ -79,8 +79,8 @@ CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT, llvm::ConstantInt::get(Int64Ty, AddressPoint) }; - llvm::Constant *Init = - llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, Idxs); + llvm::Constant *Init = llvm::ConstantExpr::getInBoundsGetElementPtr( + VTable->getValueType(), VTable, Idxs); Init = llvm::ConstantExpr::getBitCast(Init, Int8PtrTy); @@ -94,6 +94,9 @@ CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT, // Set the correct linkage. VTT->setLinkage(Linkage); + if (CGM.supportsCOMDAT() && VTT->isWeakForLinker()) + VTT->setComdat(CGM.getModule().getOrInsertComdat(VTT->getName())); + // Set the right visibility. CGM.setGlobalVisibility(VTT, RD); } @@ -174,4 +177,3 @@ CodeGenVTables::getSecondaryVirtualPointerIndex(const CXXRecordDecl *RD, return I->second; } - diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp index acb2a56..1b7f1d7 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGVTables.cpp @@ -56,6 +56,21 @@ static void setThunkVisibility(CodeGenModule &CGM, const CXXMethodDecl *MD, CGM.setGlobalVisibility(Fn, MD); } +static void setThunkProperties(CodeGenModule &CGM, const ThunkInfo &Thunk, + llvm::Function *ThunkFn, bool ForVTable, + GlobalDecl GD) { + CGM.setFunctionLinkage(GD, ThunkFn); + CGM.getCXXABI().setThunkLinkage(ThunkFn, ForVTable, GD, + !Thunk.Return.isEmpty()); + + // Set the right visibility. + const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl()); + setThunkVisibility(CGM, MD, Thunk, ThunkFn); + + if (CGM.supportsCOMDAT() && ThunkFn->isWeakForLinker()) + ThunkFn->setComdat(CGM.getModule().getOrInsertComdat(ThunkFn->getName())); +} + #ifndef NDEBUG static bool similar(const ABIArgInfo &infoL, CanQualType typeL, const ABIArgInfo &infoR, CanQualType typeR) { @@ -123,8 +138,8 @@ static RValue PerformReturnAdjustment(CodeGenFunction &CGF, // no-op thunk for the regular definition) call va_start/va_end. // There's a bit of per-call overhead for this solution, but it's // better for codesize if the definition is long. -void CodeGenFunction::GenerateVarArgsThunk( - llvm::Function *Fn, +llvm::Function * +CodeGenFunction::GenerateVarArgsThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo, GlobalDecl GD, const ThunkInfo &Thunk) { const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl()); @@ -184,6 +199,8 @@ void CodeGenFunction::GenerateVarArgsThunk( } } } + + return Fn; } void CodeGenFunction::StartThunk(llvm::Function *Fn, GlobalDecl GD, @@ -362,7 +379,7 @@ void CodeGenFunction::EmitMustTailThunk(const CXXMethodDecl *MD, FinishFunction(); } -void CodeGenFunction::GenerateThunk(llvm::Function *Fn, +void CodeGenFunction::generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo, GlobalDecl GD, const ThunkInfo &Thunk) { StartThunk(Fn, GD, FnInfo); @@ -374,13 +391,6 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn, // Make the call and return the result. EmitCallAndReturnForThunk(Callee, &Thunk); - - // Set the right linkage. - CGM.setFunctionLinkage(GD, Fn); - - // Set the right visibility. - const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl()); - setThunkVisibility(CGM, MD, Thunk, Fn); } void CodeGenVTables::emitThunk(GlobalDecl GD, const ThunkInfo &Thunk, @@ -434,8 +444,7 @@ void CodeGenVTables::emitThunk(GlobalDecl GD, const ThunkInfo &Thunk, return; } - // Change the linkage. - CGM.setFunctionLinkage(GD, ThunkFn); + setThunkProperties(CGM, Thunk, ThunkFn, ForVTable, GD); return; } @@ -447,17 +456,16 @@ void CodeGenVTables::emitThunk(GlobalDecl GD, const ThunkInfo &Thunk, // expensive/sucky at the moment, so don't generate the thunk unless // we have to. // FIXME: Do something better here; GenerateVarArgsThunk is extremely ugly. - if (!UseAvailableExternallyLinkage) { - CodeGenFunction(CGM).GenerateVarArgsThunk(ThunkFn, FnInfo, GD, Thunk); - CGM.getCXXABI().setThunkLinkage(ThunkFn, ForVTable, GD, - !Thunk.Return.isEmpty()); - } + if (UseAvailableExternallyLinkage) + return; + ThunkFn = + CodeGenFunction(CGM).GenerateVarArgsThunk(ThunkFn, FnInfo, GD, Thunk); } else { // Normal thunk body generation. - CodeGenFunction(CGM).GenerateThunk(ThunkFn, FnInfo, GD, Thunk); - CGM.getCXXABI().setThunkLinkage(ThunkFn, ForVTable, GD, - !Thunk.Return.isEmpty()); + CodeGenFunction(CGM).generateThunk(ThunkFn, FnInfo, GD, Thunk); } + + setThunkProperties(CGM, Thunk, ThunkFn, ForVTable, GD); } void CodeGenVTables::maybeEmitThunkForVTable(GlobalDecl GD, @@ -666,6 +674,8 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD, VTLayout->getNumVTableThunks(), RTTI); VTable->setInitializer(Init); + CGM.EmitVTableBitSetEntries(VTable, *VTLayout.get()); + return VTable; } @@ -697,7 +707,7 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { llvm::Function::InternalLinkage; return llvm::GlobalVariable::ExternalLinkage; - + case TSK_ImplicitInstantiation: return !Context.getLangOpts().AppleKext ? llvm::GlobalVariable::LinkOnceODRLinkage : @@ -738,7 +748,7 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { return DiscardableODRLinkage; case TSK_ExplicitInstantiationDeclaration: - llvm_unreachable("Should not have been asked to emit this"); + return llvm::GlobalVariable::ExternalLinkage; case TSK_ExplicitInstantiationDefinition: return NonDiscardableODRLinkage; @@ -747,19 +757,13 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { llvm_unreachable("Invalid TemplateSpecializationKind!"); } -/// This is a callback from Sema to tell us that it believes that a -/// particular v-table is required to be emitted in this translation -/// unit. -/// -/// The reason we don't simply trust this callback is because Sema -/// will happily report that something is used even when it's used -/// only in code that we don't actually have to emit. +/// This is a callback from Sema to tell us that that a particular v-table is +/// required to be emitted in this translation unit. /// -/// \param isRequired - if true, the v-table is mandatory, e.g. -/// because the translation unit defines the key function -void CodeGenModule::EmitVTable(CXXRecordDecl *theClass, bool isRequired) { - if (!isRequired) return; - +/// This is only called for vtables that _must_ be emitted (mainly due to key +/// functions). For weak vtables, CodeGen tracks when they are needed and +/// emits them as-needed. +void CodeGenModule::EmitVTable(CXXRecordDecl *theClass) { VTables.GenerateClassData(theClass); } @@ -840,3 +844,58 @@ void CodeGenModule::EmitDeferredVTables() { "deferred extra v-tables during v-table emission?"); DeferredVTables.clear(); } + +bool CodeGenModule::IsCFIBlacklistedRecord(const CXXRecordDecl *RD) { + // FIXME: Make this user configurable. + return RD->isInStdNamespace(); +} + +void CodeGenModule::EmitVTableBitSetEntries(llvm::GlobalVariable *VTable, + const VTableLayout &VTLayout) { + if (!LangOpts.Sanitize.has(SanitizerKind::CFIVCall) && + !LangOpts.Sanitize.has(SanitizerKind::CFINVCall) && + !LangOpts.Sanitize.has(SanitizerKind::CFIDerivedCast) && + !LangOpts.Sanitize.has(SanitizerKind::CFIUnrelatedCast)) + return; + + CharUnits PointerWidth = + Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0)); + + std::vector<llvm::MDTuple *> BitsetEntries; + // Create a bit set entry for each address point. + for (auto &&AP : VTLayout.getAddressPoints()) { + if (IsCFIBlacklistedRecord(AP.first.getBase())) + continue; + + BitsetEntries.push_back(CreateVTableBitSetEntry( + VTable, PointerWidth * AP.second, AP.first.getBase())); + } + + // Sort the bit set entries for determinism. + std::sort(BitsetEntries.begin(), BitsetEntries.end(), [](llvm::MDTuple *T1, + llvm::MDTuple *T2) { + if (T1 == T2) + return false; + + StringRef S1 = cast<llvm::MDString>(T1->getOperand(0))->getString(); + StringRef S2 = cast<llvm::MDString>(T2->getOperand(0))->getString(); + if (S1 < S2) + return true; + if (S1 != S2) + return false; + + uint64_t Offset1 = cast<llvm::ConstantInt>( + cast<llvm::ConstantAsMetadata>(T1->getOperand(2)) + ->getValue())->getZExtValue(); + uint64_t Offset2 = cast<llvm::ConstantInt>( + cast<llvm::ConstantAsMetadata>(T2->getOperand(2)) + ->getValue())->getZExtValue(); + assert(Offset1 != Offset2); + return Offset1 < Offset2; + }); + + llvm::NamedMDNode *BitsetsMD = + getModule().getOrInsertNamedMetadata("llvm.bitsets"); + for (auto BitsetEntry : BitsetEntries) + BitsetsMD->addOperand(BitsetEntry); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h b/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h index 82cd949..9205591 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CGValue.h @@ -19,6 +19,7 @@ #include "clang/AST/CharUnits.h" #include "clang/AST/Type.h" #include "llvm/IR/Value.h" +#include "llvm/IR/Type.h" namespace llvm { class Constant; @@ -299,6 +300,7 @@ public: LValue R; R.LVType = Simple; + assert(address->getType()->isPointerTy()); R.V = address; R.Initialize(type, qs, alignment, TBAAInfo); return R; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp index 12189ae..755e8aa 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenABITypes.cpp @@ -20,6 +20,8 @@ #include "CodeGenModule.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/CodeGenOptions.h" +#include "clang/Lex/HeaderSearchOptions.h" +#include "clang/Lex/PreprocessorOptions.h" using namespace clang; using namespace CodeGen; @@ -29,7 +31,10 @@ CodeGenABITypes::CodeGenABITypes(ASTContext &C, const llvm::DataLayout &TD, CoverageSourceInfo *CoverageInfo) : CGO(new CodeGenOptions), - CGM(new CodeGen::CodeGenModule(C, *CGO, M, TD, C.getDiagnostics(), + HSO(new HeaderSearchOptions), + PPO(new PreprocessorOptions), + CGM(new CodeGen::CodeGenModule(C, *HSO, *PPO, *CGO, + M, TD, C.getDiagnostics(), CoverageInfo)) { } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp index a6f6fde..0e7b6d8 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenAction.cpp @@ -46,7 +46,7 @@ namespace clang { const CodeGenOptions &CodeGenOpts; const TargetOptions &TargetOpts; const LangOptions &LangOpts; - raw_ostream *AsmOutStream; + raw_pwrite_stream *AsmOutStream; ASTContext *Context; Timer LLVMIRGeneration; @@ -56,18 +56,20 @@ namespace clang { std::unique_ptr<llvm::Module> TheModule, LinkModule; public: - BackendConsumer(BackendAction action, DiagnosticsEngine &_Diags, - const CodeGenOptions &compopts, - const TargetOptions &targetopts, - const LangOptions &langopts, bool TimePasses, - const std::string &infile, llvm::Module *LinkModule, - raw_ostream *OS, LLVMContext &C, + BackendConsumer(BackendAction Action, DiagnosticsEngine &Diags, + const HeaderSearchOptions &HeaderSearchOpts, + const PreprocessorOptions &PPOpts, + const CodeGenOptions &CodeGenOpts, + const TargetOptions &TargetOpts, + const LangOptions &LangOpts, bool TimePasses, + const std::string &InFile, llvm::Module *LinkModule, + raw_pwrite_stream *OS, LLVMContext &C, CoverageSourceInfo *CoverageInfo = nullptr) - : Diags(_Diags), Action(action), CodeGenOpts(compopts), - TargetOpts(targetopts), LangOpts(langopts), AsmOutStream(OS), + : Diags(Diags), Action(Action), CodeGenOpts(CodeGenOpts), + TargetOpts(TargetOpts), LangOpts(LangOpts), AsmOutStream(OS), Context(nullptr), LLVMIRGeneration("LLVM IR Generation Time"), - Gen(CreateLLVMCodeGen(Diags, infile, compopts, - targetopts, C, CoverageInfo)), + Gen(CreateLLVMCodeGen(Diags, InFile, HeaderSearchOpts, PPOpts, + CodeGenOpts, C, CoverageInfo)), LinkModule(LinkModule) { llvm::TimePassesIsEnabled = TimePasses; } @@ -80,6 +82,11 @@ namespace clang { } void Initialize(ASTContext &Ctx) override { + if (Context) { + assert(Context == &Ctx); + return; + } + Context = &Ctx; if (llvm::TimePassesIsEnabled) @@ -196,8 +203,8 @@ namespace clang { Gen->CompleteTentativeDefinition(D); } - void HandleVTable(CXXRecordDecl *RD, bool DefinitionRequired) override { - Gen->HandleVTable(RD, DefinitionRequired); + void HandleVTable(CXXRecordDecl *RD) override { + Gen->HandleVTable(RD); } void HandleLinkerOptionPragma(llvm::StringRef Opts) override { @@ -430,13 +437,16 @@ void BackendConsumer::EmitOptimizationMessage( FileManager &FileMgr = SourceMgr.getFileManager(); StringRef Filename; unsigned Line, Column; - D.getLocation(&Filename, &Line, &Column); SourceLocation DILoc; - const FileEntry *FE = FileMgr.getFile(Filename); - if (FE && Line > 0) { - // If -gcolumn-info was not used, Column will be 0. This upsets the - // source manager, so pass 1 if Column is not set. - DILoc = SourceMgr.translateFileLineCol(FE, Line, Column ? Column : 1); + + if (D.isLocationAvailable()) { + D.getLocation(&Filename, &Line, &Column); + const FileEntry *FE = FileMgr.getFile(Filename); + if (FE && Line > 0) { + // If -gcolumn-info was not used, Column will be 0. This upsets the + // source manager, so pass 1 if Column is not set. + DILoc = SourceMgr.translateFileLineCol(FE, Line, Column ? Column : 1); + } } // If a location isn't available, try to approximate it using the associated @@ -451,7 +461,7 @@ void BackendConsumer::EmitOptimizationMessage( << AddFlagValue(D.getPassName() ? D.getPassName() : "") << D.getMsg().str(); - if (DILoc.isInvalid()) + if (DILoc.isInvalid() && D.isLocationAvailable()) // If we were not able to translate the file:line:col information // back to a SourceLocation, at least emit a note stating that // we could not translate this location. This can happen in the @@ -602,9 +612,8 @@ llvm::LLVMContext *CodeGenAction::takeLLVMContext() { return VMContext; } -static raw_ostream *GetOutputStream(CompilerInstance &CI, - StringRef InFile, - BackendAction Action) { +static raw_pwrite_stream * +GetOutputStream(CompilerInstance &CI, StringRef InFile, BackendAction Action) { switch (Action) { case Backend_EmitAssembly: return CI.createDefaultOutputFile(false, InFile, "s"); @@ -626,7 +635,7 @@ static raw_ostream *GetOutputStream(CompilerInstance &CI, std::unique_ptr<ASTConsumer> CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { BackendAction BA = static_cast<BackendAction>(Act); - std::unique_ptr<raw_ostream> OS(GetOutputStream(CI, InFile, BA)); + raw_pwrite_stream *OS = GetOutputStream(CI, InFile, BA); if (BA != Backend_EmitNothing && !OS) return nullptr; @@ -643,14 +652,14 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { return nullptr; } - ErrorOr<llvm::Module *> ModuleOrErr = + ErrorOr<std::unique_ptr<llvm::Module>> ModuleOrErr = getLazyBitcodeModule(std::move(*BCBuf), *VMContext); if (std::error_code EC = ModuleOrErr.getError()) { CI.getDiagnostics().Report(diag::err_cannot_open_file) << LinkBCFile << EC.message(); return nullptr; } - LinkModuleToUse = ModuleOrErr.get(); + LinkModuleToUse = ModuleOrErr.get().release(); } CoverageSourceInfo *CoverageInfo = nullptr; @@ -661,19 +670,26 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { std::unique_ptr<PPCallbacks>(CoverageInfo)); } std::unique_ptr<BackendConsumer> Result(new BackendConsumer( - BA, CI.getDiagnostics(), CI.getCodeGenOpts(), CI.getTargetOpts(), + BA, CI.getDiagnostics(), CI.getHeaderSearchOpts(), + CI.getPreprocessorOpts(), CI.getCodeGenOpts(), CI.getTargetOpts(), CI.getLangOpts(), CI.getFrontendOpts().ShowTimers, InFile, - LinkModuleToUse, OS.release(), *VMContext, CoverageInfo)); + LinkModuleToUse, OS, *VMContext, CoverageInfo)); BEConsumer = Result.get(); return std::move(Result); } +static void BitcodeInlineAsmDiagHandler(const llvm::SMDiagnostic &SM, + void *Context, + unsigned LocCookie) { + SM.print(nullptr, llvm::errs()); +} + void CodeGenAction::ExecuteAction() { // If this is an IR file, we have to treat it specially. if (getCurrentFileKind() == IK_LLVM_IR) { BackendAction BA = static_cast<BackendAction>(Act); CompilerInstance &CI = getCompilerInstance(); - raw_ostream *OS = GetOutputStream(CI, getCurrentFile(), BA); + raw_pwrite_stream *OS = GetOutputStream(CI, getCurrentFile(), BA); if (BA != Backend_EmitNothing && !OS) return; @@ -710,14 +726,14 @@ void CodeGenAction::ExecuteAction() { } const TargetOptions &TargetOpts = CI.getTargetOpts(); if (TheModule->getTargetTriple() != TargetOpts.Triple) { - unsigned DiagID = CI.getDiagnostics().getCustomDiagID( - DiagnosticsEngine::Warning, - "overriding the module target triple with %0"); - - CI.getDiagnostics().Report(SourceLocation(), DiagID) << TargetOpts.Triple; + CI.getDiagnostics().Report(SourceLocation(), + diag::warn_fe_override_module) + << TargetOpts.Triple; TheModule->setTargetTriple(TargetOpts.Triple); } + LLVMContext &Ctx = TheModule->getContext(); + Ctx.setInlineAsmDiagnosticHandler(BitcodeInlineAsmDiagHandler); EmitBackendOutput(CI.getDiagnostics(), CI.getCodeGenOpts(), TargetOpts, CI.getLangOpts(), CI.getTarget().getTargetDescription(), TheModule.get(), BA, OS); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp index 826171a..ec3c75c 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "CodeGenFunction.h" +#include "CGCleanup.h" #include "CGCUDARuntime.h" #include "CGCXXABI.h" #include "CGDebugInfo.h" @@ -40,15 +41,16 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) CurFn(nullptr), CapturedStmtInfo(nullptr), SanOpts(CGM.getLangOpts().Sanitize), IsSanitizerScope(false), CurFuncIsThunk(false), AutoreleaseResult(false), SawAsmBlock(false), - BlockInfo(nullptr), BlockPointer(nullptr), + IsOutlinedSEHHelper(false), BlockInfo(nullptr), BlockPointer(nullptr), LambdaThisCaptureField(nullptr), NormalCleanupDest(nullptr), NextCleanupDestIndex(1), FirstBlockInfo(nullptr), EHResumeBlock(nullptr), ExceptionSlot(nullptr), EHSelectorSlot(nullptr), - DebugInfo(CGM.getModuleDebugInfo()), DisableDebugInfo(false), - DidCallStackSave(false), IndirectBranch(nullptr), PGO(cgm), - SwitchInsn(nullptr), SwitchWeights(nullptr), CaseRangeBlock(nullptr), - UnreachableBlock(nullptr), NumReturnExprs(0), NumSimpleReturnExprs(0), - CXXABIThisDecl(nullptr), CXXABIThisValue(nullptr), CXXThisValue(nullptr), + DebugInfo(CGM.getModuleDebugInfo()), + DisableDebugInfo(false), DidCallStackSave(false), IndirectBranch(nullptr), + PGO(cgm), SwitchInsn(nullptr), SwitchWeights(nullptr), + CaseRangeBlock(nullptr), UnreachableBlock(nullptr), NumReturnExprs(0), + NumSimpleReturnExprs(0), CXXABIThisDecl(nullptr), + CXXABIThisValue(nullptr), CXXThisValue(nullptr), CXXDefaultInitExprThis(nullptr), CXXStructorImplicitParamDecl(nullptr), CXXStructorImplicitParamValue(nullptr), OutermostConditional(nullptr), CurLexicalScope(nullptr), TerminateLandingPad(nullptr), @@ -69,6 +71,9 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) if (CGM.getCodeGenOpts().NoSignedZeros) { FMF.setNoSignedZeros(); } + if (CGM.getCodeGenOpts().ReciprocalMath) { + FMF.setAllowReciprocal(); + } Builder.SetFastMathFlags(FMF); } @@ -82,7 +87,7 @@ CodeGenFunction::~CodeGenFunction() { destroyBlockInfos(FirstBlockInfo); if (getLangOpts().OpenMP) { - CGM.getOpenMPRuntime().FunctionFinished(*this); + CGM.getOpenMPRuntime().functionFinished(*this); } } @@ -239,17 +244,18 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { // parameters. Do this in whatever block we're currently in; it's // important to do this before we enter the return block or return // edges will be *really* confused. - bool EmitRetDbgLoc = true; - if (EHStack.stable_begin() != PrologueCleanupDepth) { - PopCleanupBlocks(PrologueCleanupDepth); - + bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth; + bool HasOnlyLifetimeMarkers = + HasCleanups && EHStack.containsOnlyLifetimeMarkers(PrologueCleanupDepth); + bool EmitRetDbgLoc = !HasCleanups || HasOnlyLifetimeMarkers; + if (HasCleanups) { // Make sure the line table doesn't jump back into the body for // the ret after it's been at EndLoc. - EmitRetDbgLoc = false; - if (CGDebugInfo *DI = getDebugInfo()) if (OnlySimpleReturnStmts) DI->EmitLocation(Builder, EndLoc); + + PopCleanupBlocks(PrologueCleanupDepth); } // Emit function epilog (to return). @@ -278,6 +284,20 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { Builder.ClearInsertionPoint(); } + // If some of our locals escaped, insert a call to llvm.localescape in the + // entry block. + if (!EscapedLocals.empty()) { + // Invert the map from local to index into a simple vector. There should be + // no holes. + SmallVector<llvm::Value *, 4> EscapeArgs; + EscapeArgs.resize(EscapedLocals.size()); + for (auto &Pair : EscapedLocals) + EscapeArgs[Pair.second] = Pair.first; + llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration( + &CGM.getModule(), llvm::Intrinsic::localescape); + CGBuilderTy(AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs); + } + // Remove the AllocaInsertPt instruction, which is just a convenience for us. llvm::Instruction *Ptr = AllocaInsertPt; AllocaInsertPt = nullptr; @@ -588,6 +608,22 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, if (CGM.isInSanitizerBlacklist(Fn, Loc)) SanOpts.clear(); + if (D) { + // Apply the no_sanitize* attributes to SanOpts. + for (auto Attr : D->specific_attrs<NoSanitizeAttr>()) + SanOpts.Mask &= ~Attr->getMask(); + } + + // Apply sanitizer attributes to the function. + if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress)) + Fn->addFnAttr(llvm::Attribute::SanitizeAddress); + if (SanOpts.has(SanitizerKind::Thread)) + Fn->addFnAttr(llvm::Attribute::SanitizeThread); + if (SanOpts.has(SanitizerKind::Memory)) + Fn->addFnAttr(llvm::Attribute::SanitizeMemory); + if (SanOpts.has(SanitizerKind::SafeStack)) + Fn->addFnAttr(llvm::Attribute::SafeStack); + // Pass inline keyword to optimizer if it appears explicitly on any // declaration. Also, in the case of -fno-inline attach NoInline // attribute to all function that are not marked AlwaysInline. @@ -679,7 +715,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex(); llvm::Function::arg_iterator EI = CurFn->arg_end(); --EI; - llvm::Value *Addr = Builder.CreateStructGEP(EI, Idx); + llvm::Value *Addr = Builder.CreateStructGEP(nullptr, EI, Idx); ReturnValue = Builder.CreateLoad(Addr, "agg.result"); } else { ReturnValue = CreateIRTemp(RetTy, "retval"); @@ -753,8 +789,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, void CodeGenFunction::EmitFunctionBody(FunctionArgList &Args, const Stmt *Body) { - RegionCounter Cnt = getPGORegionCounter(Body); - Cnt.beginRegion(Builder); + incrementProfileCounter(Body); if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body)) EmitCompoundStmtWithoutScope(*S); else @@ -766,7 +801,7 @@ void CodeGenFunction::EmitFunctionBody(FunctionArgList &Args, /// emit a branch around the instrumentation code. When not instrumenting, /// this just calls EmitBlock(). void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB, - RegionCounter &Cnt) { + const Stmt *S) { llvm::BasicBlock *SkipCountBB = nullptr; if (HaveInsertPoint() && CGM.getCodeGenOpts().ProfileInstrGenerate) { // When instrumenting for profiling, the fallthrough to certain @@ -776,7 +811,9 @@ void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB, EmitBranch(SkipCountBB); } EmitBlock(BB); - Cnt.beginRegion(Builder, /*AddIncomingFallThrough=*/true); + uint64_t CurrentCount = getCurrentProfileCount(); + incrementProfileCounter(S); + setCurrentProfileCount(getCurrentProfileCount() + CurrentCount); if (SkipCountBB) EmitBlock(SkipCountBB); } @@ -801,17 +838,6 @@ static void TryMarkNoThrow(llvm::Function *F) { F->setDoesNotThrow(); } -static void EmitSizedDeallocationFunction(CodeGenFunction &CGF, - const FunctionDecl *UnsizedDealloc) { - // This is a weak discardable definition of the sized deallocation function. - CGF.CurFn->setLinkage(llvm::Function::LinkOnceAnyLinkage); - - // Call the unsized deallocation function and forward the first argument - // unchanged. - llvm::Constant *Unsized = CGF.CGM.GetAddrOfFunction(UnsizedDealloc); - CGF.Builder.CreateCall(Unsized, &*CGF.CurFn->arg_begin()); -} - void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, const CGFunctionInfo &FnInfo) { const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); @@ -832,7 +858,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, ResTy = CGM.getContext().VoidPtrTy; CGM.getCXXABI().buildThisParam(*this, Args); } - + Args.append(FD->param_begin(), FD->param_end()); if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD))) @@ -866,9 +892,9 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, else if (isa<CXXConstructorDecl>(FD)) EmitConstructorBody(Args); else if (getLangOpts().CUDA && - !CGM.getCodeGenOpts().CUDAIsDevice && + !getLangOpts().CUDAIsDevice && FD->hasAttr<CUDAGlobalAttr>()) - CGM.getCUDARuntime().EmitDeviceStubBody(*this, Args); + CGM.getCUDARuntime().emitDeviceStub(*this, Args); else if (isa<CXXConversionDecl>(FD) && cast<CXXConversionDecl>(FD)->isLambdaToBlockPointerConversion()) { // The lambda conversion to block pointer is special; the semantics can't be @@ -887,11 +913,6 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, emitImplicitAssignmentOperatorBody(Args); } else if (Stmt *Body = FD->getBody()) { EmitFunctionBody(Args, Body); - } else if (FunctionDecl *UnsizedDealloc = - FD->getCorrespondingUnsizedGlobalDeallocationFunction()) { - // Global sized deallocation functions get an implicit weak definition if - // they don't have an explicit definition. - EmitSizedDeallocationFunction(*this, UnsizedDealloc); } else llvm_unreachable("no definition for emitted function"); @@ -909,8 +930,9 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return), "missing_return", EmitCheckSourceLocation(FD->getLocation()), None); - } else if (CGM.getCodeGenOpts().OptimizationLevel == 0) - Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::trap)); + } else if (CGM.getCodeGenOpts().OptimizationLevel == 0) { + EmitTrapCall(llvm::Intrinsic::trap); + } Builder.CreateUnreachable(); Builder.ClearInsertionPoint(); } @@ -949,8 +971,8 @@ bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) { IgnoreCaseStmts = true; // Scan subexpressions for verboten labels. - for (Stmt::const_child_range I = S->children(); I; ++I) - if (ContainsLabel(*I, IgnoreCaseStmts)) + for (const Stmt *SubStmt : S->children()) + if (ContainsLabel(SubStmt, IgnoreCaseStmts)) return true; return false; @@ -973,8 +995,8 @@ bool CodeGenFunction::containsBreak(const Stmt *S) { return true; // Scan subexpressions for verboten breaks. - for (Stmt::const_child_range I = S->children(); I; ++I) - if (containsBreak(*I)) + for (const Stmt *SubStmt : S->children()) + if (containsBreak(SubStmt)) return true; return false; @@ -1028,15 +1050,13 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, // Handle X && Y in a condition. if (CondBOp->getOpcode() == BO_LAnd) { - RegionCounter Cnt = getPGORegionCounter(CondBOp); - // If we have "1 && X", simplify the code. "0 && X" would have constant // folded if the case was simple enough. bool ConstantBool = false; if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) && ConstantBool) { // br(1 && X) -> br(X). - Cnt.beginRegion(Builder); + incrementProfileCounter(CondBOp); return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, TrueCount); } @@ -1055,14 +1075,19 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true"); // The counter tells us how often we evaluate RHS, and all of TrueCount // can be propagated to that branch. - uint64_t RHSCount = Cnt.getCount(); + uint64_t RHSCount = getProfileCount(CondBOp->getRHS()); ConditionalEvaluation eval(*this); - EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount); - EmitBlock(LHSTrue); + { + ApplyDebugLocation DL(*this, Cond); + EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount); + EmitBlock(LHSTrue); + } + + incrementProfileCounter(CondBOp); + setCurrentProfileCount(getProfileCount(CondBOp->getRHS())); // Any temporaries created here are conditional. - Cnt.beginRegion(Builder); eval.begin(*this); EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, TrueCount); eval.end(*this); @@ -1071,15 +1096,13 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, } if (CondBOp->getOpcode() == BO_LOr) { - RegionCounter Cnt = getPGORegionCounter(CondBOp); - // If we have "0 || X", simplify the code. "1 || X" would have constant // folded if the case was simple enough. bool ConstantBool = false; if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) && !ConstantBool) { // br(0 || X) -> br(X). - Cnt.beginRegion(Builder); + incrementProfileCounter(CondBOp); return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, TrueCount); } @@ -1099,15 +1122,21 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, // We have the count for entry to the RHS and for the whole expression // being true, so we can divy up True count between the short circuit and // the RHS. - uint64_t LHSCount = Cnt.getParentCount() - Cnt.getCount(); + uint64_t LHSCount = + getCurrentProfileCount() - getProfileCount(CondBOp->getRHS()); uint64_t RHSCount = TrueCount - LHSCount; ConditionalEvaluation eval(*this); - EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount); - EmitBlock(LHSFalse); + { + ApplyDebugLocation DL(*this, Cond); + EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount); + EmitBlock(LHSFalse); + } + + incrementProfileCounter(CondBOp); + setCurrentProfileCount(getProfileCount(CondBOp->getRHS())); // Any temporaries created here are conditional. - Cnt.beginRegion(Builder); eval.begin(*this); EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, RHSCount); @@ -1121,7 +1150,7 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, // br(!x, t, f) -> br(x, f, t) if (CondUOp->getOpcode() == UO_LNot) { // Negate the count. - uint64_t FalseCount = PGO.getCurrentRegionCount() - TrueCount; + uint64_t FalseCount = getCurrentProfileCount() - TrueCount; // Negate the condition and swap the destination blocks. return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock, FalseCount); @@ -1133,9 +1162,9 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true"); llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false"); - RegionCounter Cnt = getPGORegionCounter(CondOp); ConditionalEvaluation cond(*this); - EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock, Cnt.getCount()); + EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock, + getProfileCount(CondOp)); // When computing PGO branch weights, we only know the overall count for // the true block. This code is essentially doing tail duplication of the @@ -1144,15 +1173,19 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, // the conditional operator. uint64_t LHSScaledTrueCount = 0; if (TrueCount) { - double LHSRatio = Cnt.getCount() / (double) Cnt.getParentCount(); + double LHSRatio = + getProfileCount(CondOp) / (double)getCurrentProfileCount(); LHSScaledTrueCount = TrueCount * LHSRatio; } cond.begin(*this); EmitBlock(LHSBlock); - Cnt.beginRegion(Builder); - EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock, - LHSScaledTrueCount); + incrementProfileCounter(CondOp); + { + ApplyDebugLocation DL(*this, Cond); + EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock, + LHSScaledTrueCount); + } cond.end(*this); cond.begin(*this); @@ -1176,12 +1209,16 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, // Create branch weights based on the number of times we get here and the // number of times the condition should be true. - uint64_t CurrentCount = std::max(PGO.getCurrentRegionCount(), TrueCount); - llvm::MDNode *Weights = PGO.createBranchWeights(TrueCount, - CurrentCount - TrueCount); + uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount); + llvm::MDNode *Weights = + createProfileWeights(TrueCount, CurrentCount - TrueCount); // Emit the code with the fully general case. - llvm::Value *CondV = EvaluateExprAsBool(Cond); + llvm::Value *CondV; + { + ApplyDebugLocation DL(*this, Cond); + CondV = EvaluateExprAsBool(Cond); + } Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights); } @@ -1231,7 +1268,8 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, /*volatile*/ false); // Go to the next element. - llvm::Value *next = Builder.CreateConstInBoundsGEP1_32(cur, 1, "vla.next"); + llvm::Value *next = Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), + cur, 1, "vla.next"); // Leave if that's the end of the VLA. llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone"); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h index 3a990d2..f2bc402 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenFunction.h @@ -27,6 +27,7 @@ #include "clang/AST/Type.h" #include "clang/Basic/ABI.h" #include "clang/Basic/CapturedStmt.h" +#include "clang/Basic/OpenMPKinds.h" #include "clang/Basic/TargetInfo.h" #include "clang/Frontend/CodeGenOptions.h" #include "llvm/ADT/ArrayRef.h" @@ -96,8 +97,8 @@ enum TypeEvaluationKind { /// CodeGenFunction - This class organizes the per-function state that is used /// while generating LLVM code. class CodeGenFunction : public CodeGenTypeCache { - CodeGenFunction(const CodeGenFunction &) LLVM_DELETED_FUNCTION; - void operator=(const CodeGenFunction &) LLVM_DELETED_FUNCTION; + CodeGenFunction(const CodeGenFunction &) = delete; + void operator=(const CodeGenFunction &) = delete; friend class CGCXXABI; public: @@ -191,26 +192,25 @@ public: CapturedRegionKind getKind() const { return Kind; } - void setContextValue(llvm::Value *V) { ThisValue = V; } + virtual void setContextValue(llvm::Value *V) { ThisValue = V; } // \brief Retrieve the value of the context parameter. - llvm::Value *getContextValue() const { return ThisValue; } + virtual llvm::Value *getContextValue() const { return ThisValue; } /// \brief Lookup the captured field decl for a variable. - const FieldDecl *lookup(const VarDecl *VD) const { + virtual const FieldDecl *lookup(const VarDecl *VD) const { return CaptureFields.lookup(VD); } - bool isCXXThisExprCaptured() const { return CXXThisFieldDecl != nullptr; } - FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; } + bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; } + virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; } static bool classof(const CGCapturedStmtInfo *) { return true; } /// \brief Emit the captured statement body. - virtual void EmitBody(CodeGenFunction &CGF, Stmt *S) { - RegionCounter Cnt = CGF.getPGORegionCounter(S); - Cnt.beginRegion(CGF.Builder); + virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) { + CGF.incrementProfileCounter(S); CGF.EmitStmt(S); } @@ -233,6 +233,20 @@ public: }; CGCapturedStmtInfo *CapturedStmtInfo; + /// \brief RAII for correct setting/restoring of CapturedStmtInfo. + class CGCapturedStmtRAII { + private: + CodeGenFunction &CGF; + CGCapturedStmtInfo *PrevCapturedStmtInfo; + public: + CGCapturedStmtRAII(CodeGenFunction &CGF, + CGCapturedStmtInfo *NewCapturedStmtInfo) + : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) { + CGF.CapturedStmtInfo = NewCapturedStmtInfo; + } + ~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; } + }; + /// BoundsChecking - Emit run-time bounds checks. Higher values mean /// potentially higher performance penalties. unsigned char BoundsChecking; @@ -262,6 +276,10 @@ public: /// potentially set the return value. bool SawAsmBlock; + /// True if the current function is an outlined SEH helper. This can be a + /// finally block or filter expression. + bool IsOutlinedSEHHelper; + const CodeGen::CGBlockInfo *BlockInfo; llvm::Value *BlockPointer; @@ -274,16 +292,17 @@ public: EHScopeStack EHStack; llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack; + llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack; /// Header for data within LifetimeExtendedCleanupStack. struct LifetimeExtendedCleanupHeader { /// The size of the following cleanup object. - unsigned Size : 29; + unsigned Size; /// The kind of cleanup to push: a value from the CleanupKind enumeration. - unsigned Kind : 3; + CleanupKind Kind; - size_t getSize() const { return size_t(Size); } - CleanupKind getKind() const { return static_cast<CleanupKind>(Kind); } + size_t getSize() const { return Size; } + CleanupKind getKind() const { return Kind; } }; /// i32s containing the indexes of the cleanup destinations. @@ -305,6 +324,14 @@ public: /// write the current selector value into this alloca. llvm::AllocaInst *EHSelectorSlot; + /// A stack of exception code slots. Entering an __except block pushes a slot + /// on the stack and leaving pops one. The __exception_code() intrinsic loads + /// a value from the top of the stack. + SmallVector<llvm::Value *, 1> SEHCodeSlotStack; + + /// Value returned by __exception_info intrinsic. + llvm::Value *SEHInfo = nullptr; + /// Emits a landing pad for the current EH stack. llvm::BasicBlock *EmitLandingPad(); @@ -343,87 +370,32 @@ public: void exit(CodeGenFunction &CGF); }; - /// pushFullExprCleanup - Push a cleanup to be run at the end of the - /// current full-expression. Safe against the possibility that - /// we're currently inside a conditionally-evaluated expression. - template <class T, class A0> - void pushFullExprCleanup(CleanupKind kind, A0 a0) { - // If we're not in a conditional branch, or if none of the - // arguments requires saving, then use the unconditional cleanup. - if (!isInConditionalBranch()) - return EHStack.pushCleanup<T>(kind, a0); - - typename DominatingValue<A0>::saved_type a0_saved = saveValueInCond(a0); - - typedef EHScopeStack::ConditionalCleanup1<T, A0> CleanupType; - EHStack.pushCleanup<CleanupType>(kind, a0_saved); - initFullExprCleanup(); - } + /// Returns true inside SEH __try blocks. + bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); } /// pushFullExprCleanup - Push a cleanup to be run at the end of the /// current full-expression. Safe against the possibility that /// we're currently inside a conditionally-evaluated expression. - template <class T, class A0, class A1> - void pushFullExprCleanup(CleanupKind kind, A0 a0, A1 a1) { + template <class T, class... As> + void pushFullExprCleanup(CleanupKind kind, As... A) { // If we're not in a conditional branch, or if none of the // arguments requires saving, then use the unconditional cleanup. if (!isInConditionalBranch()) - return EHStack.pushCleanup<T>(kind, a0, a1); - - typename DominatingValue<A0>::saved_type a0_saved = saveValueInCond(a0); - typename DominatingValue<A1>::saved_type a1_saved = saveValueInCond(a1); - - typedef EHScopeStack::ConditionalCleanup2<T, A0, A1> CleanupType; - EHStack.pushCleanup<CleanupType>(kind, a0_saved, a1_saved); - initFullExprCleanup(); - } + return EHStack.pushCleanup<T>(kind, A...); - /// pushFullExprCleanup - Push a cleanup to be run at the end of the - /// current full-expression. Safe against the possibility that - /// we're currently inside a conditionally-evaluated expression. - template <class T, class A0, class A1, class A2> - void pushFullExprCleanup(CleanupKind kind, A0 a0, A1 a1, A2 a2) { - // If we're not in a conditional branch, or if none of the - // arguments requires saving, then use the unconditional cleanup. - if (!isInConditionalBranch()) { - return EHStack.pushCleanup<T>(kind, a0, a1, a2); - } - - typename DominatingValue<A0>::saved_type a0_saved = saveValueInCond(a0); - typename DominatingValue<A1>::saved_type a1_saved = saveValueInCond(a1); - typename DominatingValue<A2>::saved_type a2_saved = saveValueInCond(a2); - - typedef EHScopeStack::ConditionalCleanup3<T, A0, A1, A2> CleanupType; - EHStack.pushCleanup<CleanupType>(kind, a0_saved, a1_saved, a2_saved); - initFullExprCleanup(); - } + // Stash values in a tuple so we can guarantee the order of saves. + typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple; + SavedTuple Saved{saveValueInCond(A)...}; - /// pushFullExprCleanup - Push a cleanup to be run at the end of the - /// current full-expression. Safe against the possibility that - /// we're currently inside a conditionally-evaluated expression. - template <class T, class A0, class A1, class A2, class A3> - void pushFullExprCleanup(CleanupKind kind, A0 a0, A1 a1, A2 a2, A3 a3) { - // If we're not in a conditional branch, or if none of the - // arguments requires saving, then use the unconditional cleanup. - if (!isInConditionalBranch()) { - return EHStack.pushCleanup<T>(kind, a0, a1, a2, a3); - } - - typename DominatingValue<A0>::saved_type a0_saved = saveValueInCond(a0); - typename DominatingValue<A1>::saved_type a1_saved = saveValueInCond(a1); - typename DominatingValue<A2>::saved_type a2_saved = saveValueInCond(a2); - typename DominatingValue<A3>::saved_type a3_saved = saveValueInCond(a3); - - typedef EHScopeStack::ConditionalCleanup4<T, A0, A1, A2, A3> CleanupType; - EHStack.pushCleanup<CleanupType>(kind, a0_saved, a1_saved, - a2_saved, a3_saved); + typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType; + EHStack.pushCleanupTuple<CleanupType>(kind, Saved); initFullExprCleanup(); } /// \brief Queue a cleanup to be pushed after finishing the current /// full-expression. - template <class T, class A0, class A1, class A2, class A3> - void pushCleanupAfterFullExpr(CleanupKind Kind, A0 a0, A1 a1, A2 a2, A3 a3) { + template <class T, class... As> + void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) { assert(!isInConditionalBranch() && "can't defer conditional cleanup"); LifetimeExtendedCleanupHeader Header = { sizeof(T), Kind }; @@ -432,9 +404,11 @@ public: LifetimeExtendedCleanupStack.resize( LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size); + static_assert(sizeof(Header) % llvm::AlignOf<T>::Alignment == 0, + "Cleanup will be allocated on misaligned address"); char *Buffer = &LifetimeExtendedCleanupStack[OldSize]; new (Buffer) LifetimeExtendedCleanupHeader(Header); - new (Buffer + sizeof(Header)) T(a0, a1, a2, a3); + new (Buffer + sizeof(Header)) T(A...); } /// Set up the last cleaup that was pushed as a conditional @@ -488,8 +462,8 @@ public: bool PerformCleanup; private: - RunCleanupsScope(const RunCleanupsScope &) LLVM_DELETED_FUNCTION; - void operator=(const RunCleanupsScope &) LLVM_DELETED_FUNCTION; + RunCleanupsScope(const RunCleanupsScope &) = delete; + void operator=(const RunCleanupsScope &) = delete; protected: CodeGenFunction& CGF; @@ -537,8 +511,8 @@ public: SmallVector<const LabelDecl*, 4> Labels; LexicalScope *ParentScope; - LexicalScope(const LexicalScope &) LLVM_DELETED_FUNCTION; - void operator=(const LexicalScope &) LLVM_DELETED_FUNCTION; + LexicalScope(const LexicalScope &) = delete; + void operator=(const LexicalScope &) = delete; public: /// \brief Enter a new cleanup scope. @@ -562,7 +536,10 @@ public: // If we should perform a cleanup, force them now. Note that // this ends the cleanup scope before rescoping any labels. - if (PerformCleanup) ForceCleanup(); + if (PerformCleanup) { + ApplyDebugLocation DL(CGF, Range.getEnd()); + ForceCleanup(); + } } /// \brief Force the emission of cleanups now, instead of waiting @@ -587,8 +564,8 @@ public: VarDeclMapTy SavedPrivates; private: - OMPPrivateScope(const OMPPrivateScope &) LLVM_DELETED_FUNCTION; - void operator=(const OMPPrivateScope &) LLVM_DELETED_FUNCTION; + OMPPrivateScope(const OMPPrivateScope &) = delete; + void operator=(const OMPPrivateScope &) = delete; public: /// \brief Enter a new OpenMP private scope. @@ -637,7 +614,10 @@ public: } /// \brief Exit scope - all the mapped variables are restored. - ~OMPPrivateScope() { ForceCleanup(); } + ~OMPPrivateScope() { + if (PerformCleanup) + ForceCleanup(); + } }; /// \brief Takes the old cleanup stack size and emits the cleanup blocks @@ -881,7 +861,8 @@ public: /// getByrefValueFieldNumber - Given a declaration, returns the LLVM field /// number that holds the value. - unsigned getByRefValueLLVMField(const ValueDecl *VD) const; + std::pair<llvm::Type *, unsigned> + getByRefValueLLVMField(const ValueDecl *VD) const; /// BuildBlockByrefAddress - Computes address location of the /// variable which is declared as __block. @@ -906,6 +887,10 @@ private: typedef llvm::DenseMap<const Decl*, llvm::Value*> DeclMapTy; DeclMapTy LocalDeclMap; + /// Track escaped local variables with auto storage. Used during SEH + /// outlining to produce a call to llvm.localescape. + llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals; + /// LabelMap - This keeps track of the LLVM basic block for each C label. llvm::DenseMap<const LabelDecl*, JumpDest> LabelMap; @@ -922,12 +907,39 @@ private: CodeGenPGO PGO; + /// Calculate branch weights appropriate for PGO data + llvm::MDNode *createProfileWeights(uint64_t TrueCount, uint64_t FalseCount); + llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights); + llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond, + uint64_t LoopCount); + public: - /// Get a counter for instrumentation of the region associated with the given - /// statement. - RegionCounter getPGORegionCounter(const Stmt *S) { - return RegionCounter(PGO, S); + /// Increment the profiler's counter for the given statement. + void incrementProfileCounter(const Stmt *S) { + if (CGM.getCodeGenOpts().ProfileInstrGenerate) + PGO.emitCounterIncrement(Builder, S); + PGO.setCurrentStmt(S); + } + + /// Get the profiler's count for the given statement. + uint64_t getProfileCount(const Stmt *S) { + Optional<uint64_t> Count = PGO.getStmtCount(S); + if (!Count.hasValue()) + return 0; + return *Count; + } + + /// Set the profiler's current count. + void setCurrentProfileCount(uint64_t Count) { + PGO.setCurrentRegionCount(Count); } + + /// Get the profiler's current count. This is generally the count for the most + /// recently incremented counter. + uint64_t getCurrentProfileCount() { + return PGO.getCurrentRegionCount(); + } + private: /// SwitchInsn - This is nearest current switch instruction. It is null if @@ -1097,6 +1109,11 @@ public: return getInvokeDestImpl(); } + bool currentFunctionUsesSEHTry() const { + const auto *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl); + return FD && FD->usesSEHTry(); + } + const TargetInfo &getTarget() const { return Target; } llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); } @@ -1248,7 +1265,7 @@ public: void EmitDestructorBody(FunctionArgList &Args); void emitImplicitAssignmentOperatorBody(FunctionArgList &Args); void EmitFunctionBody(FunctionArgList &Args, const Stmt *Body); - void EmitBlockWithFallThrough(llvm::BasicBlock *BB, RegionCounter &Cnt); + void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S); void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator, CallArgList &CallArgs); @@ -1258,15 +1275,18 @@ public: void EmitLambdaStaticInvokeFunction(const CXXMethodDecl *MD); void EmitAsanPrologueOrEpilogue(bool Prologue); - /// EmitReturnBlock - Emit the unified return block, trying to avoid its - /// emission when possible. + /// \brief Emit the unified return block, trying to avoid its emission when + /// possible. + /// \return The debug location of the user written return statement if the + /// return block is is avoided. llvm::DebugLoc EmitReturnBlock(); /// FinishFunction - Complete IR generation of the current function. It is /// legal to call this function even if there is no current insertion point. void FinishFunction(SourceLocation EndLoc=SourceLocation()); - void StartThunk(llvm::Function *Fn, GlobalDecl GD, const CGFunctionInfo &FnInfo); + void StartThunk(llvm::Function *Fn, GlobalDecl GD, + const CGFunctionInfo &FnInfo); void EmitCallAndReturnForThunk(llvm::Value *Callee, const ThunkInfo *Thunk); @@ -1274,12 +1294,13 @@ public: void EmitMustTailThunk(const CXXMethodDecl *MD, llvm::Value *AdjustedThisPtr, llvm::Value *Callee); - /// GenerateThunk - Generate a thunk for the given method. - void GenerateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo, + /// Generate a thunk for the given method. + void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo, GlobalDecl GD, const ThunkInfo &Thunk); - void GenerateVarArgsThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo, - GlobalDecl GD, const ThunkInfo &Thunk); + llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn, + const CGFunctionInfo &FnInfo, + GlobalDecl GD, const ThunkInfo &Thunk); void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type, FunctionArgList &Args); @@ -1309,6 +1330,29 @@ public: /// to by This. llvm::Value *GetVTablePtr(llvm::Value *This, llvm::Type *Ty); + enum CFITypeCheckKind { + CFITCK_VCall, + CFITCK_NVCall, + CFITCK_DerivedCast, + CFITCK_UnrelatedCast, + }; + + /// \brief Derived is the presumed address of an object of type T after a + /// cast. If T is a polymorphic class type, emit a check that the virtual + /// table for Derived belongs to a class derived from T. + void EmitVTablePtrCheckForCast(QualType T, llvm::Value *Derived, + bool MayBeNull, CFITypeCheckKind TCK, + SourceLocation Loc); + + /// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable. + /// If vptr CFI is enabled, emit a check that VTable is valid. + void EmitVTablePtrCheckForCall(const CXXMethodDecl *MD, llvm::Value *VTable, + CFITypeCheckKind TCK, SourceLocation Loc); + + /// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for + /// RD using llvm.bitset.test. + void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable, + CFITypeCheckKind TCK, SourceLocation Loc); /// CanDevirtualizeMemberFunctionCalls - Checks whether virtual calls on given /// expr can be devirtualized. @@ -1527,6 +1571,8 @@ public: void EmitAnyExprToMem(const Expr *E, llvm::Value *Location, Qualifiers Quals, bool IsInitializer); + void EmitAnyExprToExn(const Expr *E, llvm::Value *Addr); + /// EmitExprAsInit - Emits the code necessary to initialize a /// location in memory with the given initializer. void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, @@ -1552,6 +1598,15 @@ public: true); } + void EmitAggregateCopyCtor(llvm::Value *DestPtr, llvm::Value *SrcPtr, + QualType DestTy, QualType SrcTy) { + CharUnits DestTypeAlign = getContext().getTypeAlignInChars(DestTy); + CharUnits SrcTypeAlign = getContext().getTypeAlignInChars(SrcTy); + EmitAggregateCopy(DestPtr, SrcPtr, SrcTy, /*IsVolatile=*/false, + std::min(DestTypeAlign, SrcTypeAlign), + /*IsAssignment=*/false); + } + /// EmitAggregateCopy - Emit an aggregate copy. /// /// \param isVolatile - True iff either the source or the destination is @@ -1723,12 +1778,16 @@ public: llvm::Value *This); void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType, - llvm::Value *NewPtr, llvm::Value *NumElements, + llvm::Type *ElementTy, llvm::Value *NewPtr, + llvm::Value *NumElements, llvm::Value *AllocSizeWithoutCookie); void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType, llvm::Value *Ptr); + llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr); + void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr); + llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E); void EmitCXXDeleteExpr(const CXXDeleteExpr *E); @@ -1889,8 +1948,8 @@ public: llvm::Value *getObjectAddress(CodeGenFunction &CGF) const { if (!IsByRef) return Address; - return CGF.Builder.CreateStructGEP(Address, - CGF.getByRefValueLLVMField(Variable), + auto F = CGF.getByRefValueLLVMField(Variable); + return CGF.Builder.CreateStructGEP(F.first, Address, F.second, Variable->getNameAsString()); } }; @@ -1988,6 +2047,40 @@ public: void EmitCXXTryStmt(const CXXTryStmt &S); void EmitSEHTryStmt(const SEHTryStmt &S); void EmitSEHLeaveStmt(const SEHLeaveStmt &S); + void EnterSEHTryStmt(const SEHTryStmt &S); + void ExitSEHTryStmt(const SEHTryStmt &S); + + void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter, + const Stmt *OutlinedStmt); + + llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF, + const SEHExceptStmt &Except); + + llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF, + const SEHFinallyStmt &Finally); + + void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF, + llvm::Value *ParentFP, + llvm::Value *EntryEBP); + llvm::Value *EmitSEHExceptionCode(); + llvm::Value *EmitSEHExceptionInfo(); + llvm::Value *EmitSEHAbnormalTermination(); + + /// Scan the outlined statement for captures from the parent function. For + /// each capture, mark the capture as escaped and emit a call to + /// llvm.localrecover. Insert the localrecover result into the LocalDeclMap. + void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt, + bool IsFilter); + + /// Recovers the address of a local in a parent function. ParentVar is the + /// address of the variable used in the immediate parent function. It can + /// either be an alloca or a call to llvm.localrecover if there are nested + /// outlined functions. ParentFP is the frame pointer of the outermost parent + /// frame. + llvm::Value *recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, + llvm::Value *ParentVar, + llvm::Value *ParentFP); + void EmitCXXForRangeStmt(const CXXForRangeStmt &S, ArrayRef<const Attr *> Attrs = None); @@ -1997,13 +2090,107 @@ public: llvm::Function *GenerateCapturedStmtFunctionEpilog(const CapturedStmt &S); llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S); llvm::Value *GenerateCapturedStmtArgument(const CapturedStmt &S); - void EmitOMPAggregateAssign(LValue OriginalAddr, llvm::Value *PrivateAddr, - const Expr *AssignExpr, QualType Type, - const VarDecl *VDInit); - void EmitOMPFirstprivateClause(const OMPExecutableDirective &D, + /// \brief Perform element by element copying of arrays with type \a + /// OriginalType from \a SrcAddr to \a DestAddr using copying procedure + /// generated by \a CopyGen. + /// + /// \param DestAddr Address of the destination array. + /// \param SrcAddr Address of the source array. + /// \param OriginalType Type of destination and source arrays. + /// \param CopyGen Copying procedure that copies value of single array element + /// to another single array element. + void EmitOMPAggregateAssign( + llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType, + const llvm::function_ref<void(llvm::Value *, llvm::Value *)> &CopyGen); + /// \brief Emit proper copying of data from one variable to another. + /// + /// \param OriginalType Original type of the copied variables. + /// \param DestAddr Destination address. + /// \param SrcAddr Source address. + /// \param DestVD Destination variable used in \a CopyExpr (for arrays, has + /// type of the base array element). + /// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of + /// the base array element). + /// \param Copy Actual copygin expression for copying data from \a SrcVD to \a + /// DestVD. + void EmitOMPCopy(CodeGenFunction &CGF, QualType OriginalType, + llvm::Value *DestAddr, llvm::Value *SrcAddr, + const VarDecl *DestVD, const VarDecl *SrcVD, + const Expr *Copy); + /// \brief Emit atomic update code for constructs: \a X = \a X \a BO \a E or + /// \a X = \a E \a BO \a E. + /// + /// \param X Value to be updated. + /// \param E Update value. + /// \param BO Binary operation for update operation. + /// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update + /// expression, false otherwise. + /// \param AO Atomic ordering of the generated atomic instructions. + /// \param CommonGen Code generator for complex expressions that cannot be + /// expressed through atomicrmw instruction. + /// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was + /// generated, <false, RValue::get(nullptr)> otherwise. + std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr( + LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, + llvm::AtomicOrdering AO, SourceLocation Loc, + const llvm::function_ref<RValue(RValue)> &CommonGen); + bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope); void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope); + /// \brief Emit code for copyin clause in \a D directive. The next code is + /// generated at the start of outlined functions for directives: + /// \code + /// threadprivate_var1 = master_threadprivate_var1; + /// operator=(threadprivate_var2, master_threadprivate_var2); + /// ... + /// __kmpc_barrier(&loc, global_tid); + /// \endcode + /// + /// \param D OpenMP directive possibly with 'copyin' clause(s). + /// \returns true if at least one copyin variable is found, false otherwise. + bool EmitOMPCopyinClause(const OMPExecutableDirective &D); + /// \brief Emit initial code for lastprivate variables. If some variable is + /// not also firstprivate, then the default initialization is used. Otherwise + /// initialization of this variable is performed by EmitOMPFirstprivateClause + /// method. + /// + /// \param D Directive that may have 'lastprivate' directives. + /// \param PrivateScope Private scope for capturing lastprivate variables for + /// proper codegen in internal captured statement. + /// + /// \returns true if there is at least one lastprivate variable, false + /// otherwise. + bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, + OMPPrivateScope &PrivateScope); + /// \brief Emit final copying of lastprivate values to original variables at + /// the end of the worksharing or simd directive. + /// + /// \param D Directive that has at least one 'lastprivate' directives. + /// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if + /// it is the last iteration of the loop code in associated directive, or to + /// 'i1 false' otherwise. If this item is nullptr, no final check is required. + void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, + llvm::Value *IsLastIterCond = nullptr); + /// \brief Emit initial code for reduction variables. Creates reduction copies + /// and initializes them with the values according to OpenMP standard. + /// + /// \param D Directive (possibly) with the 'reduction' clause. + /// \param PrivateScope Private scope for capturing reduction variables for + /// proper codegen in internal captured statement. + /// + void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, + OMPPrivateScope &PrivateScope); + /// \brief Emit final update of reduction values to original variables at + /// the end of the directive. + /// + /// \param D Directive that has at least one 'reduction' directives. + void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D); + /// \brief Emit initial code for linear variables. Creates private copies + /// and initializes them with the values according to OpenMP standard. + /// + /// \param D Directive (possibly) with the 'linear' clause. + void EmitOMPLinearClauseInit(const OMPLoopDirective &D); void EmitOMPParallelDirective(const OMPParallelDirective &S); void EmitOMPSimdDirective(const OMPSimdDirective &S); @@ -2021,21 +2208,51 @@ public: void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S); void EmitOMPBarrierDirective(const OMPBarrierDirective &S); void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S); + void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S); void EmitOMPFlushDirective(const OMPFlushDirective &S); void EmitOMPOrderedDirective(const OMPOrderedDirective &S); void EmitOMPAtomicDirective(const OMPAtomicDirective &S); void EmitOMPTargetDirective(const OMPTargetDirective &S); void EmitOMPTeamsDirective(const OMPTeamsDirective &S); + void + EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S); + void EmitOMPCancelDirective(const OMPCancelDirective &S); + + /// \brief Emit inner loop of the worksharing/simd construct. + /// + /// \param S Directive, for which the inner loop must be emitted. + /// \param RequiresCleanup true, if directive has some associated private + /// variables. + /// \param LoopCond Bollean condition for loop continuation. + /// \param IncExpr Increment expression for loop control variable. + /// \param BodyGen Generator for the inner body of the inner loop. + /// \param PostIncGen Genrator for post-increment code (required for ordered + /// loop directvies). + void EmitOMPInnerLoop( + const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, + const Expr *IncExpr, + const llvm::function_ref<void(CodeGenFunction &)> &BodyGen, + const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen); + + JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind); private: /// Helpers for the OpenMP loop directives. - void EmitOMPLoopBody(const OMPLoopDirective &Directive, - bool SeparateIter = false); - void EmitOMPInnerLoop(const OMPLoopDirective &S, OMPPrivateScope &LoopScope, - bool SeparateIter = false); - void EmitOMPSimdFinal(const OMPLoopDirective &S); - void EmitOMPWorksharingLoop(const OMPLoopDirective &S); + void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit); + void EmitOMPSimdInit(const OMPLoopDirective &D); + void EmitOMPSimdFinal(const OMPLoopDirective &D); + /// \brief Emit code for the worksharing loop-based directive. + /// \return true, if this construct has any lastprivate clause, false - + /// otherwise. + bool EmitOMPWorksharingLoop(const OMPLoopDirective &S); + void EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind, + const OMPLoopDirective &S, + OMPPrivateScope &LoopScope, bool Ordered, + llvm::Value *LB, llvm::Value *UB, llvm::Value *ST, + llvm::Value *IL, llvm::Value *Chunk); + /// \brief Emit code for sections directive. + OpenMPDirectiveKind EmitSections(const OMPExecutableDirective &S); public: @@ -2085,17 +2302,31 @@ public: void EmitAtomicInit(Expr *E, LValue lvalue); + bool LValueIsSuitableForInlineAtomic(LValue Src); + bool typeIsSuitableForInlineAtomic(QualType Ty, bool IsVolatile) const; + + RValue EmitAtomicLoad(LValue LV, SourceLocation SL, + AggValueSlot Slot = AggValueSlot::ignored()); + RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc, + llvm::AtomicOrdering AO, bool IsVolatile = false, AggValueSlot slot = AggValueSlot::ignored()); void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit); - std::pair<RValue, RValue> EmitAtomicCompareExchange( + void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO, + bool IsVolatile, bool isInit); + + std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange( LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success = llvm::SequentiallyConsistent, llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent, bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored()); + void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, + const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile); + /// EmitToMemory - Change a scalar value from its value /// representation to its in-memory representation. llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty); @@ -2163,8 +2394,8 @@ public: /// Emit an l-value for an assignment (simple or compound) of complex type. LValue EmitComplexAssignmentLValue(const BinaryOperator *E); LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E); - LValue EmitScalarCompooundAssignWithComplex(const CompoundAssignOperator *E, - llvm::Value *&Result); + LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E, + llvm::Value *&Result); // Note: only available for agg return types LValue EmitBinaryOperatorLValue(const BinaryOperator *E); @@ -2208,7 +2439,7 @@ public: return ConstantEmission(C, false); } - LLVM_EXPLICIT operator bool() const { + explicit operator bool() const { return ValueAndIsReference.getOpaqueValue() != nullptr; } @@ -2384,19 +2615,14 @@ public: // Helper functions for EmitAArch64BuiltinExpr. llvm::Value *vectorWrapScalar8(llvm::Value *Op); llvm::Value *vectorWrapScalar16(llvm::Value *Op); - llvm::Value *emitVectorWrappedScalar8Intrinsic( - unsigned Int, SmallVectorImpl<llvm::Value *> &Ops, const char *Name); - llvm::Value *emitVectorWrappedScalar16Intrinsic( - unsigned Int, SmallVectorImpl<llvm::Value *> &Ops, const char *Name); llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E); - llvm::Value *EmitNeon64Call(llvm::Function *F, - llvm::SmallVectorImpl<llvm::Value *> &O, - const char *name); llvm::Value *BuildVector(ArrayRef<llvm::Value*> Ops); llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E); - llvm::Value *EmitR600BuiltinExpr(unsigned BuiltinID, const CallExpr *E); + llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E); + llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E); + llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E); llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E); @@ -2655,7 +2881,7 @@ public: /// \brief Create a basic block that will call a handler function in a /// sanitizer runtime with the provided arguments, and create a conditional /// branch to it. - void EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerKind>> Checked, + void EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked, StringRef CheckName, ArrayRef<llvm::Constant *> StaticArgs, ArrayRef<llvm::Value *> DynamicArgs); @@ -2663,6 +2889,15 @@ public: /// conditional branch to it, for the -ftrapv checks. void EmitTrapCheck(llvm::Value *Checked); + /// \brief Emit a call to trap or debugtrap and attach function attribute + /// "trap-func-name" if specified. + llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID); + + /// \brief Create a check for a function parameter that may potentially be + /// declared as non-null. + void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, + const FunctionDecl *FD, unsigned ParmNum); + /// EmitCallArg - Emit a single call argument. void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType); @@ -2708,31 +2943,58 @@ private: SourceLocation Loc); public: +#ifndef NDEBUG + // Determine whether the given argument is an Objective-C method + // that may have type parameters in its signature. + static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) { + const DeclContext *dc = method->getDeclContext(); + if (const ObjCInterfaceDecl *classDecl= dyn_cast<ObjCInterfaceDecl>(dc)) { + return classDecl->getTypeParamListAsWritten(); + } + + if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) { + return catDecl->getTypeParamList(); + } + + return false; + } + + template<typename T> + static bool isObjCMethodWithTypeParams(const T *) { return false; } +#endif + /// EmitCallArgs - Emit call arguments for a function. template <typename T> void EmitCallArgs(CallArgList &Args, const T *CallArgTypeInfo, CallExpr::const_arg_iterator ArgBeg, CallExpr::const_arg_iterator ArgEnd, const FunctionDecl *CalleeDecl = nullptr, - unsigned ParamsToSkip = 0, bool ForceColumnInfo = false) { + unsigned ParamsToSkip = 0) { SmallVector<QualType, 16> ArgTypes; CallExpr::const_arg_iterator Arg = ArgBeg; assert((ParamsToSkip == 0 || CallArgTypeInfo) && "Can't skip parameters if type info is not provided"); if (CallArgTypeInfo) { +#ifndef NDEBUG + bool isGenericMethod = isObjCMethodWithTypeParams(CallArgTypeInfo); +#endif + // First, use the argument types that the type info knows about for (auto I = CallArgTypeInfo->param_type_begin() + ParamsToSkip, E = CallArgTypeInfo->param_type_end(); I != E; ++I, ++Arg) { assert(Arg != ArgEnd && "Running over edge of argument list!"); - assert( - ((*I)->isVariablyModifiedType() || - getContext() - .getCanonicalType((*I).getNonReferenceType()) - .getTypePtr() == - getContext().getCanonicalType(Arg->getType()).getTypePtr()) && - "type mismatch in call argument!"); + assert((isGenericMethod || + ((*I)->isVariablyModifiedType() || + (*I).getNonReferenceType()->isObjCRetainableType() || + getContext() + .getCanonicalType((*I).getNonReferenceType()) + .getTypePtr() == + getContext() + .getCanonicalType(Arg->getType()) + .getTypePtr())) && + "type mismatch in call argument!"); ArgTypes.push_back(*I); } } @@ -2747,15 +3009,14 @@ public: for (; Arg != ArgEnd; ++Arg) ArgTypes.push_back(getVarArgType(*Arg)); - EmitCallArgs(Args, ArgTypes, ArgBeg, ArgEnd, CalleeDecl, ParamsToSkip, - ForceColumnInfo); + EmitCallArgs(Args, ArgTypes, ArgBeg, ArgEnd, CalleeDecl, ParamsToSkip); } void EmitCallArgs(CallArgList &Args, ArrayRef<QualType> ArgTypes, CallExpr::const_arg_iterator ArgBeg, CallExpr::const_arg_iterator ArgEnd, const FunctionDecl *CalleeDecl = nullptr, - unsigned ParamsToSkip = 0, bool ForceColumnInfo = false); + unsigned ParamsToSkip = 0); private: QualType getVarArgType(const Expr *Arg); diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp index d653130..a179ad4 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.cpp @@ -74,16 +74,19 @@ static CGCXXABI *createCXXABI(CodeGenModule &CGM) { llvm_unreachable("invalid C++ ABI kind"); } -CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO, - llvm::Module &M, const llvm::DataLayout &TD, +CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, + const PreprocessorOptions &PPO, + const CodeGenOptions &CGO, llvm::Module &M, + const llvm::DataLayout &TD, DiagnosticsEngine &diags, CoverageSourceInfo *CoverageInfo) - : Context(C), LangOpts(C.getLangOpts()), CodeGenOpts(CGO), TheModule(M), - Diags(diags), TheDataLayout(TD), Target(C.getTargetInfo()), - ABI(createCXXABI(*this)), VMContext(M.getContext()), TBAA(nullptr), - TheTargetCodeGenInfo(nullptr), Types(*this), VTables(*this), - ObjCRuntime(nullptr), OpenCLRuntime(nullptr), OpenMPRuntime(nullptr), - CUDARuntime(nullptr), DebugInfo(nullptr), ARCData(nullptr), + : Context(C), LangOpts(C.getLangOpts()), HeaderSearchOpts(HSO), + PreprocessorOpts(PPO), CodeGenOpts(CGO), TheModule(M), Diags(diags), + TheDataLayout(TD), Target(C.getTargetInfo()), ABI(createCXXABI(*this)), + VMContext(M.getContext()), TBAA(nullptr), TheTargetCodeGenInfo(nullptr), + Types(*this), VTables(*this), ObjCRuntime(nullptr), + OpenCLRuntime(nullptr), OpenMPRuntime(nullptr), CUDARuntime(nullptr), + DebugInfo(nullptr), ARCData(nullptr), NoObjCARCExceptionsMetadata(nullptr), RRData(nullptr), PGOReader(nullptr), CFConstantStringClassRef(nullptr), ConstantStringClassRef(nullptr), NSConstantStringType(nullptr), NSConcreteGlobalBlock(nullptr), @@ -141,12 +144,15 @@ CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO, RRData = new RREntrypoints(); if (!CodeGenOpts.InstrProfileInput.empty()) { - if (std::error_code EC = llvm::IndexedInstrProfReader::create( - CodeGenOpts.InstrProfileInput, PGOReader)) { + auto ReaderOrErr = + llvm::IndexedInstrProfReader::create(CodeGenOpts.InstrProfileInput); + if (std::error_code EC = ReaderOrErr.getError()) { unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error, - "Could not read profile: %0"); - getDiags().Report(DiagID) << EC.message(); - } + "Could not read profile %0: %1"); + getDiags().Report(DiagID) << CodeGenOpts.InstrProfileInput + << EC.message(); + } else + PGOReader = std::move(ReaderOrErr.get()); } // If coverage mapping generation is enabled, create the @@ -203,11 +209,9 @@ void CodeGenModule::addReplacement(StringRef Name, llvm::Constant *C) { } void CodeGenModule::applyReplacements() { - for (ReplacementsTy::iterator I = Replacements.begin(), - E = Replacements.end(); - I != E; ++I) { - StringRef MangledName = I->first(); - llvm::Constant *Replacement = I->second; + for (auto &I : Replacements) { + StringRef MangledName = I.first(); + llvm::Constant *Replacement = I.second; llvm::GlobalValue *Entry = GetGlobalValue(MangledName); if (!Entry) continue; @@ -259,9 +263,7 @@ void CodeGenModule::checkAliases() { // and aliases during codegen. bool Error = false; DiagnosticsEngine &Diags = getDiags(); - for (std::vector<GlobalDecl>::iterator I = Aliases.begin(), - E = Aliases.end(); I != E; ++I) { - const GlobalDecl &GD = *I; + for (const GlobalDecl &GD : Aliases) { const auto *D = cast<ValueDecl>(GD.getDecl()); const AliasAttr *AA = D->getAttr<AliasAttr>(); StringRef MangledName = getMangledName(GD); @@ -308,9 +310,7 @@ void CodeGenModule::checkAliases() { if (!Error) return; - for (std::vector<GlobalDecl>::iterator I = Aliases.begin(), - E = Aliases.end(); I != E; ++I) { - const GlobalDecl &GD = *I; + for (const GlobalDecl &GD : Aliases) { StringRef MangledName = getMangledName(GD); llvm::GlobalValue *Entry = GetGlobalValue(MangledName); auto *Alias = cast<llvm::GlobalAlias>(Entry); @@ -321,6 +321,8 @@ void CodeGenModule::checkAliases() { void CodeGenModule::clear() { DeferredDeclsToEmit.clear(); + if (OpenMPRuntime) + OpenMPRuntime->clear(); } void InstrProfStats::reportDiagnostics(DiagnosticsEngine &Diags, @@ -346,6 +348,13 @@ void CodeGenModule::Release() { if (ObjCRuntime) if (llvm::Function *ObjCInitFunction = ObjCRuntime->ModuleInitFunction()) AddGlobalCtor(ObjCInitFunction); + if (Context.getLangOpts().CUDA && !Context.getLangOpts().CUDAIsDevice && + CUDARuntime) { + if (llvm::Function *CudaCtorFunction = CUDARuntime->makeModuleCtorFunction()) + AddGlobalCtor(CudaCtorFunction); + if (llvm::Function *CudaDtorFunction = CUDARuntime->makeModuleDtorFunction()) + AddGlobalDtor(CudaDtorFunction); + } if (PGOReader && PGOStats.hasDiagnostics()) PGOStats.reportDiagnostics(getDiags(), getCodeGenOpts().MainFileName); EmitCtorList(GlobalCtors, "llvm.global_ctors"); @@ -626,15 +635,14 @@ void CodeGenModule::EmitCtorList(const CtorList &Fns, const char *GlobalName) { Int32Ty, llvm::PointerType::getUnqual(CtorFTy), VoidPtrTy, nullptr); // Construct the constructor and destructor arrays. - SmallVector<llvm::Constant*, 8> Ctors; - for (CtorList::const_iterator I = Fns.begin(), E = Fns.end(); I != E; ++I) { + SmallVector<llvm::Constant *, 8> Ctors; + for (const auto &I : Fns) { llvm::Constant *S[] = { - llvm::ConstantInt::get(Int32Ty, I->Priority, false), - llvm::ConstantExpr::getBitCast(I->Initializer, CtorPFTy), - (I->AssociatedData - ? llvm::ConstantExpr::getBitCast(I->AssociatedData, VoidPtrTy) - : llvm::Constant::getNullValue(VoidPtrTy)) - }; + llvm::ConstantInt::get(Int32Ty, I.Priority, false), + llvm::ConstantExpr::getBitCast(I.Initializer, CtorPFTy), + (I.AssociatedData + ? llvm::ConstantExpr::getBitCast(I.AssociatedData, VoidPtrTy) + : llvm::Constant::getNullValue(VoidPtrTy))}; Ctors.push_back(llvm::ConstantStruct::get(CtorStructTy, S)); } @@ -665,6 +673,25 @@ CodeGenModule::getFunctionLinkage(GlobalDecl GD) { return getLLVMLinkageForDeclarator(D, Linkage, /*isConstantVariable=*/false); } +void CodeGenModule::setFunctionDLLStorageClass(GlobalDecl GD, llvm::Function *F) { + const auto *FD = cast<FunctionDecl>(GD.getDecl()); + + if (const auto *Dtor = dyn_cast_or_null<CXXDestructorDecl>(FD)) { + if (getCXXABI().useThunkForDtorVariant(Dtor, GD.getDtorType())) { + // Don't dllexport/import destructor thunks. + F->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass); + return; + } + } + + if (FD->hasAttr<DLLImportAttr>()) + F->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass); + else if (FD->hasAttr<DLLExportAttr>()) + F->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass); + else + F->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass); +} + void CodeGenModule::setFunctionDefinitionAttributes(const FunctionDecl *D, llvm::Function *F) { setNonAliasAttributes(D, F); @@ -741,23 +768,6 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, else if (LangOpts.getStackProtector() == LangOptions::SSPReq) B.addAttribute(llvm::Attribute::StackProtectReq); - // Add sanitizer attributes if function is not blacklisted. - if (!isInSanitizerBlacklist(F, D->getLocation())) { - // When AddressSanitizer is enabled, set SanitizeAddress attribute - // unless __attribute__((no_sanitize_address)) is used. - if (LangOpts.Sanitize.has(SanitizerKind::Address) && - !D->hasAttr<NoSanitizeAddressAttr>()) - B.addAttribute(llvm::Attribute::SanitizeAddress); - // Same for ThreadSanitizer and __attribute__((no_sanitize_thread)) - if (LangOpts.Sanitize.has(SanitizerKind::Thread) && - !D->hasAttr<NoSanitizeThreadAttr>()) - B.addAttribute(llvm::Attribute::SanitizeThread); - // Same for MemorySanitizer and __attribute__((no_sanitize_memory)) - if (LangOpts.Sanitize.has(SanitizerKind::Memory) && - !D->hasAttr<NoSanitizeMemoryAttr>()) - B.addAttribute(llvm::Attribute::SanitizeMemory); - } - F->addAttributes(llvm::AttributeSet::FunctionIndex, llvm::AttributeSet::get( F->getContext(), llvm::AttributeSet::FunctionIndex, B)); @@ -823,7 +833,7 @@ void CodeGenModule::setNonAliasAttributes(const Decl *D, if (const SectionAttr *SA = D->getAttr<SectionAttr>()) GO->setSection(SA->getName()); - getTargetCodeGenInfo().SetTargetAttributes(D, GO, *this); + getTargetCodeGenInfo().setTargetAttributes(D, GO, *this); } void CodeGenModule::SetInternalFunctionAttributes(const Decl *D, @@ -865,11 +875,10 @@ static void setLinkageAndVisibilityForGV(llvm::GlobalValue *GV, void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, bool IsIncompleteFunction, bool IsThunk) { - if (unsigned IID = F->getIntrinsicID()) { + if (llvm::Intrinsic::ID IID = F->getIntrinsicID()) { // If this is an intrinsic function, set the function's attributes // to the intrinsic's attributes. - F->setAttributes(llvm::Intrinsic::getAttributes(getLLVMContext(), - (llvm::Intrinsic::ID)IID)); + F->setAttributes(llvm::Intrinsic::getAttributes(getLLVMContext(), IID)); return; } @@ -896,13 +905,6 @@ void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, setLinkageAndVisibilityForGV(F, FD); - if (const auto *Dtor = dyn_cast_or_null<CXXDestructorDecl>(FD)) { - if (getCXXABI().useThunkForDtorVariant(Dtor, GD.getDtorType())) { - // Don't dllexport/import destructor thunks. - F->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass); - } - } - if (const SectionAttr *SA = FD->getAttr<SectionAttr>()) F->setSection(SA->getName()); @@ -916,13 +918,13 @@ void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, void CodeGenModule::addUsedGlobal(llvm::GlobalValue *GV) { assert(!GV->isDeclaration() && "Only globals with definition can force usage."); - LLVMUsed.push_back(GV); + LLVMUsed.emplace_back(GV); } void CodeGenModule::addCompilerUsedGlobal(llvm::GlobalValue *GV) { assert(!GV->isDeclaration() && "Only globals with definition can force usage."); - LLVMCompilerUsed.push_back(GV); + LLVMCompilerUsed.emplace_back(GV); } static void emitUsed(CodeGenModule &CGM, StringRef Name, @@ -1024,12 +1026,9 @@ void CodeGenModule::EmitModuleLinkOptions() { SmallVector<clang::Module *, 16> Stack; // Seed the stack with imported modules. - for (llvm::SetVector<clang::Module *>::iterator M = ImportedModules.begin(), - MEnd = ImportedModules.end(); - M != MEnd; ++M) { - if (Visited.insert(*M).second) - Stack.push_back(*M); - } + for (Module *M : ImportedModules) + if (Visited.insert(M).second) + Stack.push_back(M); // Find all of the modules to import, making a little effort to prune // non-leaf modules. @@ -1065,12 +1064,9 @@ void CodeGenModule::EmitModuleLinkOptions() { // to linker options inserted by things like #pragma comment(). SmallVector<llvm::Metadata *, 16> MetadataArgs; Visited.clear(); - for (llvm::SetVector<clang::Module *>::iterator M = LinkModules.begin(), - MEnd = LinkModules.end(); - M != MEnd; ++M) { - if (Visited.insert(*M).second) - addLinkOptionsPostorder(*this, *M, MetadataArgs, Visited); - } + for (Module *M : LinkModules) + if (Visited.insert(M).second) + addLinkOptionsPostorder(*this, M, MetadataArgs, Visited); std::reverse(MetadataArgs.begin(), MetadataArgs.end()); LinkerOptionsMetadata.append(MetadataArgs.begin(), MetadataArgs.end()); @@ -1085,29 +1081,33 @@ void CodeGenModule::EmitDeferred() { // previously unused static decl may become used during the generation of code // for a static function, iterate until no changes are made. - while (true) { - if (!DeferredVTables.empty()) { - EmitDeferredVTables(); + if (!DeferredVTables.empty()) { + EmitDeferredVTables(); - // Emitting a v-table doesn't directly cause more v-tables to - // become deferred, although it can cause functions to be - // emitted that then need those v-tables. - assert(DeferredVTables.empty()); - } + // Emitting a v-table doesn't directly cause more v-tables to + // become deferred, although it can cause functions to be + // emitted that then need those v-tables. + assert(DeferredVTables.empty()); + } + + // Stop if we're out of both deferred v-tables and deferred declarations. + if (DeferredDeclsToEmit.empty()) + return; - // Stop if we're out of both deferred v-tables and deferred declarations. - if (DeferredDeclsToEmit.empty()) break; + // Grab the list of decls to emit. If EmitGlobalDefinition schedules more + // work, it will not interfere with this. + std::vector<DeferredGlobal> CurDeclsToEmit; + CurDeclsToEmit.swap(DeferredDeclsToEmit); - DeferredGlobal &G = DeferredDeclsToEmit.back(); + for (DeferredGlobal &G : CurDeclsToEmit) { GlobalDecl D = G.GD; llvm::GlobalValue *GV = G.GV; - DeferredDeclsToEmit.pop_back(); + G.GV = nullptr; assert(!GV || GV == GetGlobalValue(getMangledName(D))); if (!GV) GV = GetGlobalValue(getMangledName(D)); - // Check to see if we've already emitted this. This is necessary // for a couple of reasons: first, decls can end up in the // deferred-decls queue multiple times, and second, decls can end @@ -1119,6 +1119,14 @@ void CodeGenModule::EmitDeferred() { // Otherwise, emit the definition and move on to the next one. EmitGlobalDefinition(D, GV); + + // If we found out that we need to emit more decls, do that recursively. + // This has the advantage that the decls are emitted in a DFS and related + // ones are close together, which is convenient for testing. + if (!DeferredVTables.empty() || !DeferredDeclsToEmit.empty()) { + EmitDeferred(); + assert(DeferredVTables.empty() && DeferredDeclsToEmit.empty()); + } } } @@ -1214,8 +1222,9 @@ bool CodeGenModule::isInSanitizerBlacklist(llvm::Function *Fn, bool CodeGenModule::isInSanitizerBlacklist(llvm::GlobalVariable *GV, SourceLocation Loc, QualType Ty, StringRef Category) const { - // For now globals can be blacklisted only in ASan. - if (!LangOpts.Sanitize.has(SanitizerKind::Address)) + // For now globals can be blacklisted only in ASan and KASan. + if (!LangOpts.Sanitize.hasOneOf( + SanitizerKind::Address | SanitizerKind::KernelAddress)) return false; const auto &SanitizerBL = getContext().getSanitizerBlacklist(); if (SanitizerBL.isBlacklistedGlobal(GV->getName(), Category)) @@ -1253,6 +1262,11 @@ bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) { // Implicit template instantiations may change linkage if they are later // explicitly instantiated, so they should not be emitted eagerly. return false; + // If OpenMP is enabled and threadprivates must be generated like TLS, delay + // codegen for global variables, because they may be marked as threadprivate. + if (LangOpts.OpenMP && LangOpts.OpenMPUseTLS && + getContext().getTargetInfo().isTLSSupported() && isa<VarDecl>(Global)) + return false; return true; } @@ -1275,6 +1289,8 @@ llvm::Constant *CodeGenModule::GetAddrOfUuidDescriptor( auto *GV = new llvm::GlobalVariable( getModule(), Init->getType(), /*isConstant=*/true, llvm::GlobalValue::LinkOnceODRLinkage, Init, Name); + if (supportsCOMDAT()) + GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); return GV; } @@ -1322,7 +1338,7 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { // If this is CUDA, be selective about which declarations we emit. if (LangOpts.CUDA) { - if (CodeGenOpts.CUDAIsDevice) { + if (LangOpts.CUDAIsDevice) { if (!Global->hasAttr<CUDADeviceAttr>() && !Global->hasAttr<CUDAGlobalAttr>() && !Global->hasAttr<CUDAConstantAttr>() && @@ -1417,7 +1433,7 @@ namespace { return false; } unsigned BuiltinID = FD->getBuiltinID(); - if (!BuiltinID) + if (!BuiltinID || !BI.isLibFunction(BuiltinID)) return true; StringRef BuiltinName = BI.GetName(BuiltinID); if (BuiltinName.startswith("__builtin_") && @@ -1603,13 +1619,6 @@ CodeGenModule::GetOrCreateLLVMFunction(StringRef MangledName, addDeferredDeclToEmit(F, DDI->second); DeferredDecls.erase(DDI); - // Otherwise, if this is a sized deallocation function, emit a weak - // definition - // for it at the end of the translation unit. - } else if (D && cast<FunctionDecl>(D) - ->getCorrespondingUnsizedGlobalDeallocationFunction()) { - addDeferredDeclToEmit(F, GD); - // Otherwise, there are cases we have to worry about where we're // using a declaration for which we must emit a definition but where // we might not find a top-level definition: @@ -1766,6 +1775,8 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, // handling. GV->setConstant(isTypeConstant(D->getType(), false)); + GV->setAlignment(getContext().getDeclAlign(D).getQuantity()); + setLinkageAndVisibilityForGV(GV, D); if (D->getTLSKind()) { @@ -1829,7 +1840,11 @@ CodeGenModule::CreateOrReplaceCXXRuntimeVariable(StringRef Name, OldGV->eraseFromParent(); } - + + if (supportsCOMDAT() && GV->isWeakForLinker() && + !GV->hasAvailableExternallyLinkage()) + GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); + return GV; } @@ -1884,7 +1899,7 @@ CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const { unsigned CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D, unsigned AddrSpace) { - if (LangOpts.CUDA && CodeGenOpts.CUDAIsDevice) { + if (LangOpts.CUDA && LangOpts.CUDAIsDevice) { if (D->hasAttr<CUDAConstantAttr>()) AddrSpace = getContext().getTargetAddressSpace(LangAS::cuda_constant); else if (D->hasAttr<CUDASharedAttr>()) @@ -1929,6 +1944,38 @@ void CodeGenModule::MaybeHandleStaticInExternC(const SomeDecl *D, R.first->second = nullptr; } +static bool shouldBeInCOMDAT(CodeGenModule &CGM, const Decl &D) { + if (!CGM.supportsCOMDAT()) + return false; + + if (D.hasAttr<SelectAnyAttr>()) + return true; + + GVALinkage Linkage; + if (auto *VD = dyn_cast<VarDecl>(&D)) + Linkage = CGM.getContext().GetGVALinkageForVariable(VD); + else + Linkage = CGM.getContext().GetGVALinkageForFunction(cast<FunctionDecl>(&D)); + + switch (Linkage) { + case GVA_Internal: + case GVA_AvailableExternally: + case GVA_StrongExternal: + return false; + case GVA_DiscardableODR: + case GVA_StrongODR: + return true; + } + llvm_unreachable("No such linkage"); +} + +void CodeGenModule::maybeSetTrivialComdat(const Decl &D, + llvm::GlobalObject &GO) { + if (!shouldBeInCOMDAT(*this, D)) + return; + GO.setComdat(TheModule.getOrInsertComdat(GO.getName())); +} + void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) { llvm::Constant *Init = nullptr; QualType ASTTy = D->getType(); @@ -2072,6 +2119,8 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) { setTLSMode(GV, *D); } + maybeSetTrivialComdat(*D, *GV); + // Emit the initializer function if necessary. if (NeedsGlobalCtor || NeedsGlobalDtor) EmitCXXGlobalVarDeclInitFunc(D, GV, NeedsGlobalCtor); @@ -2085,7 +2134,8 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) { } static bool isVarDeclStrongDefinition(const ASTContext &Context, - const VarDecl *D, bool NoCommon) { + CodeGenModule &CGM, const VarDecl *D, + bool NoCommon) { // Don't give variables common linkage if -fno-common was specified unless it // was overridden by a NoCommon attribute. if ((NoCommon || D->hasAttr<NoCommonAttr>()) && !D->hasAttr<CommonAttr>()) @@ -2110,11 +2160,31 @@ static bool isVarDeclStrongDefinition(const ASTContext &Context, if (D->hasAttr<WeakImportAttr>()) return true; + // A variable cannot be both common and exist in a comdat. + if (shouldBeInCOMDAT(CGM, *D)) + return true; + // Declarations with a required alignment do not have common linakge in MSVC // mode. - if (Context.getLangOpts().MSVCCompat && - (Context.isAlignmentRequired(D->getType()) || D->hasAttr<AlignedAttr>())) - return true; + if (Context.getLangOpts().MSVCCompat) { + if (D->hasAttr<AlignedAttr>()) + return true; + QualType VarType = D->getType(); + if (Context.isAlignmentRequired(VarType)) + return true; + + if (const auto *RT = VarType->getAs<RecordType>()) { + const RecordDecl *RD = RT->getDecl(); + for (const FieldDecl *FD : RD->fields()) { + if (FD->isBitField()) + continue; + if (FD->hasAttr<AlignedAttr>()) + return true; + if (Context.isAlignmentRequired(FD->getType())) + return true; + } + } + } return false; } @@ -2162,7 +2232,7 @@ llvm::GlobalValue::LinkageTypes CodeGenModule::getLLVMLinkageForDeclarator( // C++ doesn't have tentative definitions and thus cannot have common // linkage. if (!getLangOpts().CPlusPlus && isa<VarDecl>(D) && - !isVarDeclStrongDefinition(Context, cast<VarDecl>(D), + !isVarDeclStrongDefinition(Context, *this, cast<VarDecl>(D), CodeGenOpts.NoCommon)) return llvm::GlobalVariable::CommonLinkage; @@ -2285,7 +2355,7 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, callSite->replaceAllUsesWith(newCall.getInstruction()); // Copy debug location attached to CI. - if (!callSite->getDebugLoc().isUnknown()) + if (callSite->getDebugLoc()) newCall->setDebugLoc(callSite->getDebugLoc()); callSite->eraseFromParent(); } @@ -2394,18 +2464,15 @@ void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD, // declarations). auto *Fn = cast<llvm::Function>(GV); setFunctionLinkage(GD, Fn); - if (D->hasAttr<DLLImportAttr>()) - GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass); - else if (D->hasAttr<DLLExportAttr>()) - GV->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass); - else - GV->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass); + setFunctionDLLStorageClass(GD, Fn); // FIXME: this is redundant with part of setFunctionDefinitionAttributes setGlobalVisibility(Fn, D); MaybeHandleStaticInExternC(D, Fn); + maybeSetTrivialComdat(*D, *Fn); + CodeGenFunction(*this).GenerateCode(D, Fn, FI); setFunctionDefinitionAttributes(D, Fn); @@ -2449,7 +2516,7 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) { // Create the new alias itself, but don't set a name yet. auto *GA = llvm::GlobalAlias::create( - cast<llvm::PointerType>(Aliasee->getType())->getElementType(), 0, + cast<llvm::PointerType>(Aliasee->getType()), llvm::Function::ExternalLinkage, "", Aliasee, &getModule()); if (Entry) { @@ -2497,12 +2564,10 @@ llvm::Function *CodeGenModule::getIntrinsic(unsigned IID, Tys); } -static llvm::StringMapEntry<llvm::Constant*> & -GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map, - const StringLiteral *Literal, - bool TargetIsLSB, - bool &IsUTF16, - unsigned &StringLength) { +static llvm::StringMapEntry<llvm::GlobalVariable *> & +GetConstantCFStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map, + const StringLiteral *Literal, bool TargetIsLSB, + bool &IsUTF16, unsigned &StringLength) { StringRef String = Literal->getString(); unsigned NumBytes = String.size(); @@ -2534,10 +2599,9 @@ GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map, nullptr)).first; } -static llvm::StringMapEntry<llvm::Constant*> & -GetConstantStringEntry(llvm::StringMap<llvm::Constant*> &Map, - const StringLiteral *Literal, - unsigned &StringLength) { +static llvm::StringMapEntry<llvm::GlobalVariable *> & +GetConstantStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map, + const StringLiteral *Literal, unsigned &StringLength) { StringRef String = Literal->getString(); StringLength = String.size(); return *Map.insert(std::make_pair(String, nullptr)).first; @@ -2547,10 +2611,10 @@ llvm::Constant * CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { unsigned StringLength = 0; bool isUTF16 = false; - llvm::StringMapEntry<llvm::Constant*> &Entry = - GetConstantCFStringEntry(CFConstantStringMap, Literal, - getDataLayout().isLittleEndian(), - isUTF16, StringLength); + llvm::StringMapEntry<llvm::GlobalVariable *> &Entry = + GetConstantCFStringEntry(CFConstantStringMap, Literal, + getDataLayout().isLittleEndian(), isUTF16, + StringLength); if (auto *C = Entry.second) return C; @@ -2566,7 +2630,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { llvm::Constant *GV = CreateRuntimeVariable(Ty, "__CFConstantStringClassReference"); // Decay array -> ptr - V = llvm::ConstantExpr::getGetElementPtr(GV, Zeros); + V = llvm::ConstantExpr::getGetElementPtr(Ty, GV, Zeros); CFConstantStringClassRef = V; } else @@ -2619,7 +2683,8 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { } // String. - Fields[2] = llvm::ConstantExpr::getGetElementPtr(GV, Zeros); + Fields[2] = + llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros); if (isUTF16) // Cast the UTF16 string to the correct type. @@ -2640,11 +2705,11 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { return GV; } -llvm::Constant * +llvm::GlobalVariable * CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) { unsigned StringLength = 0; - llvm::StringMapEntry<llvm::Constant*> &Entry = - GetConstantStringEntry(CFConstantStringMap, Literal, StringLength); + llvm::StringMapEntry<llvm::GlobalVariable *> &Entry = + GetConstantStringEntry(CFConstantStringMap, Literal, StringLength); if (auto *C = Entry.second) return C; @@ -2673,11 +2738,10 @@ CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) { llvm::Type *PTy = llvm::ArrayType::get(Ty, 0); GV = CreateRuntimeVariable(PTy, str); // Decay array -> ptr - V = llvm::ConstantExpr::getGetElementPtr(GV, Zeros); + V = llvm::ConstantExpr::getGetElementPtr(PTy, GV, Zeros); ConstantStringClassRef = V; } - } - else + } else V = ConstantStringClassRef; if (!NSConstantStringType) { @@ -2733,8 +2797,9 @@ CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) { // of the string is via this class initializer. CharUnits Align = getContext().getTypeAlignInChars(getContext().CharTy); GV->setAlignment(Align.getQuantity()); - Fields[1] = llvm::ConstantExpr::getGetElementPtr(GV, Zeros); - + Fields[1] = + llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros); + // String length. llvm::Type *Ty = getTypes().ConvertType(getContext().UnsignedIntTy); Fields[2] = llvm::ConstantInt::get(Ty, StringLength); @@ -2838,12 +2903,18 @@ GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT, if (CGM.getLangOpts().OpenCL) AddrSpace = CGM.getContext().getTargetAddressSpace(LangAS::opencl_constant); + llvm::Module &M = CGM.getModule(); // Create a global variable for this string auto *GV = new llvm::GlobalVariable( - CGM.getModule(), C->getType(), !CGM.getLangOpts().WritableStrings, LT, C, - GlobalName, nullptr, llvm::GlobalVariable::NotThreadLocal, AddrSpace); + M, C->getType(), !CGM.getLangOpts().WritableStrings, LT, C, GlobalName, + nullptr, llvm::GlobalVariable::NotThreadLocal, AddrSpace); GV->setAlignment(Alignment); GV->setUnnamedAddr(true); + if (GV->isWeakForLinker()) { + assert(CGM.supportsCOMDAT() && "Only COFF uses weak string literals"); + GV->setComdat(M.getOrInsertComdat(GV->getName())); + } + return GV; } @@ -3002,10 +3073,19 @@ llvm::Constant *CodeGenModule::GetAddrOfGlobalTemporary( // Create a global variable for this lifetime-extended temporary. llvm::GlobalValue::LinkageTypes Linkage = getLLVMLinkageVarDefinition(VD, Constant); - // There is no need for this temporary to have global linkage if the global - // variable has external linkage. - if (Linkage == llvm::GlobalVariable::ExternalLinkage) - Linkage = llvm::GlobalVariable::PrivateLinkage; + if (Linkage == llvm::GlobalVariable::ExternalLinkage) { + const VarDecl *InitVD; + if (VD->isStaticDataMember() && VD->getAnyInitializer(InitVD) && + isa<CXXRecordDecl>(InitVD->getLexicalDeclContext())) { + // Temporaries defined inside a class get linkonce_odr linkage because the + // class can be defined in multipe translation units. + Linkage = llvm::GlobalVariable::LinkOnceODRLinkage; + } else { + // There is no need for this temporary to have external linkage if the + // VarDecl has external linkage. + Linkage = llvm::GlobalVariable::InternalLinkage; + } + } unsigned AddrSpace = GetGlobalVarAddressSpace( VD, getContext().getTargetAddressSpace(MaterializedType)); auto *GV = new llvm::GlobalVariable( @@ -3015,6 +3095,8 @@ llvm::Constant *CodeGenModule::GetAddrOfGlobalTemporary( setGlobalVisibility(GV, VD); GV->setAlignment( getContext().getTypeAlignInChars(MaterializedType).getQuantity()); + if (supportsCOMDAT() && GV->isWeakForLinker()) + GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); if (VD->getTLSKind()) setTLSMode(GV, *VD); Slot = GV; @@ -3268,16 +3350,11 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { break; case Decl::FileScopeAsm: { + // File-scope asm is ignored during device-side CUDA compilation. + if (LangOpts.CUDA && LangOpts.CUDAIsDevice) + break; auto *AD = cast<FileScopeAsmDecl>(D); - StringRef AsmString = AD->getAsmString()->getString(); - - const std::string &S = getModule().getModuleInlineAsm(); - if (S.empty()) - getModule().setModuleInlineAsm(AsmString); - else if (S.end()[-1] == '\n') - getModule().setModuleInlineAsm(S + AsmString.str()); - else - getModule().setModuleInlineAsm(S + '\n' + AsmString.str()); + getModule().appendModuleInlineAsm(AD->getAsmString()->getString()); break; } @@ -3285,11 +3362,13 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { auto *Import = cast<ImportDecl>(D); // Ignore import declarations that come from imported modules. - if (clang::Module *Owner = Import->getOwningModule()) { + if (clang::Module *Owner = Import->getImportedOwningModule()) { if (getLangOpts().CurrentModule.empty() || Owner->getTopLevelModule()->Name == getLangOpts().CurrentModule) break; } + if (CGDebugInfo *DI = getModuleDebugInfo()) + DI->EmitImportDecl(*Import); ImportedModules.insert(Import->getImportedModule()); break; @@ -3357,7 +3436,7 @@ void CodeGenModule::ClearUnusedCoverageMapping(const Decl *D) { void CodeGenModule::EmitDeferredUnusedCoverageMappings() { std::vector<const Decl *> DeferredDecls; - for (const auto I : DeferredEmptyCoverageMappingDecls) { + for (const auto &I : DeferredEmptyCoverageMappingDecls) { if (!I.second) continue; DeferredDecls.push_back(I.first); @@ -3430,11 +3509,9 @@ static void EmitGlobalDeclMetadata(CodeGenModule &CGM, /// to such functions with an unmangled name from inline assembly within the /// same translation unit. void CodeGenModule::EmitStaticExternCAliases() { - for (StaticExternCMap::iterator I = StaticExternCValues.begin(), - E = StaticExternCValues.end(); - I != E; ++I) { - IdentifierInfo *Name = I->first; - llvm::GlobalValue *Val = I->second; + for (auto &I : StaticExternCValues) { + IdentifierInfo *Name = I.first; + llvm::GlobalValue *Val = I.second; if (Val && !getModule().getNamedValue(Name->getName())) addUsedGlobal(llvm::GlobalAlias::create(Name->getName(), Val)); } @@ -3561,6 +3638,12 @@ llvm::Constant *CodeGenModule::EmitUuidofInitializer(StringRef Uuid) { return llvm::ConstantStruct::getAnon(Fields); } +llvm::Constant * +CodeGenModule::getAddrOfCXXCatchHandlerType(QualType Ty, + QualType CatchHandlerType) { + return getCXXABI().getAddrOfCXXCatchHandlerType(Ty, CatchHandlerType); +} + llvm::Constant *CodeGenModule::GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH) { // Return a bogus pointer if RTTI is disabled, unless it's for EH. @@ -3583,11 +3666,22 @@ void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) { VD->getAnyInitializer() && !VD->getAnyInitializer()->isConstantInitializer(getContext(), /*ForRef=*/false); - if (auto InitFunction = - getOpenMPRuntime().EmitOMPThreadPrivateVarDefinition( - VD, GetAddrOfGlobalVar(VD), RefExpr->getLocStart(), - PerformInit)) + if (auto InitFunction = getOpenMPRuntime().emitThreadPrivateVarDefinition( + VD, GetAddrOfGlobalVar(VD), RefExpr->getLocStart(), PerformInit)) CXXGlobalInits.push_back(InitFunction); } } +llvm::MDTuple *CodeGenModule::CreateVTableBitSetEntry( + llvm::GlobalVariable *VTable, CharUnits Offset, const CXXRecordDecl *RD) { + std::string OutName; + llvm::raw_string_ostream Out(OutName); + getCXXABI().getMangleContext().mangleCXXVTableBitSet(RD, Out); + + llvm::Metadata *BitsetOps[] = { + llvm::MDString::get(getLLVMContext(), Out.str()), + llvm::ConstantAsMetadata::get(VTable), + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(Int64Ty, Offset.getQuantity()))}; + return llvm::MDTuple::get(getLLVMContext(), BitsetOps); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h index 2aafe7e..dd167a2 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenModule.h @@ -69,6 +69,8 @@ class ValueDecl; class VarDecl; class LangOptions; class CodeGenOptions; +class HeaderSearchOptions; +class PreprocessorOptions; class DiagnosticsEngine; class AnnotateAttr; class CXXDestructorDecl; @@ -258,8 +260,8 @@ public: /// This class organizes the cross-function state that is used while generating /// LLVM code. class CodeGenModule : public CodeGenTypeCache { - CodeGenModule(const CodeGenModule &) LLVM_DELETED_FUNCTION; - void operator=(const CodeGenModule &) LLVM_DELETED_FUNCTION; + CodeGenModule(const CodeGenModule &) = delete; + void operator=(const CodeGenModule &) = delete; public: struct Structor { @@ -278,6 +280,8 @@ public: private: ASTContext &Context; const LangOptions &LangOpts; + const HeaderSearchOptions &HeaderSearchOpts; // Only used for debug info. + const PreprocessorOptions &PreprocessorOpts; // Only used for debug info. const CodeGenOptions &CodeGenOpts; llvm::Module &TheModule; DiagnosticsEngine &Diags; @@ -329,7 +333,7 @@ private: }; std::vector<DeferredGlobal> DeferredDeclsToEmit; void addDeferredDeclToEmit(llvm::GlobalValue *GV, GlobalDecl GD) { - DeferredDeclsToEmit.push_back(DeferredGlobal(GV, GD)); + DeferredDeclsToEmit.emplace_back(GV, GD); } /// List of alias we have emitted. Used to make sure that what they point to @@ -366,7 +370,7 @@ private: /// Map used to get unique annotation strings. llvm::StringMap<llvm::Constant*> AnnotationStrings; - llvm::StringMap<llvm::Constant*> CFConstantStringMap; + llvm::StringMap<llvm::GlobalVariable *> CFConstantStringMap; llvm::DenseMap<llvm::Constant *, llvm::GlobalVariable *> ConstantStringMap; llvm::DenseMap<const Decl*, llvm::Constant *> StaticLocalDeclMap; @@ -400,7 +404,8 @@ private: /// When a C++ decl with an initializer is deferred, null is /// appended to CXXGlobalInits, and the index of that null is placed /// here so that the initializer will be performed in the correct - /// order. + /// order. Once the decl is emitted, the index is replaced with ~0U to ensure + /// that we don't re-emit the initializer. llvm::DenseMap<const Decl*, unsigned> DelayedCXXInitPosition; typedef std::pair<OrderGlobalInits, llvm::Function*> GlobalInitData; @@ -487,7 +492,10 @@ private: std::unique_ptr<CoverageMappingModuleGen> CoverageMapping; public: - CodeGenModule(ASTContext &C, const CodeGenOptions &CodeGenOpts, + CodeGenModule(ASTContext &C, + const HeaderSearchOptions &headersearchopts, + const PreprocessorOptions &ppopts, + const CodeGenOptions &CodeGenOpts, llvm::Module &M, const llvm::DataLayout &TD, DiagnosticsEngine &Diags, CoverageSourceInfo *CoverageInfo = nullptr); @@ -599,6 +607,10 @@ public: ASTContext &getContext() const { return Context; } const LangOptions &getLangOpts() const { return LangOpts; } + const HeaderSearchOptions &getHeaderSearchOpts() + const { return HeaderSearchOpts; } + const PreprocessorOptions &getPreprocessorOpts() + const { return PreprocessorOpts; } const CodeGenOptions &getCodeGenOpts() const { return CodeGenOpts; } llvm::Module &getModule() const { return TheModule; } DiagnosticsEngine &getDiags() const { return Diags; } @@ -606,6 +618,7 @@ public: const TargetInfo &getTarget() const { return Target; } const llvm::Triple &getTriple() const; bool supportsCOMDAT() const; + void maybeSetTrivialComdat(const Decl &D, llvm::GlobalObject &GO); CGCXXABI &getCXXABI() const { return *ABI; } llvm::LLVMContext &getLLVMContext() { return VMContext; } @@ -718,6 +731,9 @@ public: /// Get the address of the RTTI descriptor for the given type. llvm::Constant *GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH = false); + llvm::Constant *getAddrOfCXXCatchHandlerType(QualType Ty, + QualType CatchHandlerType); + /// Get the address of a uuid descriptor . llvm::Constant *GetAddrOfUuidDescriptor(const CXXUuidofExpr* E); @@ -727,6 +743,11 @@ public: /// Get a reference to the target of VD. llvm::Constant *GetWeakRefReference(const ValueDecl *VD); + CharUnits + computeNonVirtualBaseClassOffset(const CXXRecordDecl *DerivedClass, + CastExpr::path_const_iterator Start, + CastExpr::path_const_iterator End); + /// Returns the offset from a derived class to a class. Returns null if the /// offset is 0. llvm::Constant * @@ -782,7 +803,7 @@ public: /// Return a pointer to a constant NSString object for the given string. Or a /// user defined String object as defined via /// -fconstant-string-class=class_name option. - llvm::Constant *GetAddrOfConstantString(const StringLiteral *Literal); + llvm::GlobalVariable *GetAddrOfConstantString(const StringLiteral *Literal); /// Return a constant array for the given string. llvm::Constant *GetConstantArrayFromStringLiteral(const StringLiteral *E); @@ -871,7 +892,7 @@ public: /// Add a destructor and object to add to the C++ global destructor function. void AddCXXDtorEntry(llvm::Constant *DtorFn, llvm::Constant *Object) { - CXXGlobalDtors.push_back(std::make_pair(DtorFn, Object)); + CXXGlobalDtors.emplace_back(DtorFn, Object); } /// Create a new runtime function with the specified type and name. @@ -993,7 +1014,7 @@ public: void EmitTentativeDefinition(const VarDecl *D); - void EmitVTable(CXXRecordDecl *Class, bool DefinitionRequired); + void EmitVTable(CXXRecordDecl *Class); /// Emit the RTTI descriptors for the builtin types. void EmitFundamentalRTTIDescriptors(); @@ -1013,6 +1034,9 @@ public: F->setLinkage(getFunctionLinkage(GD)); } + /// Set the DLL storage class on F. + void setFunctionDLLStorageClass(GlobalDecl GD, llvm::Function *F); + /// Return the appropriate linkage for the vtable, VTT, and type information /// of the given class. llvm::GlobalVariable::LinkageTypes getVTableLinkage(const CXXRecordDecl *RD); @@ -1102,6 +1126,23 @@ public: /// \param D Threadprivate declaration. void EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D); + /// Returns whether the given record is blacklisted from control flow + /// integrity checks. + bool IsCFIBlacklistedRecord(const CXXRecordDecl *RD); + + /// Emit bit set entries for the given vtable using the given layout if + /// vptr CFI is enabled. + void EmitVTableBitSetEntries(llvm::GlobalVariable *VTable, + const VTableLayout &VTLayout); + + /// Create a bitset entry for the given vtable. + llvm::MDTuple *CreateVTableBitSetEntry(llvm::GlobalVariable *VTable, + CharUnits Offset, + const CXXRecordDecl *RD); + + /// \breif Get the declaration of std::terminate for the platform. + llvm::Constant *getTerminateFn(); + private: llvm::Constant * GetOrCreateLLVMFunction(StringRef MangledName, llvm::Type *Ty, GlobalDecl D, diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp index 24b035d..8dffefc 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.cpp @@ -58,12 +58,16 @@ void CodeGenPGO::setFuncName(llvm::Function *Fn) { } void CodeGenPGO::createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage) { - // Usually, we want to match the function's linkage, but - // available_externally and extern_weak both have the wrong semantics. + // We generally want to match the function's linkage, but available_externally + // and extern_weak both have the wrong semantics, and anything that doesn't + // need to link across compilation units doesn't need to be visible at all. if (Linkage == llvm::GlobalValue::ExternalWeakLinkage) Linkage = llvm::GlobalValue::LinkOnceAnyLinkage; else if (Linkage == llvm::GlobalValue::AvailableExternallyLinkage) Linkage = llvm::GlobalValue::LinkOnceODRLinkage; + else if (Linkage == llvm::GlobalValue::InternalLinkage || + Linkage == llvm::GlobalValue::ExternalLinkage) + Linkage = llvm::GlobalValue::PrivateLinkage; auto *Value = llvm::ConstantDataArray::getString(CGM.getLLVMContext(), FuncName, false); @@ -138,482 +142,468 @@ const int PGOHash::NumBitsPerType; const unsigned PGOHash::NumTypesPerWord; const unsigned PGOHash::TooBig; - /// A RecursiveASTVisitor that fills a map of statements to PGO counters. - struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { - /// The next counter value to assign. - unsigned NextCounter; - /// The function hash. - PGOHash Hash; - /// The map of statements to counters. - llvm::DenseMap<const Stmt *, unsigned> &CounterMap; - - MapRegionCounters(llvm::DenseMap<const Stmt *, unsigned> &CounterMap) - : NextCounter(0), CounterMap(CounterMap) {} - - // Blocks and lambdas are handled as separate functions, so we need not - // traverse them in the parent context. - bool TraverseBlockExpr(BlockExpr *BE) { return true; } - bool TraverseLambdaBody(LambdaExpr *LE) { return true; } - bool TraverseCapturedStmt(CapturedStmt *CS) { return true; } - - bool VisitDecl(const Decl *D) { - switch (D->getKind()) { - default: - break; - case Decl::Function: - case Decl::CXXMethod: - case Decl::CXXConstructor: - case Decl::CXXDestructor: - case Decl::CXXConversion: - case Decl::ObjCMethod: - case Decl::Block: - case Decl::Captured: - CounterMap[D->getBody()] = NextCounter++; - break; - } - return true; +/// A RecursiveASTVisitor that fills a map of statements to PGO counters. +struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { + /// The next counter value to assign. + unsigned NextCounter; + /// The function hash. + PGOHash Hash; + /// The map of statements to counters. + llvm::DenseMap<const Stmt *, unsigned> &CounterMap; + + MapRegionCounters(llvm::DenseMap<const Stmt *, unsigned> &CounterMap) + : NextCounter(0), CounterMap(CounterMap) {} + + // Blocks and lambdas are handled as separate functions, so we need not + // traverse them in the parent context. + bool TraverseBlockExpr(BlockExpr *BE) { return true; } + bool TraverseLambdaBody(LambdaExpr *LE) { return true; } + bool TraverseCapturedStmt(CapturedStmt *CS) { return true; } + + bool VisitDecl(const Decl *D) { + switch (D->getKind()) { + default: + break; + case Decl::Function: + case Decl::CXXMethod: + case Decl::CXXConstructor: + case Decl::CXXDestructor: + case Decl::CXXConversion: + case Decl::ObjCMethod: + case Decl::Block: + case Decl::Captured: + CounterMap[D->getBody()] = NextCounter++; + break; } + return true; + } - bool VisitStmt(const Stmt *S) { - auto Type = getHashType(S); - if (Type == PGOHash::None) - return true; - - CounterMap[S] = NextCounter++; - Hash.combine(Type); + bool VisitStmt(const Stmt *S) { + auto Type = getHashType(S); + if (Type == PGOHash::None) return true; + + CounterMap[S] = NextCounter++; + Hash.combine(Type); + return true; + } + PGOHash::HashType getHashType(const Stmt *S) { + switch (S->getStmtClass()) { + default: + break; + case Stmt::LabelStmtClass: + return PGOHash::LabelStmt; + case Stmt::WhileStmtClass: + return PGOHash::WhileStmt; + case Stmt::DoStmtClass: + return PGOHash::DoStmt; + case Stmt::ForStmtClass: + return PGOHash::ForStmt; + case Stmt::CXXForRangeStmtClass: + return PGOHash::CXXForRangeStmt; + case Stmt::ObjCForCollectionStmtClass: + return PGOHash::ObjCForCollectionStmt; + case Stmt::SwitchStmtClass: + return PGOHash::SwitchStmt; + case Stmt::CaseStmtClass: + return PGOHash::CaseStmt; + case Stmt::DefaultStmtClass: + return PGOHash::DefaultStmt; + case Stmt::IfStmtClass: + return PGOHash::IfStmt; + case Stmt::CXXTryStmtClass: + return PGOHash::CXXTryStmt; + case Stmt::CXXCatchStmtClass: + return PGOHash::CXXCatchStmt; + case Stmt::ConditionalOperatorClass: + return PGOHash::ConditionalOperator; + case Stmt::BinaryConditionalOperatorClass: + return PGOHash::BinaryConditionalOperator; + case Stmt::BinaryOperatorClass: { + const BinaryOperator *BO = cast<BinaryOperator>(S); + if (BO->getOpcode() == BO_LAnd) + return PGOHash::BinaryOperatorLAnd; + if (BO->getOpcode() == BO_LOr) + return PGOHash::BinaryOperatorLOr; + break; } - PGOHash::HashType getHashType(const Stmt *S) { - switch (S->getStmtClass()) { - default: - break; - case Stmt::LabelStmtClass: - return PGOHash::LabelStmt; - case Stmt::WhileStmtClass: - return PGOHash::WhileStmt; - case Stmt::DoStmtClass: - return PGOHash::DoStmt; - case Stmt::ForStmtClass: - return PGOHash::ForStmt; - case Stmt::CXXForRangeStmtClass: - return PGOHash::CXXForRangeStmt; - case Stmt::ObjCForCollectionStmtClass: - return PGOHash::ObjCForCollectionStmt; - case Stmt::SwitchStmtClass: - return PGOHash::SwitchStmt; - case Stmt::CaseStmtClass: - return PGOHash::CaseStmt; - case Stmt::DefaultStmtClass: - return PGOHash::DefaultStmt; - case Stmt::IfStmtClass: - return PGOHash::IfStmt; - case Stmt::CXXTryStmtClass: - return PGOHash::CXXTryStmt; - case Stmt::CXXCatchStmtClass: - return PGOHash::CXXCatchStmt; - case Stmt::ConditionalOperatorClass: - return PGOHash::ConditionalOperator; - case Stmt::BinaryConditionalOperatorClass: - return PGOHash::BinaryConditionalOperator; - case Stmt::BinaryOperatorClass: { - const BinaryOperator *BO = cast<BinaryOperator>(S); - if (BO->getOpcode() == BO_LAnd) - return PGOHash::BinaryOperatorLAnd; - if (BO->getOpcode() == BO_LOr) - return PGOHash::BinaryOperatorLOr; - break; - } - } - return PGOHash::None; } + return PGOHash::None; + } +}; + +/// A StmtVisitor that propagates the raw counts through the AST and +/// records the count at statements where the value may change. +struct ComputeRegionCounts : public ConstStmtVisitor<ComputeRegionCounts> { + /// PGO state. + CodeGenPGO &PGO; + + /// A flag that is set when the current count should be recorded on the + /// next statement, such as at the exit of a loop. + bool RecordNextStmtCount; + + /// The count at the current location in the traversal. + uint64_t CurrentCount; + + /// The map of statements to count values. + llvm::DenseMap<const Stmt *, uint64_t> &CountMap; + + /// BreakContinueStack - Keep counts of breaks and continues inside loops. + struct BreakContinue { + uint64_t BreakCount; + uint64_t ContinueCount; + BreakContinue() : BreakCount(0), ContinueCount(0) {} }; + SmallVector<BreakContinue, 8> BreakContinueStack; - /// A StmtVisitor that propagates the raw counts through the AST and - /// records the count at statements where the value may change. - struct ComputeRegionCounts : public ConstStmtVisitor<ComputeRegionCounts> { - /// PGO state. - CodeGenPGO &PGO; - - /// A flag that is set when the current count should be recorded on the - /// next statement, such as at the exit of a loop. - bool RecordNextStmtCount; - - /// The map of statements to count values. - llvm::DenseMap<const Stmt *, uint64_t> &CountMap; - - /// BreakContinueStack - Keep counts of breaks and continues inside loops. - struct BreakContinue { - uint64_t BreakCount; - uint64_t ContinueCount; - BreakContinue() : BreakCount(0), ContinueCount(0) {} - }; - SmallVector<BreakContinue, 8> BreakContinueStack; - - ComputeRegionCounts(llvm::DenseMap<const Stmt *, uint64_t> &CountMap, - CodeGenPGO &PGO) - : PGO(PGO), RecordNextStmtCount(false), CountMap(CountMap) {} - - void RecordStmtCount(const Stmt *S) { - if (RecordNextStmtCount) { - CountMap[S] = PGO.getCurrentRegionCount(); - RecordNextStmtCount = false; - } - } + ComputeRegionCounts(llvm::DenseMap<const Stmt *, uint64_t> &CountMap, + CodeGenPGO &PGO) + : PGO(PGO), RecordNextStmtCount(false), CountMap(CountMap) {} - void VisitStmt(const Stmt *S) { - RecordStmtCount(S); - for (Stmt::const_child_range I = S->children(); I; ++I) { - if (*I) - this->Visit(*I); - } + void RecordStmtCount(const Stmt *S) { + if (RecordNextStmtCount) { + CountMap[S] = CurrentCount; + RecordNextStmtCount = false; } + } - void VisitFunctionDecl(const FunctionDecl *D) { - // Counter tracks entry to the function body. - RegionCounter Cnt(PGO, D->getBody()); - Cnt.beginRegion(); - CountMap[D->getBody()] = PGO.getCurrentRegionCount(); - Visit(D->getBody()); - } + /// Set and return the current count. + uint64_t setCount(uint64_t Count) { + CurrentCount = Count; + return Count; + } - // Skip lambda expressions. We visit these as FunctionDecls when we're - // generating them and aren't interested in the body when generating a - // parent context. - void VisitLambdaExpr(const LambdaExpr *LE) {} - - void VisitCapturedDecl(const CapturedDecl *D) { - // Counter tracks entry to the capture body. - RegionCounter Cnt(PGO, D->getBody()); - Cnt.beginRegion(); - CountMap[D->getBody()] = PGO.getCurrentRegionCount(); - Visit(D->getBody()); - } + void VisitStmt(const Stmt *S) { + RecordStmtCount(S); + for (const Stmt *Child : S->children()) + if (Child) + this->Visit(Child); + } - void VisitObjCMethodDecl(const ObjCMethodDecl *D) { - // Counter tracks entry to the method body. - RegionCounter Cnt(PGO, D->getBody()); - Cnt.beginRegion(); - CountMap[D->getBody()] = PGO.getCurrentRegionCount(); - Visit(D->getBody()); - } + void VisitFunctionDecl(const FunctionDecl *D) { + // Counter tracks entry to the function body. + uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); + CountMap[D->getBody()] = BodyCount; + Visit(D->getBody()); + } - void VisitBlockDecl(const BlockDecl *D) { - // Counter tracks entry to the block body. - RegionCounter Cnt(PGO, D->getBody()); - Cnt.beginRegion(); - CountMap[D->getBody()] = PGO.getCurrentRegionCount(); - Visit(D->getBody()); - } + // Skip lambda expressions. We visit these as FunctionDecls when we're + // generating them and aren't interested in the body when generating a + // parent context. + void VisitLambdaExpr(const LambdaExpr *LE) {} - void VisitReturnStmt(const ReturnStmt *S) { - RecordStmtCount(S); - if (S->getRetValue()) - Visit(S->getRetValue()); - PGO.setCurrentRegionUnreachable(); - RecordNextStmtCount = true; - } + void VisitCapturedDecl(const CapturedDecl *D) { + // Counter tracks entry to the capture body. + uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); + CountMap[D->getBody()] = BodyCount; + Visit(D->getBody()); + } - void VisitGotoStmt(const GotoStmt *S) { - RecordStmtCount(S); - PGO.setCurrentRegionUnreachable(); - RecordNextStmtCount = true; - } + void VisitObjCMethodDecl(const ObjCMethodDecl *D) { + // Counter tracks entry to the method body. + uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); + CountMap[D->getBody()] = BodyCount; + Visit(D->getBody()); + } - void VisitLabelStmt(const LabelStmt *S) { - RecordNextStmtCount = false; - // Counter tracks the block following the label. - RegionCounter Cnt(PGO, S); - Cnt.beginRegion(); - CountMap[S] = PGO.getCurrentRegionCount(); - Visit(S->getSubStmt()); - } + void VisitBlockDecl(const BlockDecl *D) { + // Counter tracks entry to the block body. + uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); + CountMap[D->getBody()] = BodyCount; + Visit(D->getBody()); + } - void VisitBreakStmt(const BreakStmt *S) { - RecordStmtCount(S); - assert(!BreakContinueStack.empty() && "break not in a loop or switch!"); - BreakContinueStack.back().BreakCount += PGO.getCurrentRegionCount(); - PGO.setCurrentRegionUnreachable(); - RecordNextStmtCount = true; - } + void VisitReturnStmt(const ReturnStmt *S) { + RecordStmtCount(S); + if (S->getRetValue()) + Visit(S->getRetValue()); + CurrentCount = 0; + RecordNextStmtCount = true; + } - void VisitContinueStmt(const ContinueStmt *S) { - RecordStmtCount(S); - assert(!BreakContinueStack.empty() && "continue stmt not in a loop!"); - BreakContinueStack.back().ContinueCount += PGO.getCurrentRegionCount(); - PGO.setCurrentRegionUnreachable(); - RecordNextStmtCount = true; - } + void VisitCXXThrowExpr(const CXXThrowExpr *E) { + RecordStmtCount(E); + if (E->getSubExpr()) + Visit(E->getSubExpr()); + CurrentCount = 0; + RecordNextStmtCount = true; + } - void VisitWhileStmt(const WhileStmt *S) { - RecordStmtCount(S); - // Counter tracks the body of the loop. - RegionCounter Cnt(PGO, S); - BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first so the break/continue adjustments can be - // included when visiting the condition. - Cnt.beginRegion(); - CountMap[S->getBody()] = PGO.getCurrentRegionCount(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); - - // ...then go back and propagate counts through the condition. The count - // at the start of the condition is the sum of the incoming edges, - // the backedge from the end of the loop body, and the edges from - // continue statements. - BreakContinue BC = BreakContinueStack.pop_back_val(); - Cnt.setCurrentRegionCount(Cnt.getParentCount() + - Cnt.getAdjustedCount() + BC.ContinueCount); - CountMap[S->getCond()] = PGO.getCurrentRegionCount(); - Visit(S->getCond()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount); - RecordNextStmtCount = true; - } + void VisitGotoStmt(const GotoStmt *S) { + RecordStmtCount(S); + CurrentCount = 0; + RecordNextStmtCount = true; + } - void VisitDoStmt(const DoStmt *S) { - RecordStmtCount(S); - // Counter tracks the body of the loop. - RegionCounter Cnt(PGO, S); - BreakContinueStack.push_back(BreakContinue()); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - CountMap[S->getBody()] = PGO.getCurrentRegionCount(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); - - BreakContinue BC = BreakContinueStack.pop_back_val(); - // The count at the start of the condition is equal to the count at the - // end of the body. The adjusted count does not include either the - // fall-through count coming into the loop or the continue count, so add - // both of those separately. This is coincidentally the same equation as - // with while loops but for different reasons. - Cnt.setCurrentRegionCount(Cnt.getParentCount() + - Cnt.getAdjustedCount() + BC.ContinueCount); - CountMap[S->getCond()] = PGO.getCurrentRegionCount(); - Visit(S->getCond()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount); - RecordNextStmtCount = true; - } + void VisitLabelStmt(const LabelStmt *S) { + RecordNextStmtCount = false; + // Counter tracks the block following the label. + uint64_t BlockCount = setCount(PGO.getRegionCount(S)); + CountMap[S] = BlockCount; + Visit(S->getSubStmt()); + } - void VisitForStmt(const ForStmt *S) { - RecordStmtCount(S); - if (S->getInit()) - Visit(S->getInit()); - // Counter tracks the body of the loop. - RegionCounter Cnt(PGO, S); - BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first. (This is basically the same as a while - // loop; see further comments in VisitWhileStmt.) - Cnt.beginRegion(); - CountMap[S->getBody()] = PGO.getCurrentRegionCount(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); - - // The increment is essentially part of the body but it needs to include - // the count for all the continue statements. - if (S->getInc()) { - Cnt.setCurrentRegionCount(PGO.getCurrentRegionCount() + - BreakContinueStack.back().ContinueCount); - CountMap[S->getInc()] = PGO.getCurrentRegionCount(); - Visit(S->getInc()); - Cnt.adjustForControlFlow(); - } - - BreakContinue BC = BreakContinueStack.pop_back_val(); - - // ...then go back and propagate counts through the condition. - if (S->getCond()) { - Cnt.setCurrentRegionCount(Cnt.getParentCount() + - Cnt.getAdjustedCount() + - BC.ContinueCount); - CountMap[S->getCond()] = PGO.getCurrentRegionCount(); - Visit(S->getCond()); - Cnt.adjustForControlFlow(); - } - Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount); - RecordNextStmtCount = true; - } + void VisitBreakStmt(const BreakStmt *S) { + RecordStmtCount(S); + assert(!BreakContinueStack.empty() && "break not in a loop or switch!"); + BreakContinueStack.back().BreakCount += CurrentCount; + CurrentCount = 0; + RecordNextStmtCount = true; + } - void VisitCXXForRangeStmt(const CXXForRangeStmt *S) { - RecordStmtCount(S); - Visit(S->getRangeStmt()); - Visit(S->getBeginEndStmt()); - // Counter tracks the body of the loop. - RegionCounter Cnt(PGO, S); - BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first. (This is basically the same as a while - // loop; see further comments in VisitWhileStmt.) - Cnt.beginRegion(); - CountMap[S->getLoopVarStmt()] = PGO.getCurrentRegionCount(); - Visit(S->getLoopVarStmt()); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); - - // The increment is essentially part of the body but it needs to include - // the count for all the continue statements. - Cnt.setCurrentRegionCount(PGO.getCurrentRegionCount() + - BreakContinueStack.back().ContinueCount); - CountMap[S->getInc()] = PGO.getCurrentRegionCount(); - Visit(S->getInc()); - Cnt.adjustForControlFlow(); + void VisitContinueStmt(const ContinueStmt *S) { + RecordStmtCount(S); + assert(!BreakContinueStack.empty() && "continue stmt not in a loop!"); + BreakContinueStack.back().ContinueCount += CurrentCount; + CurrentCount = 0; + RecordNextStmtCount = true; + } - BreakContinue BC = BreakContinueStack.pop_back_val(); + void VisitWhileStmt(const WhileStmt *S) { + RecordStmtCount(S); + uint64_t ParentCount = CurrentCount; + + BreakContinueStack.push_back(BreakContinue()); + // Visit the body region first so the break/continue adjustments can be + // included when visiting the condition. + uint64_t BodyCount = setCount(PGO.getRegionCount(S)); + CountMap[S->getBody()] = CurrentCount; + Visit(S->getBody()); + uint64_t BackedgeCount = CurrentCount; + + // ...then go back and propagate counts through the condition. The count + // at the start of the condition is the sum of the incoming edges, + // the backedge from the end of the loop body, and the edges from + // continue statements. + BreakContinue BC = BreakContinueStack.pop_back_val(); + uint64_t CondCount = + setCount(ParentCount + BackedgeCount + BC.ContinueCount); + CountMap[S->getCond()] = CondCount; + Visit(S->getCond()); + setCount(BC.BreakCount + CondCount - BodyCount); + RecordNextStmtCount = true; + } - // ...then go back and propagate counts through the condition. - Cnt.setCurrentRegionCount(Cnt.getParentCount() + - Cnt.getAdjustedCount() + - BC.ContinueCount); - CountMap[S->getCond()] = PGO.getCurrentRegionCount(); - Visit(S->getCond()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount); - RecordNextStmtCount = true; - } + void VisitDoStmt(const DoStmt *S) { + RecordStmtCount(S); + uint64_t LoopCount = PGO.getRegionCount(S); + + BreakContinueStack.push_back(BreakContinue()); + // The count doesn't include the fallthrough from the parent scope. Add it. + uint64_t BodyCount = setCount(LoopCount + CurrentCount); + CountMap[S->getBody()] = BodyCount; + Visit(S->getBody()); + uint64_t BackedgeCount = CurrentCount; + + BreakContinue BC = BreakContinueStack.pop_back_val(); + // The count at the start of the condition is equal to the count at the + // end of the body, plus any continues. + uint64_t CondCount = setCount(BackedgeCount + BC.ContinueCount); + CountMap[S->getCond()] = CondCount; + Visit(S->getCond()); + setCount(BC.BreakCount + CondCount - LoopCount); + RecordNextStmtCount = true; + } - void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) { - RecordStmtCount(S); - Visit(S->getElement()); - // Counter tracks the body of the loop. - RegionCounter Cnt(PGO, S); - BreakContinueStack.push_back(BreakContinue()); - Cnt.beginRegion(); - CountMap[S->getBody()] = PGO.getCurrentRegionCount(); - Visit(S->getBody()); - BreakContinue BC = BreakContinueStack.pop_back_val(); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount); - RecordNextStmtCount = true; + void VisitForStmt(const ForStmt *S) { + RecordStmtCount(S); + if (S->getInit()) + Visit(S->getInit()); + + uint64_t ParentCount = CurrentCount; + + BreakContinueStack.push_back(BreakContinue()); + // Visit the body region first. (This is basically the same as a while + // loop; see further comments in VisitWhileStmt.) + uint64_t BodyCount = setCount(PGO.getRegionCount(S)); + CountMap[S->getBody()] = BodyCount; + Visit(S->getBody()); + uint64_t BackedgeCount = CurrentCount; + BreakContinue BC = BreakContinueStack.pop_back_val(); + + // The increment is essentially part of the body but it needs to include + // the count for all the continue statements. + if (S->getInc()) { + uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount); + CountMap[S->getInc()] = IncCount; + Visit(S->getInc()); } - void VisitSwitchStmt(const SwitchStmt *S) { - RecordStmtCount(S); + // ...then go back and propagate counts through the condition. + uint64_t CondCount = + setCount(ParentCount + BackedgeCount + BC.ContinueCount); + if (S->getCond()) { + CountMap[S->getCond()] = CondCount; Visit(S->getCond()); - PGO.setCurrentRegionUnreachable(); - BreakContinueStack.push_back(BreakContinue()); - Visit(S->getBody()); - // If the switch is inside a loop, add the continue counts. - BreakContinue BC = BreakContinueStack.pop_back_val(); - if (!BreakContinueStack.empty()) - BreakContinueStack.back().ContinueCount += BC.ContinueCount; - // Counter tracks the exit block of the switch. - RegionCounter ExitCnt(PGO, S); - ExitCnt.beginRegion(); - RecordNextStmtCount = true; } + setCount(BC.BreakCount + CondCount - BodyCount); + RecordNextStmtCount = true; + } - void VisitCaseStmt(const CaseStmt *S) { - RecordNextStmtCount = false; - // Counter for this particular case. This counts only jumps from the - // switch header and does not include fallthrough from the case before - // this one. - RegionCounter Cnt(PGO, S); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - CountMap[S] = Cnt.getCount(); - RecordNextStmtCount = true; - Visit(S->getSubStmt()); - } + void VisitCXXForRangeStmt(const CXXForRangeStmt *S) { + RecordStmtCount(S); + Visit(S->getLoopVarStmt()); + Visit(S->getRangeStmt()); + Visit(S->getBeginEndStmt()); + + uint64_t ParentCount = CurrentCount; + BreakContinueStack.push_back(BreakContinue()); + // Visit the body region first. (This is basically the same as a while + // loop; see further comments in VisitWhileStmt.) + uint64_t BodyCount = setCount(PGO.getRegionCount(S)); + CountMap[S->getBody()] = BodyCount; + Visit(S->getBody()); + uint64_t BackedgeCount = CurrentCount; + BreakContinue BC = BreakContinueStack.pop_back_val(); + + // The increment is essentially part of the body but it needs to include + // the count for all the continue statements. + uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount); + CountMap[S->getInc()] = IncCount; + Visit(S->getInc()); + + // ...then go back and propagate counts through the condition. + uint64_t CondCount = + setCount(ParentCount + BackedgeCount + BC.ContinueCount); + CountMap[S->getCond()] = CondCount; + Visit(S->getCond()); + setCount(BC.BreakCount + CondCount - BodyCount); + RecordNextStmtCount = true; + } - void VisitDefaultStmt(const DefaultStmt *S) { - RecordNextStmtCount = false; - // Counter for this default case. This does not include fallthrough from - // the previous case. - RegionCounter Cnt(PGO, S); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - CountMap[S] = Cnt.getCount(); - RecordNextStmtCount = true; - Visit(S->getSubStmt()); - } + void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) { + RecordStmtCount(S); + Visit(S->getElement()); + uint64_t ParentCount = CurrentCount; + BreakContinueStack.push_back(BreakContinue()); + // Counter tracks the body of the loop. + uint64_t BodyCount = setCount(PGO.getRegionCount(S)); + CountMap[S->getBody()] = BodyCount; + Visit(S->getBody()); + uint64_t BackedgeCount = CurrentCount; + BreakContinue BC = BreakContinueStack.pop_back_val(); + + setCount(BC.BreakCount + ParentCount + BackedgeCount + BC.ContinueCount - + BodyCount); + RecordNextStmtCount = true; + } - void VisitIfStmt(const IfStmt *S) { - RecordStmtCount(S); - // Counter tracks the "then" part of an if statement. The count for - // the "else" part, if it exists, will be calculated from this counter. - RegionCounter Cnt(PGO, S); - Visit(S->getCond()); + void VisitSwitchStmt(const SwitchStmt *S) { + RecordStmtCount(S); + Visit(S->getCond()); + CurrentCount = 0; + BreakContinueStack.push_back(BreakContinue()); + Visit(S->getBody()); + // If the switch is inside a loop, add the continue counts. + BreakContinue BC = BreakContinueStack.pop_back_val(); + if (!BreakContinueStack.empty()) + BreakContinueStack.back().ContinueCount += BC.ContinueCount; + // Counter tracks the exit block of the switch. + setCount(PGO.getRegionCount(S)); + RecordNextStmtCount = true; + } - Cnt.beginRegion(); - CountMap[S->getThen()] = PGO.getCurrentRegionCount(); - Visit(S->getThen()); - Cnt.adjustForControlFlow(); - - if (S->getElse()) { - Cnt.beginElseRegion(); - CountMap[S->getElse()] = PGO.getCurrentRegionCount(); - Visit(S->getElse()); - Cnt.adjustForControlFlow(); - } - Cnt.applyAdjustmentsToRegion(0); - RecordNextStmtCount = true; - } + void VisitSwitchCase(const SwitchCase *S) { + RecordNextStmtCount = false; + // Counter for this particular case. This counts only jumps from the + // switch header and does not include fallthrough from the case before + // this one. + uint64_t CaseCount = PGO.getRegionCount(S); + setCount(CurrentCount + CaseCount); + // We need the count without fallthrough in the mapping, so it's more useful + // for branch probabilities. + CountMap[S] = CaseCount; + RecordNextStmtCount = true; + Visit(S->getSubStmt()); + } - void VisitCXXTryStmt(const CXXTryStmt *S) { - RecordStmtCount(S); - Visit(S->getTryBlock()); - for (unsigned I = 0, E = S->getNumHandlers(); I < E; ++I) - Visit(S->getHandler(I)); - // Counter tracks the continuation block of the try statement. - RegionCounter Cnt(PGO, S); - Cnt.beginRegion(); - RecordNextStmtCount = true; - } + void VisitIfStmt(const IfStmt *S) { + RecordStmtCount(S); + uint64_t ParentCount = CurrentCount; + Visit(S->getCond()); + + // Counter tracks the "then" part of an if statement. The count for + // the "else" part, if it exists, will be calculated from this counter. + uint64_t ThenCount = setCount(PGO.getRegionCount(S)); + CountMap[S->getThen()] = ThenCount; + Visit(S->getThen()); + uint64_t OutCount = CurrentCount; + + uint64_t ElseCount = ParentCount - ThenCount; + if (S->getElse()) { + setCount(ElseCount); + CountMap[S->getElse()] = ElseCount; + Visit(S->getElse()); + OutCount += CurrentCount; + } else + OutCount += ElseCount; + setCount(OutCount); + RecordNextStmtCount = true; + } - void VisitCXXCatchStmt(const CXXCatchStmt *S) { - RecordNextStmtCount = false; - // Counter tracks the catch statement's handler block. - RegionCounter Cnt(PGO, S); - Cnt.beginRegion(); - CountMap[S] = PGO.getCurrentRegionCount(); - Visit(S->getHandlerBlock()); - } + void VisitCXXTryStmt(const CXXTryStmt *S) { + RecordStmtCount(S); + Visit(S->getTryBlock()); + for (unsigned I = 0, E = S->getNumHandlers(); I < E; ++I) + Visit(S->getHandler(I)); + // Counter tracks the continuation block of the try statement. + setCount(PGO.getRegionCount(S)); + RecordNextStmtCount = true; + } - void VisitAbstractConditionalOperator( - const AbstractConditionalOperator *E) { - RecordStmtCount(E); - // Counter tracks the "true" part of a conditional operator. The - // count in the "false" part will be calculated from this counter. - RegionCounter Cnt(PGO, E); - Visit(E->getCond()); - - Cnt.beginRegion(); - CountMap[E->getTrueExpr()] = PGO.getCurrentRegionCount(); - Visit(E->getTrueExpr()); - Cnt.adjustForControlFlow(); - - Cnt.beginElseRegion(); - CountMap[E->getFalseExpr()] = PGO.getCurrentRegionCount(); - Visit(E->getFalseExpr()); - Cnt.adjustForControlFlow(); - - Cnt.applyAdjustmentsToRegion(0); - RecordNextStmtCount = true; - } + void VisitCXXCatchStmt(const CXXCatchStmt *S) { + RecordNextStmtCount = false; + // Counter tracks the catch statement's handler block. + uint64_t CatchCount = setCount(PGO.getRegionCount(S)); + CountMap[S] = CatchCount; + Visit(S->getHandlerBlock()); + } - void VisitBinLAnd(const BinaryOperator *E) { - RecordStmtCount(E); - // Counter tracks the right hand side of a logical and operator. - RegionCounter Cnt(PGO, E); - Visit(E->getLHS()); - Cnt.beginRegion(); - CountMap[E->getRHS()] = PGO.getCurrentRegionCount(); - Visit(E->getRHS()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(0); - RecordNextStmtCount = true; - } + void VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { + RecordStmtCount(E); + uint64_t ParentCount = CurrentCount; + Visit(E->getCond()); + + // Counter tracks the "true" part of a conditional operator. The + // count in the "false" part will be calculated from this counter. + uint64_t TrueCount = setCount(PGO.getRegionCount(E)); + CountMap[E->getTrueExpr()] = TrueCount; + Visit(E->getTrueExpr()); + uint64_t OutCount = CurrentCount; + + uint64_t FalseCount = setCount(ParentCount - TrueCount); + CountMap[E->getFalseExpr()] = FalseCount; + Visit(E->getFalseExpr()); + OutCount += CurrentCount; + + setCount(OutCount); + RecordNextStmtCount = true; + } - void VisitBinLOr(const BinaryOperator *E) { - RecordStmtCount(E); - // Counter tracks the right hand side of a logical or operator. - RegionCounter Cnt(PGO, E); - Visit(E->getLHS()); - Cnt.beginRegion(); - CountMap[E->getRHS()] = PGO.getCurrentRegionCount(); - Visit(E->getRHS()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(0); - RecordNextStmtCount = true; - } - }; + void VisitBinLAnd(const BinaryOperator *E) { + RecordStmtCount(E); + uint64_t ParentCount = CurrentCount; + Visit(E->getLHS()); + // Counter tracks the right hand side of a logical and operator. + uint64_t RHSCount = setCount(PGO.getRegionCount(E)); + CountMap[E->getRHS()] = RHSCount; + Visit(E->getRHS()); + setCount(ParentCount + RHSCount - CurrentCount); + RecordNextStmtCount = true; + } + + void VisitBinLOr(const BinaryOperator *E) { + RecordStmtCount(E); + uint64_t ParentCount = CurrentCount; + Visit(E->getLHS()); + // Counter tracks the right hand side of a logical or operator. + uint64_t RHSCount = setCount(PGO.getRegionCount(E)); + CountMap[E->getRHS()] = RHSCount; + Visit(E->getRHS()); + setCount(ParentCount + RHSCount - CurrentCount); + RecordNextStmtCount = true; + } +}; } void PGOHash::combine(HashType Type) { @@ -728,12 +718,10 @@ void CodeGenPGO::emitCounterRegionMapping(const Decl *D) { } void -CodeGenPGO::emitEmptyCounterMapping(const Decl *D, StringRef FuncName, +CodeGenPGO::emitEmptyCounterMapping(const Decl *D, StringRef Name, llvm::GlobalValue::LinkageTypes Linkage) { if (SkipCoverageMapping) return; - setFuncName(FuncName, Linkage); - // Don't map the functions inside the system headers auto Loc = D->getBody()->getLocStart(); if (CGM.getContext().getSourceManager().isInSystemHeader(Loc)) @@ -750,6 +738,7 @@ CodeGenPGO::emitEmptyCounterMapping(const Decl *D, StringRef FuncName, if (CoverageMapping.empty()) return; + setFuncName(Name, Linkage); CGM.getCoverageMapping()->addFunctionMappingRecord( FuncNameVar, FuncName, FunctionHash, CoverageMapping); } @@ -783,19 +772,23 @@ CodeGenPGO::applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, // Turn on Cold attribute for cold functions. // FIXME: 1% is from preliminary tuning on SPEC, it may not be optimal. Fn->addFnAttr(llvm::Attribute::Cold); + + Fn->setEntryCount(FunctionCount); } -void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter) { +void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S) { if (!CGM.getCodeGenOpts().ProfileInstrGenerate || !RegionCounterMap) return; if (!Builder.GetInsertPoint()) return; + + unsigned Counter = (*RegionCounterMap)[S]; auto *I8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext()); - Builder.CreateCall4(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment), - llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment), + {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), Builder.getInt64(FunctionHash), Builder.getInt32(NumRegionCounters), - Builder.getInt32(Counter)); + Builder.getInt32(Counter)}); } void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader, @@ -839,8 +832,8 @@ static uint32_t scaleBranchWeight(uint64_t Weight, uint64_t Scale) { return Scaled; } -llvm::MDNode *CodeGenPGO::createBranchWeights(uint64_t TrueCount, - uint64_t FalseCount) { +llvm::MDNode *CodeGenFunction::createProfileWeights(uint64_t TrueCount, + uint64_t FalseCount) { // Check for empty weights. if (!TrueCount && !FalseCount) return nullptr; @@ -853,7 +846,8 @@ llvm::MDNode *CodeGenPGO::createBranchWeights(uint64_t TrueCount, scaleBranchWeight(FalseCount, Scale)); } -llvm::MDNode *CodeGenPGO::createBranchWeights(ArrayRef<uint64_t> Weights) { +llvm::MDNode * +CodeGenFunction::createProfileWeights(ArrayRef<uint64_t> Weights) { // We need at least two elements to create meaningful weights. if (Weights.size() < 2) return nullptr; @@ -875,17 +869,14 @@ llvm::MDNode *CodeGenPGO::createBranchWeights(ArrayRef<uint64_t> Weights) { return MDHelper.createBranchWeights(ScaledWeights); } -llvm::MDNode *CodeGenPGO::createLoopWeights(const Stmt *Cond, - RegionCounter &Cnt) { - if (!haveRegionCounts()) +llvm::MDNode *CodeGenFunction::createProfileWeightsForLoop(const Stmt *Cond, + uint64_t LoopCount) { + if (!PGO.haveRegionCounts()) return nullptr; - uint64_t LoopCount = Cnt.getCount(); - uint64_t CondCount = 0; - bool Found = getStmtCount(Cond, CondCount); - assert(Found && "missing expected loop condition count"); - (void)Found; - if (CondCount == 0) + Optional<uint64_t> CondCount = PGO.getStmtCount(Cond); + assert(CondCount.hasValue() && "missing expected loop condition count"); + if (*CondCount == 0) return nullptr; - return createBranchWeights(LoopCount, - std::max(CondCount, LoopCount) - LoopCount); + return createProfileWeights(LoopCount, + std::max(*CondCount, LoopCount) - LoopCount); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h index 431c850..de6f369 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenPGO.h @@ -24,10 +24,8 @@ namespace clang { namespace CodeGen { -class RegionCounter; -/// Per-function PGO state. This class should generally not be used directly, -/// but instead through the CodeGenFunction and RegionCounter types. +/// Per-function PGO state. class CodeGenPGO { private: CodeGenModule &CGM; @@ -62,37 +60,24 @@ public: /// exits. void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; } - /// Indicate that the current region is never reached, and thus should have a - /// counter value of zero. This is important so that subsequent regions can - /// correctly track their parent counts. - void setCurrentRegionUnreachable() { setCurrentRegionCount(0); } - /// Check if an execution count is known for a given statement. If so, return /// true and put the value in Count; else return false. - bool getStmtCount(const Stmt *S, uint64_t &Count) { + Optional<uint64_t> getStmtCount(const Stmt *S) { if (!StmtCountMap) - return false; - llvm::DenseMap<const Stmt*, uint64_t>::const_iterator - I = StmtCountMap->find(S); + return None; + auto I = StmtCountMap->find(S); if (I == StmtCountMap->end()) - return false; - Count = I->second; - return true; + return None; + return I->second; } /// If the execution count for the current statement is known, record that /// as the current count. void setCurrentStmt(const Stmt *S) { - uint64_t Count; - if (getStmtCount(S, Count)) - setCurrentRegionCount(Count); + if (auto Count = getStmtCount(S)) + setCurrentRegionCount(*Count); } - /// Calculate branch weights appropriate for PGO data - llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount); - llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights); - llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt); - /// Check if we need to emit coverage mapping for a given declaration void checkGlobalDecl(GlobalDecl GD); /// Assign counters to regions and configure them for PGO of a given @@ -117,110 +102,16 @@ private: void emitCounterVariables(); void emitCounterRegionMapping(const Decl *D); - /// Emit code to increment the counter at the given index - void emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter); +public: + void emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S); - /// Return the region counter for the given statement. This should only be - /// called on statements that have a dedicated counter. - unsigned getRegionCounter(const Stmt *S) { + /// Return the region count for the counter at the given index. + uint64_t getRegionCount(const Stmt *S) { if (!RegionCounterMap) return 0; - return (*RegionCounterMap)[S]; - } - - /// Return the region count for the counter at the given index. - uint64_t getRegionCount(unsigned Counter) { if (!haveRegionCounts()) return 0; - return RegionCounts[Counter]; - } - - friend class RegionCounter; -}; - -/// A counter for a particular region. This is the primary interface through -/// which clients manage PGO counters and their values. -class RegionCounter { - CodeGenPGO *PGO; - unsigned Counter; - uint64_t Count; - uint64_t ParentCount; - uint64_t RegionCount; - int64_t Adjust; - - RegionCounter(CodeGenPGO &PGO, unsigned CounterIndex) - : PGO(&PGO), Counter(CounterIndex), Count(PGO.getRegionCount(Counter)), - ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} - -public: - RegionCounter(CodeGenPGO &PGO, const Stmt *S) - : PGO(&PGO), Counter(PGO.getRegionCounter(S)), - Count(PGO.getRegionCount(Counter)), - ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} - - /// Get the value of the counter. In most cases this is the number of times - /// the region of the counter was entered, but for switch labels it's the - /// number of direct jumps to that label. - uint64_t getCount() const { return Count; } - - /// Get the value of the counter with adjustments applied. Adjustments occur - /// when control enters or leaves the region abnormally; i.e., if there is a - /// jump to a label within the region, or if the function can return from - /// within the region. The adjusted count, then, is the value of the counter - /// at the end of the region. - uint64_t getAdjustedCount() const { - return Count + Adjust; - } - - /// Get the value of the counter in this region's parent, i.e., the region - /// that was active when this region began. This is useful for deriving - /// counts in implicitly counted regions, like the false case of a condition - /// or the normal exits of a loop. - uint64_t getParentCount() const { return ParentCount; } - - /// Activate the counter by emitting an increment and starting to track - /// adjustments. If AddIncomingFallThrough is true, the current region count - /// will be added to the counter for the purposes of tracking the region. - void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) { - beginRegion(AddIncomingFallThrough); - PGO->emitCounterIncrement(Builder, Counter); - } - void beginRegion(bool AddIncomingFallThrough=false) { - RegionCount = Count; - if (AddIncomingFallThrough) - RegionCount += PGO->getCurrentRegionCount(); - PGO->setCurrentRegionCount(RegionCount); - } - - /// For counters on boolean branches, begins tracking adjustments for the - /// uncounted path. - void beginElseRegion() { - RegionCount = ParentCount - Count; - PGO->setCurrentRegionCount(RegionCount); - } - - /// Reset the current region count. - void setCurrentRegionCount(uint64_t CurrentCount) { - RegionCount = CurrentCount; - PGO->setCurrentRegionCount(RegionCount); - } - - /// Adjust for non-local control flow after emitting a subexpression or - /// substatement. This must be called to account for constructs such as gotos, - /// labels, and returns, so that we can ensure that our region's count is - /// correct in the code that follows. - void adjustForControlFlow() { - Adjust += PGO->getCurrentRegionCount() - RegionCount; - // Reset the region count in case this is called again later. - RegionCount = PGO->getCurrentRegionCount(); - } - - /// Commit all adjustments to the current region. If the region is a loop, - /// the LoopAdjust value should be the count of all the breaks and continues - /// from the loop, to compensate for those counts being deducted from the - /// adjustments for the body of the loop. - void applyAdjustmentsToRegion(uint64_t LoopAdjust) { - PGO->setCurrentRegionCount(ParentCount + Adjust + LoopAdjust); + return RegionCounts[(*RegionCounterMap)[S]]; } }; diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp index 67a9fbe..a4a8654 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.cpp @@ -154,14 +154,16 @@ isSafeToConvert(const RecordDecl *RD, CodeGenTypes &CGT, static bool isSafeToConvert(QualType T, CodeGenTypes &CGT, llvm::SmallPtrSet<const RecordDecl*, 16> &AlreadyChecked) { - T = T.getCanonicalType(); - + // Strip off atomic type sugar. + if (const auto *AT = T->getAs<AtomicType>()) + T = AT->getValueType(); + // If this is a record, check it. - if (const RecordType *RT = dyn_cast<RecordType>(T)) + if (const auto *RT = T->getAs<RecordType>()) return isSafeToConvert(RT->getDecl(), CGT, AlreadyChecked); - + // If this is an array, check the elements, which are embedded inline. - if (const ArrayType *AT = dyn_cast<ArrayType>(T)) + if (const auto *AT = CGT.getContext().getAsArrayType(T)) return isSafeToConvert(AT->getElementType(), CGT, AlreadyChecked); // Otherwise, there is no concern about transforming this. We only care about @@ -715,9 +717,16 @@ bool CodeGenTypes::isZeroInitializable(QualType T) { // No need to check for member pointers when not compiling C++. if (!Context.getLangOpts().CPlusPlus) return true; - - T = Context.getBaseElementType(T); - + + if (const auto *AT = Context.getAsArrayType(T)) { + if (isa<IncompleteArrayType>(AT)) + return true; + if (const auto *CAT = dyn_cast<ConstantArrayType>(AT)) + if (Context.getConstantArrayElementCount(CAT) == 0) + return true; + T = Context.getBaseElementType(T); + } + // Records are non-zero-initializable if they contain any // non-zero-initializable subobjects. if (const RecordType *RT = T->getAs<RecordType>()) { @@ -733,6 +742,6 @@ bool CodeGenTypes::isZeroInitializable(QualType T) { return true; } -bool CodeGenTypes::isZeroInitializable(const CXXRecordDecl *RD) { +bool CodeGenTypes::isZeroInitializable(const RecordDecl *RD) { return getCGRecordLayout(RD).isZeroInitializable(); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h index 64c5799..1580e21 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/CodeGenTypes.h @@ -82,6 +82,9 @@ inline StructorType getFromCtorType(CXXCtorType T) { return StructorType::Base; case Ctor_Comdat: llvm_unreachable("not expecting a COMDAT"); + case Ctor_CopyingClosure: + case Ctor_DefaultClosure: + llvm_unreachable("not expecting a closure"); } llvm_unreachable("not a CXXCtorType"); } @@ -112,8 +115,8 @@ inline StructorType getFromDtorType(CXXDtorType T) { llvm_unreachable("not a CXXDtorType"); } -/// CodeGenTypes - This class organizes the cross-module state that is used -/// while lowering AST types to LLVM types. +/// This class organizes the cross-module state that is used while lowering +/// AST types to LLVM types. class CodeGenTypes { CodeGenModule &CGM; // Some of this stuff should probably be left on the CGM. @@ -133,34 +136,32 @@ class CodeGenTypes { /// types are never refined. llvm::DenseMap<const ObjCInterfaceType*, llvm::Type *> InterfaceTypes; - /// CGRecordLayouts - This maps llvm struct type with corresponding - /// record layout info. + /// Maps clang struct type with corresponding record layout info. llvm::DenseMap<const Type*, CGRecordLayout *> CGRecordLayouts; - /// RecordDeclTypes - This contains the LLVM IR type for any converted - /// RecordDecl. + /// Contains the LLVM IR type for any converted RecordDecl. llvm::DenseMap<const Type*, llvm::StructType *> RecordDeclTypes; - /// FunctionInfos - Hold memoized CGFunctionInfo results. + /// Hold memoized CGFunctionInfo results. llvm::FoldingSet<CGFunctionInfo> FunctionInfos; - /// RecordsBeingLaidOut - This set keeps track of records that we're currently - /// converting to an IR type. For example, when converting: + /// This set keeps track of records that we're currently converting + /// to an IR type. For example, when converting: /// struct A { struct B { int x; } } when processing 'x', the 'A' and 'B' /// types will be in this set. llvm::SmallPtrSet<const Type*, 4> RecordsBeingLaidOut; llvm::SmallPtrSet<const CGFunctionInfo*, 4> FunctionsBeingProcessed; - /// SkippedLayout - True if we didn't layout a function due to a being inside + /// True if we didn't layout a function due to a being inside /// a recursive struct conversion, set this to true. bool SkippedLayout; SmallVector<const RecordDecl *, 8> DeferredRecords; private: - /// TypeCache - This map keeps cache of llvm::Types - /// and maps clang::Type to corresponding llvm::Type. + /// This map keeps cache of llvm::Types and maps clang::Type to + /// corresponding llvm::Type. llvm::DenseMap<const Type *, llvm::Type *> TypeCache; public: @@ -261,6 +262,8 @@ public: const FunctionProtoType *type, RequiredArgs required); const CGFunctionInfo &arrangeMSMemberPointerThunk(const CXXMethodDecl *MD); + const CGFunctionInfo &arrangeMSCtorClosure(const CXXConstructorDecl *CD, + CXXCtorType CT); const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionProtoType> Ty); const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionNoProtoType> Ty); @@ -305,7 +308,7 @@ public: // These are internal details of CGT that shouldn't be used externally. /// IsZeroInitializable - Return whether a record type can be /// zero-initialized (in the C++ sense) with an LLVM zeroinitializer. - bool isZeroInitializable(const CXXRecordDecl *RD); + bool isZeroInitializable(const RecordDecl *RD); bool isRecordLayoutComplete(const Type *Ty) const; bool noRecordsBeingLaidOut() const { diff --git a/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp b/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp index 6f159d4..eca9159 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/CoverageMappingGen.cpp @@ -15,6 +15,7 @@ #include "CodeGenFunction.h" #include "clang/AST/StmtVisitor.h" #include "clang/Lex/Lexer.h" +#include "llvm/ADT/Optional.h" #include "llvm/ProfileData/CoverageMapping.h" #include "llvm/ProfileData/CoverageMappingReader.h" #include "llvm/ProfileData/CoverageMappingWriter.h" @@ -33,87 +34,50 @@ namespace { /// \brief A region of source code that can be mapped to a counter. class SourceMappingRegion { -public: - enum RegionFlags { - /// \brief This region won't be emitted if it wasn't extended. - /// This is useful so that we won't emit source ranges for single tokens - /// that we don't really care that much about, like: - /// the '(' token in #define MACRO ( - IgnoreIfNotExtended = 0x0001, - }; - -private: - FileID File, MacroArgumentFile; - Counter Count; - /// \brief A statement that initiated the count of Zero. - /// - /// This initiator statement is useful to prevent merging of unreachable - /// regions with different statements that caused the counter to become - /// unreachable. - const Stmt *UnreachableInitiator; - - /// \brief A statement that separates certain mapping regions into groups. - /// - /// The group statement is sometimes useful when we are emitting the source - /// regions not in their correct lexical order, e.g. the regions for the - /// incrementation expression in the 'for' construct. By marking the regions - /// in the incrementation expression with the group statement, we avoid the - /// merging of the regions from the incrementation expression and the loop's - /// body. - const Stmt *Group; - /// \brief The region's starting location. - SourceLocation LocStart; + Optional<SourceLocation> LocStart; /// \brief The region's ending location. - SourceLocation LocEnd, AlternativeLocEnd; - unsigned Flags; + Optional<SourceLocation> LocEnd; public: - SourceMappingRegion(FileID File, FileID MacroArgumentFile, Counter Count, - const Stmt *UnreachableInitiator, const Stmt *Group, - SourceLocation LocStart, SourceLocation LocEnd, - unsigned Flags = 0) - : File(File), MacroArgumentFile(MacroArgumentFile), Count(Count), - UnreachableInitiator(UnreachableInitiator), Group(Group), - LocStart(LocStart), LocEnd(LocEnd), AlternativeLocEnd(LocStart), - Flags(Flags) {} + SourceMappingRegion(Counter Count, Optional<SourceLocation> LocStart, + Optional<SourceLocation> LocEnd) + : Count(Count), LocStart(LocStart), LocEnd(LocEnd) {} + + SourceMappingRegion(SourceMappingRegion &&Region) + : Count(std::move(Region.Count)), LocStart(std::move(Region.LocStart)), + LocEnd(std::move(Region.LocEnd)) {} - const FileID &getFile() const { return File; } + SourceMappingRegion &operator=(SourceMappingRegion &&RHS) { + Count = std::move(RHS.Count); + LocStart = std::move(RHS.LocStart); + LocEnd = std::move(RHS.LocEnd); + return *this; + } const Counter &getCounter() const { return Count; } - const SourceLocation &getStartLoc() const { return LocStart; } + void setCounter(Counter C) { Count = C; } - const SourceLocation &getEndLoc(const SourceManager &SM) const { - if (SM.getFileID(LocEnd) != File) - return AlternativeLocEnd; - return LocEnd; - } + bool hasStartLoc() const { return LocStart.hasValue(); } - bool hasFlag(RegionFlags Flag) const { return (Flags & Flag) != 0; } + void setStartLoc(SourceLocation Loc) { LocStart = Loc; } - void setFlag(RegionFlags Flag) { Flags |= Flag; } + const SourceLocation &getStartLoc() const { + assert(LocStart && "Region has no start location"); + return *LocStart; + } - void clearFlag(RegionFlags Flag) { Flags &= ~Flag; } + bool hasEndLoc() const { return LocEnd.hasValue(); } - /// \brief Return true if two regions can be merged together. - bool isMergeable(SourceMappingRegion &R) { - // FIXME: We allow merging regions with a gap in between them. Should we? - return File == R.File && MacroArgumentFile == R.MacroArgumentFile && - Count == R.Count && UnreachableInitiator == R.UnreachableInitiator && - Group == R.Group; - } + void setEndLoc(SourceLocation Loc) { LocEnd = Loc; } - /// \brief A comparison that sorts such that mergeable regions are adjacent. - friend bool operator<(const SourceMappingRegion &LHS, - const SourceMappingRegion &RHS) { - return std::tie(LHS.File, LHS.MacroArgumentFile, LHS.Count, - LHS.UnreachableInitiator, LHS.Group) < - std::tie(RHS.File, RHS.MacroArgumentFile, RHS.Count, - RHS.UnreachableInitiator, RHS.Group); + const SourceLocation &getEndLoc() const { + assert(LocEnd && "Region has no end location"); + return *LocEnd; } }; @@ -126,26 +90,11 @@ public: const LangOptions &LangOpts; private: - struct FileInfo { - /// \brief The file id that will be used by the coverage mapping system. - unsigned CovMappingFileID; - const FileEntry *Entry; - - FileInfo(unsigned CovMappingFileID, const FileEntry *Entry) - : CovMappingFileID(CovMappingFileID), Entry(Entry) {} - }; - - /// \brief This mapping maps clang's FileIDs to file ids used - /// by the coverage mapping system and clang's file entries. - llvm::SmallDenseMap<FileID, FileInfo, 8> FileIDMapping; + /// \brief Map of clang's FileIDs to IDs used for coverage mapping. + llvm::SmallDenseMap<FileID, std::pair<unsigned, SourceLocation>, 8> + FileIDMapping; public: - /// \brief The statement that corresponds to the current source group. - const Stmt *CurrentSourceGroup; - - /// \brief The statement the initiated the current unreachable region. - const Stmt *CurrentUnreachableRegionInitiator; - /// \brief The coverage mapping regions for this function llvm::SmallVector<CounterMappingRegion, 32> MappingRegions; /// \brief The source mapping regions for this function. @@ -153,60 +102,104 @@ public: CoverageMappingBuilder(CoverageMappingModuleGen &CVM, SourceManager &SM, const LangOptions &LangOpts) - : CVM(CVM), SM(SM), LangOpts(LangOpts), - CurrentSourceGroup(nullptr), - CurrentUnreachableRegionInitiator(nullptr) {} + : CVM(CVM), SM(SM), LangOpts(LangOpts) {} /// \brief Return the precise end location for the given token. SourceLocation getPreciseTokenLocEnd(SourceLocation Loc) { - return Lexer::getLocForEndOfToken(SM.getSpellingLoc(Loc), 0, SM, LangOpts); + // We avoid getLocForEndOfToken here, because it doesn't do what we want for + // macro locations, which we just treat as expanded files. + unsigned TokLen = + Lexer::MeasureTokenLength(SM.getSpellingLoc(Loc), SM, LangOpts); + return Loc.getLocWithOffset(TokLen); } - /// \brief Create the mapping that maps from the function's file ids to - /// the indices for the translation unit's filenames. - void createFileIDMapping(SmallVectorImpl<unsigned> &Mapping) { - Mapping.resize(FileIDMapping.size(), 0); - for (const auto &I : FileIDMapping) - Mapping[I.second.CovMappingFileID] = CVM.getFileID(I.second.Entry); + /// \brief Return the start location of an included file or expanded macro. + SourceLocation getStartOfFileOrMacro(SourceLocation Loc) { + if (Loc.isMacroID()) + return Loc.getLocWithOffset(-SM.getFileOffset(Loc)); + return SM.getLocForStartOfFile(SM.getFileID(Loc)); } - /// \brief Get the coverage mapping file id that corresponds to the given - /// clang file id. If such file id doesn't exist, it gets added to the - /// mapping that maps from clang's file ids to coverage mapping file ids. - /// Return true if there was an error getting the coverage mapping file id. - /// An example of an when this function fails is when the region tries - /// to get a coverage file id for a location in a built-in macro. - bool getCoverageFileID(SourceLocation LocStart, FileID File, - FileID SpellingFile, unsigned &Result) { - auto Mapping = FileIDMapping.find(File); - if (Mapping != FileIDMapping.end()) { - Result = Mapping->second.CovMappingFileID; - return false; - } + /// \brief Return the end location of an included file or expanded macro. + SourceLocation getEndOfFileOrMacro(SourceLocation Loc) { + if (Loc.isMacroID()) + return Loc.getLocWithOffset(SM.getFileIDSize(SM.getFileID(Loc)) - + SM.getFileOffset(Loc)); + return SM.getLocForEndOfFile(SM.getFileID(Loc)); + } + + /// \brief Find out where the current file is included or macro is expanded. + SourceLocation getIncludeOrExpansionLoc(SourceLocation Loc) { + return Loc.isMacroID() ? SM.getImmediateExpansionRange(Loc).first + : SM.getIncludeLoc(SM.getFileID(Loc)); + } - auto Entry = SM.getFileEntryForID(SpellingFile); - if (!Entry) - return true; + /// \brief Return true if \c Loc is a location in a built-in macro. + bool isInBuiltin(SourceLocation Loc) { + return strcmp(SM.getBufferName(SM.getSpellingLoc(Loc)), "<built-in>") == 0; + } + + /// \brief Get the start of \c S ignoring macro arguments and builtin macros. + SourceLocation getStart(const Stmt *S) { + SourceLocation Loc = S->getLocStart(); + while (SM.isMacroArgExpansion(Loc) || isInBuiltin(Loc)) + Loc = SM.getImmediateExpansionRange(Loc).first; + return Loc; + } - Result = FileIDMapping.size(); - FileIDMapping.insert(std::make_pair(File, FileInfo(Result, Entry))); - createFileExpansionRegion(LocStart, File); - return false; + /// \brief Get the end of \c S ignoring macro arguments and builtin macros. + SourceLocation getEnd(const Stmt *S) { + SourceLocation Loc = S->getLocEnd(); + while (SM.isMacroArgExpansion(Loc) || isInBuiltin(Loc)) + Loc = SM.getImmediateExpansionRange(Loc).first; + return getPreciseTokenLocEnd(Loc); } - /// \brief Get the coverage mapping file id that corresponds to the given - /// clang file id. - /// Return true if there was an error getting the coverage mapping file id. - bool getExistingCoverageFileID(FileID File, unsigned &Result) { - // Make sure that the file is valid. - if (File.isInvalid()) - return true; - auto Mapping = FileIDMapping.find(File); - if (Mapping != FileIDMapping.end()) { - Result = Mapping->second.CovMappingFileID; - return false; + /// \brief Find the set of files we have regions for and assign IDs + /// + /// Fills \c Mapping with the virtual file mapping needed to write out + /// coverage and collects the necessary file information to emit source and + /// expansion regions. + void gatherFileIDs(SmallVectorImpl<unsigned> &Mapping) { + FileIDMapping.clear(); + + SmallVector<FileID, 8> Visited; + SmallVector<std::pair<SourceLocation, unsigned>, 8> FileLocs; + for (const auto &Region : SourceRegions) { + SourceLocation Loc = Region.getStartLoc(); + FileID File = SM.getFileID(Loc); + if (std::find(Visited.begin(), Visited.end(), File) != Visited.end()) + continue; + Visited.push_back(File); + + unsigned Depth = 0; + for (SourceLocation Parent = getIncludeOrExpansionLoc(Loc); + !Parent.isInvalid(); Parent = getIncludeOrExpansionLoc(Parent)) + ++Depth; + FileLocs.push_back(std::make_pair(Loc, Depth)); + } + std::stable_sort(FileLocs.begin(), FileLocs.end(), llvm::less_second()); + + for (const auto &FL : FileLocs) { + SourceLocation Loc = FL.first; + FileID SpellingFile = SM.getDecomposedSpellingLoc(Loc).first; + auto Entry = SM.getFileEntryForID(SpellingFile); + if (!Entry) + continue; + + FileIDMapping[SM.getFileID(Loc)] = std::make_pair(Mapping.size(), Loc); + Mapping.push_back(CVM.getFileID(Entry)); } - return true; + } + + /// \brief Get the coverage mapping file ID for \c Loc. + /// + /// If such file id doesn't exist, return None. + Optional<unsigned> getCoverageFileID(SourceLocation Loc) { + auto Mapping = FileIDMapping.find(SM.getFileID(Loc)); + if (Mapping != FileIDMapping.end()) + return Mapping->second.first; + return None; } /// \brief Return true if the given clang's file id has a corresponding @@ -235,154 +228,83 @@ public: for (const auto &I : SkippedRanges) { auto LocStart = I.getBegin(); auto LocEnd = I.getEnd(); - auto FileStart = SM.getFileID(LocStart); - if (!hasExistingCoverageFileID(FileStart)) - continue; - auto ActualFileStart = SM.getDecomposedSpellingLoc(LocStart).first; - if (ActualFileStart != SM.getDecomposedSpellingLoc(LocEnd).first) - // Ignore regions that span across multiple files. - continue; + assert(SM.isWrittenInSameFile(LocStart, LocEnd) && + "region spans multiple files"); - unsigned CovFileID; - if (getCoverageFileID(LocStart, FileStart, ActualFileStart, CovFileID)) + auto CovFileID = getCoverageFileID(LocStart); + if (!CovFileID) continue; unsigned LineStart = SM.getSpellingLineNumber(LocStart); unsigned ColumnStart = SM.getSpellingColumnNumber(LocStart); unsigned LineEnd = SM.getSpellingLineNumber(LocEnd); unsigned ColumnEnd = SM.getSpellingColumnNumber(LocEnd); - CounterMappingRegion Region(Counter(), CovFileID, LineStart, ColumnStart, - LineEnd, ColumnEnd, false, - CounterMappingRegion::SkippedRegion); + auto Region = CounterMappingRegion::makeSkipped( + *CovFileID, LineStart, ColumnStart, LineEnd, ColumnEnd); // Make sure that we only collect the regions that are inside // the souce code of this function. - if (Region.LineStart >= FileLineRanges[CovFileID].first && - Region.LineEnd <= FileLineRanges[CovFileID].second) + if (Region.LineStart >= FileLineRanges[*CovFileID].first && + Region.LineEnd <= FileLineRanges[*CovFileID].second) MappingRegions.push_back(Region); } } - /// \brief Create a mapping region that correponds to an expansion of - /// a macro or an embedded include. - void createFileExpansionRegion(SourceLocation Loc, FileID ExpandedFile) { - SourceLocation LocStart; - if (Loc.isMacroID()) - LocStart = SM.getImmediateExpansionRange(Loc).first; - else { - LocStart = SM.getIncludeLoc(ExpandedFile); - if (LocStart.isInvalid()) - return; // This file has no expansion region. - } - - auto File = SM.getFileID(LocStart); - auto SpellingFile = SM.getDecomposedSpellingLoc(LocStart).first; - unsigned CovFileID, ExpandedFileID; - if (getExistingCoverageFileID(ExpandedFile, ExpandedFileID)) - return; - if (getCoverageFileID(LocStart, File, SpellingFile, CovFileID)) - return; - unsigned LineStart = SM.getSpellingLineNumber(LocStart); - unsigned ColumnStart = SM.getSpellingColumnNumber(LocStart); - unsigned LineEnd = LineStart; - // Compute the end column manually as Lexer::getLocForEndOfToken doesn't - // give the correct result in all cases. - unsigned ColumnEnd = - ColumnStart + - Lexer::MeasureTokenLength(SM.getSpellingLoc(LocStart), SM, LangOpts); - - MappingRegions.push_back(CounterMappingRegion( - Counter(), CovFileID, LineStart, ColumnStart, LineEnd, ColumnEnd, - false, CounterMappingRegion::ExpansionRegion)); - MappingRegions.back().ExpandedFileID = ExpandedFileID; - } - - /// \brief Enter a source region group that is identified by the given - /// statement. - /// It's not possible to enter a group when there is already - /// another group present. - void beginSourceRegionGroup(const Stmt *Group) { - assert(!CurrentSourceGroup); - CurrentSourceGroup = Group; - } - - /// \brief Exit the current source region group. - void endSourceRegionGroup() { CurrentSourceGroup = nullptr; } - - /// \brief Associate a counter with a given source code range. - void mapSourceCodeRange(SourceLocation LocStart, SourceLocation LocEnd, - Counter Count, const Stmt *UnreachableInitiator, - const Stmt *SourceGroup, unsigned Flags = 0, - FileID MacroArgumentFile = FileID()) { - if (SM.isMacroArgExpansion(LocStart)) { - // Map the code range with the macro argument's value. - mapSourceCodeRange(SM.getImmediateSpellingLoc(LocStart), - SM.getImmediateSpellingLoc(LocEnd), Count, - UnreachableInitiator, SourceGroup, Flags, - SM.getFileID(LocStart)); - // Map the code range where the macro argument is referenced. - SourceLocation RefLocStart(SM.getImmediateExpansionRange(LocStart).first); - SourceLocation RefLocEnd(RefLocStart); - if (SM.isMacroArgExpansion(RefLocStart)) - mapSourceCodeRange(RefLocStart, RefLocEnd, Count, UnreachableInitiator, - SourceGroup, 0, SM.getFileID(RefLocStart)); - else - mapSourceCodeRange(RefLocStart, RefLocEnd, Count, UnreachableInitiator, - SourceGroup); - return; - } - auto File = SM.getFileID(LocStart); - // Make sure that the file id is valid. - if (File.isInvalid()) - return; - SourceRegions.emplace_back(File, MacroArgumentFile, Count, - UnreachableInitiator, SourceGroup, LocStart, - LocEnd, Flags); - } - - void mapSourceCodeRange(SourceLocation LocStart, SourceLocation LocEnd, - Counter Count, unsigned Flags = 0) { - mapSourceCodeRange(LocStart, LocEnd, Count, - CurrentUnreachableRegionInitiator, CurrentSourceGroup, - Flags); - } - /// \brief Generate the coverage counter mapping regions from collected /// source regions. void emitSourceRegions() { - std::sort(SourceRegions.begin(), SourceRegions.end()); - - for (auto I = SourceRegions.begin(), E = SourceRegions.end(); I != E; ++I) { - // Keep the original start location of this region. - SourceLocation LocStart = I->getStartLoc(); - SourceLocation LocEnd = I->getEndLoc(SM); - - bool Ignore = I->hasFlag(SourceMappingRegion::IgnoreIfNotExtended); - // We need to handle mergeable regions together. - for (auto Next = I + 1; Next != E && Next->isMergeable(*I); ++Next) { - ++I; - LocStart = std::min(LocStart, I->getStartLoc()); - LocEnd = std::max(LocEnd, I->getEndLoc(SM)); - // FIXME: Should we && together the Ignore flag of multiple regions? - Ignore = false; - } - if (Ignore) + for (const auto &Region : SourceRegions) { + assert(Region.hasEndLoc() && "incomplete region"); + + SourceLocation LocStart = Region.getStartLoc(); + assert(!SM.getFileID(LocStart).isInvalid() && "region in invalid file"); + + auto CovFileID = getCoverageFileID(LocStart); + // Ignore regions that don't have a file, such as builtin macros. + if (!CovFileID) continue; + SourceLocation LocEnd = Region.getEndLoc(); + assert(SM.isWrittenInSameFile(LocStart, LocEnd) && + "region spans multiple files"); + // Find the spilling locations for the mapping region. - LocEnd = getPreciseTokenLocEnd(LocEnd); unsigned LineStart = SM.getSpellingLineNumber(LocStart); unsigned ColumnStart = SM.getSpellingColumnNumber(LocStart); unsigned LineEnd = SM.getSpellingLineNumber(LocEnd); unsigned ColumnEnd = SM.getSpellingColumnNumber(LocEnd); - auto SpellingFile = SM.getDecomposedSpellingLoc(LocStart).first; - unsigned CovFileID; - if (getCoverageFileID(LocStart, I->getFile(), SpellingFile, CovFileID)) + assert(LineStart <= LineEnd && "region start and end out of order"); + MappingRegions.push_back(CounterMappingRegion::makeRegion( + Region.getCounter(), *CovFileID, LineStart, ColumnStart, LineEnd, + ColumnEnd)); + } + } + + /// \brief Generate expansion regions for each virtual file we've seen. + void emitExpansionRegions() { + for (const auto &FM : FileIDMapping) { + SourceLocation ExpandedLoc = FM.second.second; + SourceLocation ParentLoc = getIncludeOrExpansionLoc(ExpandedLoc); + if (ParentLoc.isInvalid()) + continue; + + auto ParentFileID = getCoverageFileID(ParentLoc); + if (!ParentFileID) continue; + auto ExpandedFileID = getCoverageFileID(ExpandedLoc); + assert(ExpandedFileID && "expansion in uncovered file"); + + SourceLocation LocEnd = getPreciseTokenLocEnd(ParentLoc); + assert(SM.isWrittenInSameFile(ParentLoc, LocEnd) && + "region spans multiple files"); + + unsigned LineStart = SM.getSpellingLineNumber(ParentLoc); + unsigned ColumnStart = SM.getSpellingColumnNumber(ParentLoc); + unsigned LineEnd = SM.getSpellingLineNumber(LocEnd); + unsigned ColumnEnd = SM.getSpellingColumnNumber(LocEnd); - assert(LineStart <= LineEnd); - MappingRegions.push_back(CounterMappingRegion( - I->getCounter(), CovFileID, LineStart, ColumnStart, LineEnd, - ColumnEnd, false, CounterMappingRegion::CodeRegion)); + MappingRegions.push_back(CounterMappingRegion::makeExpansion( + *ParentFileID, *ExpandedFileID, LineStart, ColumnStart, LineEnd, + ColumnEnd)); } } }; @@ -398,14 +320,14 @@ struct EmptyCoverageMappingBuilder : public CoverageMappingBuilder { if (!D->hasBody()) return; auto Body = D->getBody(); - mapSourceCodeRange(Body->getLocStart(), Body->getLocEnd(), Counter()); + SourceRegions.emplace_back(Counter(), getStart(Body), getEnd(Body)); } /// \brief Write the mapping data to the output stream void write(llvm::raw_ostream &OS) { - emitSourceRegions(); SmallVector<unsigned, 16> FileIDMapping; - createFileIDMapping(FileIDMapping); + gatherFileIDs(FileIDMapping); + emitSourceRegions(); CoverageMappingWriter Writer(FileIDMapping, None, MappingRegions); Writer.write(OS); @@ -420,136 +342,216 @@ struct CounterCoverageMappingBuilder /// \brief The map of statements to count values. llvm::DenseMap<const Stmt *, unsigned> &CounterMap; - Counter CurrentRegionCount; + /// \brief A stack of currently live regions. + std::vector<SourceMappingRegion> RegionStack; CounterExpressionBuilder Builder; - /// \brief Return a counter that represents the - /// expression that subracts rhs from lhs. + /// \brief A location in the most recently visited file or macro. + /// + /// This is used to adjust the active source regions appropriately when + /// expressions cross file or macro boundaries. + SourceLocation MostRecentLocation; + + /// \brief Return a counter for the subtraction of \c RHS from \c LHS Counter subtractCounters(Counter LHS, Counter RHS) { return Builder.subtract(LHS, RHS); } - /// \brief Return a counter that represents the - /// the exression that adds lhs and rhs. + /// \brief Return a counter for the sum of \c LHS and \c RHS. Counter addCounters(Counter LHS, Counter RHS) { return Builder.add(LHS, RHS); } + Counter addCounters(Counter C1, Counter C2, Counter C3) { + return addCounters(addCounters(C1, C2), C3); + } + + Counter addCounters(Counter C1, Counter C2, Counter C3, Counter C4) { + return addCounters(addCounters(C1, C2, C3), C4); + } + /// \brief Return the region counter for the given statement. + /// /// This should only be called on statements that have a dedicated counter. - unsigned getRegionCounter(const Stmt *S) { return CounterMap[S]; } + Counter getRegionCounter(const Stmt *S) { + return Counter::getCounter(CounterMap[S]); + } + + /// \brief Push a region onto the stack. + /// + /// Returns the index on the stack where the region was pushed. This can be + /// used with popRegions to exit a "scope", ending the region that was pushed. + size_t pushRegion(Counter Count, Optional<SourceLocation> StartLoc = None, + Optional<SourceLocation> EndLoc = None) { + if (StartLoc) + MostRecentLocation = *StartLoc; + RegionStack.emplace_back(Count, StartLoc, EndLoc); - /// \brief Return the region count for the counter at the given index. - Counter getRegionCount(unsigned CounterId) { - return Counter::getCounter(CounterId); + return RegionStack.size() - 1; } - /// \brief Return the counter value of the current region. - Counter getCurrentRegionCount() { return CurrentRegionCount; } + /// \brief Pop regions from the stack into the function's list of regions. + /// + /// Adds all regions from \c ParentIndex to the top of the stack to the + /// function's \c SourceRegions. + void popRegions(size_t ParentIndex) { + assert(RegionStack.size() >= ParentIndex && "parent not in stack"); + while (RegionStack.size() > ParentIndex) { + SourceMappingRegion &Region = RegionStack.back(); + if (Region.hasStartLoc()) { + SourceLocation StartLoc = Region.getStartLoc(); + SourceLocation EndLoc = Region.hasEndLoc() + ? Region.getEndLoc() + : RegionStack[ParentIndex].getEndLoc(); + while (!SM.isWrittenInSameFile(StartLoc, EndLoc)) { + // The region ends in a nested file or macro expansion. Create a + // separate region for each expansion. + SourceLocation NestedLoc = getStartOfFileOrMacro(EndLoc); + assert(SM.isWrittenInSameFile(NestedLoc, EndLoc)); + + SourceRegions.emplace_back(Region.getCounter(), NestedLoc, EndLoc); + + EndLoc = getPreciseTokenLocEnd(getIncludeOrExpansionLoc(EndLoc)); + assert(!EndLoc.isInvalid() && + "File exit was not handled before popRegions"); + } + Region.setEndLoc(EndLoc); + + MostRecentLocation = EndLoc; + // If this region happens to span an entire expansion, we need to make + // sure we don't overlap the parent region with it. + if (StartLoc == getStartOfFileOrMacro(StartLoc) && + EndLoc == getEndOfFileOrMacro(EndLoc)) + MostRecentLocation = getIncludeOrExpansionLoc(EndLoc); + + assert(SM.isWrittenInSameFile(Region.getStartLoc(), EndLoc)); + SourceRegions.push_back(std::move(Region)); + } + RegionStack.pop_back(); + } + } - /// \brief Set the counter value for the current region. - /// This is used to keep track of changes to the most recent counter - /// from control flow and non-local exits. - void setCurrentRegionCount(Counter Count) { - CurrentRegionCount = Count; - CurrentUnreachableRegionInitiator = nullptr; + /// \brief Return the currently active region. + SourceMappingRegion &getRegion() { + assert(!RegionStack.empty() && "statement has no region"); + return RegionStack.back(); } - /// \brief Indicate that the current region is never reached, - /// and thus should have a counter value of zero. - /// This is important so that subsequent regions can correctly track - /// their parent counts. - void setCurrentRegionUnreachable(const Stmt *Initiator) { - CurrentRegionCount = Counter::getZero(); - CurrentUnreachableRegionInitiator = Initiator; + /// \brief Propagate counts through the children of \c S. + Counter propagateCounts(Counter TopCount, const Stmt *S) { + size_t Index = pushRegion(TopCount, getStart(S), getEnd(S)); + Visit(S); + Counter ExitCount = getRegion().getCounter(); + popRegions(Index); + return ExitCount; } - /// \brief A counter for a particular region. - /// This is the primary interface through - /// which the coverage mapping builder manages counters and their values. - class RegionMapper { - CounterCoverageMappingBuilder &Mapping; - Counter Count; - Counter ParentCount; - Counter RegionCount; - Counter Adjust; - - public: - RegionMapper(CounterCoverageMappingBuilder *Mapper, const Stmt *S) - : Mapping(*Mapper), - Count(Mapper->getRegionCount(Mapper->getRegionCounter(S))), - ParentCount(Mapper->getCurrentRegionCount()) {} - - /// Get the value of the counter. In most cases this is the number of times - /// the region of the counter was entered, but for switch labels it's the - /// number of direct jumps to that label. - Counter getCount() const { return Count; } - - /// Get the value of the counter with adjustments applied. Adjustments occur - /// when control enters or leaves the region abnormally; i.e., if there is a - /// jump to a label within the region, or if the function can return from - /// within the region. The adjusted count, then, is the value of the counter - /// at the end of the region. - Counter getAdjustedCount() const { - return Mapping.addCounters(Count, Adjust); - } + /// \brief Adjust the most recently visited location to \c EndLoc. + /// + /// This should be used after visiting any statements in non-source order. + void adjustForOutOfOrderTraversal(SourceLocation EndLoc) { + MostRecentLocation = EndLoc; + // Avoid adding duplicate regions if we have a completed region on the top + // of the stack and are adjusting to the end of a virtual file. + if (getRegion().hasEndLoc() && + MostRecentLocation == getEndOfFileOrMacro(MostRecentLocation)) + MostRecentLocation = getIncludeOrExpansionLoc(MostRecentLocation); + } + + /// \brief Check whether \c Loc is included or expanded from \c Parent. + bool isNestedIn(SourceLocation Loc, FileID Parent) { + do { + Loc = getIncludeOrExpansionLoc(Loc); + if (Loc.isInvalid()) + return false; + } while (!SM.isInFileID(Loc, Parent)); + return true; + } - /// Get the value of the counter in this region's parent, i.e., the region - /// that was active when this region began. This is useful for deriving - /// counts in implicitly counted regions, like the false case of a condition - /// or the normal exits of a loop. - Counter getParentCount() const { return ParentCount; } - - /// Activate the counter by emitting an increment and starting to track - /// adjustments. If AddIncomingFallThrough is true, the current region count - /// will be added to the counter for the purposes of tracking the region. - void beginRegion(bool AddIncomingFallThrough = false) { - RegionCount = Count; - if (AddIncomingFallThrough) - RegionCount = - Mapping.addCounters(RegionCount, Mapping.getCurrentRegionCount()); - Mapping.setCurrentRegionCount(RegionCount); - } + /// \brief Adjust regions and state when \c NewLoc exits a file. + /// + /// If moving from our most recently tracked location to \c NewLoc exits any + /// files, this adjusts our current region stack and creates the file regions + /// for the exited file. + void handleFileExit(SourceLocation NewLoc) { + if (NewLoc.isInvalid() || + SM.isWrittenInSameFile(MostRecentLocation, NewLoc)) + return; - /// For counters on boolean branches, begins tracking adjustments for the - /// uncounted path. - void beginElseRegion() { - RegionCount = Mapping.subtractCounters(ParentCount, Count); - Mapping.setCurrentRegionCount(RegionCount); + // If NewLoc is not in a file that contains MostRecentLocation, walk up to + // find the common ancestor. + SourceLocation LCA = NewLoc; + FileID ParentFile = SM.getFileID(LCA); + while (!isNestedIn(MostRecentLocation, ParentFile)) { + LCA = getIncludeOrExpansionLoc(LCA); + if (LCA.isInvalid() || SM.isWrittenInSameFile(LCA, MostRecentLocation)) { + // Since there isn't a common ancestor, no file was exited. We just need + // to adjust our location to the new file. + MostRecentLocation = NewLoc; + return; + } + ParentFile = SM.getFileID(LCA); } - /// Reset the current region count. - void setCurrentRegionCount(Counter CurrentCount) { - RegionCount = CurrentCount; - Mapping.setCurrentRegionCount(RegionCount); - } + llvm::SmallSet<SourceLocation, 8> StartLocs; + Optional<Counter> ParentCounter; + for (auto I = RegionStack.rbegin(), E = RegionStack.rend(); I != E; ++I) { + if (!I->hasStartLoc()) + continue; + SourceLocation Loc = I->getStartLoc(); + if (!isNestedIn(Loc, ParentFile)) { + ParentCounter = I->getCounter(); + break; + } - /// Adjust for non-local control flow after emitting a subexpression or - /// substatement. This must be called to account for constructs such as - /// gotos, - /// labels, and returns, so that we can ensure that our region's count is - /// correct in the code that follows. - void adjustForControlFlow() { - Adjust = Mapping.addCounters( - Adjust, Mapping.subtractCounters(Mapping.getCurrentRegionCount(), - RegionCount)); - // Reset the region count in case this is called again later. - RegionCount = Mapping.getCurrentRegionCount(); + while (!SM.isInFileID(Loc, ParentFile)) { + // The most nested region for each start location is the one with the + // correct count. We avoid creating redundant regions by stopping once + // we've seen this region. + if (StartLocs.insert(Loc).second) + SourceRegions.emplace_back(I->getCounter(), Loc, + getEndOfFileOrMacro(Loc)); + Loc = getIncludeOrExpansionLoc(Loc); + } + I->setStartLoc(getPreciseTokenLocEnd(Loc)); } - /// Commit all adjustments to the current region. If the region is a loop, - /// the LoopAdjust value should be the count of all the breaks and continues - /// from the loop, to compensate for those counts being deducted from the - /// adjustments for the body of the loop. - void applyAdjustmentsToRegion() { - Mapping.setCurrentRegionCount(Mapping.addCounters(ParentCount, Adjust)); - } - void applyAdjustmentsToRegion(Counter LoopAdjust) { - Mapping.setCurrentRegionCount(Mapping.addCounters( - Mapping.addCounters(ParentCount, Adjust), LoopAdjust)); + if (ParentCounter) { + // If the file is contained completely by another region and doesn't + // immediately start its own region, the whole file gets a region + // corresponding to the parent. + SourceLocation Loc = MostRecentLocation; + while (isNestedIn(Loc, ParentFile)) { + SourceLocation FileStart = getStartOfFileOrMacro(Loc); + if (StartLocs.insert(FileStart).second) + SourceRegions.emplace_back(*ParentCounter, FileStart, + getEndOfFileOrMacro(Loc)); + Loc = getIncludeOrExpansionLoc(Loc); + } } - }; + + MostRecentLocation = NewLoc; + } + + /// \brief Ensure that \c S is included in the current region. + void extendRegion(const Stmt *S) { + SourceMappingRegion &Region = getRegion(); + SourceLocation StartLoc = getStart(S); + + handleFileExit(StartLoc); + if (!Region.hasStartLoc()) + Region.setStartLoc(StartLoc); + } + + /// \brief Mark \c S as a terminator, starting a zero region. + void terminateRegion(const Stmt *S) { + extendRegion(S); + SourceMappingRegion &Region = getRegion(); + if (!Region.hasEndLoc()) + Region.setEndLoc(getEnd(S)); + pushRegion(Counter::getZero()); + } /// \brief Keep counts of breaks and continues inside loops. struct BreakContinue { @@ -566,437 +568,320 @@ struct CounterCoverageMappingBuilder /// \brief Write the mapping data to the output stream void write(llvm::raw_ostream &OS) { - emitSourceRegions(); llvm::SmallVector<unsigned, 8> VirtualFileMapping; - createFileIDMapping(VirtualFileMapping); + gatherFileIDs(VirtualFileMapping); + emitSourceRegions(); + emitExpansionRegions(); gatherSkippedRegions(); - CoverageMappingWriter Writer( - VirtualFileMapping, Builder.getExpressions(), MappingRegions); + CoverageMappingWriter Writer(VirtualFileMapping, Builder.getExpressions(), + MappingRegions); Writer.write(OS); } - /// \brief Associate the source code range with the current region count. - void mapSourceCodeRange(SourceLocation LocStart, SourceLocation LocEnd, - unsigned Flags = 0) { - CoverageMappingBuilder::mapSourceCodeRange(LocStart, LocEnd, - CurrentRegionCount, Flags); - } - - void mapSourceCodeRange(SourceLocation LocStart) { - CoverageMappingBuilder::mapSourceCodeRange(LocStart, LocStart, - CurrentRegionCount); - } - - /// \brief Associate the source range of a token with the current region - /// count. - /// Ignore the source range for this token if it produces a distinct - /// mapping region with no other source ranges. - void mapToken(SourceLocation LocStart) { - CoverageMappingBuilder::mapSourceCodeRange( - LocStart, LocStart, CurrentRegionCount, - SourceMappingRegion::IgnoreIfNotExtended); - } - void VisitStmt(const Stmt *S) { - mapSourceCodeRange(S->getLocStart()); - for (Stmt::const_child_range I = S->children(); I; ++I) { - if (*I) - this->Visit(*I); - } + if (!S->getLocStart().isInvalid()) + extendRegion(S); + for (const Stmt *Child : S->children()) + if (Child) + this->Visit(Child); + handleFileExit(getEnd(S)); } void VisitDecl(const Decl *D) { - if (!D->hasBody()) - return; - // Counter tracks entry to the function body. - auto Body = D->getBody(); - RegionMapper Cnt(this, Body); - Cnt.beginRegion(); - Visit(Body); - } - - void VisitDeclStmt(const DeclStmt *S) { - mapSourceCodeRange(S->getLocStart()); - for (Stmt::const_child_range I = static_cast<const Stmt *>(S)->children(); - I; ++I) { - if (*I) - this->Visit(*I); - } - } - - void VisitCompoundStmt(const CompoundStmt *S) { - mapSourceCodeRange(S->getLBracLoc()); - mapSourceCodeRange(S->getRBracLoc()); - for (Stmt::const_child_range I = S->children(); I; ++I) { - if (*I) - this->Visit(*I); - } + Stmt *Body = D->getBody(); + propagateCounts(getRegionCounter(Body), Body); } void VisitReturnStmt(const ReturnStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); if (S->getRetValue()) Visit(S->getRetValue()); - setCurrentRegionUnreachable(S); + terminateRegion(S); } - void VisitGotoStmt(const GotoStmt *S) { - mapSourceCodeRange(S->getLocStart()); - mapToken(S->getLabelLoc()); - setCurrentRegionUnreachable(S); + void VisitCXXThrowExpr(const CXXThrowExpr *E) { + extendRegion(E); + if (E->getSubExpr()) + Visit(E->getSubExpr()); + terminateRegion(E); } + void VisitGotoStmt(const GotoStmt *S) { terminateRegion(S); } + void VisitLabelStmt(const LabelStmt *S) { - // Counter tracks the block following the label. - RegionMapper Cnt(this, S); - Cnt.beginRegion(); - mapSourceCodeRange(S->getLocStart()); - // Can't map the ':' token as its location isn't known. + SourceLocation Start = getStart(S); + // We can't extendRegion here or we risk overlapping with our new region. + handleFileExit(Start); + pushRegion(getRegionCounter(S), Start); Visit(S->getSubStmt()); } void VisitBreakStmt(const BreakStmt *S) { - mapSourceCodeRange(S->getLocStart()); assert(!BreakContinueStack.empty() && "break not in a loop or switch!"); BreakContinueStack.back().BreakCount = addCounters( - BreakContinueStack.back().BreakCount, getCurrentRegionCount()); - setCurrentRegionUnreachable(S); + BreakContinueStack.back().BreakCount, getRegion().getCounter()); + terminateRegion(S); } void VisitContinueStmt(const ContinueStmt *S) { - mapSourceCodeRange(S->getLocStart()); assert(!BreakContinueStack.empty() && "continue stmt not in a loop!"); BreakContinueStack.back().ContinueCount = addCounters( - BreakContinueStack.back().ContinueCount, getCurrentRegionCount()); - setCurrentRegionUnreachable(S); + BreakContinueStack.back().ContinueCount, getRegion().getCounter()); + terminateRegion(S); } void VisitWhileStmt(const WhileStmt *S) { - mapSourceCodeRange(S->getLocStart()); - // Counter tracks the body of the loop. - RegionMapper Cnt(this, S); + extendRegion(S); + + Counter ParentCount = getRegion().getCounter(); + Counter BodyCount = getRegionCounter(S); + + // Handle the body first so that we can get the backedge count. BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first so the break/continue adjustments can be - // included when visiting the condition. - Cnt.beginRegion(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); - - // ...then go back and propagate counts through the condition. The count - // at the start of the condition is the sum of the incoming edges, - // the backedge from the end of the loop body, and the edges from - // continue statements. + extendRegion(S->getBody()); + Counter BackedgeCount = propagateCounts(BodyCount, S->getBody()); BreakContinue BC = BreakContinueStack.pop_back_val(); - Cnt.setCurrentRegionCount( - addCounters(Cnt.getParentCount(), - addCounters(Cnt.getAdjustedCount(), BC.ContinueCount))); - beginSourceRegionGroup(S->getCond()); - Visit(S->getCond()); - endSourceRegionGroup(); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(addCounters(BC.BreakCount, BC.ContinueCount)); + + // Go back to handle the condition. + Counter CondCount = + addCounters(ParentCount, BackedgeCount, BC.ContinueCount); + propagateCounts(CondCount, S->getCond()); + adjustForOutOfOrderTraversal(getEnd(S)); + + Counter OutCount = + addCounters(BC.BreakCount, subtractCounters(CondCount, BodyCount)); + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitDoStmt(const DoStmt *S) { - mapSourceCodeRange(S->getLocStart()); - // Counter tracks the body of the loop. - RegionMapper Cnt(this, S); - BreakContinueStack.push_back(BreakContinue()); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); + extendRegion(S); + + Counter ParentCount = getRegion().getCounter(); + Counter BodyCount = getRegionCounter(S); + BreakContinueStack.push_back(BreakContinue()); + extendRegion(S->getBody()); + Counter BackedgeCount = + propagateCounts(addCounters(ParentCount, BodyCount), S->getBody()); BreakContinue BC = BreakContinueStack.pop_back_val(); - // The count at the start of the condition is equal to the count at the - // end of the body. The adjusted count does not include either the - // fall-through count coming into the loop or the continue count, so add - // both of those separately. This is coincidentally the same equation as - // with while loops but for different reasons. - Cnt.setCurrentRegionCount( - addCounters(Cnt.getParentCount(), - addCounters(Cnt.getAdjustedCount(), BC.ContinueCount))); - Visit(S->getCond()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(addCounters(BC.BreakCount, BC.ContinueCount)); + + Counter CondCount = addCounters(BackedgeCount, BC.ContinueCount); + propagateCounts(CondCount, S->getCond()); + + Counter OutCount = + addCounters(BC.BreakCount, subtractCounters(CondCount, BodyCount)); + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitForStmt(const ForStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); if (S->getInit()) Visit(S->getInit()); - // Counter tracks the body of the loop. - RegionMapper Cnt(this, S); + Counter ParentCount = getRegion().getCounter(); + Counter BodyCount = getRegionCounter(S); + + // Handle the body first so that we can get the backedge count. BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first. (This is basically the same as a while - // loop; see further comments in VisitWhileStmt.) - Cnt.beginRegion(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); + extendRegion(S->getBody()); + Counter BackedgeCount = propagateCounts(BodyCount, S->getBody()); + BreakContinue BC = BreakContinueStack.pop_back_val(); // The increment is essentially part of the body but it needs to include // the count for all the continue statements. - if (S->getInc()) { - Cnt.setCurrentRegionCount(addCounters( - getCurrentRegionCount(), BreakContinueStack.back().ContinueCount)); - beginSourceRegionGroup(S->getInc()); - Visit(S->getInc()); - endSourceRegionGroup(); - Cnt.adjustForControlFlow(); + if (const Stmt *Inc = S->getInc()) + propagateCounts(addCounters(BackedgeCount, BC.ContinueCount), Inc); + + // Go back to handle the condition. + Counter CondCount = + addCounters(ParentCount, BackedgeCount, BC.ContinueCount); + if (const Expr *Cond = S->getCond()) { + propagateCounts(CondCount, Cond); + adjustForOutOfOrderTraversal(getEnd(S)); } - BreakContinue BC = BreakContinueStack.pop_back_val(); - - // ...then go back and propagate counts through the condition. - if (S->getCond()) { - Cnt.setCurrentRegionCount( - addCounters(addCounters(Cnt.getParentCount(), Cnt.getAdjustedCount()), - BC.ContinueCount)); - beginSourceRegionGroup(S->getCond()); - Visit(S->getCond()); - endSourceRegionGroup(); - Cnt.adjustForControlFlow(); - } - Cnt.applyAdjustmentsToRegion(addCounters(BC.BreakCount, BC.ContinueCount)); + Counter OutCount = + addCounters(BC.BreakCount, subtractCounters(CondCount, BodyCount)); + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitCXXForRangeStmt(const CXXForRangeStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); + Visit(S->getLoopVarStmt()); Visit(S->getRangeStmt()); - Visit(S->getBeginEndStmt()); - // Counter tracks the body of the loop. - RegionMapper Cnt(this, S); + + Counter ParentCount = getRegion().getCounter(); + Counter BodyCount = getRegionCounter(S); + BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first. (This is basically the same as a while - // loop; see further comments in VisitWhileStmt.) - Cnt.beginRegion(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); + extendRegion(S->getBody()); + Counter BackedgeCount = propagateCounts(BodyCount, S->getBody()); BreakContinue BC = BreakContinueStack.pop_back_val(); - Cnt.applyAdjustmentsToRegion(addCounters(BC.BreakCount, BC.ContinueCount)); + + Counter LoopCount = + addCounters(ParentCount, BackedgeCount, BC.ContinueCount); + Counter OutCount = + addCounters(BC.BreakCount, subtractCounters(LoopCount, BodyCount)); + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); Visit(S->getElement()); - // Counter tracks the body of the loop. - RegionMapper Cnt(this, S); + + Counter ParentCount = getRegion().getCounter(); + Counter BodyCount = getRegionCounter(S); + BreakContinueStack.push_back(BreakContinue()); - Cnt.beginRegion(); - Visit(S->getBody()); + extendRegion(S->getBody()); + Counter BackedgeCount = propagateCounts(BodyCount, S->getBody()); BreakContinue BC = BreakContinueStack.pop_back_val(); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(addCounters(BC.BreakCount, BC.ContinueCount)); + + Counter LoopCount = + addCounters(ParentCount, BackedgeCount, BC.ContinueCount); + Counter OutCount = + addCounters(BC.BreakCount, subtractCounters(LoopCount, BodyCount)); + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitSwitchStmt(const SwitchStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); Visit(S->getCond()); + BreakContinueStack.push_back(BreakContinue()); - // Map the '}' for the body to have the same count as the regions after - // the switch. - SourceLocation RBracLoc; - if (const auto *CS = dyn_cast<CompoundStmt>(S->getBody())) { - mapSourceCodeRange(CS->getLBracLoc()); - setCurrentRegionUnreachable(S); - for (Stmt::const_child_range I = CS->children(); I; ++I) { - if (*I) - this->Visit(*I); + + const Stmt *Body = S->getBody(); + extendRegion(Body); + if (const auto *CS = dyn_cast<CompoundStmt>(Body)) { + if (!CS->body_empty()) { + // The body of the switch needs a zero region so that fallthrough counts + // behave correctly, but it would be misleading to include the braces of + // the compound statement in the zeroed area, so we need to handle this + // specially. + size_t Index = + pushRegion(Counter::getZero(), getStart(CS->body_front()), + getEnd(CS->body_back())); + for (const auto *Child : CS->children()) + Visit(Child); + popRegions(Index); } - RBracLoc = CS->getRBracLoc(); - } else { - setCurrentRegionUnreachable(S); - Visit(S->getBody()); - } - // If the switch is inside a loop, add the continue counts. + } else + propagateCounts(Counter::getZero(), Body); BreakContinue BC = BreakContinueStack.pop_back_val(); + if (!BreakContinueStack.empty()) BreakContinueStack.back().ContinueCount = addCounters( BreakContinueStack.back().ContinueCount, BC.ContinueCount); - // Counter tracks the exit block of the switch. - RegionMapper ExitCnt(this, S); - ExitCnt.beginRegion(); - if (RBracLoc.isValid()) - mapSourceCodeRange(RBracLoc); - } - void VisitCaseStmt(const CaseStmt *S) { - // Counter for this particular case. This counts only jumps from the - // switch header and does not include fallthrough from the case before - // this one. - RegionMapper Cnt(this, S); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - mapSourceCodeRange(S->getLocStart()); - mapToken(S->getColonLoc()); - Visit(S->getSubStmt()); + Counter ExitCount = getRegionCounter(S); + pushRegion(ExitCount); } - void VisitDefaultStmt(const DefaultStmt *S) { - // Counter for this default case. This does not include fallthrough from - // the previous case. - RegionMapper Cnt(this, S); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - mapSourceCodeRange(S->getLocStart()); - mapToken(S->getColonLoc()); + void VisitSwitchCase(const SwitchCase *S) { + extendRegion(S); + + SourceMappingRegion &Parent = getRegion(); + + Counter Count = addCounters(Parent.getCounter(), getRegionCounter(S)); + // Reuse the existing region if it starts at our label. This is typical of + // the first case in a switch. + if (Parent.hasStartLoc() && Parent.getStartLoc() == getStart(S)) + Parent.setCounter(Count); + else + pushRegion(Count, getStart(S)); + + if (const CaseStmt *CS = dyn_cast<CaseStmt>(S)) { + Visit(CS->getLHS()); + if (const Expr *RHS = CS->getRHS()) + Visit(RHS); + } Visit(S->getSubStmt()); } void VisitIfStmt(const IfStmt *S) { - mapSourceCodeRange(S->getLocStart()); - Visit(S->getCond()); - mapToken(S->getElseLoc()); - - // Counter tracks the "then" part of an if statement. The count for - // the "else" part, if it exists, will be calculated from this counter. - RegionMapper Cnt(this, S); - Cnt.beginRegion(); - Visit(S->getThen()); - Cnt.adjustForControlFlow(); - - if (S->getElse()) { - Cnt.beginElseRegion(); - Visit(S->getElse()); - Cnt.adjustForControlFlow(); - } - Cnt.applyAdjustmentsToRegion(); + extendRegion(S); + // Extend into the condition before we propagate through it below - this is + // needed to handle macros that generate the "if" but not the condition. + extendRegion(S->getCond()); + + Counter ParentCount = getRegion().getCounter(); + Counter ThenCount = getRegionCounter(S); + + // Emitting a counter for the condition makes it easier to interpret the + // counter for the body when looking at the coverage. + propagateCounts(ParentCount, S->getCond()); + + extendRegion(S->getThen()); + Counter OutCount = propagateCounts(ThenCount, S->getThen()); + + Counter ElseCount = subtractCounters(ParentCount, ThenCount); + if (const Stmt *Else = S->getElse()) { + extendRegion(S->getElse()); + OutCount = addCounters(OutCount, propagateCounts(ElseCount, Else)); + } else + OutCount = addCounters(OutCount, ElseCount); + + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitCXXTryStmt(const CXXTryStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); Visit(S->getTryBlock()); for (unsigned I = 0, E = S->getNumHandlers(); I < E; ++I) Visit(S->getHandler(I)); - // Counter tracks the continuation block of the try statement. - RegionMapper Cnt(this, S); - Cnt.beginRegion(); + + Counter ExitCount = getRegionCounter(S); + pushRegion(ExitCount); } void VisitCXXCatchStmt(const CXXCatchStmt *S) { - mapSourceCodeRange(S->getLocStart()); - // Counter tracks the catch statement's handler block. - RegionMapper Cnt(this, S); - Cnt.beginRegion(); - Visit(S->getHandlerBlock()); + extendRegion(S); + propagateCounts(getRegionCounter(S), S->getHandlerBlock()); } void VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { - Visit(E->getCond()); - mapToken(E->getQuestionLoc()); - mapToken(E->getColonLoc()); + extendRegion(E); - // Counter tracks the "true" part of a conditional operator. The - // count in the "false" part will be calculated from this counter. - RegionMapper Cnt(this, E); - Cnt.beginRegion(); - Visit(E->getTrueExpr()); - Cnt.adjustForControlFlow(); + Counter ParentCount = getRegion().getCounter(); + Counter TrueCount = getRegionCounter(E); - Cnt.beginElseRegion(); - Visit(E->getFalseExpr()); - Cnt.adjustForControlFlow(); + Visit(E->getCond()); - Cnt.applyAdjustmentsToRegion(); + if (!isa<BinaryConditionalOperator>(E)) { + extendRegion(E->getTrueExpr()); + propagateCounts(TrueCount, E->getTrueExpr()); + } + extendRegion(E->getFalseExpr()); + propagateCounts(subtractCounters(ParentCount, TrueCount), + E->getFalseExpr()); } void VisitBinLAnd(const BinaryOperator *E) { + extendRegion(E); Visit(E->getLHS()); - mapToken(E->getOperatorLoc()); - // Counter tracks the right hand side of a logical and operator. - RegionMapper Cnt(this, E); - Cnt.beginRegion(); - Visit(E->getRHS()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(); - } - - void VisitBinLOr(const BinaryOperator *E) { - Visit(E->getLHS()); - mapToken(E->getOperatorLoc()); - // Counter tracks the right hand side of a logical or operator. - RegionMapper Cnt(this, E); - Cnt.beginRegion(); - Visit(E->getRHS()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(); - } - void VisitParenExpr(const ParenExpr *E) { - mapToken(E->getLParen()); - Visit(E->getSubExpr()); - mapToken(E->getRParen()); + extendRegion(E->getRHS()); + propagateCounts(getRegionCounter(E), E->getRHS()); } - void VisitBinaryOperator(const BinaryOperator *E) { - Visit(E->getLHS()); - mapToken(E->getOperatorLoc()); - Visit(E->getRHS()); - } - - void VisitUnaryOperator(const UnaryOperator *E) { - bool Postfix = E->isPostfix(); - if (!Postfix) - mapToken(E->getOperatorLoc()); - Visit(E->getSubExpr()); - if (Postfix) - mapToken(E->getOperatorLoc()); - } - - void VisitMemberExpr(const MemberExpr *E) { - Visit(E->getBase()); - mapToken(E->getMemberLoc()); - } - - void VisitCallExpr(const CallExpr *E) { - Visit(E->getCallee()); - for (const auto &Arg : E->arguments()) - Visit(Arg); - mapToken(E->getRParenLoc()); - } - - void VisitArraySubscriptExpr(const ArraySubscriptExpr *E) { + void VisitBinLOr(const BinaryOperator *E) { + extendRegion(E); Visit(E->getLHS()); - Visit(E->getRHS()); - mapToken(E->getRBracketLoc()); - } - - void VisitCStyleCastExpr(const CStyleCastExpr *E) { - mapToken(E->getLParenLoc()); - mapToken(E->getRParenLoc()); - Visit(E->getSubExpr()); - } - - // Map literals as tokens so that the macros like #define PI 3.14 - // won't generate coverage mapping regions. - - void VisitIntegerLiteral(const IntegerLiteral *E) { - mapToken(E->getLocStart()); - } - - void VisitFloatingLiteral(const FloatingLiteral *E) { - mapToken(E->getLocStart()); - } - void VisitCharacterLiteral(const CharacterLiteral *E) { - mapToken(E->getLocStart()); + extendRegion(E->getRHS()); + propagateCounts(getRegionCounter(E), E->getRHS()); } - void VisitStringLiteral(const StringLiteral *E) { - mapToken(E->getLocStart()); - } - - void VisitImaginaryLiteral(const ImaginaryLiteral *E) { - mapToken(E->getLocStart()); - } - - void VisitObjCMessageExpr(const ObjCMessageExpr *E) { - mapToken(E->getLeftLoc()); - for (Stmt::const_child_range I = static_cast<const Stmt*>(E)->children(); I; - ++I) { - if (*I) - this->Visit(*I); - } - mapToken(E->getRightLoc()); + void VisitLambdaExpr(const LambdaExpr *LE) { + // Lambdas are treated as their own functions for now, so we shouldn't + // propagate counts into them. } }; } @@ -1009,10 +894,12 @@ static StringRef getCoverageSection(const CodeGenModule &CGM) { return isMachO(CGM) ? "__DATA,__llvm_covmap" : "__llvm_covmap"; } -static void dump(llvm::raw_ostream &OS, const CoverageMappingRecord &Function) { - OS << Function.FunctionName << ":\n"; - CounterMappingContext Ctx(Function.Expressions); - for (const auto &R : Function.MappingRegions) { +static void dump(llvm::raw_ostream &OS, StringRef FunctionName, + ArrayRef<CounterExpression> Expressions, + ArrayRef<CounterMappingRegion> Regions) { + OS << FunctionName << ":\n"; + CounterMappingContext Ctx(Expressions); + for (const auto &R : Regions) { OS.indent(2); switch (R.Kind) { case CounterMappingRegion::CodeRegion: @@ -1025,15 +912,12 @@ static void dump(llvm::raw_ostream &OS, const CoverageMappingRecord &Function) { break; } - OS << "File " << R.FileID << ", " << R.LineStart << ":" - << R.ColumnStart << " -> " << R.LineEnd << ":" << R.ColumnEnd - << " = "; - Ctx.dump(R.Count); - OS << " (HasCodeBefore = " << R.HasCodeBefore; + OS << "File " << R.FileID << ", " << R.LineStart << ":" << R.ColumnStart + << " -> " << R.LineEnd << ":" << R.ColumnEnd << " = "; + Ctx.dump(R.Count, OS); if (R.Kind == CounterMappingRegion::ExpansionRegion) - OS << ", Expanded file = " << R.ExpandedFileID; - - OS << ")\n"; + OS << " (Expanded file = " << R.ExpandedFileID << ")"; + OS << "\n"; } } @@ -1047,7 +931,8 @@ void CoverageMappingModuleGen::addFunctionMappingRecord( if (!FunctionRecordTy) { llvm::Type *FunctionRecordTypes[] = {Int8PtrTy, Int32Ty, Int32Ty, Int64Ty}; FunctionRecordTy = - llvm::StructType::get(Ctx, makeArrayRef(FunctionRecordTypes)); + llvm::StructType::get(Ctx, makeArrayRef(FunctionRecordTypes), + /*isPacked=*/true); } llvm::Constant *FunctionRecordVals[] = { @@ -1072,13 +957,11 @@ void CoverageMappingModuleGen::addFunctionMappingRecord( FilenameRefs.resize(FileEntries.size()); for (const auto &Entry : FileEntries) FilenameRefs[Entry.second] = Entry.first->getName(); - RawCoverageMappingReader Reader(FunctionNameValue, CoverageMapping, - FilenameRefs, - Filenames, Expressions, Regions); - CoverageMappingRecord FunctionRecord; - if (Reader.read(FunctionRecord)) + RawCoverageMappingReader Reader(CoverageMapping, FilenameRefs, Filenames, + Expressions, Regions); + if (Reader.read()) return; - dump(llvm::outs(), FunctionRecord); + dump(llvm::outs(), FunctionNameValue, Expressions, Regions); } } @@ -1098,7 +981,7 @@ void CoverageMappingModuleGen::emit() { llvm::sys::fs::make_absolute(Path); auto I = Entry.second; - FilenameStrs[I] = std::move(std::string(Path.begin(), Path.end())); + FilenameStrs[I] = std::string(Path.begin(), Path.end()); FilenameRefs[I] = FilenameStrs[I]; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h b/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h index e695848..a795188 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/EHScopeStack.h @@ -17,6 +17,7 @@ #define LLVM_CLANG_LIB_CODEGEN_EHSCOPESTACK_H #include "clang/Basic/LLVM.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Instructions.h" @@ -75,8 +76,14 @@ template <class T> struct DominatingPointer<T,false> : InvariantValue<T*> {}; template <class T> struct DominatingValue<T*> : DominatingPointer<T> {}; enum CleanupKind : unsigned { + /// Denotes a cleanup that should run when a scope is exited using exceptional + /// control flow (a throw statement leading to stack unwinding, ). EHCleanup = 0x1, + + /// Denotes a cleanup that should run when a scope is exited using normal + /// control flow (falling off the end of the scope, return, goto, ...). NormalCleanup = 0x2, + NormalAndEHCleanup = EHCleanup | NormalCleanup, InactiveCleanup = 0x4, @@ -175,84 +182,28 @@ public: virtual void Emit(CodeGenFunction &CGF, Flags flags) = 0; }; - /// ConditionalCleanupN stores the saved form of its N parameters, + /// ConditionalCleanup stores the saved form of its parameters, /// then restores them and performs the cleanup. - template <class T, class A0> - class ConditionalCleanup1 : public Cleanup { - typedef typename DominatingValue<A0>::saved_type A0_saved; - A0_saved a0_saved; - - void Emit(CodeGenFunction &CGF, Flags flags) override { - A0 a0 = DominatingValue<A0>::restore(CGF, a0_saved); - T(a0).Emit(CGF, flags); - } - - public: - ConditionalCleanup1(A0_saved a0) - : a0_saved(a0) {} - }; - - template <class T, class A0, class A1> - class ConditionalCleanup2 : public Cleanup { - typedef typename DominatingValue<A0>::saved_type A0_saved; - typedef typename DominatingValue<A1>::saved_type A1_saved; - A0_saved a0_saved; - A1_saved a1_saved; - - void Emit(CodeGenFunction &CGF, Flags flags) override { - A0 a0 = DominatingValue<A0>::restore(CGF, a0_saved); - A1 a1 = DominatingValue<A1>::restore(CGF, a1_saved); - T(a0, a1).Emit(CGF, flags); + template <class T, class... As> class ConditionalCleanup : public Cleanup { + typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple; + SavedTuple Saved; + + template <std::size_t... Is> + T restore(CodeGenFunction &CGF, llvm::index_sequence<Is...>) { + // It's important that the restores are emitted in order. The braced init + // list guarentees that. + return T{DominatingValue<As>::restore(CGF, std::get<Is>(Saved))...}; } - public: - ConditionalCleanup2(A0_saved a0, A1_saved a1) - : a0_saved(a0), a1_saved(a1) {} - }; - - template <class T, class A0, class A1, class A2> - class ConditionalCleanup3 : public Cleanup { - typedef typename DominatingValue<A0>::saved_type A0_saved; - typedef typename DominatingValue<A1>::saved_type A1_saved; - typedef typename DominatingValue<A2>::saved_type A2_saved; - A0_saved a0_saved; - A1_saved a1_saved; - A2_saved a2_saved; - void Emit(CodeGenFunction &CGF, Flags flags) override { - A0 a0 = DominatingValue<A0>::restore(CGF, a0_saved); - A1 a1 = DominatingValue<A1>::restore(CGF, a1_saved); - A2 a2 = DominatingValue<A2>::restore(CGF, a2_saved); - T(a0, a1, a2).Emit(CGF, flags); + restore(CGF, llvm::index_sequence_for<As...>()).Emit(CGF, flags); } public: - ConditionalCleanup3(A0_saved a0, A1_saved a1, A2_saved a2) - : a0_saved(a0), a1_saved(a1), a2_saved(a2) {} - }; - - template <class T, class A0, class A1, class A2, class A3> - class ConditionalCleanup4 : public Cleanup { - typedef typename DominatingValue<A0>::saved_type A0_saved; - typedef typename DominatingValue<A1>::saved_type A1_saved; - typedef typename DominatingValue<A2>::saved_type A2_saved; - typedef typename DominatingValue<A3>::saved_type A3_saved; - A0_saved a0_saved; - A1_saved a1_saved; - A2_saved a2_saved; - A3_saved a3_saved; + ConditionalCleanup(typename DominatingValue<As>::saved_type... A) + : Saved(A...) {} - void Emit(CodeGenFunction &CGF, Flags flags) override { - A0 a0 = DominatingValue<A0>::restore(CGF, a0_saved); - A1 a1 = DominatingValue<A1>::restore(CGF, a1_saved); - A2 a2 = DominatingValue<A2>::restore(CGF, a2_saved); - A3 a3 = DominatingValue<A3>::restore(CGF, a3_saved); - T(a0, a1, a2, a3).Emit(CGF, flags); - } - - public: - ConditionalCleanup4(A0_saved a0, A1_saved a1, A2_saved a2, A3_saved a3) - : a0_saved(a0), a1_saved(a1), a2_saved(a2), a3_saved(a3) {} + ConditionalCleanup(SavedTuple Tuple) : Saved(std::move(Tuple)) {} }; private: @@ -306,53 +257,18 @@ public: InnermostEHScope(stable_end()) {} ~EHScopeStack() { delete[] StartOfBuffer; } - // Variadic templates would make this not terrible. - /// Push a lazily-created cleanup on the stack. - template <class T> - void pushCleanup(CleanupKind Kind) { + template <class T, class... As> void pushCleanup(CleanupKind Kind, As... A) { void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(); + Cleanup *Obj = new (Buffer) T(A...); (void) Obj; } - /// Push a lazily-created cleanup on the stack. - template <class T, class A0> - void pushCleanup(CleanupKind Kind, A0 a0) { - void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(a0); - (void) Obj; - } - - /// Push a lazily-created cleanup on the stack. - template <class T, class A0, class A1> - void pushCleanup(CleanupKind Kind, A0 a0, A1 a1) { - void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(a0, a1); - (void) Obj; - } - - /// Push a lazily-created cleanup on the stack. - template <class T, class A0, class A1, class A2> - void pushCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2) { + /// Push a lazily-created cleanup on the stack. Tuple version. + template <class T, class... As> + void pushCleanupTuple(CleanupKind Kind, std::tuple<As...> A) { void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(a0, a1, a2); - (void) Obj; - } - - /// Push a lazily-created cleanup on the stack. - template <class T, class A0, class A1, class A2, class A3> - void pushCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2, A3 a3) { - void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(a0, a1, a2, a3); - (void) Obj; - } - - /// Push a lazily-created cleanup on the stack. - template <class T, class A0, class A1, class A2, class A3, class A4> - void pushCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2, A3 a3, A4 a4) { - void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(a0, a1, a2, a3, a4); + Cleanup *Obj = new (Buffer) T(std::move(A)); (void) Obj; } @@ -369,10 +285,10 @@ public: /// /// The pointer returned from this method is valid until the cleanup /// stack is modified. - template <class T, class A0, class A1, class A2> - T *pushCleanupWithExtra(CleanupKind Kind, size_t N, A0 a0, A1 a1, A2 a2) { + template <class T, class... As> + T *pushCleanupWithExtra(CleanupKind Kind, size_t N, As... A) { void *Buffer = pushCleanup(Kind, sizeof(T) + T::getExtraSize(N)); - return new (Buffer) T(N, a0, a1, a2); + return new (Buffer) T(N, A...); } void pushCopyOfCleanup(CleanupKind Kind, const void *Cleanup, size_t Size) { @@ -403,6 +319,10 @@ public: /// Pops a terminate handler off the stack. void popTerminate(); + // Returns true iff the current scope is either empty or contains only + // lifetime markers, i.e. no real cleanup code + bool containsOnlyLifetimeMarkers(stable_iterator Old) const; + /// Determines whether the exception-scopes stack is empty. bool empty() const { return StartOfData == EndOfBuffer; } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp index 9e88b1e..2be9ceb 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -19,14 +19,18 @@ //===----------------------------------------------------------------------===// #include "CGCXXABI.h" +#include "CGCleanup.h" #include "CGRecordLayout.h" #include "CGVTables.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" +#include "TargetInfo.h" #include "clang/AST/Mangle.h" #include "clang/AST/Type.h" +#include "clang/AST/StmtCXX.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Value.h" @@ -90,7 +94,7 @@ public: llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT) override; - llvm::Constant *EmitMemberPointer(const CXXMethodDecl *MD) override; + llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD) override; llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT, CharUnits offset) override; llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT) override; @@ -111,10 +115,22 @@ public: const CXXDestructorDecl *Dtor) override; void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override; + void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override; + + void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override; + + llvm::CallInst * + emitTerminateForUnexpectedException(CodeGenFunction &CGF, + llvm::Value *Exn) override; void EmitFundamentalRTTIDescriptor(QualType Type); void EmitFundamentalRTTIDescriptors(); llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override; + llvm::Constant * + getAddrOfCXXCatchHandlerType(QualType Ty, + QualType CatchHandlerType) override { + return getAddrOfRTTIDescriptor(Ty); + } bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override; void EmitBadTypeidCall(CodeGenFunction &CGF) override; @@ -188,7 +204,8 @@ public: llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, llvm::Value *This, - llvm::Type *Ty) override; + llvm::Type *Ty, + SourceLocation Loc) override; llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, @@ -202,7 +219,7 @@ public: bool ReturnAdjustment) override { // Allow inlining of thunks by emitting them with available_externally // linkage together with vtables when needed. - if (ForVTable) + if (ForVTable && !Thunk->hasLocalLinkage()) Thunk->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage); } @@ -639,7 +656,8 @@ ItaniumCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT, return llvm::ConstantInt::get(CGM.PtrDiffTy, offset.getQuantity()); } -llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) { +llvm::Constant * +ItaniumCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) { return BuildMemberPointer(MD, CharUnits::Zero()); } @@ -849,7 +867,7 @@ bool ItaniumCXXABI::classifyReturnType(CGFunctionInfo &FI) const { /// The Itanium ABI requires non-zero initialization only for data /// member pointers, for which '0' is a valid offset. bool ItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) { - return MPT->getPointeeType()->isFunctionType(); + return MPT->isMemberFunctionPointer(); } /// The Itanium ABI always places an offset to the complete object @@ -906,6 +924,59 @@ void ItaniumCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) { CGF.EmitRuntimeCallOrInvoke(Fn); } +static llvm::Constant *getAllocateExceptionFn(CodeGenModule &CGM) { + // void *__cxa_allocate_exception(size_t thrown_size); + + llvm::FunctionType *FTy = + llvm::FunctionType::get(CGM.Int8PtrTy, CGM.SizeTy, /*IsVarArgs=*/false); + + return CGM.CreateRuntimeFunction(FTy, "__cxa_allocate_exception"); +} + +static llvm::Constant *getThrowFn(CodeGenModule &CGM) { + // void __cxa_throw(void *thrown_exception, std::type_info *tinfo, + // void (*dest) (void *)); + + llvm::Type *Args[3] = { CGM.Int8PtrTy, CGM.Int8PtrTy, CGM.Int8PtrTy }; + llvm::FunctionType *FTy = + llvm::FunctionType::get(CGM.VoidTy, Args, /*IsVarArgs=*/false); + + return CGM.CreateRuntimeFunction(FTy, "__cxa_throw"); +} + +void ItaniumCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { + QualType ThrowType = E->getSubExpr()->getType(); + // Now allocate the exception object. + llvm::Type *SizeTy = CGF.ConvertType(getContext().getSizeType()); + uint64_t TypeSize = getContext().getTypeSizeInChars(ThrowType).getQuantity(); + + llvm::Constant *AllocExceptionFn = getAllocateExceptionFn(CGM); + llvm::CallInst *ExceptionPtr = CGF.EmitNounwindRuntimeCall( + AllocExceptionFn, llvm::ConstantInt::get(SizeTy, TypeSize), "exception"); + + CGF.EmitAnyExprToExn(E->getSubExpr(), ExceptionPtr); + + // Now throw the exception. + llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType, + /*ForEH=*/true); + + // The address of the destructor. If the exception type has a + // trivial destructor (or isn't a record), we just pass null. + llvm::Constant *Dtor = nullptr; + if (const RecordType *RecordTy = ThrowType->getAs<RecordType>()) { + CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordTy->getDecl()); + if (!Record->hasTrivialDestructor()) { + CXXDestructorDecl *DtorD = Record->getDestructor(); + Dtor = CGM.getAddrOfCXXStructor(DtorD, StructorType::Complete); + Dtor = llvm::ConstantExpr::getBitCast(Dtor, CGM.Int8PtrTy); + } + } + if (!Dtor) Dtor = llvm::Constant::getNullValue(CGM.Int8PtrTy); + + llvm::Value *args[] = { ExceptionPtr, TypeInfo, Dtor }; + CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(CGM), args); +} + static llvm::Constant *getItaniumDynamicCastFn(CodeGenFunction &CGF) { // void *__dynamic_cast(const void *sub, // const abi::__class_type_info *src, @@ -1259,6 +1330,9 @@ void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, // Set the correct linkage. VTable->setLinkage(Linkage); + if (CGM.supportsCOMDAT() && VTable->isWeakForLinker()) + VTable->setComdat(CGM.getModule().getOrInsertComdat(VTable->getName())); + // Set the right visibility. CGM.setGlobalVisibility(VTable, RD); @@ -1278,6 +1352,8 @@ void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, cast<NamespaceDecl>(DC)->getIdentifier()->isStr("__cxxabiv1") && DC->getParent()->isTranslationUnit()) EmitFundamentalRTTIDescriptors(); + + CGM.EmitVTableBitSetEntries(VTable, VTLayout); } llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor( @@ -1314,7 +1390,7 @@ llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor( llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr( BaseSubobject Base, const CXXRecordDecl *VTableClass) { - llvm::Constant *VTable = getAddrOfVTable(VTableClass, CharUnits()); + auto *VTable = getAddrOfVTable(VTableClass, CharUnits()); // Find the appropriate vtable within the vtable group. uint64_t AddressPoint = CGM.getItaniumVTableContext() @@ -1325,7 +1401,8 @@ llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr( llvm::ConstantInt::get(CGM.Int64Ty, AddressPoint) }; - return llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, Indices); + return llvm::ConstantExpr::getInBoundsGetElementPtr(VTable->getValueType(), + VTable, Indices); } llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, @@ -1364,11 +1441,16 @@ llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, llvm::Value *ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, llvm::Value *This, - llvm::Type *Ty) { + llvm::Type *Ty, + SourceLocation Loc) { GD = GD.getCanonicalDecl(); Ty = Ty->getPointerTo()->getPointerTo(); llvm::Value *VTable = CGF.GetVTablePtr(This, Ty); + if (CGF.SanOpts.has(SanitizerKind::CFIVCall)) + CGF.EmitVTablePtrCheckForCall(cast<CXXMethodDecl>(GD.getDecl()), VTable, + CodeGenFunction::CFITCK_VCall, Loc); + uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn"); @@ -1385,7 +1467,8 @@ llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall( Dtor, getFromDtorType(DtorType)); llvm::Type *Ty = CGF.CGM.getTypes().GetFunctionType(*FInfo); llvm::Value *Callee = - getVirtualFunctionPointer(CGF, GlobalDecl(Dtor, DtorType), This, Ty); + getVirtualFunctionPointer(CGF, GlobalDecl(Dtor, DtorType), This, Ty, + CE ? CE->getLocStart() : SourceLocation()); CGF.EmitCXXMemberOrOperatorCall(Dtor, Callee, ReturnValueSlot(), This, /*ImplicitParam=*/nullptr, QualType(), CE); @@ -1591,7 +1674,7 @@ llvm::Value *ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, CGF.Builder.CreateStore(elementSize, cookie); // The second element is the element count. - cookie = CGF.Builder.CreateConstInBoundsGEP1_32(cookie, 1); + cookie = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.SizeTy, cookie, 1); CGF.Builder.CreateStore(numElements, cookie); // Finally, compute a pointer to the actual data buffer by skipping @@ -1714,11 +1797,12 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, // The ABI says: It is suggested that it be emitted in the same COMDAT group // as the associated data object - if (!D.isLocalVarDecl() && var->isWeakForLinker() && CGM.supportsCOMDAT()) { - llvm::Comdat *C = CGM.getModule().getOrInsertComdat(var->getName()); + llvm::Comdat *C = var->getComdat(); + if (!D.isLocalVarDecl() && C) { guard->setComdat(C); - var->setComdat(C); CGF.CurFn->setComdat(C); + } else if (CGM.supportsCOMDAT() && guard->isWeakForLinker()) { + guard->setComdat(CGM.getModule().getOrInsertComdat(guard->getName())); } CGM.setStaticLocalDeclGuardAddress(&D, guard); @@ -2336,10 +2420,13 @@ static bool ShouldUseExternalRTTIDescriptor(CodeGenModule &CGM, // FIXME: this may need to be reconsidered if the key function // changes. + // N.B. We must always emit the RTTI data ourselves if there exists a key + // function. + bool IsDLLImport = RD->hasAttr<DLLImportAttr>(); if (CGM.getVTables().isVTableExternal(RD)) - return true; + return IsDLLImport ? false : true; - if (RD->hasAttr<DLLImportAttr>()) + if (IsDLLImport) return true; } @@ -2528,7 +2615,8 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty) { // The vtable address point is 2. llvm::Constant *Two = llvm::ConstantInt::get(PtrDiffTy, 2); - VTable = llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, Two); + VTable = + llvm::ConstantExpr::getInBoundsGetElementPtr(CGM.Int8PtrTy, VTable, Two); VTable = llvm::ConstantExpr::getBitCast(VTable, CGM.Int8PtrTy); Fields.push_back(VTable); @@ -2568,8 +2656,15 @@ static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(CodeGenModule &CGM, const CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl()); if (RD->hasAttr<WeakAttr>()) return llvm::GlobalValue::WeakODRLinkage; - if (RD->isDynamicClass()) - return CGM.getVTableLinkage(RD); + if (RD->isDynamicClass()) { + llvm::GlobalValue::LinkageTypes LT = CGM.getVTableLinkage(RD); + // MinGW won't export the RTTI information when there is a key function. + // Make sure we emit our own copy instead of attempting to dllimport it. + if (RD->hasAttr<DLLImportAttr>() && + llvm::GlobalValue::isAvailableExternallyLinkage(LT)) + LT = llvm::GlobalValue::LinkOnceODRLinkage; + return LT; + } } return llvm::GlobalValue::LinkOnceODRLinkage; @@ -2718,9 +2813,13 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) { llvm::Constant *Init = llvm::ConstantStruct::getAnon(Fields); + llvm::Module &M = CGM.getModule(); llvm::GlobalVariable *GV = - new llvm::GlobalVariable(CGM.getModule(), Init->getType(), - /*Constant=*/true, Linkage, Init, Name); + new llvm::GlobalVariable(M, Init->getType(), + /*Constant=*/true, Linkage, Init, Name); + + if (CGM.supportsCOMDAT() && GV->isWeakForLinker()) + GV->setComdat(M.getOrInsertComdat(GV->getName())); // If there's already an old global variable, replace it with the new one. if (OldGV) { @@ -3136,8 +3235,8 @@ static void emitConstructorDestructorAlias(CodeGenModule &CGM, llvm::PointerType *AliasType = Aliasee->getType(); // Create the alias with no name. - auto *Alias = llvm::GlobalAlias::create( - AliasType->getElementType(), 0, Linkage, "", Aliasee, &CGM.getModule()); + auto *Alias = llvm::GlobalAlias::create(AliasType, Linkage, "", Aliasee, + &CGM.getModule()); // Switch any previous uses to the alias. if (Entry) { @@ -3204,5 +3303,352 @@ void ItaniumCXXABI::emitCXXStructor(const CXXMethodDecl *MD, getMangleContext().mangleCXXCtorComdat(CD, Out); llvm::Comdat *C = CGM.getModule().getOrInsertComdat(Out.str()); Fn->setComdat(C); + } else { + CGM.maybeSetTrivialComdat(*MD, *Fn); + } +} + +static llvm::Constant *getBeginCatchFn(CodeGenModule &CGM) { + // void *__cxa_begin_catch(void*); + llvm::FunctionType *FTy = llvm::FunctionType::get( + CGM.Int8PtrTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); + + return CGM.CreateRuntimeFunction(FTy, "__cxa_begin_catch"); +} + +static llvm::Constant *getEndCatchFn(CodeGenModule &CGM) { + // void __cxa_end_catch(); + llvm::FunctionType *FTy = + llvm::FunctionType::get(CGM.VoidTy, /*IsVarArgs=*/false); + + return CGM.CreateRuntimeFunction(FTy, "__cxa_end_catch"); +} + +static llvm::Constant *getGetExceptionPtrFn(CodeGenModule &CGM) { + // void *__cxa_get_exception_ptr(void*); + llvm::FunctionType *FTy = llvm::FunctionType::get( + CGM.Int8PtrTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); + + return CGM.CreateRuntimeFunction(FTy, "__cxa_get_exception_ptr"); +} + +namespace { + /// A cleanup to call __cxa_end_catch. In many cases, the caught + /// exception type lets us state definitively that the thrown exception + /// type does not have a destructor. In particular: + /// - Catch-alls tell us nothing, so we have to conservatively + /// assume that the thrown exception might have a destructor. + /// - Catches by reference behave according to their base types. + /// - Catches of non-record types will only trigger for exceptions + /// of non-record types, which never have destructors. + /// - Catches of record types can trigger for arbitrary subclasses + /// of the caught type, so we have to assume the actual thrown + /// exception type might have a throwing destructor, even if the + /// caught type's destructor is trivial or nothrow. + struct CallEndCatch : EHScopeStack::Cleanup { + CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {} + bool MightThrow; + + void Emit(CodeGenFunction &CGF, Flags flags) override { + if (!MightThrow) { + CGF.EmitNounwindRuntimeCall(getEndCatchFn(CGF.CGM)); + return; + } + + CGF.EmitRuntimeCallOrInvoke(getEndCatchFn(CGF.CGM)); + } + }; +} + +/// Emits a call to __cxa_begin_catch and enters a cleanup to call +/// __cxa_end_catch. +/// +/// \param EndMightThrow - true if __cxa_end_catch might throw +static llvm::Value *CallBeginCatch(CodeGenFunction &CGF, + llvm::Value *Exn, + bool EndMightThrow) { + llvm::CallInst *call = + CGF.EmitNounwindRuntimeCall(getBeginCatchFn(CGF.CGM), Exn); + + CGF.EHStack.pushCleanup<CallEndCatch>(NormalAndEHCleanup, EndMightThrow); + + return call; +} + +/// A "special initializer" callback for initializing a catch +/// parameter during catch initialization. +static void InitCatchParam(CodeGenFunction &CGF, + const VarDecl &CatchParam, + llvm::Value *ParamAddr, + SourceLocation Loc) { + // Load the exception from where the landing pad saved it. + llvm::Value *Exn = CGF.getExceptionFromSlot(); + + CanQualType CatchType = + CGF.CGM.getContext().getCanonicalType(CatchParam.getType()); + llvm::Type *LLVMCatchTy = CGF.ConvertTypeForMem(CatchType); + + // If we're catching by reference, we can just cast the object + // pointer to the appropriate pointer. + if (isa<ReferenceType>(CatchType)) { + QualType CaughtType = cast<ReferenceType>(CatchType)->getPointeeType(); + bool EndCatchMightThrow = CaughtType->isRecordType(); + + // __cxa_begin_catch returns the adjusted object pointer. + llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndCatchMightThrow); + + // We have no way to tell the personality function that we're + // catching by reference, so if we're catching a pointer, + // __cxa_begin_catch will actually return that pointer by value. + if (const PointerType *PT = dyn_cast<PointerType>(CaughtType)) { + QualType PointeeType = PT->getPointeeType(); + + // When catching by reference, generally we should just ignore + // this by-value pointer and use the exception object instead. + if (!PointeeType->isRecordType()) { + + // Exn points to the struct _Unwind_Exception header, which + // we have to skip past in order to reach the exception data. + unsigned HeaderSize = + CGF.CGM.getTargetCodeGenInfo().getSizeOfUnwindException(); + AdjustedExn = CGF.Builder.CreateConstGEP1_32(Exn, HeaderSize); + + // However, if we're catching a pointer-to-record type that won't + // work, because the personality function might have adjusted + // the pointer. There's actually no way for us to fully satisfy + // the language/ABI contract here: we can't use Exn because it + // might have the wrong adjustment, but we can't use the by-value + // pointer because it's off by a level of abstraction. + // + // The current solution is to dump the adjusted pointer into an + // alloca, which breaks language semantics (because changing the + // pointer doesn't change the exception) but at least works. + // The better solution would be to filter out non-exact matches + // and rethrow them, but this is tricky because the rethrow + // really needs to be catchable by other sites at this landing + // pad. The best solution is to fix the personality function. + } else { + // Pull the pointer for the reference type off. + llvm::Type *PtrTy = + cast<llvm::PointerType>(LLVMCatchTy)->getElementType(); + + // Create the temporary and write the adjusted pointer into it. + llvm::Value *ExnPtrTmp = CGF.CreateTempAlloca(PtrTy, "exn.byref.tmp"); + llvm::Value *Casted = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy); + CGF.Builder.CreateStore(Casted, ExnPtrTmp); + + // Bind the reference to the temporary. + AdjustedExn = ExnPtrTmp; + } + } + + llvm::Value *ExnCast = + CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.byref"); + CGF.Builder.CreateStore(ExnCast, ParamAddr); + return; + } + + // Scalars and complexes. + TypeEvaluationKind TEK = CGF.getEvaluationKind(CatchType); + if (TEK != TEK_Aggregate) { + llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, false); + + // If the catch type is a pointer type, __cxa_begin_catch returns + // the pointer by value. + if (CatchType->hasPointerRepresentation()) { + llvm::Value *CastExn = + CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.casted"); + + switch (CatchType.getQualifiers().getObjCLifetime()) { + case Qualifiers::OCL_Strong: + CastExn = CGF.EmitARCRetainNonBlock(CastExn); + // fallthrough + + case Qualifiers::OCL_None: + case Qualifiers::OCL_ExplicitNone: + case Qualifiers::OCL_Autoreleasing: + CGF.Builder.CreateStore(CastExn, ParamAddr); + return; + + case Qualifiers::OCL_Weak: + CGF.EmitARCInitWeak(ParamAddr, CastExn); + return; + } + llvm_unreachable("bad ownership qualifier!"); + } + + // Otherwise, it returns a pointer into the exception object. + + llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok + llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy); + + LValue srcLV = CGF.MakeNaturalAlignAddrLValue(Cast, CatchType); + LValue destLV = CGF.MakeAddrLValue(ParamAddr, CatchType, + CGF.getContext().getDeclAlign(&CatchParam)); + switch (TEK) { + case TEK_Complex: + CGF.EmitStoreOfComplex(CGF.EmitLoadOfComplex(srcLV, Loc), destLV, + /*init*/ true); + return; + case TEK_Scalar: { + llvm::Value *ExnLoad = CGF.EmitLoadOfScalar(srcLV, Loc); + CGF.EmitStoreOfScalar(ExnLoad, destLV, /*init*/ true); + return; + } + case TEK_Aggregate: + llvm_unreachable("evaluation kind filtered out!"); + } + llvm_unreachable("bad evaluation kind"); + } + + assert(isa<RecordType>(CatchType) && "unexpected catch type!"); + + llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok + + // Check for a copy expression. If we don't have a copy expression, + // that means a trivial copy is okay. + const Expr *copyExpr = CatchParam.getInit(); + if (!copyExpr) { + llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, true); + llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy); + CGF.EmitAggregateCopy(ParamAddr, adjustedExn, CatchType); + return; + } + + // We have to call __cxa_get_exception_ptr to get the adjusted + // pointer before copying. + llvm::CallInst *rawAdjustedExn = + CGF.EmitNounwindRuntimeCall(getGetExceptionPtrFn(CGF.CGM), Exn); + + // Cast that to the appropriate type. + llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy); + + // The copy expression is defined in terms of an OpaqueValueExpr. + // Find it and map it to the adjusted expression. + CodeGenFunction::OpaqueValueMapping + opaque(CGF, OpaqueValueExpr::findInCopyConstruct(copyExpr), + CGF.MakeAddrLValue(adjustedExn, CatchParam.getType())); + + // Call the copy ctor in a terminate scope. + CGF.EHStack.pushTerminate(); + + // Perform the copy construction. + CharUnits Alignment = CGF.getContext().getDeclAlign(&CatchParam); + CGF.EmitAggExpr(copyExpr, + AggValueSlot::forAddr(ParamAddr, Alignment, Qualifiers(), + AggValueSlot::IsNotDestructed, + AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased)); + + // Leave the terminate scope. + CGF.EHStack.popTerminate(); + + // Undo the opaque value mapping. + opaque.pop(); + + // Finally we can call __cxa_begin_catch. + CallBeginCatch(CGF, Exn, true); +} + +/// Begins a catch statement by initializing the catch variable and +/// calling __cxa_begin_catch. +void ItaniumCXXABI::emitBeginCatch(CodeGenFunction &CGF, + const CXXCatchStmt *S) { + // We have to be very careful with the ordering of cleanups here: + // C++ [except.throw]p4: + // The destruction [of the exception temporary] occurs + // immediately after the destruction of the object declared in + // the exception-declaration in the handler. + // + // So the precise ordering is: + // 1. Construct catch variable. + // 2. __cxa_begin_catch + // 3. Enter __cxa_end_catch cleanup + // 4. Enter dtor cleanup + // + // We do this by using a slightly abnormal initialization process. + // Delegation sequence: + // - ExitCXXTryStmt opens a RunCleanupsScope + // - EmitAutoVarAlloca creates the variable and debug info + // - InitCatchParam initializes the variable from the exception + // - CallBeginCatch calls __cxa_begin_catch + // - CallBeginCatch enters the __cxa_end_catch cleanup + // - EmitAutoVarCleanups enters the variable destructor cleanup + // - EmitCXXTryStmt emits the code for the catch body + // - EmitCXXTryStmt close the RunCleanupsScope + + VarDecl *CatchParam = S->getExceptionDecl(); + if (!CatchParam) { + llvm::Value *Exn = CGF.getExceptionFromSlot(); + CallBeginCatch(CGF, Exn, true); + return; + } + + // Emit the local. + CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam); + InitCatchParam(CGF, *CatchParam, var.getObjectAddress(CGF), S->getLocStart()); + CGF.EmitAutoVarCleanups(var); +} + +/// Get or define the following function: +/// void @__clang_call_terminate(i8* %exn) nounwind noreturn +/// This code is used only in C++. +static llvm::Constant *getClangCallTerminateFn(CodeGenModule &CGM) { + llvm::FunctionType *fnTy = + llvm::FunctionType::get(CGM.VoidTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); + llvm::Constant *fnRef = + CGM.CreateRuntimeFunction(fnTy, "__clang_call_terminate"); + + llvm::Function *fn = dyn_cast<llvm::Function>(fnRef); + if (fn && fn->empty()) { + fn->setDoesNotThrow(); + fn->setDoesNotReturn(); + + // What we really want is to massively penalize inlining without + // forbidding it completely. The difference between that and + // 'noinline' is negligible. + fn->addFnAttr(llvm::Attribute::NoInline); + + // Allow this function to be shared across translation units, but + // we don't want it to turn into an exported symbol. + fn->setLinkage(llvm::Function::LinkOnceODRLinkage); + fn->setVisibility(llvm::Function::HiddenVisibility); + if (CGM.supportsCOMDAT()) + fn->setComdat(CGM.getModule().getOrInsertComdat(fn->getName())); + + // Set up the function. + llvm::BasicBlock *entry = + llvm::BasicBlock::Create(CGM.getLLVMContext(), "", fn); + CGBuilderTy builder(entry); + + // Pull the exception pointer out of the parameter list. + llvm::Value *exn = &*fn->arg_begin(); + + // Call __cxa_begin_catch(exn). + llvm::CallInst *catchCall = builder.CreateCall(getBeginCatchFn(CGM), exn); + catchCall->setDoesNotThrow(); + catchCall->setCallingConv(CGM.getRuntimeCC()); + + // Call std::terminate(). + llvm::CallInst *termCall = builder.CreateCall(CGM.getTerminateFn()); + termCall->setDoesNotThrow(); + termCall->setDoesNotReturn(); + termCall->setCallingConv(CGM.getRuntimeCC()); + + // std::terminate cannot return. + builder.CreateUnreachable(); + } + + return fnRef; +} + +llvm::CallInst * +ItaniumCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF, + llvm::Value *Exn) { + // In C++, we want to call __cxa_begin_catch() before terminating. + if (Exn) { + assert(CGF.CGM.getLangOpts().CPlusPlus); + return CGF.EmitNounwindRuntimeCall(getClangCallTerminateFn(CGF.CGM), Exn); } + return CGF.EmitNounwindRuntimeCall(CGF.CGM.getTerminateFn()); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp index c067fab..3433990 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -17,12 +17,16 @@ #include "CGCXXABI.h" #include "CGVTables.h" #include "CodeGenModule.h" +#include "CodeGenTypes.h" +#include "TargetInfo.h" #include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" +#include "clang/AST/StmtCXX.h" #include "clang/AST/VTableBuilder.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSet.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/Intrinsics.h" using namespace clang; using namespace CodeGen; @@ -40,7 +44,8 @@ public: MicrosoftCXXABI(CodeGenModule &CGM) : CGCXXABI(CGM), BaseClassDescriptorType(nullptr), ClassHierarchyDescriptorType(nullptr), - CompleteObjectLocatorType(nullptr) {} + CompleteObjectLocatorType(nullptr), CatchableTypeType(nullptr), + ThrowInfoType(nullptr), CatchHandlerTypeType(nullptr) {} bool HasThisReturn(GlobalDecl GD) const override; bool hasMostDerivedReturn(GlobalDecl GD) const override; @@ -71,11 +76,16 @@ public: const CXXDestructorDecl *Dtor) override; void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override; + void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override; + + void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override; llvm::GlobalVariable *getMSCompleteObjectLocator(const CXXRecordDecl *RD, const VPtrInfo *Info); llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override; + llvm::Constant * + getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) override; bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override; void EmitBadTypeidCall(CodeGenFunction &CGF) override; @@ -195,6 +205,9 @@ public: CXXDtorType Type, bool ForVirtualBase, bool Delegating, llvm::Value *This) override; + void emitVTableBitSetEntries(VPtrInfo *Info, const CXXRecordDecl *RD, + llvm::GlobalVariable *VTable); + void emitVTableDefinitions(CodeGenVTables &CGVT, const CXXRecordDecl *RD) override; @@ -211,8 +224,8 @@ public: CharUnits VPtrOffset) override; llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, - llvm::Value *This, - llvm::Type *Ty) override; + llvm::Value *This, llvm::Type *Ty, + SourceLocation Loc) override; llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, @@ -225,7 +238,7 @@ public: assert(GD.getDtorType() == Dtor_Deleting && "Only deleting destructor thunks are available in this ABI"); CallArgs.add(RValue::get(getStructorImplicitParamValue(CGF)), - CGM.getContext().IntTy); + getContext().IntTy); } void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override; @@ -234,6 +247,50 @@ public: getAddrOfVBTable(const VPtrInfo &VBT, const CXXRecordDecl *RD, llvm::GlobalVariable::LinkageTypes Linkage); + llvm::GlobalVariable * + getAddrOfVirtualDisplacementMap(const CXXRecordDecl *SrcRD, + const CXXRecordDecl *DstRD) { + SmallString<256> OutName; + llvm::raw_svector_ostream Out(OutName); + getMangleContext().mangleCXXVirtualDisplacementMap(SrcRD, DstRD, Out); + Out.flush(); + StringRef MangledName = OutName.str(); + + if (auto *VDispMap = CGM.getModule().getNamedGlobal(MangledName)) + return VDispMap; + + MicrosoftVTableContext &VTContext = CGM.getMicrosoftVTableContext(); + unsigned NumEntries = 1 + SrcRD->getNumVBases(); + SmallVector<llvm::Constant *, 4> Map(NumEntries, + llvm::UndefValue::get(CGM.IntTy)); + Map[0] = llvm::ConstantInt::get(CGM.IntTy, 0); + bool AnyDifferent = false; + for (const auto &I : SrcRD->vbases()) { + const CXXRecordDecl *VBase = I.getType()->getAsCXXRecordDecl(); + if (!DstRD->isVirtuallyDerivedFrom(VBase)) + continue; + + unsigned SrcVBIndex = VTContext.getVBTableIndex(SrcRD, VBase); + unsigned DstVBIndex = VTContext.getVBTableIndex(DstRD, VBase); + Map[SrcVBIndex] = llvm::ConstantInt::get(CGM.IntTy, DstVBIndex * 4); + AnyDifferent |= SrcVBIndex != DstVBIndex; + } + // This map would be useless, don't use it. + if (!AnyDifferent) + return nullptr; + + llvm::ArrayType *VDispMapTy = llvm::ArrayType::get(CGM.IntTy, Map.size()); + llvm::Constant *Init = llvm::ConstantArray::get(VDispMapTy, Map); + llvm::GlobalValue::LinkageTypes Linkage = + SrcRD->isExternallyVisible() && DstRD->isExternallyVisible() + ? llvm::GlobalValue::LinkOnceODRLinkage + : llvm::GlobalValue::InternalLinkage; + auto *VDispMap = new llvm::GlobalVariable( + CGM.getModule(), VDispMapTy, /*Constant=*/true, Linkage, + /*Initializer=*/Init, MangledName); + return VDispMap; + } + void emitVBTableDefinition(const VPtrInfo &VBT, const CXXRecordDecl *RD, llvm::GlobalVariable *GV) const; @@ -410,6 +467,9 @@ public: if (!isImageRelative()) return PtrVal; + if (PtrVal->isNullValue()) + return llvm::Constant::getNullValue(CGM.IntTy); + llvm::Constant *ImageBaseAsInt = llvm::ConstantExpr::getPtrToInt(getImageBase(), CGM.IntPtrTy); llvm::Constant *PtrValAsInt = @@ -467,6 +527,10 @@ private: return GetVBaseOffsetFromVBPtr(CGF, Base, VBPOffset, VBTOffset, VBPtr); } + std::pair<llvm::Value *, llvm::Value *> + performBaseAdjustment(CodeGenFunction &CGF, llvm::Value *Value, + QualType SrcRecordTy); + /// \brief Performs a full virtual base adjustment. Used to dereference /// pointers to members of virtual bases. llvm::Value *AdjustVirtualBase(CodeGenFunction &CGF, const Expr *E, @@ -479,11 +543,8 @@ private: llvm::Constant *EmitFullMemberPointer(llvm::Constant *FirstField, bool IsMemberFunction, const CXXRecordDecl *RD, - CharUnits NonVirtualBaseAdjustment); - - llvm::Constant *BuildMemberPointer(const CXXRecordDecl *RD, - const CXXMethodDecl *MD, - CharUnits NonVirtualBaseAdjustment); + CharUnits NonVirtualBaseAdjustment, + unsigned VBTableIndex); bool MemberPointerConstantIsNull(const MemberPointerType *MPT, llvm::Constant *MP); @@ -524,7 +585,7 @@ public: llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT, CharUnits offset) override; - llvm::Constant *EmitMemberPointer(const CXXMethodDecl *MD) override; + llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD) override; llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT) override; llvm::Value *EmitMemberPointerComparison(CodeGenFunction &CGF, @@ -542,6 +603,12 @@ public: llvm::Value *Base, llvm::Value *MemPtr, const MemberPointerType *MPT) override; + llvm::Value *EmitNonNullMemberPointerConversion( + const MemberPointerType *SrcTy, const MemberPointerType *DstTy, + CastKind CK, CastExpr::path_const_iterator PathBegin, + CastExpr::path_const_iterator PathEnd, llvm::Value *Src, + CGBuilderTy &Builder); + llvm::Value *EmitMemberPointerConversion(CodeGenFunction &CGF, const CastExpr *E, llvm::Value *Src) override; @@ -549,6 +616,11 @@ public: llvm::Constant *EmitMemberPointerConversion(const CastExpr *E, llvm::Constant *Src) override; + llvm::Constant *EmitMemberPointerConversion( + const MemberPointerType *SrcTy, const MemberPointerType *DstTy, + CastKind CK, CastExpr::path_const_iterator PathBegin, + CastExpr::path_const_iterator PathEnd, llvm::Constant *Src); + llvm::Value * EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF, const Expr *E, llvm::Value *&This, llvm::Value *MemPtr, @@ -556,6 +628,94 @@ public: void emitCXXStructor(const CXXMethodDecl *MD, StructorType Type) override; + llvm::StructType *getCatchHandlerTypeType() { + if (!CatchHandlerTypeType) { + llvm::Type *FieldTypes[] = { + CGM.IntTy, // Flags + CGM.Int8PtrTy, // TypeDescriptor + }; + CatchHandlerTypeType = llvm::StructType::create( + CGM.getLLVMContext(), FieldTypes, "eh.CatchHandlerType"); + } + return CatchHandlerTypeType; + } + + llvm::StructType *getCatchableTypeType() { + if (CatchableTypeType) + return CatchableTypeType; + llvm::Type *FieldTypes[] = { + CGM.IntTy, // Flags + getImageRelativeType(CGM.Int8PtrTy), // TypeDescriptor + CGM.IntTy, // NonVirtualAdjustment + CGM.IntTy, // OffsetToVBPtr + CGM.IntTy, // VBTableIndex + CGM.IntTy, // Size + getImageRelativeType(CGM.Int8PtrTy) // CopyCtor + }; + CatchableTypeType = llvm::StructType::create( + CGM.getLLVMContext(), FieldTypes, "eh.CatchableType"); + return CatchableTypeType; + } + + llvm::StructType *getCatchableTypeArrayType(uint32_t NumEntries) { + llvm::StructType *&CatchableTypeArrayType = + CatchableTypeArrayTypeMap[NumEntries]; + if (CatchableTypeArrayType) + return CatchableTypeArrayType; + + llvm::SmallString<23> CTATypeName("eh.CatchableTypeArray."); + CTATypeName += llvm::utostr(NumEntries); + llvm::Type *CTType = + getImageRelativeType(getCatchableTypeType()->getPointerTo()); + llvm::Type *FieldTypes[] = { + CGM.IntTy, // NumEntries + llvm::ArrayType::get(CTType, NumEntries) // CatchableTypes + }; + CatchableTypeArrayType = + llvm::StructType::create(CGM.getLLVMContext(), FieldTypes, CTATypeName); + return CatchableTypeArrayType; + } + + llvm::StructType *getThrowInfoType() { + if (ThrowInfoType) + return ThrowInfoType; + llvm::Type *FieldTypes[] = { + CGM.IntTy, // Flags + getImageRelativeType(CGM.Int8PtrTy), // CleanupFn + getImageRelativeType(CGM.Int8PtrTy), // ForwardCompat + getImageRelativeType(CGM.Int8PtrTy) // CatchableTypeArray + }; + ThrowInfoType = llvm::StructType::create(CGM.getLLVMContext(), FieldTypes, + "eh.ThrowInfo"); + return ThrowInfoType; + } + + llvm::Constant *getThrowFn() { + // _CxxThrowException is passed an exception object and a ThrowInfo object + // which describes the exception. + llvm::Type *Args[] = {CGM.Int8PtrTy, getThrowInfoType()->getPointerTo()}; + llvm::FunctionType *FTy = + llvm::FunctionType::get(CGM.VoidTy, Args, /*IsVarArgs=*/false); + auto *Fn = cast<llvm::Function>( + CGM.CreateRuntimeFunction(FTy, "_CxxThrowException")); + // _CxxThrowException is stdcall on 32-bit x86 platforms. + if (CGM.getTarget().getTriple().getArch() == llvm::Triple::x86) + Fn->setCallingConv(llvm::CallingConv::X86_StdCall); + return Fn; + } + + llvm::Function *getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD, + CXXCtorType CT); + + llvm::Constant *getCatchableType(QualType T, + uint32_t NVOffset = 0, + int32_t VBPtrOffset = -1, + uint32_t VBIndex = 0); + + llvm::GlobalVariable *getCatchableTypeArray(QualType T); + + llvm::GlobalVariable *getThrowInfo(QualType T) override; + private: typedef std::pair<const CXXRecordDecl *, CharUnits> VFTableIdTy; typedef llvm::DenseMap<VFTableIdTy, llvm::GlobalVariable *> VTablesMapTy; @@ -582,11 +742,20 @@ private: /// Map from DeclContext to the current guard variable. We assume that the /// AST is visited in source code order. llvm::DenseMap<const DeclContext *, GuardInfo> GuardVariableMap; + llvm::DenseMap<const DeclContext *, GuardInfo> ThreadLocalGuardVariableMap; + llvm::DenseMap<const DeclContext *, unsigned> ThreadSafeGuardNumMap; llvm::DenseMap<size_t, llvm::StructType *> TypeDescriptorTypeMap; llvm::StructType *BaseClassDescriptorType; llvm::StructType *ClassHierarchyDescriptorType; llvm::StructType *CompleteObjectLocatorType; + + llvm::DenseMap<QualType, llvm::GlobalVariable *> CatchableTypeArrays; + + llvm::StructType *CatchableTypeType; + llvm::DenseMap<uint32_t, llvm::StructType *> CatchableTypeArrayTypeMap; + llvm::StructType *ThrowInfoType; + llvm::StructType *CatchHandlerTypeType; }; } @@ -667,55 +836,73 @@ void MicrosoftCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF, CGF.EmitDeleteCall(DE->getOperatorDelete(), MDThis, ElementType); } -static llvm::Function *getRethrowFn(CodeGenModule &CGM) { - // _CxxThrowException takes two pointer width arguments: a value and a context - // object which points to a TypeInfo object. - llvm::Type *ArgTypes[] = {CGM.Int8PtrTy, CGM.Int8PtrTy}; - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false); - auto *Fn = cast<llvm::Function>( - CGM.CreateRuntimeFunction(FTy, "_CxxThrowException")); - // _CxxThrowException is stdcall on 32-bit x86 platforms. - if (CGM.getTarget().getTriple().getArch() == llvm::Triple::x86) - Fn->setCallingConv(llvm::CallingConv::X86_StdCall); - return Fn; -} - void MicrosoftCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) { - llvm::Value *Args[] = {llvm::ConstantPointerNull::get(CGM.Int8PtrTy), - llvm::ConstantPointerNull::get(CGM.Int8PtrTy)}; - auto *Fn = getRethrowFn(CGM); + llvm::Value *Args[] = { + llvm::ConstantPointerNull::get(CGM.Int8PtrTy), + llvm::ConstantPointerNull::get(getThrowInfoType()->getPointerTo())}; + auto *Fn = getThrowFn(); if (isNoReturn) CGF.EmitNoreturnRuntimeCallOrInvoke(Fn, Args); else CGF.EmitRuntimeCallOrInvoke(Fn, Args); } -/// \brief Gets the offset to the virtual base that contains the vfptr for -/// MS-ABI polymorphic types. -static llvm::Value *getPolymorphicOffset(CodeGenFunction &CGF, - const CXXRecordDecl *RD, - llvm::Value *Value) { - const ASTContext &Context = RD->getASTContext(); - for (const CXXBaseSpecifier &Base : RD->vbases()) - if (Context.getASTRecordLayout(Base.getType()->getAsCXXRecordDecl()) - .hasExtendableVFPtr()) - return CGF.CGM.getCXXABI().GetVirtualBaseClassOffset( - CGF, Value, RD, Base.getType()->getAsCXXRecordDecl()); - llvm_unreachable("One of our vbases should be polymorphic."); +namespace { +struct CallEndCatchMSVC : EHScopeStack::Cleanup { + CallEndCatchMSVC() {} + void Emit(CodeGenFunction &CGF, Flags flags) override { + CGF.EmitNounwindRuntimeCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_endcatch)); + } +}; +} + +void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF, + const CXXCatchStmt *S) { + // In the MS ABI, the runtime handles the copy, and the catch handler is + // responsible for destruction. + VarDecl *CatchParam = S->getExceptionDecl(); + llvm::Value *Exn = CGF.getExceptionFromSlot(); + llvm::Function *BeginCatch = + CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_begincatch); + + // If this is a catch-all or the catch parameter is unnamed, we don't need to + // emit an alloca to the object. + if (!CatchParam || !CatchParam->getDeclName()) { + llvm::Value *Args[2] = {Exn, llvm::Constant::getNullValue(CGF.Int8PtrTy)}; + CGF.EmitNounwindRuntimeCall(BeginCatch, Args); + CGF.EHStack.pushCleanup<CallEndCatchMSVC>(NormalCleanup); + return; + } + + CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam); + llvm::Value *ParamAddr = + CGF.Builder.CreateBitCast(var.getObjectAddress(CGF), CGF.Int8PtrTy); + llvm::Value *Args[2] = {Exn, ParamAddr}; + CGF.EmitNounwindRuntimeCall(BeginCatch, Args); + CGF.EHStack.pushCleanup<CallEndCatchMSVC>(NormalCleanup); + CGF.EmitAutoVarCleanups(var); } -static std::pair<llvm::Value *, llvm::Value *> -performBaseAdjustment(CodeGenFunction &CGF, llvm::Value *Value, - QualType SrcRecordTy) { +std::pair<llvm::Value *, llvm::Value *> +MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, llvm::Value *Value, + QualType SrcRecordTy) { Value = CGF.Builder.CreateBitCast(Value, CGF.Int8PtrTy); const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl(); + const ASTContext &Context = getContext(); - if (CGF.getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr()) + if (Context.getASTRecordLayout(SrcDecl).hasExtendableVFPtr()) return std::make_pair(Value, llvm::ConstantInt::get(CGF.Int32Ty, 0)); // Perform a base adjustment. - llvm::Value *Offset = getPolymorphicOffset(CGF, SrcDecl, Value); + const CXXBaseSpecifier *PolymorphicBase = std::find_if( + SrcDecl->vbases_begin(), SrcDecl->vbases_end(), + [&](const CXXBaseSpecifier &Base) { + const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); + return Context.getASTRecordLayout(BaseDecl).hasExtendableVFPtr(); + }); + llvm::Value *Offset = GetVirtualBaseClassOffset( + CGF, Value, SrcDecl, PolymorphicBase->getType()->getAsCXXRecordDecl()); Value = CGF.Builder.CreateInBoundsGEP(Value, Offset); Offset = CGF.Builder.CreateTrunc(Offset, CGF.Int32Ty); return std::make_pair(Value, Offset); @@ -725,7 +912,7 @@ bool MicrosoftCXXABI::shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) { const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl(); return IsDeref && - !CGM.getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr(); + !getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr(); } static llvm::CallSite emitRTtypeidCall(CodeGenFunction &CGF, @@ -759,7 +946,7 @@ bool MicrosoftCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr, QualType SrcRecordTy) { const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl(); return SrcIsPtr && - !CGM.getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr(); + !getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr(); } llvm::Value *MicrosoftCXXABI::EmitDynamicCastCall( @@ -817,10 +1004,11 @@ bool MicrosoftCXXABI::EmitBadCastCall(CodeGenFunction &CGF) { llvm::Value *MicrosoftCXXABI::GetVirtualBaseClassOffset( CodeGenFunction &CGF, llvm::Value *This, const CXXRecordDecl *ClassDecl, const CXXRecordDecl *BaseClassDecl) { + const ASTContext &Context = getContext(); int64_t VBPtrChars = - getContext().getASTRecordLayout(ClassDecl).getVBPtrOffset().getQuantity(); + Context.getASTRecordLayout(ClassDecl).getVBPtrOffset().getQuantity(); llvm::Value *VBPtrOffset = llvm::ConstantInt::get(CGM.PtrDiffTy, VBPtrChars); - CharUnits IntSize = getContext().getTypeSizeInChars(getContext().IntTy); + CharUnits IntSize = Context.getTypeSizeInChars(Context.IntTy); CharUnits VBTableChars = IntSize * CGM.getMicrosoftVTableContext().getVBTableIndex(ClassDecl, BaseClassDecl); @@ -947,30 +1135,52 @@ void MicrosoftCXXABI::initializeHiddenVirtualInheritanceMembers( } } +static bool hasDefaultCXXMethodCC(ASTContext &Context, + const CXXMethodDecl *MD) { + CallingConv ExpectedCallingConv = Context.getDefaultCallingConvention( + /*IsVariadic=*/false, /*IsCXXMethod=*/true); + CallingConv ActualCallingConv = + MD->getType()->getAs<FunctionProtoType>()->getCallConv(); + return ExpectedCallingConv == ActualCallingConv; +} + void MicrosoftCXXABI::EmitCXXConstructors(const CXXConstructorDecl *D) { // There's only one constructor type in this ABI. CGM.EmitGlobal(GlobalDecl(D, Ctor_Complete)); + + // Exported default constructors either have a simple call-site where they use + // the typical calling convention and have a single 'this' pointer for an + // argument -or- they get a wrapper function which appropriately thunks to the + // real default constructor. This thunk is the default constructor closure. + if (D->hasAttr<DLLExportAttr>() && D->isDefaultConstructor()) + if (!hasDefaultCXXMethodCC(getContext(), D) || D->getNumParams() != 0) { + llvm::Function *Fn = getAddrOfCXXCtorClosure(D, Ctor_DefaultClosure); + Fn->setLinkage(llvm::GlobalValue::WeakODRLinkage); + Fn->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); + } } void MicrosoftCXXABI::EmitVBPtrStores(CodeGenFunction &CGF, const CXXRecordDecl *RD) { llvm::Value *ThisInt8Ptr = CGF.Builder.CreateBitCast(getThisValue(CGF), CGM.Int8PtrTy, "this.int8"); - const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD); + const ASTContext &Context = getContext(); + const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD); const VBTableGlobals &VBGlobals = enumerateVBTables(RD); for (unsigned I = 0, E = VBGlobals.VBTables->size(); I != E; ++I) { const VPtrInfo *VBT = (*VBGlobals.VBTables)[I]; llvm::GlobalVariable *GV = VBGlobals.Globals[I]; const ASTRecordLayout &SubobjectLayout = - CGM.getContext().getASTRecordLayout(VBT->BaseWithVPtr); + Context.getASTRecordLayout(VBT->BaseWithVPtr); CharUnits Offs = VBT->NonVirtualOffset; Offs += SubobjectLayout.getVBPtrOffset(); if (VBT->getVBaseWithVPtr()) Offs += Layout.getVBaseClassOffset(VBT->getVBaseWithVPtr()); llvm::Value *VBPtr = CGF.Builder.CreateConstInBoundsGEP1_64(ThisInt8Ptr, Offs.getQuantity()); - llvm::Value *GVPtr = CGF.Builder.CreateConstInBoundsGEP2_32(GV, 0, 0); + llvm::Value *GVPtr = + CGF.Builder.CreateConstInBoundsGEP2_32(GV->getValueType(), GV, 0, 0); VBPtr = CGF.Builder.CreateBitCast(VBPtr, GVPtr->getType()->getPointerTo(0), "vbptr." + VBT->ReusingBase->getName()); CGF.Builder.CreateStore(GVPtr, VBPtr); @@ -983,7 +1193,7 @@ MicrosoftCXXABI::buildStructorSignature(const CXXMethodDecl *MD, StructorType T, // TODO: 'for base' flag if (T == StructorType::Deleting) { // The scalar deleting destructor takes an implicit int parameter. - ArgTys.push_back(CGM.getContext().IntTy); + ArgTys.push_back(getContext().IntTy); } auto *CD = dyn_cast<CXXConstructorDecl>(MD); if (!CD) @@ -996,9 +1206,9 @@ MicrosoftCXXABI::buildStructorSignature(const CXXMethodDecl *MD, StructorType T, const FunctionProtoType *FPT = CD->getType()->castAs<FunctionProtoType>(); if (Class->getNumVBases()) { if (FPT->isVariadic()) - ArgTys.insert(ArgTys.begin() + 1, CGM.getContext().IntTy); + ArgTys.insert(ArgTys.begin() + 1, getContext().IntTy); else - ArgTys.push_back(CGM.getContext().IntTy); + ArgTys.push_back(getContext().IntTy); } } @@ -1038,7 +1248,7 @@ MicrosoftCXXABI::getVirtualFunctionPrologueThisAdjustment(GlobalDecl GD) { if (ML.VBase) { const ASTRecordLayout &DerivedLayout = - CGM.getContext().getASTRecordLayout(MD->getParent()); + getContext().getASTRecordLayout(MD->getParent()); Adjustment += DerivedLayout.getVBaseClassOffset(ML.VBase); } @@ -1104,7 +1314,7 @@ llvm::Value *MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall( // FIXME: Update the code that emits this adjustment in thunks prologues. This = CGF.Builder.CreateConstGEP1_32(This, StaticOffset.getQuantity()); } else { - This = CGF.Builder.CreateConstInBoundsGEP1_32(This, + This = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, This, StaticOffset.getQuantity()); } } @@ -1159,8 +1369,8 @@ llvm::Value *MicrosoftCXXABI::adjustThisParameterInVirtualFunctionPrologue( This = CGF.Builder.CreateBitCast(This, charPtrTy); assert(Adjustment.isPositive()); - This = - CGF.Builder.CreateConstInBoundsGEP1_32(This, -Adjustment.getQuantity()); + This = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, This, + -Adjustment.getQuantity()); return CGF.Builder.CreateBitCast(This, thisTy); } @@ -1245,6 +1455,66 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF, getFromDtorType(Type)); } +void MicrosoftCXXABI::emitVTableBitSetEntries(VPtrInfo *Info, + const CXXRecordDecl *RD, + llvm::GlobalVariable *VTable) { + if (!getContext().getLangOpts().Sanitize.has(SanitizerKind::CFIVCall) && + !getContext().getLangOpts().Sanitize.has(SanitizerKind::CFINVCall) && + !getContext().getLangOpts().Sanitize.has(SanitizerKind::CFIDerivedCast) && + !getContext().getLangOpts().Sanitize.has(SanitizerKind::CFIUnrelatedCast)) + return; + + llvm::NamedMDNode *BitsetsMD = + CGM.getModule().getOrInsertNamedMetadata("llvm.bitsets"); + + // The location of the first virtual function pointer in the virtual table, + // aka the "address point" on Itanium. This is at offset 0 if RTTI is + // disabled, or sizeof(void*) if RTTI is enabled. + CharUnits AddressPoint = + getContext().getLangOpts().RTTIData + ? getContext().toCharUnitsFromBits( + getContext().getTargetInfo().getPointerWidth(0)) + : CharUnits::Zero(); + + if (Info->PathToBaseWithVPtr.empty()) { + if (!CGM.IsCFIBlacklistedRecord(RD)) + BitsetsMD->addOperand( + CGM.CreateVTableBitSetEntry(VTable, AddressPoint, RD)); + return; + } + + // Add a bitset entry for the least derived base belonging to this vftable. + if (!CGM.IsCFIBlacklistedRecord(Info->PathToBaseWithVPtr.back())) + BitsetsMD->addOperand(CGM.CreateVTableBitSetEntry( + VTable, AddressPoint, Info->PathToBaseWithVPtr.back())); + + // Add a bitset entry for each derived class that is laid out at the same + // offset as the least derived base. + for (unsigned I = Info->PathToBaseWithVPtr.size() - 1; I != 0; --I) { + const CXXRecordDecl *DerivedRD = Info->PathToBaseWithVPtr[I - 1]; + const CXXRecordDecl *BaseRD = Info->PathToBaseWithVPtr[I]; + + const ASTRecordLayout &Layout = + getContext().getASTRecordLayout(DerivedRD); + CharUnits Offset; + auto VBI = Layout.getVBaseOffsetsMap().find(BaseRD); + if (VBI == Layout.getVBaseOffsetsMap().end()) + Offset = Layout.getBaseClassOffset(BaseRD); + else + Offset = VBI->second.VBaseOffset; + if (!Offset.isZero()) + return; + if (!CGM.IsCFIBlacklistedRecord(DerivedRD)) + BitsetsMD->addOperand( + CGM.CreateVTableBitSetEntry(VTable, AddressPoint, DerivedRD)); + } + + // Finally do the same for the most derived class. + if (Info->FullOffsetInMDC.isZero() && !CGM.IsCFIBlacklistedRecord(RD)) + BitsetsMD->addOperand( + CGM.CreateVTableBitSetEntry(VTable, AddressPoint, RD)); +} + void MicrosoftCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, const CXXRecordDecl *RD) { MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext(); @@ -1267,6 +1537,8 @@ void MicrosoftCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, VTLayout.getNumVTableThunks(), RTTI); VTable->setInitializer(Init); + + emitVTableBitSetEntries(Info, RD, VTable); } } @@ -1280,7 +1552,7 @@ llvm::Value *MicrosoftCXXABI::getVTableAddressPointInStructor( llvm::GlobalValue *VTableAddressPoint = VFTablesMap[ID]; if (!VTableAddressPoint) { assert(Base.getBase()->getNumVBases() && - !CGM.getContext().getASTRecordLayout(Base.getBase()).hasOwnVFPtr()); + !getContext().getASTRecordLayout(Base.getBase()).hasOwnVFPtr()); } return VTableAddressPoint; } @@ -1336,105 +1608,147 @@ llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, #endif } - for (size_t J = 0, F = VFPtrs.size(); J != F; ++J) { - if (VFPtrs[J]->FullOffsetInMDC != VPtrOffset) - continue; - SmallString<256> VFTableName; - mangleVFTableName(getMangleContext(), RD, VFPtrs[J], VFTableName); - StringRef VTableName = VFTableName; - - uint64_t NumVTableSlots = - VTContext.getVFTableLayout(RD, VFPtrs[J]->FullOffsetInMDC) - .getNumVTableComponents(); - llvm::GlobalValue::LinkageTypes VTableLinkage = - llvm::GlobalValue::ExternalLinkage; - llvm::ArrayType *VTableType = - llvm::ArrayType::get(CGM.Int8PtrTy, NumVTableSlots); - if (getContext().getLangOpts().RTTIData) { - VTableLinkage = llvm::GlobalValue::PrivateLinkage; - VTableName = ""; + VPtrInfo *const *VFPtrI = + std::find_if(VFPtrs.begin(), VFPtrs.end(), [&](VPtrInfo *VPI) { + return VPI->FullOffsetInMDC == VPtrOffset; + }); + if (VFPtrI == VFPtrs.end()) { + VFTablesMap[ID] = nullptr; + return nullptr; + } + VPtrInfo *VFPtr = *VFPtrI; + + SmallString<256> VFTableName; + mangleVFTableName(getMangleContext(), RD, VFPtr, VFTableName); + + llvm::GlobalValue::LinkageTypes VFTableLinkage = CGM.getVTableLinkage(RD); + bool VFTableComesFromAnotherTU = + llvm::GlobalValue::isAvailableExternallyLinkage(VFTableLinkage) || + llvm::GlobalValue::isExternalLinkage(VFTableLinkage); + bool VTableAliasIsRequred = + !VFTableComesFromAnotherTU && getContext().getLangOpts().RTTIData; + + if (llvm::GlobalValue *VFTable = + CGM.getModule().getNamedGlobal(VFTableName)) { + VFTablesMap[ID] = VFTable; + return VTableAliasIsRequred + ? cast<llvm::GlobalVariable>( + cast<llvm::GlobalAlias>(VFTable)->getBaseObject()) + : cast<llvm::GlobalVariable>(VFTable); + } + + uint64_t NumVTableSlots = + VTContext.getVFTableLayout(RD, VFPtr->FullOffsetInMDC) + .getNumVTableComponents(); + llvm::GlobalValue::LinkageTypes VTableLinkage = + VTableAliasIsRequred ? llvm::GlobalValue::PrivateLinkage : VFTableLinkage; + + StringRef VTableName = VTableAliasIsRequred ? StringRef() : VFTableName.str(); + + llvm::ArrayType *VTableType = + llvm::ArrayType::get(CGM.Int8PtrTy, NumVTableSlots); + + // Create a backing variable for the contents of VTable. The VTable may + // or may not include space for a pointer to RTTI data. + llvm::GlobalValue *VFTable; + VTable = new llvm::GlobalVariable(CGM.getModule(), VTableType, + /*isConstant=*/true, VTableLinkage, + /*Initializer=*/nullptr, VTableName); + VTable->setUnnamedAddr(true); + + llvm::Comdat *C = nullptr; + if (!VFTableComesFromAnotherTU && + (llvm::GlobalValue::isWeakForLinker(VFTableLinkage) || + (llvm::GlobalValue::isLocalLinkage(VFTableLinkage) && + VTableAliasIsRequred))) + C = CGM.getModule().getOrInsertComdat(VFTableName.str()); + + // Only insert a pointer into the VFTable for RTTI data if we are not + // importing it. We never reference the RTTI data directly so there is no + // need to make room for it. + if (VTableAliasIsRequred) { + llvm::Value *GEPIndices[] = {llvm::ConstantInt::get(CGM.IntTy, 0), + llvm::ConstantInt::get(CGM.IntTy, 1)}; + // Create a GEP which points just after the first entry in the VFTable, + // this should be the location of the first virtual method. + llvm::Constant *VTableGEP = llvm::ConstantExpr::getInBoundsGetElementPtr( + VTable->getValueType(), VTable, GEPIndices); + if (llvm::GlobalValue::isWeakForLinker(VFTableLinkage)) { + VFTableLinkage = llvm::GlobalValue::ExternalLinkage; + if (C) + C->setSelectionKind(llvm::Comdat::Largest); } + VFTable = llvm::GlobalAlias::create( + cast<llvm::PointerType>(VTableGEP->getType()), VFTableLinkage, + VFTableName.str(), VTableGEP, &CGM.getModule()); + VFTable->setUnnamedAddr(true); + } else { + // We don't need a GlobalAlias to be a symbol for the VTable if we won't + // be referencing any RTTI data. + // The GlobalVariable will end up being an appropriate definition of the + // VFTable. + VFTable = VTable; + } + if (C) + VTable->setComdat(C); - VTable = CGM.getModule().getNamedGlobal(VFTableName); - if (!VTable) { - // Create a backing variable for the contents of VTable. The VTable may - // or may not include space for a pointer to RTTI data. - llvm::GlobalValue *VFTable = VTable = new llvm::GlobalVariable( - CGM.getModule(), VTableType, /*isConstant=*/true, VTableLinkage, - /*Initializer=*/nullptr, VTableName); - VTable->setUnnamedAddr(true); - - // Only insert a pointer into the VFTable for RTTI data if we are not - // importing it. We never reference the RTTI data directly so there is no - // need to make room for it. - if (getContext().getLangOpts().RTTIData && - !RD->hasAttr<DLLImportAttr>()) { - llvm::Value *GEPIndices[] = {llvm::ConstantInt::get(CGM.IntTy, 0), - llvm::ConstantInt::get(CGM.IntTy, 1)}; - // Create a GEP which points just after the first entry in the VFTable, - // this should be the location of the first virtual method. - llvm::Constant *VTableGEP = - llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, GEPIndices); - // The symbol for the VFTable is an alias to the GEP. It is - // transparent, to other modules, what the nature of this symbol is; all - // that matters is that the alias be the address of the first virtual - // method. - VFTable = llvm::GlobalAlias::create( - cast<llvm::SequentialType>(VTableGEP->getType())->getElementType(), - /*AddressSpace=*/0, llvm::GlobalValue::ExternalLinkage, - VFTableName.str(), VTableGEP, &CGM.getModule()); - } else { - // We don't need a GlobalAlias to be a symbol for the VTable if we won't - // be referencing any RTTI data. The GlobalVariable will end up being - // an appropriate definition of the VFTable. - VTable->setName(VFTableName.str()); - } + if (RD->hasAttr<DLLImportAttr>()) + VFTable->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + else if (RD->hasAttr<DLLExportAttr>()) + VFTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); - VFTable->setUnnamedAddr(true); - if (RD->hasAttr<DLLImportAttr>()) - VFTable->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); - else if (RD->hasAttr<DLLExportAttr>()) - VFTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); - - llvm::GlobalValue::LinkageTypes VFTableLinkage = CGM.getVTableLinkage(RD); - if (VFTable != VTable) { - if (llvm::GlobalValue::isAvailableExternallyLinkage(VFTableLinkage)) { - // AvailableExternally implies that we grabbed the data from another - // executable. No need to stick the alias in a Comdat. - } else if (llvm::GlobalValue::isInternalLinkage(VFTableLinkage) || - llvm::GlobalValue::isWeakODRLinkage(VFTableLinkage) || - llvm::GlobalValue::isLinkOnceODRLinkage(VFTableLinkage)) { - // The alias is going to be dropped into a Comdat, no need to make it - // weak. - if (!llvm::GlobalValue::isInternalLinkage(VFTableLinkage)) - VFTableLinkage = llvm::GlobalValue::ExternalLinkage; - llvm::Comdat *C = - CGM.getModule().getOrInsertComdat(VFTable->getName()); - // We must indicate which VFTable is larger to support linking between - // translation units which do and do not have RTTI data. The largest - // VFTable contains the RTTI data; translation units which reference - // the smaller VFTable always reference it relative to the first - // virtual method. - C->setSelectionKind(llvm::Comdat::Largest); - VTable->setComdat(C); - } else { - llvm_unreachable("unexpected linkage for vftable!"); - } - } - VFTable->setLinkage(VFTableLinkage); - CGM.setGlobalVisibility(VFTable, RD); - VFTablesMap[ID] = VFTable; + VFTablesMap[ID] = VFTable; + return VTable; +} + +// Compute the identity of the most derived class whose virtual table is located +// at the given offset into RD. +static const CXXRecordDecl *getClassAtVTableLocation(ASTContext &Ctx, + const CXXRecordDecl *RD, + CharUnits Offset) { + if (Offset.isZero()) + return RD; + + const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD); + const CXXRecordDecl *MaxBase = nullptr; + CharUnits MaxBaseOffset; + for (auto &&B : RD->bases()) { + const CXXRecordDecl *Base = B.getType()->getAsCXXRecordDecl(); + CharUnits BaseOffset = Layout.getBaseClassOffset(Base); + if (BaseOffset <= Offset && BaseOffset >= MaxBaseOffset) { + MaxBase = Base; + MaxBaseOffset = BaseOffset; } - break; } + for (auto &&B : RD->vbases()) { + const CXXRecordDecl *Base = B.getType()->getAsCXXRecordDecl(); + CharUnits BaseOffset = Layout.getVBaseClassOffset(Base); + if (BaseOffset <= Offset && BaseOffset >= MaxBaseOffset) { + MaxBase = Base; + MaxBaseOffset = BaseOffset; + } + } + assert(MaxBase); + return getClassAtVTableLocation(Ctx, MaxBase, Offset - MaxBaseOffset); +} - return VTable; +// Compute the identity of the most derived class whose virtual table is located +// at the MethodVFTableLocation ML. +static const CXXRecordDecl * +getClassAtVTableLocation(ASTContext &Ctx, GlobalDecl GD, + MicrosoftVTableContext::MethodVFTableLocation &ML) { + const CXXRecordDecl *RD = ML.VBase; + if (!RD) + RD = cast<CXXMethodDecl>(GD.getDecl())->getParent(); + + return getClassAtVTableLocation(Ctx, RD, ML.VFPtrOffset); } llvm::Value *MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, llvm::Value *This, - llvm::Type *Ty) { + llvm::Type *Ty, + SourceLocation Loc) { GD = GD.getCanonicalDecl(); CGBuilderTy &Builder = CGF.Builder; @@ -1445,6 +1759,10 @@ llvm::Value *MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, MicrosoftVTableContext::MethodVFTableLocation ML = CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD); + if (CGF.SanOpts.has(SanitizerKind::CFIVCall)) + CGF.EmitVTablePtrCheck(getClassAtVTableLocation(getContext(), GD, ML), + VTable, CodeGenFunction::CFITCK_VCall, Loc); + llvm::Value *VFuncPtr = Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn"); return Builder.CreateLoad(VFuncPtr); @@ -1462,9 +1780,10 @@ llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( const CGFunctionInfo *FInfo = &CGM.getTypes().arrangeCXXStructorDeclaration( Dtor, StructorType::Deleting); llvm::Type *Ty = CGF.CGM.getTypes().GetFunctionType(*FInfo); - llvm::Value *Callee = getVirtualFunctionPointer(CGF, GD, This, Ty); + llvm::Value *Callee = getVirtualFunctionPointer( + CGF, GD, This, Ty, CE ? CE->getLocStart() : SourceLocation()); - ASTContext &Context = CGF.getContext(); + ASTContext &Context = getContext(); llvm::Value *ImplicitParam = llvm::ConstantInt::get( llvm::IntegerType::getInt32Ty(CGF.getLLVMContext()), DtorType == Dtor_Deleting); @@ -1530,6 +1849,8 @@ llvm::Function *MicrosoftCXXABI::EmitVirtualMemPtrThunk( ThunkFn->setLinkage(MD->isExternallyVisible() ? llvm::GlobalValue::LinkOnceODRLinkage : llvm::GlobalValue::InternalLinkage); + if (MD->isExternallyVisible()) + ThunkFn->setComdat(CGM.getModule().getOrInsertComdat(ThunkFn->getName())); CGM.SetLLVMFunctionAttributes(MD, FnInfo, ThunkFn); CGM.SetLLVMFunctionAttributesForDefinition(MD, ThunkFn); @@ -1576,7 +1897,8 @@ void MicrosoftCXXABI::emitVirtualInheritanceTables(const CXXRecordDecl *RD) { for (unsigned I = 0, E = VBGlobals.VBTables->size(); I != E; ++I) { const VPtrInfo *VBT = (*VBGlobals.VBTables)[I]; llvm::GlobalVariable *GV = VBGlobals.Globals[I]; - emitVBTableDefinition(*VBT, RD, GV); + if (GV->isDeclaration()) + emitVBTableDefinition(*VBT, RD, GV); } } @@ -1603,6 +1925,9 @@ MicrosoftCXXABI::getAddrOfVBTable(const VPtrInfo &VBT, const CXXRecordDecl *RD, else if (RD->hasAttr<DLLExportAttr>()) GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); + if (!GV->hasExternalLinkage()) + emitVBTableDefinition(VBT, RD, GV); + return GV; } @@ -1615,9 +1940,8 @@ void MicrosoftCXXABI::emitVBTableDefinition(const VPtrInfo &VBT, "should only emit vbtables for classes with vbtables"); const ASTRecordLayout &BaseLayout = - CGM.getContext().getASTRecordLayout(VBT.BaseWithVPtr); - const ASTRecordLayout &DerivedLayout = - CGM.getContext().getASTRecordLayout(RD); + getContext().getASTRecordLayout(VBT.BaseWithVPtr); + const ASTRecordLayout &DerivedLayout = getContext().getASTRecordLayout(RD); SmallVector<llvm::Constant *, 4> Offsets(1 + ReusingBase->getNumVBases(), nullptr); @@ -1651,9 +1975,6 @@ void MicrosoftCXXABI::emitVBTableDefinition(const VPtrInfo &VBT, llvm::ArrayType::get(CGM.IntTy, Offsets.size()); llvm::Constant *Init = llvm::ConstantArray::get(VBTableType, Offsets); GV->setInitializer(Init); - - // Set the right visibility. - CGM.setGlobalVisibility(GV, RD); } llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, @@ -1709,8 +2030,8 @@ MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret, if (RA.Virtual.Microsoft.VBIndex) { assert(RA.Virtual.Microsoft.VBIndex > 0); - int32_t IntSize = - getContext().getTypeSizeInChars(getContext().IntTy).getQuantity(); + const ASTContext &Context = getContext(); + int32_t IntSize = Context.getTypeSizeInChars(Context.IntTy).getQuantity(); llvm::Value *VBPtr; llvm::Value *VBaseOffset = GetVBaseOffsetFromVBPtr(CGF, V, RA.Virtual.Microsoft.VBPtrOffset, @@ -1719,7 +2040,7 @@ MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret, } if (RA.NonVirtual) - V = CGF.Builder.CreateConstInBoundsGEP1_32(V, RA.NonVirtual); + V = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, V, RA.NonVirtual); // Cast back to the original type. return CGF.Builder.CreateBitCast(V, Ret->getType()); @@ -1835,18 +2156,10 @@ void MicrosoftCXXABI::EmitThreadLocalInitFuncs( llvm::Function *F = CXXThreadLocalInits[I]; // If the GV is already in a comdat group, then we have to join it. - llvm::Comdat *C = GV->getComdat(); - - // LinkOnce and Weak linkage are lowered down to a single-member comdat - // group. - // Make an explicit group so we can join it. - if (!C && (GV->hasWeakLinkage() || GV->hasLinkOnceLinkage())) { - C = CGM.getModule().getOrInsertComdat(GV->getName()); - GV->setComdat(C); + if (llvm::Comdat *C = GV->getComdat()) AddToXDU(F)->setComdat(C); - } else { + else NonComdatInits.push_back(F); - } } if (!NonComdatInits.empty()) { @@ -1868,6 +2181,81 @@ LValue MicrosoftCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, return LValue(); } +static llvm::GlobalVariable *getInitThreadEpochPtr(CodeGenModule &CGM) { + StringRef VarName("_Init_thread_epoch"); + if (auto *GV = CGM.getModule().getNamedGlobal(VarName)) + return GV; + auto *GV = new llvm::GlobalVariable( + CGM.getModule(), CGM.IntTy, + /*Constant=*/false, llvm::GlobalVariable::ExternalLinkage, + /*Initializer=*/nullptr, VarName, + /*InsertBefore=*/nullptr, llvm::GlobalVariable::GeneralDynamicTLSModel); + GV->setAlignment(CGM.getTarget().getIntAlign() / 8); + return GV; +} + +static llvm::Constant *getInitThreadHeaderFn(CodeGenModule &CGM) { + llvm::FunctionType *FTy = + llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()), + CGM.IntTy->getPointerTo(), /*isVarArg=*/false); + return CGM.CreateRuntimeFunction( + FTy, "_Init_thread_header", + llvm::AttributeSet::get(CGM.getLLVMContext(), + llvm::AttributeSet::FunctionIndex, + llvm::Attribute::NoUnwind)); +} + +static llvm::Constant *getInitThreadFooterFn(CodeGenModule &CGM) { + llvm::FunctionType *FTy = + llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()), + CGM.IntTy->getPointerTo(), /*isVarArg=*/false); + return CGM.CreateRuntimeFunction( + FTy, "_Init_thread_footer", + llvm::AttributeSet::get(CGM.getLLVMContext(), + llvm::AttributeSet::FunctionIndex, + llvm::Attribute::NoUnwind)); +} + +static llvm::Constant *getInitThreadAbortFn(CodeGenModule &CGM) { + llvm::FunctionType *FTy = + llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()), + CGM.IntTy->getPointerTo(), /*isVarArg=*/false); + return CGM.CreateRuntimeFunction( + FTy, "_Init_thread_abort", + llvm::AttributeSet::get(CGM.getLLVMContext(), + llvm::AttributeSet::FunctionIndex, + llvm::Attribute::NoUnwind)); +} + +namespace { +struct ResetGuardBit : EHScopeStack::Cleanup { + llvm::GlobalVariable *Guard; + unsigned GuardNum; + ResetGuardBit(llvm::GlobalVariable *Guard, unsigned GuardNum) + : Guard(Guard), GuardNum(GuardNum) {} + + void Emit(CodeGenFunction &CGF, Flags flags) override { + // Reset the bit in the mask so that the static variable may be + // reinitialized. + CGBuilderTy &Builder = CGF.Builder; + llvm::LoadInst *LI = Builder.CreateLoad(Guard); + llvm::ConstantInt *Mask = + llvm::ConstantInt::get(CGF.IntTy, ~(1U << GuardNum)); + Builder.CreateStore(Builder.CreateAnd(LI, Mask), Guard); + } +}; + +struct CallInitThreadAbort : EHScopeStack::Cleanup { + llvm::GlobalVariable *Guard; + CallInitThreadAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {} + + void Emit(CodeGenFunction &CGF, Flags flags) override { + // Calling _Init_thread_abort will reset the guard's state. + CGF.EmitNounwindRuntimeCall(getInitThreadAbortFn(CGF.CGM), Guard); + } +}; +} + void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, llvm::GlobalVariable *GV, bool PerformInit) { @@ -1875,91 +2263,161 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, if (!D.isStaticLocal()) { assert(GV->hasWeakLinkage() || GV->hasLinkOnceLinkage()); // GlobalOpt is allowed to discard the initializer, so use linkonce_odr. - CGF.CurFn->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage); + llvm::Function *F = CGF.CurFn; + F->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage); + F->setComdat(CGM.getModule().getOrInsertComdat(F->getName())); CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); return; } - // MSVC always uses an i32 bitfield to guard initialization, which is *not* - // threadsafe. Since the user may be linking in inline functions compiled by - // cl.exe, there's no reason to provide a false sense of security by using - // critical sections here. + bool ThreadlocalStatic = D.getTLSKind(); + bool ThreadsafeStatic = getContext().getLangOpts().ThreadsafeStatics; - if (D.getTLSKind()) - CGM.ErrorUnsupported(&D, "dynamic TLS initialization"); + // Thread-safe static variables which aren't thread-specific have a + // per-variable guard. + bool HasPerVariableGuard = ThreadsafeStatic && !ThreadlocalStatic; CGBuilderTy &Builder = CGF.Builder; llvm::IntegerType *GuardTy = CGF.Int32Ty; llvm::ConstantInt *Zero = llvm::ConstantInt::get(GuardTy, 0); // Get the guard variable for this function if we have one already. - GuardInfo *GI = &GuardVariableMap[D.getDeclContext()]; - - unsigned BitIndex; - if (D.isStaticLocal() && D.isExternallyVisible()) { + GuardInfo *GI = nullptr; + if (ThreadlocalStatic) + GI = &ThreadLocalGuardVariableMap[D.getDeclContext()]; + else if (!ThreadsafeStatic) + GI = &GuardVariableMap[D.getDeclContext()]; + + llvm::GlobalVariable *GuardVar = GI ? GI->Guard : nullptr; + unsigned GuardNum; + if (D.isExternallyVisible()) { // Externally visible variables have to be numbered in Sema to properly // handle unreachable VarDecls. - BitIndex = getContext().getStaticLocalNumber(&D); - assert(BitIndex > 0); - BitIndex--; + GuardNum = getContext().getStaticLocalNumber(&D); + assert(GuardNum > 0); + GuardNum--; + } else if (HasPerVariableGuard) { + GuardNum = ThreadSafeGuardNumMap[D.getDeclContext()]++; } else { // Non-externally visible variables are numbered here in CodeGen. - BitIndex = GI->BitIndex++; + GuardNum = GI->BitIndex++; } - if (BitIndex >= 32) { + if (!HasPerVariableGuard && GuardNum >= 32) { if (D.isExternallyVisible()) ErrorUnsupportedABI(CGF, "more than 32 guarded initializations"); - BitIndex %= 32; - GI->Guard = nullptr; + GuardNum %= 32; + GuardVar = nullptr; } - // Lazily create the i32 bitfield for this function. - if (!GI->Guard) { + if (!GuardVar) { // Mangle the name for the guard. SmallString<256> GuardName; { llvm::raw_svector_ostream Out(GuardName); - getMangleContext().mangleStaticGuardVariable(&D, Out); + if (HasPerVariableGuard) + getMangleContext().mangleThreadSafeStaticGuardVariable(&D, GuardNum, + Out); + else + getMangleContext().mangleStaticGuardVariable(&D, Out); Out.flush(); } // Create the guard variable with a zero-initializer. Just absorb linkage, // visibility and dll storage class from the guarded variable. - GI->Guard = - new llvm::GlobalVariable(CGM.getModule(), GuardTy, false, + GuardVar = + new llvm::GlobalVariable(CGM.getModule(), GuardTy, /*isConstant=*/false, GV->getLinkage(), Zero, GuardName.str()); - GI->Guard->setVisibility(GV->getVisibility()); - GI->Guard->setDLLStorageClass(GV->getDLLStorageClass()); - } else { - assert(GI->Guard->getLinkage() == GV->getLinkage() && - "static local from the same function had different linkage"); + GuardVar->setVisibility(GV->getVisibility()); + GuardVar->setDLLStorageClass(GV->getDLLStorageClass()); + if (GuardVar->isWeakForLinker()) + GuardVar->setComdat( + CGM.getModule().getOrInsertComdat(GuardVar->getName())); + if (D.getTLSKind()) + GuardVar->setThreadLocal(true); + if (GI && !HasPerVariableGuard) + GI->Guard = GuardVar; } - // Pseudo code for the test: - // if (!(GuardVar & MyGuardBit)) { - // GuardVar |= MyGuardBit; - // ... initialize the object ...; - // } - - // Test our bit from the guard variable. - llvm::ConstantInt *Bit = llvm::ConstantInt::get(GuardTy, 1U << BitIndex); - llvm::LoadInst *LI = Builder.CreateLoad(GI->Guard); - llvm::Value *IsInitialized = - Builder.CreateICmpNE(Builder.CreateAnd(LI, Bit), Zero); - llvm::BasicBlock *InitBlock = CGF.createBasicBlock("init"); - llvm::BasicBlock *EndBlock = CGF.createBasicBlock("init.end"); - Builder.CreateCondBr(IsInitialized, EndBlock, InitBlock); + assert(GuardVar->getLinkage() == GV->getLinkage() && + "static local from the same function had different linkage"); + + if (!HasPerVariableGuard) { + // Pseudo code for the test: + // if (!(GuardVar & MyGuardBit)) { + // GuardVar |= MyGuardBit; + // ... initialize the object ...; + // } + + // Test our bit from the guard variable. + llvm::ConstantInt *Bit = llvm::ConstantInt::get(GuardTy, 1U << GuardNum); + llvm::LoadInst *LI = Builder.CreateLoad(GuardVar); + llvm::Value *IsInitialized = + Builder.CreateICmpNE(Builder.CreateAnd(LI, Bit), Zero); + llvm::BasicBlock *InitBlock = CGF.createBasicBlock("init"); + llvm::BasicBlock *EndBlock = CGF.createBasicBlock("init.end"); + Builder.CreateCondBr(IsInitialized, EndBlock, InitBlock); + + // Set our bit in the guard variable and emit the initializer and add a global + // destructor if appropriate. + CGF.EmitBlock(InitBlock); + Builder.CreateStore(Builder.CreateOr(LI, Bit), GuardVar); + CGF.EHStack.pushCleanup<ResetGuardBit>(EHCleanup, GuardVar, GuardNum); + CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); + CGF.PopCleanupBlock(); + Builder.CreateBr(EndBlock); - // Set our bit in the guard variable and emit the initializer and add a global - // destructor if appropriate. - CGF.EmitBlock(InitBlock); - Builder.CreateStore(Builder.CreateOr(LI, Bit), GI->Guard); - CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); - Builder.CreateBr(EndBlock); + // Continue. + CGF.EmitBlock(EndBlock); + } else { + // Pseudo code for the test: + // if (TSS > _Init_thread_epoch) { + // _Init_thread_header(&TSS); + // if (TSS == -1) { + // ... initialize the object ...; + // _Init_thread_footer(&TSS); + // } + // } + // + // The algorithm is almost identical to what can be found in the appendix + // found in N2325. + + unsigned IntAlign = CGM.getTarget().getIntAlign() / 8; + + // This BasicBLock determines whether or not we have any work to do. + llvm::LoadInst *FirstGuardLoad = + Builder.CreateAlignedLoad(GuardVar, IntAlign); + FirstGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered); + llvm::LoadInst *InitThreadEpoch = + Builder.CreateLoad(getInitThreadEpochPtr(CGM)); + llvm::Value *IsUninitialized = + Builder.CreateICmpSGT(FirstGuardLoad, InitThreadEpoch); + llvm::BasicBlock *AttemptInitBlock = CGF.createBasicBlock("init.attempt"); + llvm::BasicBlock *EndBlock = CGF.createBasicBlock("init.end"); + Builder.CreateCondBr(IsUninitialized, AttemptInitBlock, EndBlock); + + // This BasicBlock attempts to determine whether or not this thread is + // responsible for doing the initialization. + CGF.EmitBlock(AttemptInitBlock); + CGF.EmitNounwindRuntimeCall(getInitThreadHeaderFn(CGM), GuardVar); + llvm::LoadInst *SecondGuardLoad = + Builder.CreateAlignedLoad(GuardVar, IntAlign); + SecondGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered); + llvm::Value *ShouldDoInit = + Builder.CreateICmpEQ(SecondGuardLoad, getAllOnesInt()); + llvm::BasicBlock *InitBlock = CGF.createBasicBlock("init"); + Builder.CreateCondBr(ShouldDoInit, InitBlock, EndBlock); + + // Ok, we ended up getting selected as the initializing thread. + CGF.EmitBlock(InitBlock); + CGF.EHStack.pushCleanup<CallInitThreadAbort>(EHCleanup, GuardVar); + CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); + CGF.PopCleanupBlock(); + CGF.EmitNounwindRuntimeCall(getInitThreadFooterFn(CGM), GuardVar); + Builder.CreateBr(EndBlock); - // Continue. - CGF.EmitBlock(EndBlock); + CGF.EmitBlock(EndBlock); + } } bool MicrosoftCXXABI::isZeroInitializable(const MemberPointerType *MPT) { @@ -2040,8 +2498,8 @@ llvm::Constant * MicrosoftCXXABI::EmitFullMemberPointer(llvm::Constant *FirstField, bool IsMemberFunction, const CXXRecordDecl *RD, - CharUnits NonVirtualBaseAdjustment) -{ + CharUnits NonVirtualBaseAdjustment, + unsigned VBTableIndex) { MSInheritanceAttr::Spelling Inheritance = RD->getMSInheritanceModel(); // Single inheritance class member pointer are represented as scalars instead @@ -2058,14 +2516,14 @@ MicrosoftCXXABI::EmitFullMemberPointer(llvm::Constant *FirstField, if (MSInheritanceAttr::hasVBPtrOffsetField(Inheritance)) { CharUnits Offs = CharUnits::Zero(); - if (RD->getNumVBases()) + if (VBTableIndex) Offs = getContext().getASTRecordLayout(RD).getVBPtrOffset(); fields.push_back(llvm::ConstantInt::get(CGM.IntTy, Offs.getQuantity())); } // The rest of the fields are adjusted by conversions to a more derived class. if (MSInheritanceAttr::hasVBTableOffsetField(Inheritance)) - fields.push_back(getZeroInt()); + fields.push_back(llvm::ConstantInt::get(CGM.IntTy, VBTableIndex)); return llvm::ConstantStruct::getAnon(fields); } @@ -2074,45 +2532,79 @@ llvm::Constant * MicrosoftCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT, CharUnits offset) { const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl(); + if (RD->getMSInheritanceModel() == + MSInheritanceAttr::Keyword_virtual_inheritance) + offset -= getContext().getOffsetOfBaseWithVBPtr(RD); llvm::Constant *FirstField = llvm::ConstantInt::get(CGM.IntTy, offset.getQuantity()); return EmitFullMemberPointer(FirstField, /*IsMemberFunction=*/false, RD, - CharUnits::Zero()); -} - -llvm::Constant *MicrosoftCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) { - return BuildMemberPointer(MD->getParent(), MD, CharUnits::Zero()); + CharUnits::Zero(), /*VBTableIndex=*/0); } llvm::Constant *MicrosoftCXXABI::EmitMemberPointer(const APValue &MP, QualType MPType) { - const MemberPointerType *MPT = MPType->castAs<MemberPointerType>(); + const MemberPointerType *DstTy = MPType->castAs<MemberPointerType>(); const ValueDecl *MPD = MP.getMemberPointerDecl(); if (!MPD) - return EmitNullMemberPointer(MPT); + return EmitNullMemberPointer(DstTy); - CharUnits ThisAdjustment = getMemberPointerPathAdjustment(MP); + ASTContext &Ctx = getContext(); + ArrayRef<const CXXRecordDecl *> MemberPointerPath = MP.getMemberPointerPath(); - // FIXME PR15713: Support virtual inheritance paths. + llvm::Constant *C; + if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MPD)) { + C = EmitMemberFunctionPointer(MD); + } else { + CharUnits FieldOffset = Ctx.toCharUnitsFromBits(Ctx.getFieldOffset(MPD)); + C = EmitMemberDataPointer(DstTy, FieldOffset); + } - if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MPD)) - return BuildMemberPointer(MPT->getMostRecentCXXRecordDecl(), MD, - ThisAdjustment); + if (!MemberPointerPath.empty()) { + const CXXRecordDecl *SrcRD = cast<CXXRecordDecl>(MPD->getDeclContext()); + const Type *SrcRecTy = Ctx.getTypeDeclType(SrcRD).getTypePtr(); + const MemberPointerType *SrcTy = + Ctx.getMemberPointerType(DstTy->getPointeeType(), SrcRecTy) + ->castAs<MemberPointerType>(); + + bool DerivedMember = MP.isMemberPointerToDerivedMember(); + SmallVector<const CXXBaseSpecifier *, 4> DerivedToBasePath; + const CXXRecordDecl *PrevRD = SrcRD; + for (const CXXRecordDecl *PathElem : MemberPointerPath) { + const CXXRecordDecl *Base = nullptr; + const CXXRecordDecl *Derived = nullptr; + if (DerivedMember) { + Base = PathElem; + Derived = PrevRD; + } else { + Base = PrevRD; + Derived = PathElem; + } + for (const CXXBaseSpecifier &BS : Derived->bases()) + if (BS.getType()->getAsCXXRecordDecl()->getCanonicalDecl() == + Base->getCanonicalDecl()) + DerivedToBasePath.push_back(&BS); + PrevRD = PathElem; + } + assert(DerivedToBasePath.size() == MemberPointerPath.size()); - CharUnits FieldOffset = - getContext().toCharUnitsFromBits(getContext().getFieldOffset(MPD)); - return EmitMemberDataPointer(MPT, ThisAdjustment + FieldOffset); + CastKind CK = DerivedMember ? CK_DerivedToBaseMemberPointer + : CK_BaseToDerivedMemberPointer; + C = EmitMemberPointerConversion(SrcTy, DstTy, CK, DerivedToBasePath.begin(), + DerivedToBasePath.end(), C); + } + return C; } llvm::Constant * -MicrosoftCXXABI::BuildMemberPointer(const CXXRecordDecl *RD, - const CXXMethodDecl *MD, - CharUnits NonVirtualBaseAdjustment) { +MicrosoftCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) { assert(MD->isInstance() && "Member function must not be static!"); + MD = MD->getCanonicalDecl(); - RD = RD->getMostRecentDecl(); + CharUnits NonVirtualBaseAdjustment = CharUnits::Zero(); + const CXXRecordDecl *RD = MD->getParent()->getMostRecentDecl(); CodeGenTypes &Types = CGM.getTypes(); + unsigned VBTableIndex = 0; llvm::Constant *FirstField; const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>(); if (!MD->isVirtual()) { @@ -2127,33 +2619,26 @@ MicrosoftCXXABI::BuildMemberPointer(const CXXRecordDecl *RD, Ty = CGM.PtrDiffTy; } FirstField = CGM.GetAddrOfFunction(MD, Ty); - FirstField = llvm::ConstantExpr::getBitCast(FirstField, CGM.VoidPtrTy); } else { + auto &VTableContext = CGM.getMicrosoftVTableContext(); MicrosoftVTableContext::MethodVFTableLocation ML = - CGM.getMicrosoftVTableContext().getMethodVFTableLocation(MD); - if (!CGM.getTypes().isFuncTypeConvertible( - MD->getType()->castAs<FunctionType>())) { - CGM.ErrorUnsupported(MD, "pointer to virtual member function with " - "incomplete return or parameter type"); - FirstField = llvm::Constant::getNullValue(CGM.VoidPtrTy); - } else if (FPT->getCallConv() == CC_X86FastCall) { - CGM.ErrorUnsupported(MD, "pointer to fastcall virtual member function"); - FirstField = llvm::Constant::getNullValue(CGM.VoidPtrTy); - } else if (ML.VBase) { - CGM.ErrorUnsupported(MD, "pointer to virtual member function overriding " - "member function in virtual base class"); - FirstField = llvm::Constant::getNullValue(CGM.VoidPtrTy); - } else { - llvm::Function *Thunk = EmitVirtualMemPtrThunk(MD, ML); - FirstField = llvm::ConstantExpr::getBitCast(Thunk, CGM.VoidPtrTy); - // Include the vfptr adjustment if the method is in a non-primary vftable. - NonVirtualBaseAdjustment += ML.VFPtrOffset; - } + VTableContext.getMethodVFTableLocation(MD); + FirstField = EmitVirtualMemPtrThunk(MD, ML); + // Include the vfptr adjustment if the method is in a non-primary vftable. + NonVirtualBaseAdjustment += ML.VFPtrOffset; + if (ML.VBase) + VBTableIndex = VTableContext.getVBTableIndex(RD, ML.VBase) * 4; } + if (VBTableIndex == 0 && + RD->getMSInheritanceModel() == + MSInheritanceAttr::Keyword_virtual_inheritance) + NonVirtualBaseAdjustment -= getContext().getOffsetOfBaseWithVBPtr(RD); + // The rest of the fields are common with data member pointers. + FirstField = llvm::ConstantExpr::getBitCast(FirstField, CGM.VoidPtrTy); return EmitFullMemberPointer(FirstField, /*IsMemberFunction=*/true, RD, - NonVirtualBaseAdjustment); + NonVirtualBaseAdjustment, VBTableIndex); } /// Member pointers are the same if they're either bitwise identical *or* both @@ -2409,11 +2894,6 @@ llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress( return Builder.CreateBitCast(Addr, PType); } -static MSInheritanceAttr::Spelling -getInheritanceFromMemptr(const MemberPointerType *MPT) { - return MPT->getMostRecentCXXRecordDecl()->getMSInheritanceModel(); -} - llvm::Value * MicrosoftCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF, const CastExpr *E, @@ -2465,12 +2945,37 @@ MicrosoftCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF, Builder.CreateCondBr(IsNotNull, ConvertBB, ContinueBB); CGF.EmitBlock(ConvertBB); + llvm::Value *Dst = EmitNonNullMemberPointerConversion( + SrcTy, DstTy, E->getCastKind(), E->path_begin(), E->path_end(), Src, + Builder); + + Builder.CreateBr(ContinueBB); + + // In the continuation, choose between DstNull and Dst. + CGF.EmitBlock(ContinueBB); + llvm::PHINode *Phi = Builder.CreatePHI(DstNull->getType(), 2, "memptr.converted"); + Phi->addIncoming(DstNull, OriginalBB); + Phi->addIncoming(Dst, ConvertBB); + return Phi; +} + +llvm::Value *MicrosoftCXXABI::EmitNonNullMemberPointerConversion( + const MemberPointerType *SrcTy, const MemberPointerType *DstTy, CastKind CK, + CastExpr::path_const_iterator PathBegin, + CastExpr::path_const_iterator PathEnd, llvm::Value *Src, + CGBuilderTy &Builder) { + const CXXRecordDecl *SrcRD = SrcTy->getMostRecentCXXRecordDecl(); + const CXXRecordDecl *DstRD = DstTy->getMostRecentCXXRecordDecl(); + MSInheritanceAttr::Spelling SrcInheritance = SrcRD->getMSInheritanceModel(); + MSInheritanceAttr::Spelling DstInheritance = DstRD->getMSInheritanceModel(); + bool IsFunc = SrcTy->isMemberFunctionPointer(); + bool IsConstant = isa<llvm::Constant>(Src); + // Decompose src. llvm::Value *FirstField = Src; - llvm::Value *NonVirtualBaseAdjustment = nullptr; - llvm::Value *VirtualBaseAdjustmentOffset = nullptr; - llvm::Value *VBPtrOffset = nullptr; - MSInheritanceAttr::Spelling SrcInheritance = SrcRD->getMSInheritanceModel(); + llvm::Value *NonVirtualBaseAdjustment = getZeroInt(); + llvm::Value *VirtualBaseAdjustmentOffset = getZeroInt(); + llvm::Value *VBPtrOffset = getZeroInt(); if (!MSInheritanceAttr::hasOnlyOneField(IsFunc, SrcInheritance)) { // We need to extract values. unsigned I = 0; @@ -2483,59 +2988,138 @@ MicrosoftCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF, VirtualBaseAdjustmentOffset = Builder.CreateExtractValue(Src, I++); } + bool IsDerivedToBase = (CK == CK_DerivedToBaseMemberPointer); + const MemberPointerType *DerivedTy = IsDerivedToBase ? SrcTy : DstTy; + const CXXRecordDecl *DerivedClass = DerivedTy->getMostRecentCXXRecordDecl(); + // For data pointers, we adjust the field offset directly. For functions, we // have a separate field. - llvm::Constant *Adj = getMemberPointerAdjustment(E); - if (Adj) { - Adj = llvm::ConstantExpr::getTruncOrBitCast(Adj, CGM.IntTy); - llvm::Value *&NVAdjustField = IsFunc ? NonVirtualBaseAdjustment : FirstField; - bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer); - if (!NVAdjustField) // If this field didn't exist in src, it's zero. - NVAdjustField = getZeroInt(); - if (isDerivedToBase) - NVAdjustField = Builder.CreateNSWSub(NVAdjustField, Adj, "adj"); - else - NVAdjustField = Builder.CreateNSWAdd(NVAdjustField, Adj, "adj"); + llvm::Value *&NVAdjustField = IsFunc ? NonVirtualBaseAdjustment : FirstField; + + // The virtual inheritance model has a quirk: the virtual base table is always + // referenced when dereferencing a member pointer even if the member pointer + // is non-virtual. This is accounted for by adjusting the non-virtual offset + // to point backwards to the top of the MDC from the first VBase. Undo this + // adjustment to normalize the member pointer. + llvm::Value *SrcVBIndexEqZero = + Builder.CreateICmpEQ(VirtualBaseAdjustmentOffset, getZeroInt()); + if (SrcInheritance == MSInheritanceAttr::Keyword_virtual_inheritance) { + if (int64_t SrcOffsetToFirstVBase = + getContext().getOffsetOfBaseWithVBPtr(SrcRD).getQuantity()) { + llvm::Value *UndoSrcAdjustment = Builder.CreateSelect( + SrcVBIndexEqZero, + llvm::ConstantInt::get(CGM.IntTy, SrcOffsetToFirstVBase), + getZeroInt()); + NVAdjustField = Builder.CreateNSWAdd(NVAdjustField, UndoSrcAdjustment); + } + } + + // A non-zero vbindex implies that we are dealing with a source member in a + // floating virtual base in addition to some non-virtual offset. If the + // vbindex is zero, we are dealing with a source that exists in a non-virtual, + // fixed, base. The difference between these two cases is that the vbindex + + // nvoffset *always* point to the member regardless of what context they are + // evaluated in so long as the vbindex is adjusted. A member inside a fixed + // base requires explicit nv adjustment. + llvm::Constant *BaseClassOffset = llvm::ConstantInt::get( + CGM.IntTy, + CGM.computeNonVirtualBaseClassOffset(DerivedClass, PathBegin, PathEnd) + .getQuantity()); + + llvm::Value *NVDisp; + if (IsDerivedToBase) + NVDisp = Builder.CreateNSWSub(NVAdjustField, BaseClassOffset, "adj"); + else + NVDisp = Builder.CreateNSWAdd(NVAdjustField, BaseClassOffset, "adj"); + + NVAdjustField = Builder.CreateSelect(SrcVBIndexEqZero, NVDisp, getZeroInt()); + + // Update the vbindex to an appropriate value in the destination because + // SrcRD's vbtable might not be a strict prefix of the one in DstRD. + llvm::Value *DstVBIndexEqZero = SrcVBIndexEqZero; + if (MSInheritanceAttr::hasVBTableOffsetField(DstInheritance) && + MSInheritanceAttr::hasVBTableOffsetField(SrcInheritance)) { + if (llvm::GlobalVariable *VDispMap = + getAddrOfVirtualDisplacementMap(SrcRD, DstRD)) { + llvm::Value *VBIndex = Builder.CreateExactUDiv( + VirtualBaseAdjustmentOffset, llvm::ConstantInt::get(CGM.IntTy, 4)); + if (IsConstant) { + llvm::Constant *Mapping = VDispMap->getInitializer(); + VirtualBaseAdjustmentOffset = + Mapping->getAggregateElement(cast<llvm::Constant>(VBIndex)); + } else { + llvm::Value *Idxs[] = {getZeroInt(), VBIndex}; + VirtualBaseAdjustmentOffset = + Builder.CreateLoad(Builder.CreateInBoundsGEP(VDispMap, Idxs)); + } + + DstVBIndexEqZero = + Builder.CreateICmpEQ(VirtualBaseAdjustmentOffset, getZeroInt()); + } + } + + // Set the VBPtrOffset to zero if the vbindex is zero. Otherwise, initialize + // it to the offset of the vbptr. + if (MSInheritanceAttr::hasVBPtrOffsetField(DstInheritance)) { + llvm::Value *DstVBPtrOffset = llvm::ConstantInt::get( + CGM.IntTy, + getContext().getASTRecordLayout(DstRD).getVBPtrOffset().getQuantity()); + VBPtrOffset = + Builder.CreateSelect(DstVBIndexEqZero, getZeroInt(), DstVBPtrOffset); } - // FIXME PR15713: Support conversions through virtually derived classes. + // Likewise, apply a similar adjustment so that dereferencing the member + // pointer correctly accounts for the distance between the start of the first + // virtual base and the top of the MDC. + if (DstInheritance == MSInheritanceAttr::Keyword_virtual_inheritance) { + if (int64_t DstOffsetToFirstVBase = + getContext().getOffsetOfBaseWithVBPtr(DstRD).getQuantity()) { + llvm::Value *DoDstAdjustment = Builder.CreateSelect( + DstVBIndexEqZero, + llvm::ConstantInt::get(CGM.IntTy, DstOffsetToFirstVBase), + getZeroInt()); + NVAdjustField = Builder.CreateNSWSub(NVAdjustField, DoDstAdjustment); + } + } // Recompose dst from the null struct and the adjusted fields from src. - MSInheritanceAttr::Spelling DstInheritance = DstRD->getMSInheritanceModel(); llvm::Value *Dst; if (MSInheritanceAttr::hasOnlyOneField(IsFunc, DstInheritance)) { Dst = FirstField; } else { - Dst = llvm::UndefValue::get(DstNull->getType()); + Dst = llvm::UndefValue::get(ConvertMemberPointerType(DstTy)); unsigned Idx = 0; Dst = Builder.CreateInsertValue(Dst, FirstField, Idx++); if (MSInheritanceAttr::hasNVOffsetField(IsFunc, DstInheritance)) - Dst = Builder.CreateInsertValue( - Dst, getValueOrZeroInt(NonVirtualBaseAdjustment), Idx++); + Dst = Builder.CreateInsertValue(Dst, NonVirtualBaseAdjustment, Idx++); if (MSInheritanceAttr::hasVBPtrOffsetField(DstInheritance)) - Dst = Builder.CreateInsertValue( - Dst, getValueOrZeroInt(VBPtrOffset), Idx++); + Dst = Builder.CreateInsertValue(Dst, VBPtrOffset, Idx++); if (MSInheritanceAttr::hasVBTableOffsetField(DstInheritance)) - Dst = Builder.CreateInsertValue( - Dst, getValueOrZeroInt(VirtualBaseAdjustmentOffset), Idx++); + Dst = Builder.CreateInsertValue(Dst, VirtualBaseAdjustmentOffset, Idx++); } - Builder.CreateBr(ContinueBB); - - // In the continuation, choose between DstNull and Dst. - CGF.EmitBlock(ContinueBB); - llvm::PHINode *Phi = Builder.CreatePHI(DstNull->getType(), 2, "memptr.converted"); - Phi->addIncoming(DstNull, OriginalBB); - Phi->addIncoming(Dst, ConvertBB); - return Phi; + return Dst; } llvm::Constant * MicrosoftCXXABI::EmitMemberPointerConversion(const CastExpr *E, llvm::Constant *Src) { const MemberPointerType *SrcTy = - E->getSubExpr()->getType()->castAs<MemberPointerType>(); + E->getSubExpr()->getType()->castAs<MemberPointerType>(); const MemberPointerType *DstTy = E->getType()->castAs<MemberPointerType>(); + CastKind CK = E->getCastKind(); + + return EmitMemberPointerConversion(SrcTy, DstTy, CK, E->path_begin(), + E->path_end(), Src); +} + +llvm::Constant *MicrosoftCXXABI::EmitMemberPointerConversion( + const MemberPointerType *SrcTy, const MemberPointerType *DstTy, CastKind CK, + CastExpr::path_const_iterator PathBegin, + CastExpr::path_const_iterator PathEnd, llvm::Constant *Src) { + assert(CK == CK_DerivedToBaseMemberPointer || + CK == CK_BaseToDerivedMemberPointer || + CK == CK_ReinterpretMemberPointer); // If src is null, emit a new null for dst. We can't return src because dst // might have a new representation. if (MemberPointerConstantIsNull(SrcTy, Src)) @@ -2544,61 +3128,14 @@ MicrosoftCXXABI::EmitMemberPointerConversion(const CastExpr *E, // We don't need to do anything for reinterpret_casts of non-null member // pointers. We should only get here when the two type representations have // the same size. - if (E->getCastKind() == CK_ReinterpretMemberPointer) + if (CK == CK_ReinterpretMemberPointer) return Src; - MSInheritanceAttr::Spelling SrcInheritance = getInheritanceFromMemptr(SrcTy); - MSInheritanceAttr::Spelling DstInheritance = getInheritanceFromMemptr(DstTy); - - // Decompose src. - llvm::Constant *FirstField = Src; - llvm::Constant *NonVirtualBaseAdjustment = nullptr; - llvm::Constant *VirtualBaseAdjustmentOffset = nullptr; - llvm::Constant *VBPtrOffset = nullptr; - bool IsFunc = SrcTy->isMemberFunctionPointer(); - if (!MSInheritanceAttr::hasOnlyOneField(IsFunc, SrcInheritance)) { - // We need to extract values. - unsigned I = 0; - FirstField = Src->getAggregateElement(I++); - if (MSInheritanceAttr::hasNVOffsetField(IsFunc, SrcInheritance)) - NonVirtualBaseAdjustment = Src->getAggregateElement(I++); - if (MSInheritanceAttr::hasVBPtrOffsetField(SrcInheritance)) - VBPtrOffset = Src->getAggregateElement(I++); - if (MSInheritanceAttr::hasVBTableOffsetField(SrcInheritance)) - VirtualBaseAdjustmentOffset = Src->getAggregateElement(I++); - } + CGBuilderTy Builder(CGM.getLLVMContext()); + auto *Dst = cast<llvm::Constant>(EmitNonNullMemberPointerConversion( + SrcTy, DstTy, CK, PathBegin, PathEnd, Src, Builder)); - // For data pointers, we adjust the field offset directly. For functions, we - // have a separate field. - llvm::Constant *Adj = getMemberPointerAdjustment(E); - if (Adj) { - Adj = llvm::ConstantExpr::getTruncOrBitCast(Adj, CGM.IntTy); - llvm::Constant *&NVAdjustField = - IsFunc ? NonVirtualBaseAdjustment : FirstField; - bool IsDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer); - if (!NVAdjustField) // If this field didn't exist in src, it's zero. - NVAdjustField = getZeroInt(); - if (IsDerivedToBase) - NVAdjustField = llvm::ConstantExpr::getNSWSub(NVAdjustField, Adj); - else - NVAdjustField = llvm::ConstantExpr::getNSWAdd(NVAdjustField, Adj); - } - - // FIXME PR15713: Support conversions through virtually derived classes. - - // Recompose dst from the null struct and the adjusted fields from src. - if (MSInheritanceAttr::hasOnlyOneField(IsFunc, DstInheritance)) - return FirstField; - - llvm::SmallVector<llvm::Constant *, 4> Fields; - Fields.push_back(FirstField); - if (MSInheritanceAttr::hasNVOffsetField(IsFunc, DstInheritance)) - Fields.push_back(getConstantOrZeroInt(NonVirtualBaseAdjustment)); - if (MSInheritanceAttr::hasVBPtrOffsetField(DstInheritance)) - Fields.push_back(getConstantOrZeroInt(VBPtrOffset)); - if (MSInheritanceAttr::hasVBTableOffsetField(DstInheritance)) - Fields.push_back(getConstantOrZeroInt(VirtualBaseAdjustmentOffset)); - return llvm::ConstantStruct::getAnon(Fields); + return Dst; } llvm::Value *MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer( @@ -2867,7 +3404,11 @@ llvm::GlobalVariable *MSRTTIBuilder::getClassHierarchyDescriptor() { auto Type = ABI.getClassHierarchyDescriptorType(); auto CHD = new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage, /*Initializer=*/nullptr, - MangledName.c_str()); + StringRef(MangledName)); + if (CHD->isWeakForLinker()) + CHD->setComdat(CGM.getModule().getOrInsertComdat(CHD->getName())); + + auto *Bases = getBaseClassArray(Classes); // Initialize the base class ClassHierarchyDescriptor. llvm::Constant *Fields[] = { @@ -2875,7 +3416,7 @@ llvm::GlobalVariable *MSRTTIBuilder::getClassHierarchyDescriptor() { llvm::ConstantInt::get(CGM.IntTy, Flags), llvm::ConstantInt::get(CGM.IntTy, Classes.size()), ABI.getImageRelativeConstant(llvm::ConstantExpr::getInBoundsGetElementPtr( - getBaseClassArray(Classes), + Bases->getValueType(), Bases, llvm::ArrayRef<llvm::Value *>(GEPIndices))), }; CHD->setInitializer(llvm::ConstantStruct::get(Type, Fields)); @@ -2898,9 +3439,12 @@ MSRTTIBuilder::getBaseClassArray(SmallVectorImpl<MSRTTIClass> &Classes) { llvm::Type *PtrType = ABI.getImageRelativeType( ABI.getBaseClassDescriptorType()->getPointerTo()); auto *ArrType = llvm::ArrayType::get(PtrType, Classes.size() + 1); - auto *BCA = new llvm::GlobalVariable( - Module, ArrType, - /*Constant=*/true, Linkage, /*Initializer=*/nullptr, MangledName.c_str()); + auto *BCA = + new llvm::GlobalVariable(Module, ArrType, + /*Constant=*/true, Linkage, + /*Initializer=*/nullptr, StringRef(MangledName)); + if (BCA->isWeakForLinker()) + BCA->setComdat(CGM.getModule().getOrInsertComdat(BCA->getName())); // Initialize the BaseClassArray. SmallVector<llvm::Constant *, 8> BaseClassArrayData; @@ -2938,9 +3482,11 @@ MSRTTIBuilder::getBaseClassDescriptor(const MSRTTIClass &Class) { // Forward-declare the base class descriptor. auto Type = ABI.getBaseClassDescriptorType(); - auto BCD = new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage, - /*Initializer=*/nullptr, - MangledName.c_str()); + auto BCD = + new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage, + /*Initializer=*/nullptr, StringRef(MangledName)); + if (BCD->isWeakForLinker()) + BCD->setComdat(CGM.getModule().getOrInsertComdat(BCD->getName())); // Initialize the BaseClassDescriptor. llvm::Constant *Fields[] = { @@ -2984,7 +3530,7 @@ MSRTTIBuilder::getCompleteObjectLocator(const VPtrInfo *Info) { // Forward-declare the complete object locator. llvm::StructType *Type = ABI.getCompleteObjectLocatorType(); auto COL = new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage, - /*Initializer=*/nullptr, MangledName.c_str()); + /*Initializer=*/nullptr, StringRef(MangledName)); // Initialize the CompleteObjectLocator. llvm::Constant *Fields[] = { @@ -3000,15 +3546,91 @@ MSRTTIBuilder::getCompleteObjectLocator(const VPtrInfo *Info) { if (!ABI.isImageRelative()) FieldsRef = FieldsRef.drop_back(); COL->setInitializer(llvm::ConstantStruct::get(Type, FieldsRef)); + if (COL->isWeakForLinker()) + COL->setComdat(CGM.getModule().getOrInsertComdat(COL->getName())); return COL; } +static QualType decomposeTypeForEH(ASTContext &Context, QualType T, + bool &IsConst, bool &IsVolatile) { + T = Context.getExceptionObjectType(T); + + // C++14 [except.handle]p3: + // A handler is a match for an exception object of type E if [...] + // - the handler is of type cv T or const T& where T is a pointer type and + // E is a pointer type that can be converted to T by [...] + // - a qualification conversion + IsConst = false; + IsVolatile = false; + QualType PointeeType = T->getPointeeType(); + if (!PointeeType.isNull()) { + IsConst = PointeeType.isConstQualified(); + IsVolatile = PointeeType.isVolatileQualified(); + } + + // Member pointer types like "const int A::*" are represented by having RTTI + // for "int A::*" and separately storing the const qualifier. + if (const auto *MPTy = T->getAs<MemberPointerType>()) + T = Context.getMemberPointerType(PointeeType.getUnqualifiedType(), + MPTy->getClass()); + + // Pointer types like "const int * const *" are represented by having RTTI + // for "const int **" and separately storing the const qualifier. + if (T->isPointerType()) + T = Context.getPointerType(PointeeType.getUnqualifiedType()); + + return T; +} + +llvm::Constant * +MicrosoftCXXABI::getAddrOfCXXCatchHandlerType(QualType Type, + QualType CatchHandlerType) { + // TypeDescriptors for exceptions never have qualified pointer types, + // qualifiers are stored seperately in order to support qualification + // conversions. + bool IsConst, IsVolatile; + Type = decomposeTypeForEH(getContext(), Type, IsConst, IsVolatile); + + bool IsReference = CatchHandlerType->isReferenceType(); + + uint32_t Flags = 0; + if (IsConst) + Flags |= 1; + if (IsVolatile) + Flags |= 2; + if (IsReference) + Flags |= 8; + + SmallString<256> MangledName; + { + llvm::raw_svector_ostream Out(MangledName); + getMangleContext().mangleCXXCatchHandlerType(Type, Flags, Out); + } + + if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(MangledName)) + return llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy); + + llvm::Constant *Fields[] = { + llvm::ConstantInt::get(CGM.IntTy, Flags), // Flags + getAddrOfRTTIDescriptor(Type), // TypeDescriptor + }; + llvm::StructType *CatchHandlerTypeType = getCatchHandlerTypeType(); + auto *Var = new llvm::GlobalVariable( + CGM.getModule(), CatchHandlerTypeType, /*Constant=*/true, + llvm::GlobalValue::PrivateLinkage, + llvm::ConstantStruct::get(CatchHandlerTypeType, Fields), + StringRef(MangledName)); + Var->setUnnamedAddr(true); + Var->setSection("llvm.metadata"); + return Var; +} + /// \brief Gets a TypeDescriptor. Returns a llvm::Constant * rather than a /// llvm::GlobalVariable * because different type descriptors have different /// types, and need to be abstracted. They are abstracting by casting the /// address to an Int8PtrTy. llvm::Constant *MicrosoftCXXABI::getAddrOfRTTIDescriptor(QualType Type) { - SmallString<256> MangledName, TypeInfoString; + SmallString<256> MangledName; { llvm::raw_svector_ostream Out(MangledName); getMangleContext().mangleCXXRTTI(Type, Out); @@ -3019,6 +3641,7 @@ llvm::Constant *MicrosoftCXXABI::getAddrOfRTTIDescriptor(QualType Type) { return llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy); // Compute the fields for the TypeDescriptor. + SmallString<256> TypeInfoString; { llvm::raw_svector_ostream Out(TypeInfoString); getMangleContext().mangleCXXRTTIName(Type, Out); @@ -3031,13 +3654,14 @@ llvm::Constant *MicrosoftCXXABI::getAddrOfRTTIDescriptor(QualType Type) { llvm::ConstantDataArray::getString(CGM.getLLVMContext(), TypeInfoString)}; llvm::StructType *TypeDescriptorType = getTypeDescriptorType(TypeInfoString); - return llvm::ConstantExpr::getBitCast( - new llvm::GlobalVariable( - CGM.getModule(), TypeDescriptorType, /*Constant=*/false, - getLinkageForRTTI(Type), - llvm::ConstantStruct::get(TypeDescriptorType, Fields), - MangledName.c_str()), - CGM.Int8PtrTy); + auto *Var = new llvm::GlobalVariable( + CGM.getModule(), TypeDescriptorType, /*Constant=*/false, + getLinkageForRTTI(Type), + llvm::ConstantStruct::get(TypeDescriptorType, Fields), + StringRef(MangledName)); + if (Var->isWeakForLinker()) + Var->setComdat(CGM.getModule().getOrInsertComdat(Var->getName())); + return llvm::ConstantExpr::getBitCast(Var, CGM.Int8PtrTy); } /// \brief Gets or a creates a Microsoft CompleteObjectLocator. @@ -3051,7 +3675,8 @@ static void emitCXXConstructor(CodeGenModule &CGM, const CXXConstructorDecl *ctor, StructorType ctorType) { // There are no constructor variants, always emit the complete destructor. - CGM.codegenCXXStructor(ctor, StructorType::Complete); + llvm::Function *Fn = CGM.codegenCXXStructor(ctor, StructorType::Complete); + CGM.maybeSetTrivialComdat(*ctor, *Fn); } static void emitCXXDestructor(CodeGenModule &CGM, const CXXDestructorDecl *dtor, @@ -3077,7 +3702,9 @@ static void emitCXXDestructor(CodeGenModule &CGM, const CXXDestructorDecl *dtor, if (dtorType == StructorType::Base && !CGM.TryEmitBaseDestructorAsAlias(dtor)) return; - CGM.codegenCXXStructor(dtor, dtorType); + llvm::Function *Fn = CGM.codegenCXXStructor(dtor, dtorType); + if (Fn->isWeakForLinker()) + Fn->setComdat(CGM.getModule().getOrInsertComdat(Fn->getName())); } void MicrosoftCXXABI::emitCXXStructor(const CXXMethodDecl *MD, @@ -3088,3 +3715,402 @@ void MicrosoftCXXABI::emitCXXStructor(const CXXMethodDecl *MD, } emitCXXDestructor(CGM, cast<CXXDestructorDecl>(MD), Type); } + +llvm::Function * +MicrosoftCXXABI::getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD, + CXXCtorType CT) { + assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure); + + // Calculate the mangled name. + SmallString<256> ThunkName; + llvm::raw_svector_ostream Out(ThunkName); + getMangleContext().mangleCXXCtor(CD, CT, Out); + Out.flush(); + + // If the thunk has been generated previously, just return it. + if (llvm::GlobalValue *GV = CGM.getModule().getNamedValue(ThunkName)) + return cast<llvm::Function>(GV); + + // Create the llvm::Function. + const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeMSCtorClosure(CD, CT); + llvm::FunctionType *ThunkTy = CGM.getTypes().GetFunctionType(FnInfo); + const CXXRecordDecl *RD = CD->getParent(); + QualType RecordTy = getContext().getRecordType(RD); + llvm::Function *ThunkFn = llvm::Function::Create( + ThunkTy, getLinkageForRTTI(RecordTy), ThunkName.str(), &CGM.getModule()); + ThunkFn->setCallingConv(static_cast<llvm::CallingConv::ID>( + FnInfo.getEffectiveCallingConvention())); + if (ThunkFn->isWeakForLinker()) + ThunkFn->setComdat(CGM.getModule().getOrInsertComdat(ThunkFn->getName())); + bool IsCopy = CT == Ctor_CopyingClosure; + + // Start codegen. + CodeGenFunction CGF(CGM); + CGF.CurGD = GlobalDecl(CD, Ctor_Complete); + + // Build FunctionArgs. + FunctionArgList FunctionArgs; + + // A constructor always starts with a 'this' pointer as its first argument. + buildThisParam(CGF, FunctionArgs); + + // Following the 'this' pointer is a reference to the source object that we + // are copying from. + ImplicitParamDecl SrcParam( + getContext(), nullptr, SourceLocation(), &getContext().Idents.get("src"), + getContext().getLValueReferenceType(RecordTy, + /*SpelledAsLValue=*/true)); + if (IsCopy) + FunctionArgs.push_back(&SrcParam); + + // Constructors for classes which utilize virtual bases have an additional + // parameter which indicates whether or not it is being delegated to by a more + // derived constructor. + ImplicitParamDecl IsMostDerived(getContext(), nullptr, SourceLocation(), + &getContext().Idents.get("is_most_derived"), + getContext().IntTy); + // Only add the parameter to the list if thie class has virtual bases. + if (RD->getNumVBases() > 0) + FunctionArgs.push_back(&IsMostDerived); + + // Start defining the function. + CGF.StartFunction(GlobalDecl(), FnInfo.getReturnType(), ThunkFn, FnInfo, + FunctionArgs, CD->getLocation(), SourceLocation()); + EmitThisParam(CGF); + llvm::Value *This = getThisValue(CGF); + + llvm::Value *SrcVal = + IsCopy ? CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&SrcParam), "src") + : nullptr; + + CallArgList Args; + + // Push the this ptr. + Args.add(RValue::get(This), CD->getThisType(getContext())); + + // Push the src ptr. + if (SrcVal) + Args.add(RValue::get(SrcVal), SrcParam.getType()); + + // Add the rest of the default arguments. + std::vector<Stmt *> ArgVec; + for (unsigned I = IsCopy ? 1 : 0, E = CD->getNumParams(); I != E; ++I) { + Stmt *DefaultArg = getContext().getDefaultArgExprForConstructor(CD, I); + assert(DefaultArg && "sema forgot to instantiate default args"); + ArgVec.push_back(DefaultArg); + } + + CodeGenFunction::RunCleanupsScope Cleanups(CGF); + + const auto *FPT = CD->getType()->castAs<FunctionProtoType>(); + ConstExprIterator ArgBegin(ArgVec.data()), + ArgEnd(ArgVec.data() + ArgVec.size()); + CGF.EmitCallArgs(Args, FPT, ArgBegin, ArgEnd, CD, IsCopy ? 1 : 0); + + // Insert any ABI-specific implicit constructor arguments. + unsigned ExtraArgs = addImplicitConstructorArgs(CGF, CD, Ctor_Complete, + /*ForVirtualBase=*/false, + /*Delegating=*/false, Args); + + // Call the destructor with our arguments. + llvm::Value *CalleeFn = CGM.getAddrOfCXXStructor(CD, StructorType::Complete); + const CGFunctionInfo &CalleeInfo = CGM.getTypes().arrangeCXXConstructorCall( + Args, CD, Ctor_Complete, ExtraArgs); + CGF.EmitCall(CalleeInfo, CalleeFn, ReturnValueSlot(), Args, CD); + + Cleanups.ForceCleanup(); + + // Emit the ret instruction, remove any temporary instructions created for the + // aid of CodeGen. + CGF.FinishFunction(SourceLocation()); + + return ThunkFn; +} + +llvm::Constant *MicrosoftCXXABI::getCatchableType(QualType T, + uint32_t NVOffset, + int32_t VBPtrOffset, + uint32_t VBIndex) { + assert(!T->isReferenceType()); + + CXXRecordDecl *RD = T->getAsCXXRecordDecl(); + const CXXConstructorDecl *CD = + RD ? CGM.getContext().getCopyConstructorForExceptionObject(RD) : nullptr; + CXXCtorType CT = Ctor_Complete; + if (CD) + if (!hasDefaultCXXMethodCC(getContext(), CD) || CD->getNumParams() != 1) + CT = Ctor_CopyingClosure; + + uint32_t Size = getContext().getTypeSizeInChars(T).getQuantity(); + SmallString<256> MangledName; + { + llvm::raw_svector_ostream Out(MangledName); + getMangleContext().mangleCXXCatchableType(T, CD, CT, Size, NVOffset, + VBPtrOffset, VBIndex, Out); + } + if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(MangledName)) + return getImageRelativeConstant(GV); + + // The TypeDescriptor is used by the runtime to determine if a catch handler + // is appropriate for the exception object. + llvm::Constant *TD = getImageRelativeConstant(getAddrOfRTTIDescriptor(T)); + + // The runtime is responsible for calling the copy constructor if the + // exception is caught by value. + llvm::Constant *CopyCtor; + if (CD) { + if (CT == Ctor_CopyingClosure) + CopyCtor = getAddrOfCXXCtorClosure(CD, Ctor_CopyingClosure); + else + CopyCtor = CGM.getAddrOfCXXStructor(CD, StructorType::Complete); + + CopyCtor = llvm::ConstantExpr::getBitCast(CopyCtor, CGM.Int8PtrTy); + } else { + CopyCtor = llvm::Constant::getNullValue(CGM.Int8PtrTy); + } + CopyCtor = getImageRelativeConstant(CopyCtor); + + bool IsScalar = !RD; + bool HasVirtualBases = false; + bool IsStdBadAlloc = false; // std::bad_alloc is special for some reason. + QualType PointeeType = T; + if (T->isPointerType()) + PointeeType = T->getPointeeType(); + if (const CXXRecordDecl *RD = PointeeType->getAsCXXRecordDecl()) { + HasVirtualBases = RD->getNumVBases() > 0; + if (IdentifierInfo *II = RD->getIdentifier()) + IsStdBadAlloc = II->isStr("bad_alloc") && RD->isInStdNamespace(); + } + + // Encode the relevant CatchableType properties into the Flags bitfield. + // FIXME: Figure out how bits 2 or 8 can get set. + uint32_t Flags = 0; + if (IsScalar) + Flags |= 1; + if (HasVirtualBases) + Flags |= 4; + if (IsStdBadAlloc) + Flags |= 16; + + llvm::Constant *Fields[] = { + llvm::ConstantInt::get(CGM.IntTy, Flags), // Flags + TD, // TypeDescriptor + llvm::ConstantInt::get(CGM.IntTy, NVOffset), // NonVirtualAdjustment + llvm::ConstantInt::get(CGM.IntTy, VBPtrOffset), // OffsetToVBPtr + llvm::ConstantInt::get(CGM.IntTy, VBIndex), // VBTableIndex + llvm::ConstantInt::get(CGM.IntTy, Size), // Size + CopyCtor // CopyCtor + }; + llvm::StructType *CTType = getCatchableTypeType(); + auto *GV = new llvm::GlobalVariable( + CGM.getModule(), CTType, /*Constant=*/true, getLinkageForRTTI(T), + llvm::ConstantStruct::get(CTType, Fields), StringRef(MangledName)); + GV->setUnnamedAddr(true); + GV->setSection(".xdata"); + if (GV->isWeakForLinker()) + GV->setComdat(CGM.getModule().getOrInsertComdat(GV->getName())); + return getImageRelativeConstant(GV); +} + +llvm::GlobalVariable *MicrosoftCXXABI::getCatchableTypeArray(QualType T) { + assert(!T->isReferenceType()); + + // See if we've already generated a CatchableTypeArray for this type before. + llvm::GlobalVariable *&CTA = CatchableTypeArrays[T]; + if (CTA) + return CTA; + + // Ensure that we don't have duplicate entries in our CatchableTypeArray by + // using a SmallSetVector. Duplicates may arise due to virtual bases + // occurring more than once in the hierarchy. + llvm::SmallSetVector<llvm::Constant *, 2> CatchableTypes; + + // C++14 [except.handle]p3: + // A handler is a match for an exception object of type E if [...] + // - the handler is of type cv T or cv T& and T is an unambiguous public + // base class of E, or + // - the handler is of type cv T or const T& where T is a pointer type and + // E is a pointer type that can be converted to T by [...] + // - a standard pointer conversion (4.10) not involving conversions to + // pointers to private or protected or ambiguous classes + const CXXRecordDecl *MostDerivedClass = nullptr; + bool IsPointer = T->isPointerType(); + if (IsPointer) + MostDerivedClass = T->getPointeeType()->getAsCXXRecordDecl(); + else + MostDerivedClass = T->getAsCXXRecordDecl(); + + // Collect all the unambiguous public bases of the MostDerivedClass. + if (MostDerivedClass) { + const ASTContext &Context = getContext(); + const ASTRecordLayout &MostDerivedLayout = + Context.getASTRecordLayout(MostDerivedClass); + MicrosoftVTableContext &VTableContext = CGM.getMicrosoftVTableContext(); + SmallVector<MSRTTIClass, 8> Classes; + serializeClassHierarchy(Classes, MostDerivedClass); + Classes.front().initialize(/*Parent=*/nullptr, /*Specifier=*/nullptr); + detectAmbiguousBases(Classes); + for (const MSRTTIClass &Class : Classes) { + // Skip any ambiguous or private bases. + if (Class.Flags & + (MSRTTIClass::IsPrivateOnPath | MSRTTIClass::IsAmbiguous)) + continue; + // Write down how to convert from a derived pointer to a base pointer. + uint32_t OffsetInVBTable = 0; + int32_t VBPtrOffset = -1; + if (Class.VirtualRoot) { + OffsetInVBTable = + VTableContext.getVBTableIndex(MostDerivedClass, Class.VirtualRoot)*4; + VBPtrOffset = MostDerivedLayout.getVBPtrOffset().getQuantity(); + } + + // Turn our record back into a pointer if the exception object is a + // pointer. + QualType RTTITy = QualType(Class.RD->getTypeForDecl(), 0); + if (IsPointer) + RTTITy = Context.getPointerType(RTTITy); + CatchableTypes.insert(getCatchableType(RTTITy, Class.OffsetInVBase, + VBPtrOffset, OffsetInVBTable)); + } + } + + // C++14 [except.handle]p3: + // A handler is a match for an exception object of type E if + // - The handler is of type cv T or cv T& and E and T are the same type + // (ignoring the top-level cv-qualifiers) + CatchableTypes.insert(getCatchableType(T)); + + // C++14 [except.handle]p3: + // A handler is a match for an exception object of type E if + // - the handler is of type cv T or const T& where T is a pointer type and + // E is a pointer type that can be converted to T by [...] + // - a standard pointer conversion (4.10) not involving conversions to + // pointers to private or protected or ambiguous classes + // + // C++14 [conv.ptr]p2: + // A prvalue of type "pointer to cv T," where T is an object type, can be + // converted to a prvalue of type "pointer to cv void". + if (IsPointer && T->getPointeeType()->isObjectType()) + CatchableTypes.insert(getCatchableType(getContext().VoidPtrTy)); + + // C++14 [except.handle]p3: + // A handler is a match for an exception object of type E if [...] + // - the handler is of type cv T or const T& where T is a pointer or + // pointer to member type and E is std::nullptr_t. + // + // We cannot possibly list all possible pointer types here, making this + // implementation incompatible with the standard. However, MSVC includes an + // entry for pointer-to-void in this case. Let's do the same. + if (T->isNullPtrType()) + CatchableTypes.insert(getCatchableType(getContext().VoidPtrTy)); + + uint32_t NumEntries = CatchableTypes.size(); + llvm::Type *CTType = + getImageRelativeType(getCatchableTypeType()->getPointerTo()); + llvm::ArrayType *AT = llvm::ArrayType::get(CTType, NumEntries); + llvm::StructType *CTAType = getCatchableTypeArrayType(NumEntries); + llvm::Constant *Fields[] = { + llvm::ConstantInt::get(CGM.IntTy, NumEntries), // NumEntries + llvm::ConstantArray::get( + AT, llvm::makeArrayRef(CatchableTypes.begin(), + CatchableTypes.end())) // CatchableTypes + }; + SmallString<256> MangledName; + { + llvm::raw_svector_ostream Out(MangledName); + getMangleContext().mangleCXXCatchableTypeArray(T, NumEntries, Out); + } + CTA = new llvm::GlobalVariable( + CGM.getModule(), CTAType, /*Constant=*/true, getLinkageForRTTI(T), + llvm::ConstantStruct::get(CTAType, Fields), StringRef(MangledName)); + CTA->setUnnamedAddr(true); + CTA->setSection(".xdata"); + if (CTA->isWeakForLinker()) + CTA->setComdat(CGM.getModule().getOrInsertComdat(CTA->getName())); + return CTA; +} + +llvm::GlobalVariable *MicrosoftCXXABI::getThrowInfo(QualType T) { + bool IsConst, IsVolatile; + T = decomposeTypeForEH(getContext(), T, IsConst, IsVolatile); + + // The CatchableTypeArray enumerates the various (CV-unqualified) types that + // the exception object may be caught as. + llvm::GlobalVariable *CTA = getCatchableTypeArray(T); + // The first field in a CatchableTypeArray is the number of CatchableTypes. + // This is used as a component of the mangled name which means that we need to + // know what it is in order to see if we have previously generated the + // ThrowInfo. + uint32_t NumEntries = + cast<llvm::ConstantInt>(CTA->getInitializer()->getAggregateElement(0U)) + ->getLimitedValue(); + + SmallString<256> MangledName; + { + llvm::raw_svector_ostream Out(MangledName); + getMangleContext().mangleCXXThrowInfo(T, IsConst, IsVolatile, NumEntries, + Out); + } + + // Reuse a previously generated ThrowInfo if we have generated an appropriate + // one before. + if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(MangledName)) + return GV; + + // The RTTI TypeDescriptor uses an unqualified type but catch clauses must + // be at least as CV qualified. Encode this requirement into the Flags + // bitfield. + uint32_t Flags = 0; + if (IsConst) + Flags |= 1; + if (IsVolatile) + Flags |= 2; + + // The cleanup-function (a destructor) must be called when the exception + // object's lifetime ends. + llvm::Constant *CleanupFn = llvm::Constant::getNullValue(CGM.Int8PtrTy); + if (const CXXRecordDecl *RD = T->getAsCXXRecordDecl()) + if (CXXDestructorDecl *DtorD = RD->getDestructor()) + if (!DtorD->isTrivial()) + CleanupFn = llvm::ConstantExpr::getBitCast( + CGM.getAddrOfCXXStructor(DtorD, StructorType::Complete), + CGM.Int8PtrTy); + // This is unused as far as we can tell, initialize it to null. + llvm::Constant *ForwardCompat = + getImageRelativeConstant(llvm::Constant::getNullValue(CGM.Int8PtrTy)); + llvm::Constant *PointerToCatchableTypes = getImageRelativeConstant( + llvm::ConstantExpr::getBitCast(CTA, CGM.Int8PtrTy)); + llvm::StructType *TIType = getThrowInfoType(); + llvm::Constant *Fields[] = { + llvm::ConstantInt::get(CGM.IntTy, Flags), // Flags + getImageRelativeConstant(CleanupFn), // CleanupFn + ForwardCompat, // ForwardCompat + PointerToCatchableTypes // CatchableTypeArray + }; + auto *GV = new llvm::GlobalVariable( + CGM.getModule(), TIType, /*Constant=*/true, getLinkageForRTTI(T), + llvm::ConstantStruct::get(TIType, Fields), StringRef(MangledName)); + GV->setUnnamedAddr(true); + GV->setSection(".xdata"); + if (GV->isWeakForLinker()) + GV->setComdat(CGM.getModule().getOrInsertComdat(GV->getName())); + return GV; +} + +void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { + const Expr *SubExpr = E->getSubExpr(); + QualType ThrowType = SubExpr->getType(); + // The exception object lives on the stack and it's address is passed to the + // runtime function. + llvm::AllocaInst *AI = CGF.CreateMemTemp(ThrowType); + CGF.EmitAnyExprToMem(SubExpr, AI, ThrowType.getQualifiers(), + /*IsInit=*/true); + + // The so-called ThrowInfo is used to describe how the exception object may be + // caught. + llvm::GlobalVariable *TI = getThrowInfo(ThrowType); + + // Call into the runtime to throw the exception. + llvm::Value *Args[] = {CGF.Builder.CreateBitCast(AI, CGM.Int8PtrTy), TI}; + CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(), Args); +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp index 4f1a82e..def56a9 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/ModuleBuilder.cpp @@ -32,6 +32,8 @@ namespace { DiagnosticsEngine &Diags; std::unique_ptr<const llvm::DataLayout> TD; ASTContext *Ctx; + const HeaderSearchOptions &HeaderSearchOpts; // Only used for debug info. + const PreprocessorOptions &PreprocessorOpts; // Only used for debug info. const CodeGenOptions CodeGenOpts; // Intentionally copied in. unsigned HandlingTopLevelDecls; @@ -56,14 +58,17 @@ namespace { SmallVector<CXXMethodDecl *, 8> DeferredInlineMethodDefinitions; public: - CodeGeneratorImpl(DiagnosticsEngine &diags, const std::string& ModuleName, - const CodeGenOptions &CGO, llvm::LLVMContext& C, + CodeGeneratorImpl(DiagnosticsEngine &diags, const std::string &ModuleName, + const HeaderSearchOptions &HSO, + const PreprocessorOptions &PPO, const CodeGenOptions &CGO, + llvm::LLVMContext &C, CoverageSourceInfo *CoverageInfo = nullptr) - : Diags(diags), Ctx(nullptr), CodeGenOpts(CGO), HandlingTopLevelDecls(0), - CoverageInfo(CoverageInfo), - M(new llvm::Module(ModuleName, C)) {} + : Diags(diags), Ctx(nullptr), HeaderSearchOpts(HSO), + PreprocessorOpts(PPO), CodeGenOpts(CGO), HandlingTopLevelDecls(0), + CoverageInfo(CoverageInfo), + M(new llvm::Module(ModuleName, C)) {} - virtual ~CodeGeneratorImpl() { + ~CodeGeneratorImpl() override { // There should normally not be any leftover inline method definitions. assert(DeferredInlineMethodDefinitions.empty() || Diags.hasErrorOccurred()); @@ -95,8 +100,12 @@ namespace { M->setTargetTriple(Ctx->getTargetInfo().getTriple().getTriple()); M->setDataLayout(Ctx->getTargetInfo().getTargetDescription()); - TD.reset(new llvm::DataLayout(Ctx->getTargetInfo().getTargetDescription())); - Builder.reset(new CodeGen::CodeGenModule(Context, CodeGenOpts, *M, *TD, + TD.reset( + new llvm::DataLayout(Ctx->getTargetInfo().getTargetDescription())); + Builder.reset(new CodeGen::CodeGenModule(Context, + HeaderSearchOpts, + PreprocessorOpts, + CodeGenOpts, *M, *TD, Diags, CoverageInfo)); for (size_t i = 0, e = CodeGenOpts.DependentLibraries.size(); i < e; ++i) @@ -211,11 +220,11 @@ namespace { Builder->EmitTentativeDefinition(D); } - void HandleVTable(CXXRecordDecl *RD, bool DefinitionRequired) override { + void HandleVTable(CXXRecordDecl *RD) override { if (Diags.hasErrorOccurred()) return; - Builder->EmitVTable(RD, DefinitionRequired); + Builder->EmitVTable(RD); } void HandleLinkerOptionPragma(llvm::StringRef Opts) override { @@ -235,11 +244,11 @@ namespace { void CodeGenerator::anchor() { } -CodeGenerator *clang::CreateLLVMCodeGen(DiagnosticsEngine &Diags, - const std::string& ModuleName, - const CodeGenOptions &CGO, - const TargetOptions &/*TO*/, - llvm::LLVMContext& C, - CoverageSourceInfo *CoverageInfo) { - return new CodeGeneratorImpl(Diags, ModuleName, CGO, C, CoverageInfo); +CodeGenerator *clang::CreateLLVMCodeGen( + DiagnosticsEngine &Diags, const std::string &ModuleName, + const HeaderSearchOptions &HeaderSearchOpts, + const PreprocessorOptions &PreprocessorOpts, const CodeGenOptions &CGO, + llvm::LLVMContext &C, CoverageSourceInfo *CoverageInfo) { + return new CodeGeneratorImpl(Diags, ModuleName, HeaderSearchOpts, + PreprocessorOpts, CGO, C, CoverageInfo); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp b/contrib/llvm/tools/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp new file mode 100644 index 0000000..9c9b123 --- /dev/null +++ b/contrib/llvm/tools/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp @@ -0,0 +1,193 @@ +//===--- ObjectFilePCHContainerOperations.cpp -----------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" +#include "CGDebugInfo.h" +#include "CodeGenModule.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/DeclObjC.h" +#include "clang/AST/Expr.h" +#include "clang/AST/RecursiveASTVisitor.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/TargetInfo.h" +#include "clang/CodeGen/BackendUtil.h" +#include "clang/Frontend/CodeGenOptions.h" +#include "clang/Serialization/ASTWriter.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Bitcode/BitstreamReader.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/COFF.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/TargetRegistry.h" +#include <memory> +using namespace clang; + +#define DEBUG_TYPE "pchcontainer" + +namespace { +class PCHContainerGenerator : public ASTConsumer { + DiagnosticsEngine &Diags; + const std::string MainFileName; + ASTContext *Ctx; + const HeaderSearchOptions &HeaderSearchOpts; + const PreprocessorOptions &PreprocessorOpts; + CodeGenOptions CodeGenOpts; + const TargetOptions TargetOpts; + const LangOptions LangOpts; + std::unique_ptr<llvm::LLVMContext> VMContext; + std::unique_ptr<llvm::Module> M; + std::unique_ptr<CodeGen::CodeGenModule> Builder; + raw_pwrite_stream *OS; + std::shared_ptr<PCHBuffer> Buffer; + +public: + PCHContainerGenerator(DiagnosticsEngine &diags, + const HeaderSearchOptions &HSO, + const PreprocessorOptions &PPO, const TargetOptions &TO, + const LangOptions &LO, const std::string &MainFileName, + const std::string &OutputFileName, + raw_pwrite_stream *OS, + std::shared_ptr<PCHBuffer> Buffer) + : Diags(diags), HeaderSearchOpts(HSO), PreprocessorOpts(PPO), + TargetOpts(TO), LangOpts(LO), OS(OS), Buffer(Buffer) { + // The debug info output isn't affected by CodeModel and + // ThreadModel, but the backend expects them to be nonempty. + CodeGenOpts.CodeModel = "default"; + CodeGenOpts.ThreadModel = "single"; + CodeGenOpts.setDebugInfo(CodeGenOptions::FullDebugInfo); + CodeGenOpts.SplitDwarfFile = OutputFileName; + } + + virtual ~PCHContainerGenerator() {} + + void Initialize(ASTContext &Context) override { + Ctx = &Context; + VMContext.reset(new llvm::LLVMContext()); + M.reset(new llvm::Module(MainFileName, *VMContext)); + M->setDataLayout(Ctx->getTargetInfo().getTargetDescription()); + Builder.reset(new CodeGen::CodeGenModule(*Ctx, HeaderSearchOpts, + PreprocessorOpts, CodeGenOpts, *M, + M->getDataLayout(), Diags)); + } + + /// Emit a container holding the serialized AST. + void HandleTranslationUnit(ASTContext &Ctx) override { + assert(M && VMContext && Builder); + // Delete these on function exit. + std::unique_ptr<llvm::LLVMContext> VMContext = std::move(this->VMContext); + std::unique_ptr<llvm::Module> M = std::move(this->M); + std::unique_ptr<CodeGen::CodeGenModule> Builder = std::move(this->Builder); + + if (Diags.hasErrorOccurred()) + return; + + M->setTargetTriple(Ctx.getTargetInfo().getTriple().getTriple()); + M->setDataLayout(Ctx.getTargetInfo().getTargetDescription()); + + // Finalize the Builder. + if (Builder) + Builder->Release(); + + // Ensure the target exists. + std::string Error; + auto Triple = Ctx.getTargetInfo().getTriple(); + if (!llvm::TargetRegistry::lookupTarget(Triple.getTriple(), Error)) + llvm::report_fatal_error(Error); + + // Emit the serialized Clang AST into its own section. + assert(Buffer->IsComplete && "serialization did not complete"); + auto &SerializedAST = Buffer->Data; + auto Size = SerializedAST.size(); + auto Int8Ty = llvm::Type::getInt8Ty(*VMContext); + auto *Ty = llvm::ArrayType::get(Int8Ty, Size); + auto *Data = llvm::ConstantDataArray::getString( + *VMContext, StringRef(SerializedAST.data(), Size), + /*AddNull=*/false); + auto *ASTSym = new llvm::GlobalVariable( + *M, Ty, /*constant*/ true, llvm::GlobalVariable::InternalLinkage, Data, + "__clang_ast"); + // The on-disk hashtable needs to be aligned. + ASTSym->setAlignment(8); + + // Mach-O also needs a segment name. + if (Triple.isOSBinFormatMachO()) + ASTSym->setSection("__CLANG,__clangast"); + // COFF has an eight character length limit. + else if (Triple.isOSBinFormatCOFF()) + ASTSym->setSection("clangast"); + else + ASTSym->setSection("__clangast"); + + DEBUG({ + // Print the IR for the PCH container to the debug output. + llvm::SmallString<0> Buffer; + llvm::raw_svector_ostream OS(Buffer); + clang::EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts, + Ctx.getTargetInfo().getTargetDescription(), + M.get(), BackendAction::Backend_EmitLL, &OS); + OS.flush(); + llvm::dbgs() << Buffer; + }); + + // Use the LLVM backend to emit the pch container. + clang::EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts, + Ctx.getTargetInfo().getTargetDescription(), + M.get(), BackendAction::Backend_EmitObj, OS); + + // Make sure the pch container hits disk. + OS->flush(); + + // Free the memory for the temporary buffer. + llvm::SmallVector<char, 0> Empty; + SerializedAST = std::move(Empty); + } +}; + +} // namespace + +std::unique_ptr<ASTConsumer> +ObjectFilePCHContainerWriter::CreatePCHContainerGenerator( + DiagnosticsEngine &Diags, const HeaderSearchOptions &HSO, + const PreprocessorOptions &PPO, const TargetOptions &TO, + const LangOptions &LO, const std::string &MainFileName, + const std::string &OutputFileName, llvm::raw_pwrite_stream *OS, + std::shared_ptr<PCHBuffer> Buffer) const { + return llvm::make_unique<PCHContainerGenerator>( + Diags, HSO, PPO, TO, LO, MainFileName, OutputFileName, OS, Buffer); +} + +void ObjectFilePCHContainerReader::ExtractPCH( + llvm::MemoryBufferRef Buffer, llvm::BitstreamReader &StreamFile) const { + if (auto OF = llvm::object::ObjectFile::createObjectFile(Buffer)) { + auto *Obj = OF.get().get(); + bool IsCOFF = isa<llvm::object::COFFObjectFile>(Obj); + // Find the clang AST section in the container. + for (auto &Section : OF->get()->sections()) { + StringRef Name; + Section.getName(Name); + if ((!IsCOFF && Name == "__clangast") || + ( IsCOFF && Name == "clangast")) { + StringRef Buf; + Section.getContents(Buf); + StreamFile.init((const unsigned char *)Buf.begin(), + (const unsigned char *)Buf.end()); + return; + } + } + } + + // As a fallback, treat the buffer as a raw AST. + StreamFile.init((const unsigned char *)Buffer.getBufferStart(), + (const unsigned char *)Buffer.getBufferEnd()); + return; +} diff --git a/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp b/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp index 7c38b28..2a338ba 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.cpp @@ -25,7 +25,8 @@ void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV, SourceLocation Loc, StringRef Name, QualType Ty, bool IsDynInit, bool IsBlacklisted) { - if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (!CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address | + SanitizerKind::KernelAddress)) return; IsDynInit &= !CGM.isInSanitizerBlacklist(GV, Loc, Ty, "init"); IsBlacklisted |= CGM.isInSanitizerBlacklist(GV, Loc, Ty); @@ -56,7 +57,8 @@ void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV, void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV, const VarDecl &D, bool IsDynInit) { - if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (!CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address | + SanitizerKind::KernelAddress)) return; std::string QualName; llvm::raw_string_ostream OS(QualName); @@ -67,7 +69,8 @@ void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV, void SanitizerMetadata::disableSanitizerForGlobal(llvm::GlobalVariable *GV) { // For now, just make sure the global is not modified by the ASan // instrumentation. - if (CGM.getLangOpts().Sanitize.has(SanitizerKind::Address)) + if (CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address | + SanitizerKind::KernelAddress)) reportGlobalToASan(GV, SourceLocation(), "", QualType(), false, true); } diff --git a/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.h b/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.h index d2f0651..166f0e6 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/SanitizerMetadata.h @@ -31,8 +31,8 @@ namespace CodeGen { class CodeGenModule; class SanitizerMetadata { - SanitizerMetadata(const SanitizerMetadata &) LLVM_DELETED_FUNCTION; - void operator=(const SanitizerMetadata &) LLVM_DELETED_FUNCTION; + SanitizerMetadata(const SanitizerMetadata &) = delete; + void operator=(const SanitizerMetadata &) = delete; CodeGenModule &CGM; public: diff --git a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp index c05b23a..48a8b37 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp +++ b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.cpp @@ -37,7 +37,8 @@ static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder, unsigned LastIndex) { // Alternatively, we could emit this as a loop in the source. for (unsigned I = FirstIndex; I <= LastIndex; ++I) { - llvm::Value *Cell = Builder.CreateConstInBoundsGEP1_32(Array, I); + llvm::Value *Cell = + Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I); Builder.CreateStore(Value, Cell); } } @@ -107,6 +108,10 @@ bool ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base, return false; } +bool ABIInfo::shouldSignExtUnsignedType(QualType Ty) const { + return false; +} + void ABIArgInfo::dump() const { raw_ostream &OS = llvm::errs(); OS << "(ABIArgInfo Kind="; @@ -238,7 +243,7 @@ static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays) { /// \return The field declaration for the single non-empty field, if /// it exists. static const Type *isSingleElementStruct(QualType T, ASTContext &Context) { - const RecordType *RT = T->getAsStructureType(); + const RecordType *RT = T->getAs<RecordType>(); if (!RT) return nullptr; @@ -339,9 +344,15 @@ static bool canExpandIndirectArgument(QualType Ty, ASTContext &Context) { // // FIXME: This needs to be generalized to handle classes as well. const RecordDecl *RD = RT->getDecl(); - if (!RD->isStruct() || isa<CXXRecordDecl>(RD)) + if (!RD->isStruct()) return false; + // We try to expand CLike CXXRecordDecl. + if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) { + if (!CXXRD->isCLike()) + return false; + } + uint64_t Size = 0; for (const auto *FD : RD->fields()) { @@ -399,8 +410,16 @@ llvm::Value *DefaultABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, } ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const { - if (isAggregateTypeForABI(Ty)) + Ty = useFirstFieldIfTransparentUnion(Ty); + + if (isAggregateTypeForABI(Ty)) { + // Records with non-trivial destructors/copy-constructors should not be + // passed by value. + if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) + return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); + return ABIArgInfo::getIndirect(0); + } // Treat an enum type as its underlying type. if (const EnumType *EnumTy = Ty->getAs<EnumType>()) @@ -630,7 +649,7 @@ public: static bool isStructReturnInRegABI( const llvm::Triple &Triple, const CodeGenOptions &Opts); - void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const override; int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override { @@ -807,7 +826,8 @@ ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(CCState &State) const { return ABIArgInfo::getIndirect(/*Align=*/0, /*ByVal=*/false); } -ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy, CCState &State) const { +ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy, + CCState &State) const { if (RetTy->isVoidType()) return ABIArgInfo::getIgnore(); @@ -1311,7 +1331,7 @@ bool X86_32TargetCodeGenInfo::isStructReturnInRegABI( } } -void X86_32TargetCodeGenInfo::SetTargetAttributes(const Decl *D, +void X86_32TargetCodeGenInfo::setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const { if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) { @@ -1352,7 +1372,8 @@ bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable( } else { // 9 is %eflags, which doesn't get a size on Darwin for some // reason. - Builder.CreateStore(Four8, Builder.CreateConstInBoundsGEP1_32(Address, 9)); + Builder.CreateStore( + Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9)); // 11-16 are st(0..5). Not sure why we stop at 5. // These have size 12, which is sizeof(long double) on @@ -1370,6 +1391,26 @@ bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable( namespace { +/// The AVX ABI level for X86 targets. +enum class X86AVXABILevel { + None, + AVX, + AVX512 +}; + +/// \p returns the size in bits of the largest (native) vector for \p AVXLevel. +static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) { + switch (AVXLevel) { + case X86AVXABILevel::AVX512: + return 512; + case X86AVXABILevel::AVX: + return 256; + case X86AVXABILevel::None: + return 128; + } + llvm_unreachable("Unknown AVXLevel"); +} + /// X86_64ABIInfo - The X86_64 ABI information. class X86_64ABIInfo : public ABIInfo { enum Class { @@ -1475,14 +1516,14 @@ class X86_64ABIInfo : public ABIInfo { return !getTarget().getTriple().isOSDarwin(); } - bool HasAVX; + X86AVXABILevel AVXLevel; // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on // 64-bit hardware. bool Has64BitPointers; public: - X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, bool hasavx) : - ABIInfo(CGT), HasAVX(hasavx), + X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) : + ABIInfo(CGT), AVXLevel(AVXLevel), Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) { } @@ -1503,6 +1544,10 @@ public: llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, CodeGenFunction &CGF) const override; + + bool has64BitPointers() const { + return Has64BitPointers; + } }; /// WinX86_64ABIInfo - The Windows X86_64 ABI information. @@ -1532,10 +1577,9 @@ public: }; class X86_64TargetCodeGenInfo : public TargetCodeGenInfo { - bool HasAVX; public: - X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) - : TargetCodeGenInfo(new X86_64ABIInfo(CGT, HasAVX)), HasAVX(HasAVX) {} + X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) + : TargetCodeGenInfo(new X86_64ABIInfo(CGT, AVXLevel)) {} const X86_64ABIInfo &getABIInfo() const { return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo()); @@ -1588,24 +1632,43 @@ public: llvm::Constant * getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override { - unsigned Sig = (0xeb << 0) | // jmp rel8 - (0x0a << 8) | // .+0x0c - ('F' << 16) | - ('T' << 24); + unsigned Sig; + if (getABIInfo().has64BitPointers()) + Sig = (0xeb << 0) | // jmp rel8 + (0x0a << 8) | // .+0x0c + ('F' << 16) | + ('T' << 24); + else + Sig = (0xeb << 0) | // jmp rel8 + (0x06 << 8) | // .+0x08 + ('F' << 16) | + ('T' << 24); return llvm::ConstantInt::get(CGM.Int32Ty, Sig); } +}; - unsigned getOpenMPSimdDefaultAlignment(QualType) const override { - return HasAVX ? 32 : 16; +class PS4TargetCodeGenInfo : public X86_64TargetCodeGenInfo { +public: + PS4TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) + : X86_64TargetCodeGenInfo(CGT, AVXLevel) {} + + void getDependentLibraryOption(llvm::StringRef Lib, + llvm::SmallString<24> &Opt) const override { + Opt = "\01"; + Opt += Lib; } }; static std::string qualifyWindowsLibrary(llvm::StringRef Lib) { - // If the argument does not end in .lib, automatically add the suffix. This - // matches the behavior of MSVC. - std::string ArgStr = Lib; + // If the argument does not end in .lib, automatically add the suffix. + // If the argument contains a space, enclose it in quotes. + // This matches the behavior of MSVC. + bool Quote = (Lib.find(" ") != StringRef::npos); + std::string ArgStr = Quote ? "\"" : ""; + ArgStr += Lib; if (!Lib.endswith_lower(".lib")) ArgStr += ".lib"; + ArgStr += Quote ? "\"" : ""; return ArgStr; } @@ -1615,6 +1678,9 @@ public: bool d, bool p, bool w, unsigned RegParms) : X86_32TargetCodeGenInfo(CGT, d, p, w, RegParms) {} + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const override; + void getDependentLibraryOption(llvm::StringRef Lib, llvm::SmallString<24> &Opt) const override { Opt = "/DEFAULTLIB:"; @@ -1628,11 +1694,35 @@ public: } }; +static void addStackProbeSizeTargetAttribute(const Decl *D, + llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) { + if (isa<FunctionDecl>(D)) { + if (CGM.getCodeGenOpts().StackProbeSize != 4096) { + llvm::Function *Fn = cast<llvm::Function>(GV); + + Fn->addFnAttr("stack-probe-size", + llvm::utostr(CGM.getCodeGenOpts().StackProbeSize)); + } + } +} + +void WinX86_32TargetCodeGenInfo::setTargetAttributes(const Decl *D, + llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const { + X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM); + + addStackProbeSizeTargetAttribute(D, GV, CGM); +} + class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo { - bool HasAVX; public: - WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) - : TargetCodeGenInfo(new WinX86_64ABIInfo(CGT)), HasAVX(HasAVX) {} + WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, + X86AVXABILevel AVXLevel) + : TargetCodeGenInfo(new WinX86_64ABIInfo(CGT)) {} + + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const override; int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override { return 7; @@ -1659,12 +1749,15 @@ public: llvm::SmallString<32> &Opt) const override { Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\""; } - - unsigned getOpenMPSimdDefaultAlignment(QualType) const override { - return HasAVX ? 32 : 16; - } }; +void WinX86_64TargetCodeGenInfo::setTargetAttributes(const Decl *D, + llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const { + TargetCodeGenInfo::setTargetAttributes(D, GV, CGM); + + addStackProbeSizeTargetAttribute(D, GV, CGM); +} } void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo, @@ -1765,13 +1858,20 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Hi = Integer; } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) { Current = Integer; - } else if ((k == BuiltinType::Float || k == BuiltinType::Double) || - (k == BuiltinType::LongDouble && - getTarget().getTriple().isOSNaCl())) { + } else if (k == BuiltinType::Float || k == BuiltinType::Double) { Current = SSE; } else if (k == BuiltinType::LongDouble) { - Lo = X87; - Hi = X87Up; + const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat(); + if (LDF == &llvm::APFloat::IEEEquad) { + Lo = SSE; + Hi = SSEUp; + } else if (LDF == &llvm::APFloat::x87DoubleExtended) { + Lo = X87; + Hi = X87Up; + } else if (LDF == &llvm::APFloat::IEEEdouble) { + Current = SSE; + } else + llvm_unreachable("unexpected long double representation!"); } // FIXME: _Decimal32 and _Decimal64 are SSE. // FIXME: _float128 and _Decimal128 are (SSE, SSEUp). @@ -1843,7 +1943,8 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, // split. if (OffsetBase && OffsetBase != 64) Hi = Lo; - } else if (Size == 128 || (HasAVX && isNamedArg && Size == 256)) { + } else if (Size == 128 || + (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) { // Arguments of 256-bits are split into four eightbyte chunks. The // least significant one belongs to class SSE and all the others to class // SSEUP. The original Lo and Hi design considers that types can't be @@ -1855,6 +1956,9 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in // registers if they are "named", i.e. not part of the "..." of a // variadic function. + // + // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are + // split into eight eightbyte chunks, one SSE and seven SSEUP. Lo = SSE; Hi = SSEUp; } @@ -1870,14 +1974,21 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Current = Integer; else if (Size <= 128) Lo = Hi = Integer; - } else if (ET == getContext().FloatTy) + } else if (ET == getContext().FloatTy) { Current = SSE; - else if (ET == getContext().DoubleTy || - (ET == getContext().LongDoubleTy && - getTarget().getTriple().isOSNaCl())) + } else if (ET == getContext().DoubleTy) { Lo = Hi = SSE; - else if (ET == getContext().LongDoubleTy) - Current = ComplexX87; + } else if (ET == getContext().LongDoubleTy) { + const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat(); + if (LDF == &llvm::APFloat::IEEEquad) + Current = Memory; + else if (LDF == &llvm::APFloat::x87DoubleExtended) + Current = ComplexX87; + else if (LDF == &llvm::APFloat::IEEEdouble) + Lo = Hi = SSE; + else + llvm_unreachable("unexpected long double representation!"); + } // If this complex type crosses an eightbyte boundary then it // should be split. @@ -1976,8 +2087,10 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg); Lo = merge(Lo, FieldLo); Hi = merge(Hi, FieldHi); - if (Lo == Memory || Hi == Memory) - break; + if (Lo == Memory || Hi == Memory) { + postMerge(Size, Lo, Hi); + return; + } } } @@ -1997,11 +2110,13 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, // if (Size > 128 && getContext().getTypeSize(i->getType()) != 256) { Lo = Memory; + postMerge(Size, Lo, Hi); return; } // Note, skip this test for bit-fields, see below. if (!BitField && Offset % getContext().getTypeAlign(i->getType())) { Lo = Memory; + postMerge(Size, Lo, Hi); return; } @@ -2065,7 +2180,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const { bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const { if (const VectorType *VecTy = Ty->getAs<VectorType>()) { uint64_t Size = getContext().getTypeSize(VecTy); - unsigned LargestVector = HasAVX ? 256 : 128; + unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel); if (Size <= 64 || Size > LargestVector) return true; } @@ -2142,20 +2257,17 @@ llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const { Ty = QualType(InnerTy, 0); llvm::Type *IRType = CGT.ConvertType(Ty); + if (isa<llvm::VectorType>(IRType) || + IRType->getTypeID() == llvm::Type::FP128TyID) + return IRType; - // If the preferred type is a 16-byte vector, prefer to pass it. - if (llvm::VectorType *VT = dyn_cast<llvm::VectorType>(IRType)){ - llvm::Type *EltTy = VT->getElementType(); - unsigned BitWidth = VT->getBitWidth(); - if ((BitWidth >= 128 && BitWidth <= 256) && - (EltTy->isFloatTy() || EltTy->isDoubleTy() || - EltTy->isIntegerTy(8) || EltTy->isIntegerTy(16) || - EltTy->isIntegerTy(32) || EltTy->isIntegerTy(64) || - EltTy->isIntegerTy(128))) - return VT; - } + // We couldn't find the preferred IR vector type for 'Ty'. + uint64_t Size = getContext().getTypeSize(Ty); + assert((Size == 128 || Size == 256) && "Invalid type found!"); - return llvm::VectorType::get(llvm::Type::getDoubleTy(getVMContext()), 2); + // Return a LLVM IR vector type based on the size of 'Ty'. + return llvm::VectorType::get(llvm::Type::getDoubleTy(getVMContext()), + Size / 64); } /// BitsContainNoUserData - Return true if the specified [start,end) bit range @@ -2394,13 +2506,16 @@ GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi, // of the second element because it might make us access off the end of the // struct. if (HiStart != 8) { - // There are only two sorts of types the ABI generation code can produce for - // the low part of a pair that aren't 8 bytes in size: float or i8/i16/i32. + // There are usually two sorts of types the ABI generation code can produce + // for the low part of a pair that aren't 8 bytes in size: float or + // i8/i16/i32. This can also include pointers when they are 32-bit (X32 and + // NaCl). // Promote these to a larger type. if (Lo->isFloatTy()) Lo = llvm::Type::getDoubleTy(Lo->getContext()); else { - assert(Lo->isIntegerTy() && "Invalid/unknown lo type"); + assert((Lo->isIntegerTy() || Lo->isPointerTy()) + && "Invalid/unknown lo type"); Lo = llvm::Type::getInt64Ty(Lo->getContext()); } } @@ -2717,8 +2832,8 @@ void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { static llvm::Value *EmitVAArgFromMemory(llvm::Value *VAListAddr, QualType Ty, CodeGenFunction &CGF) { - llvm::Value *overflow_arg_area_p = - CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p"); + llvm::Value *overflow_arg_area_p = CGF.Builder.CreateStructGEP( + nullptr, VAListAddr, 2, "overflow_arg_area_p"); llvm::Value *overflow_arg_area = CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area"); @@ -2775,7 +2890,7 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, unsigned neededInt, neededSSE; Ty = CGF.getContext().getCanonicalType(Ty); - ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE, + ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE, /*isNamedArg*/false); // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed @@ -2798,14 +2913,16 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, llvm::Value *gp_offset_p = nullptr, *gp_offset = nullptr; llvm::Value *fp_offset_p = nullptr, *fp_offset = nullptr; if (neededInt) { - gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p"); + gp_offset_p = + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 0, "gp_offset_p"); gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset"); InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8); InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp"); } if (neededSSE) { - fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p"); + fp_offset_p = + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 1, "fp_offset_p"); fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset"); llvm::Value *FitsInFP = llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16); @@ -2833,9 +2950,8 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, // simple assembling of a structure from scattered addresses has many more // loads than necessary. Can we clean this up? llvm::Type *LTy = CGF.ConvertTypeForMem(Ty); - llvm::Value *RegAddr = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(VAListAddr, 3), - "reg_save_area"); + llvm::Value *RegAddr = CGF.Builder.CreateLoad( + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3), "reg_save_area"); if (neededInt && neededSSE) { // FIXME: Cleanup. assert(AI.isDirect() && "Unexpected ABI info for mixed regs"); @@ -2855,9 +2971,9 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr; llvm::Value *V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegLoAddr, PTyLo)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0)); + CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 0)); V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegHiAddr, PTyHi)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1)); + CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 1)); RegAddr = CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(LTy)); @@ -2894,10 +3010,10 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, Tmp = CGF.Builder.CreateBitCast(Tmp, ST->getPointerTo()); V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrLo, DblPtrTy)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0)); + CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 0)); V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrHi, DblPtrTy)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1)); + CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 1)); RegAddr = CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(LTy)); } @@ -2984,7 +3100,7 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs, return ABIArgInfo::getDirect(); } - if (RT || Ty->isMemberPointerType()) { + if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) { // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is // not 1, 2, 4, or 8 bytes, must be passed by reference." if (Width > 64 || !llvm::isPowerOf2_64(Width)) @@ -3040,48 +3156,6 @@ llvm::Value *WinX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, return AddrTyped; } -namespace { - -class NaClX86_64ABIInfo : public ABIInfo { - public: - NaClX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) - : ABIInfo(CGT), PInfo(CGT), NInfo(CGT, HasAVX) {} - void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; - private: - PNaClABIInfo PInfo; // Used for generating calls with pnaclcall callingconv. - X86_64ABIInfo NInfo; // Used for everything else. -}; - -class NaClX86_64TargetCodeGenInfo : public TargetCodeGenInfo { - bool HasAVX; - public: - NaClX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) - : TargetCodeGenInfo(new NaClX86_64ABIInfo(CGT, HasAVX)), HasAVX(HasAVX) { - } - unsigned getOpenMPSimdDefaultAlignment(QualType) const override { - return HasAVX ? 32 : 16; - } -}; - -} - -void NaClX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { - if (FI.getASTCallingConvention() == CC_PnaclCall) - PInfo.computeInfo(FI); - else - NInfo.computeInfo(FI); -} - -llvm::Value *NaClX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - // Always use the native convention; calling pnacl-style varargs functions - // is unuspported. - return NInfo.EmitVAArg(VAListAddr, Ty, CGF); -} - - // PowerPC-32 namespace { /// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information. @@ -3095,7 +3169,8 @@ public: class PPC32TargetCodeGenInfo : public TargetCodeGenInfo { public: - PPC32TargetCodeGenInfo(CodeGenTypes &CGT) : TargetCodeGenInfo(new PPC32_SVR4_ABIInfo(CGT)) {} + PPC32TargetCodeGenInfo(CodeGenTypes &CGT) + : TargetCodeGenInfo(new PPC32_SVR4_ABIInfo(CGT)) {} int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { // This is recovered from gcc output. @@ -3104,10 +3179,6 @@ public: bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const override; - - unsigned getOpenMPSimdDefaultAlignment(QualType) const override { - return 16; // Natural alignment for Altivec vectors. - } }; } @@ -3122,19 +3193,25 @@ llvm::Value *PPC32_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, } bool isI64 = Ty->isIntegerType() && getContext().getTypeSize(Ty) == 64; - bool isInt = Ty->isIntegerType() || Ty->isPointerType() || Ty->isAggregateType(); + bool isInt = + Ty->isIntegerType() || Ty->isPointerType() || Ty->isAggregateType(); llvm::Type *CharPtr = CGF.Int8PtrTy; llvm::Type *CharPtrPtr = CGF.Int8PtrPtrTy; CGBuilderTy &Builder = CGF.Builder; llvm::Value *GPRPtr = Builder.CreateBitCast(VAListAddr, CharPtr, "gprptr"); llvm::Value *GPRPtrAsInt = Builder.CreatePtrToInt(GPRPtr, CGF.Int32Ty); - llvm::Value *FPRPtrAsInt = Builder.CreateAdd(GPRPtrAsInt, Builder.getInt32(1)); + llvm::Value *FPRPtrAsInt = + Builder.CreateAdd(GPRPtrAsInt, Builder.getInt32(1)); llvm::Value *FPRPtr = Builder.CreateIntToPtr(FPRPtrAsInt, CharPtr); - llvm::Value *OverflowAreaPtrAsInt = Builder.CreateAdd(FPRPtrAsInt, Builder.getInt32(3)); - llvm::Value *OverflowAreaPtr = Builder.CreateIntToPtr(OverflowAreaPtrAsInt, CharPtrPtr); - llvm::Value *RegsaveAreaPtrAsInt = Builder.CreateAdd(OverflowAreaPtrAsInt, Builder.getInt32(4)); - llvm::Value *RegsaveAreaPtr = Builder.CreateIntToPtr(RegsaveAreaPtrAsInt, CharPtrPtr); + llvm::Value *OverflowAreaPtrAsInt = + Builder.CreateAdd(FPRPtrAsInt, Builder.getInt32(3)); + llvm::Value *OverflowAreaPtr = + Builder.CreateIntToPtr(OverflowAreaPtrAsInt, CharPtrPtr); + llvm::Value *RegsaveAreaPtrAsInt = + Builder.CreateAdd(OverflowAreaPtrAsInt, Builder.getInt32(4)); + llvm::Value *RegsaveAreaPtr = + Builder.CreateIntToPtr(RegsaveAreaPtrAsInt, CharPtrPtr); llvm::Value *GPR = Builder.CreateLoad(GPRPtr, false, "gpr"); // Align GPR when TY is i64. if (isI64) { @@ -3144,18 +3221,23 @@ llvm::Value *PPC32_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, GPR = Builder.CreateSelect(CC64, GPRPlusOne, GPR); } llvm::Value *FPR = Builder.CreateLoad(FPRPtr, false, "fpr"); - llvm::Value *OverflowArea = Builder.CreateLoad(OverflowAreaPtr, false, "overflow_area"); - llvm::Value *OverflowAreaAsInt = Builder.CreatePtrToInt(OverflowArea, CGF.Int32Ty); - llvm::Value *RegsaveArea = Builder.CreateLoad(RegsaveAreaPtr, false, "regsave_area"); - llvm::Value *RegsaveAreaAsInt = Builder.CreatePtrToInt(RegsaveArea, CGF.Int32Ty); + llvm::Value *OverflowArea = + Builder.CreateLoad(OverflowAreaPtr, false, "overflow_area"); + llvm::Value *OverflowAreaAsInt = + Builder.CreatePtrToInt(OverflowArea, CGF.Int32Ty); + llvm::Value *RegsaveArea = + Builder.CreateLoad(RegsaveAreaPtr, false, "regsave_area"); + llvm::Value *RegsaveAreaAsInt = + Builder.CreatePtrToInt(RegsaveArea, CGF.Int32Ty); - llvm::Value *CC = Builder.CreateICmpULT(isInt ? GPR : FPR, - Builder.getInt8(8), "cond"); + llvm::Value *CC = + Builder.CreateICmpULT(isInt ? GPR : FPR, Builder.getInt8(8), "cond"); - llvm::Value *RegConstant = Builder.CreateMul(isInt ? GPR : FPR, - Builder.getInt8(isInt ? 4 : 8)); + llvm::Value *RegConstant = + Builder.CreateMul(isInt ? GPR : FPR, Builder.getInt8(isInt ? 4 : 8)); - llvm::Value *OurReg = Builder.CreateAdd(RegsaveAreaAsInt, Builder.CreateSExt(RegConstant, CGF.Int32Ty)); + llvm::Value *OurReg = Builder.CreateAdd( + RegsaveAreaAsInt, Builder.CreateSExt(RegConstant, CGF.Int32Ty)); if (Ty->isFloatingType()) OurReg = Builder.CreateAdd(OurReg, Builder.getInt32(32)); @@ -3184,8 +3266,10 @@ llvm::Value *PPC32_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, // Increase the overflow area. llvm::Value *Result2 = Builder.CreateIntToPtr(OverflowAreaAsInt, PTy); - OverflowAreaAsInt = Builder.CreateAdd(OverflowAreaAsInt, Builder.getInt32(isInt ? 4 : 8)); - Builder.CreateStore(Builder.CreateIntToPtr(OverflowAreaAsInt, CharPtr), OverflowAreaPtr); + OverflowAreaAsInt = + Builder.CreateAdd(OverflowAreaAsInt, Builder.getInt32(isInt ? 4 : 8)); + Builder.CreateStore(Builder.CreateIntToPtr(OverflowAreaAsInt, CharPtr), + OverflowAreaPtr); CGF.EmitBranch(Cont); CGF.EmitBlock(Cont); @@ -3195,7 +3279,7 @@ llvm::Value *PPC32_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, Result->addIncoming(Result2, UsingOverflow); if (Ty->isAggregateType()) { - llvm::Value *AGGPtr = Builder.CreateBitCast(Result, CharPtrPtr, "aggrptr") ; + llvm::Value *AGGPtr = Builder.CreateBitCast(Result, CharPtrPtr, "aggrptr"); return Builder.CreateLoad(AGGPtr, false, "aggr"); } @@ -3257,13 +3341,42 @@ public: private: static const unsigned GPRBits = 64; ABIKind Kind; + bool HasQPX; + + // A vector of float or double will be promoted to <4 x f32> or <4 x f64> and + // will be passed in a QPX register. + bool IsQPXVectorTy(const Type *Ty) const { + if (!HasQPX) + return false; + + if (const VectorType *VT = Ty->getAs<VectorType>()) { + unsigned NumElements = VT->getNumElements(); + if (NumElements == 1) + return false; + + if (VT->getElementType()->isSpecificBuiltinType(BuiltinType::Double)) { + if (getContext().getTypeSize(Ty) <= 256) + return true; + } else if (VT->getElementType()-> + isSpecificBuiltinType(BuiltinType::Float)) { + if (getContext().getTypeSize(Ty) <= 128) + return true; + } + } + + return false; + } + + bool IsQPXVectorTy(QualType Ty) const { + return IsQPXVectorTy(Ty.getTypePtr()); + } public: - PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind) - : DefaultABIInfo(CGT), Kind(Kind) {} + PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind, bool HasQPX) + : DefaultABIInfo(CGT), Kind(Kind), HasQPX(HasQPX) {} bool isPromotableTypeForABI(QualType Ty) const; - bool isAlignedParamType(QualType Ty) const; + bool isAlignedParamType(QualType Ty, bool &Align32) const; ABIArgInfo classifyReturnType(QualType RetTy) const; ABIArgInfo classifyArgumentType(QualType Ty) const; @@ -3288,7 +3401,8 @@ public: const Type *T = isSingleElementStruct(I.type, getContext()); if (T) { const BuiltinType *BT = T->getAs<BuiltinType>(); - if ((T->isVectorType() && getContext().getTypeSize(T) == 128) || + if (IsQPXVectorTy(T) || + (T->isVectorType() && getContext().getTypeSize(T) == 128) || (BT && BT->isFloatingPoint())) { QualType QT(T, 0); I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT)); @@ -3304,10 +3418,11 @@ public: }; class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo { + public: PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT, - PPC64_SVR4_ABIInfo::ABIKind Kind) - : TargetCodeGenInfo(new PPC64_SVR4_ABIInfo(CGT, Kind)) {} + PPC64_SVR4_ABIInfo::ABIKind Kind, bool HasQPX) + : TargetCodeGenInfo(new PPC64_SVR4_ABIInfo(CGT, Kind, HasQPX)) {} int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { // This is recovered from gcc output. @@ -3316,10 +3431,6 @@ public: bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const override; - - unsigned getOpenMPSimdDefaultAlignment(QualType) const override { - return 16; // Natural alignment for Altivec and VSX vectors. - } }; class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo { @@ -3333,10 +3444,6 @@ public: bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const override; - - unsigned getOpenMPSimdDefaultAlignment(QualType) const override { - return 16; // Natural alignment for Altivec vectors. - } }; } @@ -3370,15 +3477,23 @@ PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const { /// isAlignedParamType - Determine whether a type requires 16-byte /// alignment in the parameter area. bool -PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty) const { +PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty, bool &Align32) const { + Align32 = false; + // Complex types are passed just like their elements. if (const ComplexType *CTy = Ty->getAs<ComplexType>()) Ty = CTy->getElementType(); // Only vector types of size 16 bytes need alignment (larger types are // passed via reference, smaller types are not aligned). - if (Ty->isVectorType()) + if (IsQPXVectorTy(Ty)) { + if (getContext().getTypeSize(Ty) > 128) + Align32 = true; + + return true; + } else if (Ty->isVectorType()) { return getContext().getTypeSize(Ty) == 128; + } // For single-element float/vector structs, we consider the whole type // to have the same alignment requirements as its single element. @@ -3386,7 +3501,7 @@ PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty) const { const Type *EltType = isSingleElementStruct(Ty, getContext()); if (EltType) { const BuiltinType *BT = EltType->getAs<BuiltinType>(); - if ((EltType->isVectorType() && + if (IsQPXVectorTy(EltType) || (EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) || (BT && BT->isFloatingPoint())) AlignAsType = EltType; @@ -3400,13 +3515,22 @@ PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty) const { AlignAsType = Base; // With special case aggregates, only vector base types need alignment. - if (AlignAsType) + if (AlignAsType && IsQPXVectorTy(AlignAsType)) { + if (getContext().getTypeSize(AlignAsType) > 128) + Align32 = true; + + return true; + } else if (AlignAsType) { return AlignAsType->isVectorType(); + } // Otherwise, we only need alignment for any aggregate type that // has an alignment requirement of >= 16 bytes. - if (isAggregateTypeForABI(Ty) && getContext().getTypeAlign(Ty) >= 128) + if (isAggregateTypeForABI(Ty) && getContext().getTypeAlign(Ty) >= 128) { + if (HasQPX && getContext().getTypeAlign(Ty) >= 256) + Align32 = true; return true; + } return false; } @@ -3512,7 +3636,7 @@ bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const { return true; } if (const VectorType *VT = Ty->getAs<VectorType>()) { - if (getContext().getTypeSize(VT) == 128) + if (getContext().getTypeSize(VT) == 128 || IsQPXVectorTy(Ty)) return true; } return false; @@ -3538,7 +3662,7 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const { // Non-Altivec vector types are passed in GPRs (smaller than 16 bytes) // or via reference (larger than 16 bytes). - if (Ty->isVectorType()) { + if (Ty->isVectorType() && !IsQPXVectorTy(Ty)) { uint64_t Size = getContext().getTypeSize(Ty); if (Size > 128) return ABIArgInfo::getIndirect(0, /*ByVal=*/false); @@ -3552,7 +3676,9 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const { if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); - uint64_t ABIAlign = isAlignedParamType(Ty)? 16 : 8; + bool Align32; + uint64_t ABIAlign = isAlignedParamType(Ty, Align32) ? + (Align32 ? 32 : 16) : 8; uint64_t TyAlign = getContext().getTypeAlign(Ty) / 8; // ELFv2 homogeneous aggregates are passed as array types. @@ -3609,7 +3735,7 @@ PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const { // Non-Altivec vector types are returned in GPRs (smaller than 16 bytes) // or via reference (larger than 16 bytes). - if (RetTy->isVectorType()) { + if (RetTy->isVectorType() && !IsQPXVectorTy(RetTy)) { uint64_t Size = getContext().getTypeSize(RetTy); if (Size > 128) return ABIArgInfo::getIndirect(0); @@ -3666,10 +3792,13 @@ llvm::Value *PPC64_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); // Handle types that require 16-byte alignment in the parameter save area. - if (isAlignedParamType(Ty)) { + bool Align32; + if (isAlignedParamType(Ty, Align32)) { llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int64Ty); - AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt64(15)); - AddrAsInt = Builder.CreateAnd(AddrAsInt, Builder.getInt64(-16)); + AddrAsInt = Builder.CreateAdd(AddrAsInt, + Builder.getInt64(Align32 ? 31 : 15)); + AddrAsInt = Builder.CreateAnd(AddrAsInt, + Builder.getInt64(Align32 ? -32 : -16)); Addr = Builder.CreateIntToPtr(AddrAsInt, BP, "ap.align"); } @@ -3704,8 +3833,10 @@ llvm::Value *PPC64_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, llvm::Value *RealAddr = Builder.CreatePtrToInt(Addr, CGF.Int64Ty); llvm::Value *ImagAddr = RealAddr; if (CGF.CGM.getDataLayout().isBigEndian()) { - RealAddr = Builder.CreateAdd(RealAddr, Builder.getInt64(8 - CplxBaseSize)); - ImagAddr = Builder.CreateAdd(ImagAddr, Builder.getInt64(16 - CplxBaseSize)); + RealAddr = + Builder.CreateAdd(RealAddr, Builder.getInt64(8 - CplxBaseSize)); + ImagAddr = + Builder.CreateAdd(ImagAddr, Builder.getInt64(16 - CplxBaseSize)); } else { ImagAddr = Builder.CreateAdd(ImagAddr, Builder.getInt64(8)); } @@ -3714,10 +3845,12 @@ llvm::Value *PPC64_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, ImagAddr = Builder.CreateIntToPtr(ImagAddr, PBaseTy); llvm::Value *Real = Builder.CreateLoad(RealAddr, false, ".vareal"); llvm::Value *Imag = Builder.CreateLoad(ImagAddr, false, ".vaimag"); - llvm::Value *Ptr = CGF.CreateTempAlloca(CGT.ConvertTypeForMem(Ty), - "vacplx"); - llvm::Value *RealPtr = Builder.CreateStructGEP(Ptr, 0, ".real"); - llvm::Value *ImagPtr = Builder.CreateStructGEP(Ptr, 1, ".imag"); + llvm::AllocaInst *Ptr = + CGF.CreateTempAlloca(CGT.ConvertTypeForMem(Ty), "vacplx"); + llvm::Value *RealPtr = + Builder.CreateStructGEP(Ptr->getAllocatedType(), Ptr, 0, ".real"); + llvm::Value *ImagPtr = + Builder.CreateStructGEP(Ptr->getAllocatedType(), Ptr, 1, ".imag"); Builder.CreateStore(Real, RealPtr, false); Builder.CreateStore(Imag, ImagPtr, false); return Ptr; @@ -3837,8 +3970,8 @@ private: llvm::Value *EmitAAPCSVAArg(llvm::Value *VAListAddr, QualType Ty, CodeGenFunction &CGF) const; - virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override { + llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, + CodeGenFunction &CGF) const override { return isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF) : EmitAAPCSVAArg(VAListAddr, Ty, CGF); } @@ -3849,13 +3982,15 @@ public: AArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIInfo::ABIKind Kind) : TargetCodeGenInfo(new AArch64ABIInfo(CGT, Kind)) {} - StringRef getARCRetainAutoreleasedReturnValueMarker() const { + StringRef getARCRetainAutoreleasedReturnValueMarker() const override { return "mov\tfp, fp\t\t; marker for objc_retainAutoreleaseReturnValue"; } - int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const { return 31; } + int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { + return 31; + } - virtual bool doesReturnSlotInterfereWithArgs() const { return false; } + bool doesReturnSlotInterfereWithArgs() const override { return false; } }; } @@ -3964,7 +4099,15 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy) const { // Aggregates <= 16 bytes are returned directly in registers or on the stack. uint64_t Size = getContext().getTypeSize(RetTy); if (Size <= 128) { + unsigned Alignment = getContext().getTypeAlign(RetTy); Size = 64 * ((Size + 63) / 64); // round up to multiple of 8 bytes + + // We use a pair of i64 for 16-byte aggregate with 8-byte alignment. + // For aggregates with 16-byte alignment, we use i128. + if (Alignment < 128 && Size == 128) { + llvm::Type *BaseTy = llvm::Type::getInt64Ty(getVMContext()); + return ABIArgInfo::getDirect(llvm::ArrayType::get(BaseTy, Size / 64)); + } return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size)); } @@ -4047,13 +4190,15 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, int RegSize = IsIndirect ? 8 : getContext().getTypeSize(Ty) / 8; if (!IsFPR) { // 3 is the field number of __gr_offs - reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 3, "gr_offs_p"); + reg_offs_p = + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3, "gr_offs_p"); reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs"); reg_top_index = 1; // field number for __gr_top RegSize = llvm::RoundUpToAlignment(RegSize, 8); } else { // 4 is the field number of __vr_offs. - reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 4, "vr_offs_p"); + reg_offs_p = + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 4, "vr_offs_p"); reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs"); reg_top_index = 2; // field number for __vr_top RegSize = 16 * NumRegs; @@ -4114,8 +4259,8 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, CGF.EmitBlock(InRegBlock); llvm::Value *reg_top_p = nullptr, *reg_top = nullptr; - reg_top_p = - CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index, "reg_top_p"); + reg_top_p = CGF.Builder.CreateStructGEP(nullptr, VAListAddr, reg_top_index, + "reg_top_p"); reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top"); llvm::Value *BaseAddr = CGF.Builder.CreateGEP(reg_top, reg_offs); llvm::Value *RegAddr = nullptr; @@ -4138,7 +4283,7 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, assert(!IsIndirect && "Homogeneous aggregates should be passed directly"); llvm::Type *BaseTy = CGF.ConvertType(QualType(Base, 0)); llvm::Type *HFATy = llvm::ArrayType::get(BaseTy, NumMembers); - llvm::Value *Tmp = CGF.CreateTempAlloca(HFATy); + llvm::AllocaInst *Tmp = CGF.CreateTempAlloca(HFATy); int Offset = 0; if (CGF.CGM.getDataLayout().isBigEndian() && Ctx.getTypeSize(Base) < 128) @@ -4149,7 +4294,8 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, llvm::Value *LoadAddr = CGF.Builder.CreateGEP(BaseAddr, BaseOffset); LoadAddr = CGF.Builder.CreateBitCast( LoadAddr, llvm::PointerType::getUnqual(BaseTy)); - llvm::Value *StoreAddr = CGF.Builder.CreateStructGEP(Tmp, i); + llvm::Value *StoreAddr = + CGF.Builder.CreateStructGEP(Tmp->getAllocatedType(), Tmp, i); llvm::Value *Elem = CGF.Builder.CreateLoad(LoadAddr); CGF.Builder.CreateStore(Elem, StoreAddr); @@ -4182,7 +4328,7 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, CGF.EmitBlock(OnStackBlock); llvm::Value *stack_p = nullptr, *OnStackAddr = nullptr; - stack_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "stack_p"); + stack_p = CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 0, "stack_p"); OnStackAddr = CGF.Builder.CreateLoad(stack_p, "stack"); // Again, stack arguments may need realigmnent. In this case both integer and @@ -4248,8 +4394,9 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, return ResAddr; } -llvm::Value *AArch64ABIInfo::EmitDarwinVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { +llvm::Value *AArch64ABIInfo::EmitDarwinVAArg(llvm::Value *VAListAddr, + QualType Ty, + CodeGenFunction &CGF) const { // We do not support va_arg for aggregates or illegal vector types. // Lower VAArg here for these cases and use the LLVM va_arg instruction for // other cases. @@ -4324,17 +4471,10 @@ public: private: ABIKind Kind; - mutable int VFPRegs[16]; - const unsigned NumVFPs; - const unsigned NumGPRs; - mutable unsigned AllocatedGPRs; - mutable unsigned AllocatedVFPs; public: - ARMABIInfo(CodeGenTypes &CGT, ABIKind _Kind) : ABIInfo(CGT), Kind(_Kind), - NumVFPs(16), NumGPRs(4) { + ARMABIInfo(CodeGenTypes &CGT, ABIKind _Kind) : ABIInfo(CGT), Kind(_Kind) { setCCs(); - resetAllocatedRegs(); } bool isEABI() const { @@ -4364,8 +4504,7 @@ public: private: ABIArgInfo classifyReturnType(QualType RetTy, bool isVariadic) const; - ABIArgInfo classifyArgumentType(QualType RetTy, bool isVariadic, - bool &IsCPRC) const; + ABIArgInfo classifyArgumentType(QualType RetTy, bool isVariadic) const; bool isIllegalVectorType(QualType Ty) const; bool isHomogeneousAggregateBaseType(QualType Ty) const override; @@ -4380,10 +4519,6 @@ private: llvm::CallingConv::ID getLLVMDefaultCC() const; llvm::CallingConv::ID getABIDefaultCC() const; void setCCs(); - - void markAllocatedGPRs(unsigned Alignment, unsigned NumRequired) const; - void markAllocatedVFPs(unsigned Alignment, unsigned NumRequired) const; - void resetAllocatedRegs(void) const; }; class ARMTargetCodeGenInfo : public TargetCodeGenInfo { @@ -4417,7 +4552,7 @@ public: return TargetCodeGenInfo::getSizeOfUnwindException(); } - void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const override { const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); if (!FD) @@ -4456,55 +4591,44 @@ public: } }; +class WindowsARMTargetCodeGenInfo : public ARMTargetCodeGenInfo { + void addStackProbeSizeTargetAttribute(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const; + +public: + WindowsARMTargetCodeGenInfo(CodeGenTypes &CGT, ARMABIInfo::ABIKind K) + : ARMTargetCodeGenInfo(CGT, K) {} + + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const override; +}; + +void WindowsARMTargetCodeGenInfo::addStackProbeSizeTargetAttribute( + const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const { + if (!isa<FunctionDecl>(D)) + return; + if (CGM.getCodeGenOpts().StackProbeSize == 4096) + return; + + llvm::Function *F = cast<llvm::Function>(GV); + F->addFnAttr("stack-probe-size", + llvm::utostr(CGM.getCodeGenOpts().StackProbeSize)); +} + +void WindowsARMTargetCodeGenInfo::setTargetAttributes( + const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const { + ARMTargetCodeGenInfo::setTargetAttributes(D, GV, CGM); + addStackProbeSizeTargetAttribute(D, GV, CGM); +} } void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const { - // To correctly handle Homogeneous Aggregate, we need to keep track of the - // VFP registers allocated so far. - // C.1.vfp If the argument is a VFP CPRC and there are sufficient consecutive - // VFP registers of the appropriate type unallocated then the argument is - // allocated to the lowest-numbered sequence of such registers. - // C.2.vfp If the argument is a VFP CPRC then any VFP registers that are - // unallocated are marked as unavailable. - resetAllocatedRegs(); - - if (getCXXABI().classifyReturnType(FI)) { - if (FI.getReturnInfo().isIndirect()) - markAllocatedGPRs(1, 1); - } else { - FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), FI.isVariadic()); - } - for (auto &I : FI.arguments()) { - unsigned PreAllocationVFPs = AllocatedVFPs; - unsigned PreAllocationGPRs = AllocatedGPRs; - bool IsCPRC = false; - // 6.1.2.3 There is one VFP co-processor register class using registers - // s0-s15 (d0-d7) for passing arguments. - I.info = classifyArgumentType(I.type, FI.isVariadic(), IsCPRC); - - // If we have allocated some arguments onto the stack (due to running - // out of VFP registers), we cannot split an argument between GPRs and - // the stack. If this situation occurs, we add padding to prevent the - // GPRs from being used. In this situation, the current argument could - // only be allocated by rule C.8, so rule C.6 would mark these GPRs as - // unusable anyway. - // We do not have to do this if the argument is being passed ByVal, as the - // backend can handle that situation correctly. - const bool StackUsed = PreAllocationGPRs > NumGPRs || PreAllocationVFPs > NumVFPs; - const bool IsByVal = I.info.isIndirect() && I.info.getIndirectByVal(); - if (!IsCPRC && PreAllocationGPRs < NumGPRs && AllocatedGPRs > NumGPRs && - StackUsed && !IsByVal) { - llvm::Type *PaddingTy = llvm::ArrayType::get( - llvm::Type::getInt32Ty(getVMContext()), NumGPRs - PreAllocationGPRs); - if (I.info.canHaveCoerceToType()) { - I.info = ABIArgInfo::getDirect(I.info.getCoerceToType() /* type */, - 0 /* offset */, PaddingTy, true); - } else { - I.info = ABIArgInfo::getDirect(nullptr /* type */, 0 /* offset */, - PaddingTy, true); - } - } - } + if (!getCXXABI().classifyReturnType(FI)) + FI.getReturnInfo() = + classifyReturnType(FI.getReturnType(), FI.isVariadic()); + + for (auto &I : FI.arguments()) + I.info = classifyArgumentType(I.type, FI.isVariadic()); // Always honor user-specified calling convention. if (FI.getCallingConvention() != llvm::CallingConv::C) @@ -4512,7 +4636,7 @@ void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const { llvm::CallingConv::ID cc = getRuntimeCC(); if (cc != llvm::CallingConv::C) - FI.setEffectiveCallingConvention(cc); + FI.setEffectiveCallingConvention(cc); } /// Return the default calling convention that LLVM will use. @@ -4550,64 +4674,8 @@ void ARMABIInfo::setCCs() { llvm::CallingConv::ARM_APCS : llvm::CallingConv::ARM_AAPCS); } -/// markAllocatedVFPs - update VFPRegs according to the alignment and -/// number of VFP registers (unit is S register) requested. -void ARMABIInfo::markAllocatedVFPs(unsigned Alignment, - unsigned NumRequired) const { - // Early Exit. - if (AllocatedVFPs >= 16) { - // We use AllocatedVFP > 16 to signal that some CPRCs were allocated on - // the stack. - AllocatedVFPs = 17; - return; - } - // C.1.vfp If the argument is a VFP CPRC and there are sufficient consecutive - // VFP registers of the appropriate type unallocated then the argument is - // allocated to the lowest-numbered sequence of such registers. - for (unsigned I = 0; I < 16; I += Alignment) { - bool FoundSlot = true; - for (unsigned J = I, JEnd = I + NumRequired; J < JEnd; J++) - if (J >= 16 || VFPRegs[J]) { - FoundSlot = false; - break; - } - if (FoundSlot) { - for (unsigned J = I, JEnd = I + NumRequired; J < JEnd; J++) - VFPRegs[J] = 1; - AllocatedVFPs += NumRequired; - return; - } - } - // C.2.vfp If the argument is a VFP CPRC then any VFP registers that are - // unallocated are marked as unavailable. - for (unsigned I = 0; I < 16; I++) - VFPRegs[I] = 1; - AllocatedVFPs = 17; // We do not have enough VFP registers. -} - -/// Update AllocatedGPRs to record the number of general purpose registers -/// which have been allocated. It is valid for AllocatedGPRs to go above 4, -/// this represents arguments being stored on the stack. -void ARMABIInfo::markAllocatedGPRs(unsigned Alignment, - unsigned NumRequired) const { - assert((Alignment == 1 || Alignment == 2) && "Alignment must be 4 or 8 bytes"); - - if (Alignment == 2 && AllocatedGPRs & 0x1) - AllocatedGPRs += 1; - - AllocatedGPRs += NumRequired; -} - -void ARMABIInfo::resetAllocatedRegs(void) const { - AllocatedGPRs = 0; - AllocatedVFPs = 0; - for (unsigned i = 0; i < NumVFPs; ++i) - VFPRegs[i] = 0; -} - -ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, - bool &IsCPRC) const { - // We update number of allocated VFPs according to +ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, + bool isVariadic) const { // 6.1.2.1 The following argument types are VFP CPRCs: // A single-precision floating-point type (including promoted // half-precision types); A double-precision floating-point type; @@ -4625,58 +4693,20 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, if (Size <= 32) { llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext()); - markAllocatedGPRs(1, 1); return ABIArgInfo::getDirect(ResType); } if (Size == 64) { llvm::Type *ResType = llvm::VectorType::get( llvm::Type::getInt32Ty(getVMContext()), 2); - if (getABIKind() == ARMABIInfo::AAPCS || isVariadic){ - markAllocatedGPRs(2, 2); - } else { - markAllocatedVFPs(2, 2); - IsCPRC = true; - } return ABIArgInfo::getDirect(ResType); } if (Size == 128) { llvm::Type *ResType = llvm::VectorType::get( llvm::Type::getInt32Ty(getVMContext()), 4); - if (getABIKind() == ARMABIInfo::AAPCS || isVariadic) { - markAllocatedGPRs(2, 4); - } else { - markAllocatedVFPs(4, 4); - IsCPRC = true; - } return ABIArgInfo::getDirect(ResType); } - markAllocatedGPRs(1, 1); return ABIArgInfo::getIndirect(0, /*ByVal=*/false); } - // Update VFPRegs for legal vector types. - if (getABIKind() == ARMABIInfo::AAPCS_VFP && !isVariadic) { - if (const VectorType *VT = Ty->getAs<VectorType>()) { - uint64_t Size = getContext().getTypeSize(VT); - // Size of a legal vector should be power of 2 and above 64. - markAllocatedVFPs(Size >= 128 ? 4 : 2, Size / 32); - IsCPRC = true; - } - } - // Update VFPRegs for floating point types. - if (getABIKind() == ARMABIInfo::AAPCS_VFP && !isVariadic) { - if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) { - if (BT->getKind() == BuiltinType::Half || - BT->getKind() == BuiltinType::Float) { - markAllocatedVFPs(1, 1); - IsCPRC = true; - } - if (BT->getKind() == BuiltinType::Double || - BT->getKind() == BuiltinType::LongDouble) { - markAllocatedVFPs(2, 2); - IsCPRC = true; - } - } - } if (!isAggregateTypeForABI(Ty)) { // Treat an enum type as its underlying type. @@ -4684,15 +4714,11 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, Ty = EnumTy->getDecl()->getIntegerType(); } - unsigned Size = getContext().getTypeSize(Ty); - if (!IsCPRC) - markAllocatedGPRs(Size > 32 ? 2 : 1, (Size + 31) / 32); return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend() : ABIArgInfo::getDirect()); } if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) { - markAllocatedGPRs(1, 1); return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); } @@ -4708,19 +4734,6 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, if (isHomogeneousAggregate(Ty, Base, Members)) { assert(Base && "Base class should be set for homogeneous aggregate"); // Base can be a floating-point or a vector. - if (Base->isVectorType()) { - // ElementSize is in number of floats. - unsigned ElementSize = getContext().getTypeSize(Base) == 64 ? 2 : 4; - markAllocatedVFPs(ElementSize, - Members * ElementSize); - } else if (Base->isSpecificBuiltinType(BuiltinType::Float)) - markAllocatedVFPs(1, Members); - else { - assert(Base->isSpecificBuiltinType(BuiltinType::Double) || - Base->isSpecificBuiltinType(BuiltinType::LongDouble)); - markAllocatedVFPs(2, Members * 2); - } - IsCPRC = true; return ABIArgInfo::getDirect(nullptr, 0, nullptr, false); } } @@ -4732,15 +4745,11 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, uint64_t ABIAlign = 4; uint64_t TyAlign = getContext().getTypeAlign(Ty) / 8; if (getABIKind() == ARMABIInfo::AAPCS_VFP || - getABIKind() == ARMABIInfo::AAPCS) + getABIKind() == ARMABIInfo::AAPCS) ABIAlign = std::min(std::max(TyAlign, (uint64_t)4), (uint64_t)8); + if (getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(64)) { - // Update Allocated GPRs. Since this is only used when the size of the - // argument is greater than 64 bytes, this will always use up any available - // registers (of which there are 4). We also don't care about getting the - // alignment right, because general-purpose registers cannot be back-filled. - markAllocatedGPRs(1, 4); - return ABIArgInfo::getIndirect(TyAlign, /*ByVal=*/true, + return ABIArgInfo::getIndirect(ABIAlign, /*ByVal=*/true, /*Realign=*/TyAlign > ABIAlign); } @@ -4752,11 +4761,9 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, if (getContext().getTypeAlign(Ty) <= 32) { ElemTy = llvm::Type::getInt32Ty(getVMContext()); SizeRegs = (getContext().getTypeSize(Ty) + 31) / 32; - markAllocatedGPRs(1, SizeRegs); } else { ElemTy = llvm::Type::getInt64Ty(getVMContext()); SizeRegs = (getContext().getTypeSize(Ty) + 63) / 64; - markAllocatedGPRs(2, SizeRegs * 2); } return ABIArgInfo::getDirect(llvm::ArrayType::get(ElemTy, SizeRegs)); @@ -4856,7 +4863,6 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, // Large vector types should be returned via memory. if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128) { - markAllocatedGPRs(1, 1); return ABIArgInfo::getIndirect(0); } @@ -4894,7 +4900,6 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, } // Otherwise return in memory. - markAllocatedGPRs(1, 1); return ABIArgInfo::getIndirect(0); } @@ -4930,7 +4935,6 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext())); } - markAllocatedGPRs(1, 1); return ABIArgInfo::getIndirect(0); } @@ -5046,42 +5050,6 @@ llvm::Value *ARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, return AddrTyped; } -namespace { - -class NaClARMABIInfo : public ABIInfo { - public: - NaClARMABIInfo(CodeGen::CodeGenTypes &CGT, ARMABIInfo::ABIKind Kind) - : ABIInfo(CGT), PInfo(CGT), NInfo(CGT, Kind) {} - void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; - private: - PNaClABIInfo PInfo; // Used for generating calls with pnaclcall callingconv. - ARMABIInfo NInfo; // Used for everything else. -}; - -class NaClARMTargetCodeGenInfo : public TargetCodeGenInfo { - public: - NaClARMTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, ARMABIInfo::ABIKind Kind) - : TargetCodeGenInfo(new NaClARMABIInfo(CGT, Kind)) {} -}; - -} - -void NaClARMABIInfo::computeInfo(CGFunctionInfo &FI) const { - if (FI.getASTCallingConvention() == CC_PnaclCall) - PInfo.computeInfo(FI); - else - static_cast<const ABIInfo&>(NInfo).computeInfo(FI); -} - -llvm::Value *NaClARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - // Always use the native convention; calling pnacl-style varargs functions - // is unsupported. - return static_cast<const ABIInfo&>(NInfo).EmitVAArg(VAListAddr, Ty, CGF); -} - //===----------------------------------------------------------------------===// // NVPTX ABI Implementation //===----------------------------------------------------------------------===// @@ -5105,7 +5073,7 @@ public: NVPTXTargetCodeGenInfo(CodeGenTypes &CGT) : TargetCodeGenInfo(new NVPTXABIInfo(CGT)) {} - void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const override; private: // Adds a NamedMDNode with F, Name, and Operand as operands, and adds the @@ -5161,7 +5129,7 @@ llvm::Value *NVPTXABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, } void NVPTXTargetCodeGenInfo:: -SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, +setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const{ const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); if (!FD) return; @@ -5190,18 +5158,22 @@ SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, // Create !{<func-ref>, metadata !"kernel", i32 1} node addNVVMMetadata(F, "kernel", 1); } - if (FD->hasAttr<CUDALaunchBoundsAttr>()) { + if (CUDALaunchBoundsAttr *Attr = FD->getAttr<CUDALaunchBoundsAttr>()) { // Create !{<func-ref>, metadata !"maxntidx", i32 <val>} node - addNVVMMetadata(F, "maxntidx", - FD->getAttr<CUDALaunchBoundsAttr>()->getMaxThreads()); - // min blocks is a default argument for CUDALaunchBoundsAttr, so getting a - // zero value from getMinBlocks either means it was not specified in - // __launch_bounds__ or the user specified a 0 value. In both cases, we - // don't have to add a PTX directive. - int MinCTASM = FD->getAttr<CUDALaunchBoundsAttr>()->getMinBlocks(); - if (MinCTASM > 0) { - // Create !{<func-ref>, metadata !"minctasm", i32 <val>} node - addNVVMMetadata(F, "minctasm", MinCTASM); + llvm::APSInt MaxThreads(32); + MaxThreads = Attr->getMaxThreads()->EvaluateKnownConstInt(M.getContext()); + if (MaxThreads > 0) + addNVVMMetadata(F, "maxntidx", MaxThreads.getExtValue()); + + // min blocks is an optional argument for CUDALaunchBoundsAttr. If it was + // not specified in __launch_bounds__ or if the user specified a 0 value, + // we don't have to add a PTX directive. + if (Attr->getMinBlocks()) { + llvm::APSInt MinBlocks(32); + MinBlocks = Attr->getMinBlocks()->EvaluateKnownConstInt(M.getContext()); + if (MinBlocks > 0) + // Create !{<func-ref>, metadata !"minctasm", i32 <val>} node + addNVVMMetadata(F, "minctasm", MinBlocks.getExtValue()); } } } @@ -5231,12 +5203,17 @@ void NVPTXTargetCodeGenInfo::addNVVMMetadata(llvm::Function *F, StringRef Name, namespace { class SystemZABIInfo : public ABIInfo { + bool HasVector; + public: - SystemZABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {} + SystemZABIInfo(CodeGenTypes &CGT, bool HV) + : ABIInfo(CGT), HasVector(HV) {} bool isPromotableIntegerType(QualType Ty) const; bool isCompoundType(QualType Ty) const; + bool isVectorArgumentType(QualType Ty) const; bool isFPArgumentType(QualType Ty) const; + QualType GetSingleElementType(QualType Ty) const; ABIArgInfo classifyReturnType(QualType RetTy) const; ABIArgInfo classifyArgumentType(QualType ArgTy) const; @@ -5254,8 +5231,8 @@ public: class SystemZTargetCodeGenInfo : public TargetCodeGenInfo { public: - SystemZTargetCodeGenInfo(CodeGenTypes &CGT) - : TargetCodeGenInfo(new SystemZABIInfo(CGT)) {} + SystemZTargetCodeGenInfo(CodeGenTypes &CGT, bool HasVector) + : TargetCodeGenInfo(new SystemZABIInfo(CGT, HasVector)) {} }; } @@ -5282,7 +5259,15 @@ bool SystemZABIInfo::isPromotableIntegerType(QualType Ty) const { } bool SystemZABIInfo::isCompoundType(QualType Ty) const { - return Ty->isAnyComplexType() || isAggregateTypeForABI(Ty); + return (Ty->isAnyComplexType() || + Ty->isVectorType() || + isAggregateTypeForABI(Ty)); +} + +bool SystemZABIInfo::isVectorArgumentType(QualType Ty) const { + return (HasVector && + Ty->isVectorType() && + getContext().getTypeSize(Ty) <= 128); } bool SystemZABIInfo::isFPArgumentType(QualType Ty) const { @@ -5295,9 +5280,13 @@ bool SystemZABIInfo::isFPArgumentType(QualType Ty) const { return false; } + return false; +} + +QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const { if (const RecordType *RT = Ty->getAsStructureType()) { const RecordDecl *RD = RT->getDecl(); - bool Found = false; + QualType Found; // If this is a C++ record, check the bases first. if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) @@ -5308,36 +5297,34 @@ bool SystemZABIInfo::isFPArgumentType(QualType Ty) const { if (isEmptyRecord(getContext(), Base, true)) continue; - if (Found) - return false; - Found = isFPArgumentType(Base); - if (!Found) - return false; + if (!Found.isNull()) + return Ty; + Found = GetSingleElementType(Base); } // Check the fields. for (const auto *FD : RD->fields()) { - // Empty bitfields don't affect things either way. + // For compatibility with GCC, ignore empty bitfields in C++ mode. // Unlike isSingleElementStruct(), empty structure and array fields // do count. So do anonymous bitfields that aren't zero-sized. - if (FD->isBitField() && FD->getBitWidthValue(getContext()) == 0) - return true; + if (getContext().getLangOpts().CPlusPlus && + FD->isBitField() && FD->getBitWidthValue(getContext()) == 0) + continue; // Unlike isSingleElementStruct(), arrays do not count. - // Nested isFPArgumentType structures still do though. - if (Found) - return false; - Found = isFPArgumentType(FD->getType()); - if (!Found) - return false; + // Nested structures still do though. + if (!Found.isNull()) + return Ty; + Found = GetSingleElementType(FD->getType()); } // Unlike isSingleElementStruct(), trailing padding is allowed. // An 8-byte aligned struct s { float f; } is passed as a double. - return Found; + if (!Found.isNull()) + return Found; } - return false; + return Ty; } llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, @@ -5350,26 +5337,56 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, // i8 *__reg_save_area; // }; - // Every argument occupies 8 bytes and is passed by preference in either - // GPRs or FPRs. + // Every non-vector argument occupies 8 bytes and is passed by preference + // in either GPRs or FPRs. Vector arguments occupy 8 or 16 bytes and are + // always passed on the stack. Ty = CGF.getContext().getCanonicalType(Ty); + llvm::Type *ArgTy = CGF.ConvertTypeForMem(Ty); + llvm::Type *APTy = llvm::PointerType::getUnqual(ArgTy); ABIArgInfo AI = classifyArgumentType(Ty); - bool InFPRs = isFPArgumentType(Ty); - - llvm::Type *APTy = llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty)); bool IsIndirect = AI.isIndirect(); + bool InFPRs = false; + bool IsVector = false; unsigned UnpaddedBitSize; if (IsIndirect) { APTy = llvm::PointerType::getUnqual(APTy); UnpaddedBitSize = 64; - } else + } else { + if (AI.getCoerceToType()) + ArgTy = AI.getCoerceToType(); + InFPRs = ArgTy->isFloatTy() || ArgTy->isDoubleTy(); + IsVector = ArgTy->isVectorTy(); UnpaddedBitSize = getContext().getTypeSize(Ty); - unsigned PaddedBitSize = 64; + } + unsigned PaddedBitSize = (IsVector && UnpaddedBitSize > 64) ? 128 : 64; assert((UnpaddedBitSize <= PaddedBitSize) && "Invalid argument size."); unsigned PaddedSize = PaddedBitSize / 8; unsigned Padding = (PaddedBitSize - UnpaddedBitSize) / 8; + llvm::Type *IndexTy = CGF.Int64Ty; + llvm::Value *PaddedSizeV = llvm::ConstantInt::get(IndexTy, PaddedSize); + + if (IsVector) { + // Work out the address of a vector argument on the stack. + // Vector arguments are always passed in the high bits of a + // single (8 byte) or double (16 byte) stack slot. + llvm::Value *OverflowArgAreaPtr = + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 2, + "overflow_arg_area_ptr"); + llvm::Value *OverflowArgArea = + CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"); + llvm::Value *MemAddr = + CGF.Builder.CreateBitCast(OverflowArgArea, APTy, "mem_addr"); + + // Update overflow_arg_area_ptr pointer + llvm::Value *NewOverflowArgArea = + CGF.Builder.CreateGEP(OverflowArgArea, PaddedSizeV, "overflow_arg_area"); + CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); + + return MemAddr; + } + unsigned MaxRegs, RegCountField, RegSaveIndex, RegPadding; if (InFPRs) { MaxRegs = 4; // Maximum of 4 FPR arguments @@ -5383,10 +5400,9 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, RegPadding = Padding; // values are passed in the low bits of a GPR } - llvm::Value *RegCountPtr = - CGF.Builder.CreateStructGEP(VAListAddr, RegCountField, "reg_count_ptr"); + llvm::Value *RegCountPtr = CGF.Builder.CreateStructGEP( + nullptr, VAListAddr, RegCountField, "reg_count_ptr"); llvm::Value *RegCount = CGF.Builder.CreateLoad(RegCountPtr, "reg_count"); - llvm::Type *IndexTy = RegCount->getType(); llvm::Value *MaxRegsV = llvm::ConstantInt::get(IndexTy, MaxRegs); llvm::Value *InRegs = CGF.Builder.CreateICmpULT(RegCount, MaxRegsV, "fits_in_regs"); @@ -5400,7 +5416,6 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, CGF.EmitBlock(InRegBlock); // Work out the address of an argument register. - llvm::Value *PaddedSizeV = llvm::ConstantInt::get(IndexTy, PaddedSize); llvm::Value *ScaledRegCount = CGF.Builder.CreateMul(RegCount, PaddedSizeV, "scaled_reg_count"); llvm::Value *RegBase = @@ -5408,7 +5423,7 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, llvm::Value *RegOffset = CGF.Builder.CreateAdd(ScaledRegCount, RegBase, "reg_offset"); llvm::Value *RegSaveAreaPtr = - CGF.Builder.CreateStructGEP(VAListAddr, 3, "reg_save_area_ptr"); + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3, "reg_save_area_ptr"); llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(RegSaveAreaPtr, "reg_save_area"); llvm::Value *RawRegAddr = @@ -5427,8 +5442,8 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, CGF.EmitBlock(InMemBlock); // Work out the address of a stack argument. - llvm::Value *OverflowArgAreaPtr = - CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_ptr"); + llvm::Value *OverflowArgAreaPtr = CGF.Builder.CreateStructGEP( + nullptr, VAListAddr, 2, "overflow_arg_area_ptr"); llvm::Value *OverflowArgArea = CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"); llvm::Value *PaddingV = llvm::ConstantInt::get(IndexTy, Padding); @@ -5458,6 +5473,8 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy) const { if (RetTy->isVoidType()) return ABIArgInfo::getIgnore(); + if (isVectorArgumentType(RetTy)) + return ABIArgInfo::getDirect(); if (isCompoundType(RetTy) || getContext().getTypeSize(RetTy) > 64) return ABIArgInfo::getIndirect(0); return (isPromotableIntegerType(RetTy) ? @@ -5473,8 +5490,16 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { if (isPromotableIntegerType(Ty)) return ABIArgInfo::getExtend(); - // Values that are not 1, 2, 4 or 8 bytes in size are passed indirectly. + // Handle vector types and vector-like structure types. Note that + // as opposed to float-like structure types, we do not allow any + // padding for vector-like structures, so verify the sizes match. uint64_t Size = getContext().getTypeSize(Ty); + QualType SingleElementTy = GetSingleElementType(Ty); + if (isVectorArgumentType(SingleElementTy) && + getContext().getTypeSize(SingleElementTy) == Size) + return ABIArgInfo::getDirect(CGT.ConvertType(SingleElementTy)); + + // Values that are not 1, 2, 4 or 8 bytes in size are passed indirectly. if (Size != 8 && Size != 16 && Size != 32 && Size != 64) return ABIArgInfo::getIndirect(0, /*ByVal=*/false); @@ -5488,7 +5513,7 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { // The structure is passed as an unextended integer, a float, or a double. llvm::Type *PassTy; - if (isFPArgumentType(Ty)) { + if (isFPArgumentType(SingleElementTy)) { assert(Size == 32 || Size == 64); if (Size == 32) PassTy = llvm::Type::getFloatTy(getVMContext()); @@ -5516,13 +5541,13 @@ class MSP430TargetCodeGenInfo : public TargetCodeGenInfo { public: MSP430TargetCodeGenInfo(CodeGenTypes &CGT) : TargetCodeGenInfo(new DefaultABIInfo(CGT)) {} - void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const override; }; } -void MSP430TargetCodeGenInfo::SetTargetAttributes(const Decl *D, +void MSP430TargetCodeGenInfo::setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) { @@ -5568,6 +5593,7 @@ public: void computeInfo(CGFunctionInfo &FI) const override; llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, CodeGenFunction &CGF) const override; + bool shouldSignExtUnsignedType(QualType Ty) const override; }; class MIPSTargetCodeGenInfo : public TargetCodeGenInfo { @@ -5581,7 +5607,7 @@ public: return 29; } - void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const override { const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); if (!FD) return; @@ -5603,8 +5629,8 @@ public: }; } -void MipsABIInfo::CoerceToIntArgs(uint64_t TySize, - SmallVectorImpl<llvm::Type *> &ArgList) const { +void MipsABIInfo::CoerceToIntArgs( + uint64_t TySize, SmallVectorImpl<llvm::Type *> &ArgList) const { llvm::IntegerType *IntTy = llvm::IntegerType::get(getVMContext(), MinABIStackAlignInBytes * 8); @@ -5643,7 +5669,7 @@ llvm::Type* MipsABIInfo::HandleAggregates(QualType Ty, uint64_t TySize) const { const RecordDecl *RD = RT->getDecl(); const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD); assert(!(TySize % 8) && "Size of structure must be multiple of 8."); - + uint64_t LastOffset = 0; unsigned idx = 0; llvm::IntegerType *I64 = llvm::IntegerType::get(getVMContext(), 64); @@ -5745,7 +5771,7 @@ MipsABIInfo::returnAggregateInRegs(QualType RetTy, uint64_t Size) const { // 1. The size of the struct/class is no larger than 128-bit. // 2. The struct/class has one or two fields all of which are floating // point types. - // 3. The offset of the first field is zero (this follows what gcc does). + // 3. The offset of the first field is zero (this follows what gcc does). // // Any other composite results are returned in integer registers. // @@ -5815,7 +5841,7 @@ void MipsABIInfo::computeInfo(CGFunctionInfo &FI) const { if (!getCXXABI().classifyReturnType(FI)) RetInfo = classifyReturnType(FI.getReturnType()); - // Check if a pointer to an aggregate is passed as a hidden argument. + // Check if a pointer to an aggregate is passed as a hidden argument. uint64_t Offset = RetInfo.isIndirect() ? MinABIStackAlignInBytes : 0; for (auto &I : FI.arguments()) @@ -5837,7 +5863,7 @@ llvm::Value* MipsABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, Ty = CGF.getContext().getIntTypeForBitwidth(SlotSizeInBits, Ty->isSignedIntegerType()); } - + CGBuilderTy &Builder = CGF.Builder; llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap"); llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); @@ -5856,7 +5882,7 @@ llvm::Value* MipsABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, AddrTyped = CGF.Builder.CreateIntToPtr(And, PTy); } else - AddrTyped = Builder.CreateBitCast(Addr, PTy); + AddrTyped = Builder.CreateBitCast(Addr, PTy); llvm::Value *AlignedAddr = Builder.CreateBitCast(AddrTyped, BP); TypeAlign = std::max((unsigned)TypeAlign, MinABIStackAlignInBytes); @@ -5866,10 +5892,20 @@ llvm::Value* MipsABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, Builder.CreateGEP(AlignedAddr, llvm::ConstantInt::get(IntTy, Offset), "ap.next"); Builder.CreateStore(NextAddr, VAListAddrAsBPP); - + return AddrTyped; } +bool MipsABIInfo::shouldSignExtUnsignedType(QualType Ty) const { + int TySize = getContext().getTypeSize(Ty); + + // MIPS64 ABI requires unsigned 32 bit integers to be sign extended. + if (Ty->isUnsignedIntegerOrEnumerationType() && TySize == 32) + return true; + + return false; +} + bool MIPSTargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const { @@ -5900,7 +5936,7 @@ MIPSTargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, //===----------------------------------------------------------------------===// // TCE ABI Implementation (see http://tce.cs.tut.fi). Uses mostly the defaults. -// Currently subclassed only to implement custom OpenCL C function attribute +// Currently subclassed only to implement custom OpenCL C function attribute // handling. //===----------------------------------------------------------------------===// @@ -5911,18 +5947,17 @@ public: TCETargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {} - void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const override; }; -void TCETargetCodeGenInfo::SetTargetAttributes(const Decl *D, - llvm::GlobalValue *GV, - CodeGen::CodeGenModule &M) const { +void TCETargetCodeGenInfo::setTargetAttributes( + const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); if (!FD) return; llvm::Function *F = cast<llvm::Function>(GV); - + if (M.getLangOpts().OpenCL) { if (FD->hasAttr<OpenCLKernelAttr>()) { // OpenCL C Kernel functions are not subject to inlining @@ -5931,8 +5966,9 @@ void TCETargetCodeGenInfo::SetTargetAttributes(const Decl *D, if (Attr) { // Convert the reqd_work_group_size() attributes to metadata. llvm::LLVMContext &Context = F->getContext(); - llvm::NamedMDNode *OpenCLMetadata = - M.getModule().getOrInsertNamedMetadata("opencl.kernel_wg_size_info"); + llvm::NamedMDNode *OpenCLMetadata = + M.getModule().getOrInsertNamedMetadata( + "opencl.kernel_wg_size_info"); SmallVector<llvm::Metadata *, 5> Operands; Operands.push_back(llvm::ConstantAsMetadata::get(F)); @@ -5947,9 +5983,9 @@ void TCETargetCodeGenInfo::SetTargetAttributes(const Decl *D, llvm::ConstantAsMetadata::get(llvm::Constant::getIntegerValue( M.Int32Ty, llvm::APInt(32, Attr->getZDim())))); - // Add a boolean constant operand for "required" (true) or "hint" (false) - // for implementing the work_group_size_hint attr later. Currently - // always true as the hint is not yet implemented. + // Add a boolean constant operand for "required" (true) or "hint" + // (false) for implementing the work_group_size_hint attr later. + // Currently always true as the hint is not yet implemented. Operands.push_back( llvm::ConstantAsMetadata::get(llvm::ConstantInt::getTrue(Context))); OpenCLMetadata->addOperand(llvm::MDNode::get(Context, Operands)); @@ -6103,13 +6139,13 @@ class AMDGPUTargetCodeGenInfo : public TargetCodeGenInfo { public: AMDGPUTargetCodeGenInfo(CodeGenTypes &CGT) : TargetCodeGenInfo(new DefaultABIInfo(CGT)) {} - void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const override; }; } -void AMDGPUTargetCodeGenInfo::SetTargetAttributes( +void AMDGPUTargetCodeGenInfo::setTargetAttributes( const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const { @@ -6264,12 +6300,7 @@ private: // Check if Ty is a usable substitute for the coercion type. bool isUsableType(llvm::StructType *Ty) const { - if (Ty->getNumElements() != Elems.size()) - return false; - for (unsigned i = 0, e = Elems.size(); i != e; ++i) - if (Elems[i] != Ty->getElementType(i)) - return false; - return true; + return llvm::makeArrayRef(Elems) == Ty->elements(); } // Get the coercion type as a literal struct type. @@ -6430,7 +6461,7 @@ SparcV9TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, // FSR = 70 // CSR = 71 AssignToArrayRange(Builder, Address, Eight8, 64, 71); - + // 72-87: d0-15, the 8-byte floating-point registers AssignToArrayRange(Builder, Address, Eight8, 72, 87); @@ -6703,7 +6734,7 @@ StringRef TypeStringCache::lookupStr(const IdentifierInfo *ID) { /// /// The TypeString carries type, qualifier, name, size & value details. /// Please see 'Tools Development Guide' section 2.16.2 for format details: -/// <https://www.xmos.com/download/public/Tools-Development-Guide%28X9114A%29.pdf> +/// https://www.xmos.com/download/public/Tools-Development-Guide%28X9114A%29.pdf /// The output is tested by test/CodeGen/xcore-stringtype.c. /// static bool getTypeString(SmallStringEnc &Enc, const Decl *D, @@ -6729,7 +6760,8 @@ static bool appendType(SmallStringEnc &Enc, QualType QType, TypeStringCache &TSC); /// Helper function for appendRecordType(). -/// Builds a SmallVector containing the encoded field types in declaration order. +/// Builds a SmallVector containing the encoded field types in declaration +/// order. static bool extractFieldType(SmallVectorImpl<FieldEncoding> &FE, const RecordDecl *RD, const CodeGen::CodeGenModule &CGM, @@ -6752,7 +6784,7 @@ static bool extractFieldType(SmallVectorImpl<FieldEncoding> &FE, if (Field->isBitField()) Enc += ')'; Enc += '}'; - FE.push_back(FieldEncoding(!Field->getName().empty(), Enc)); + FE.emplace_back(!Field->getName().empty(), Enc); } return true; } @@ -7082,6 +7114,8 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { return *(TheTargetCodeGenInfo = new PNaClTargetCodeGenInfo(Types)); case llvm::Triple::mips: case llvm::Triple::mipsel: + if (Triple.getOS() == llvm::Triple::NaCl) + return *(TheTargetCodeGenInfo = new PNaClTargetCodeGenInfo(Types)); return *(TheTargetCodeGenInfo = new MIPSTargetCodeGenInfo(Types, true)); case llvm::Triple::mips64: @@ -7102,6 +7136,12 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { case llvm::Triple::thumb: case llvm::Triple::thumbeb: { + if (Triple.getOS() == llvm::Triple::Win32) { + TheTargetCodeGenInfo = + new WindowsARMTargetCodeGenInfo(Types, ARMABIInfo::AAPCS_VFP); + return *TheTargetCodeGenInfo; + } + ARMABIInfo::ABIKind Kind = ARMABIInfo::AAPCS; if (getTarget().getABI() == "apcs-gnu") Kind = ARMABIInfo::APCS; @@ -7110,14 +7150,7 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { Triple.getEnvironment() == llvm::Triple::GNUEABIHF)) Kind = ARMABIInfo::AAPCS_VFP; - switch (Triple.getOS()) { - case llvm::Triple::NaCl: - return *(TheTargetCodeGenInfo = - new NaClARMTargetCodeGenInfo(Types, Kind)); - default: - return *(TheTargetCodeGenInfo = - new ARMTargetCodeGenInfo(Types, Kind)); - } + return *(TheTargetCodeGenInfo = new ARMTargetCodeGenInfo(Types, Kind)); } case llvm::Triple::ppc: @@ -7127,19 +7160,21 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv1; if (getTarget().getABI() == "elfv2") Kind = PPC64_SVR4_ABIInfo::ELFv2; + bool HasQPX = getTarget().getABI() == "elfv1-qpx"; return *(TheTargetCodeGenInfo = - new PPC64_SVR4_TargetCodeGenInfo(Types, Kind)); + new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, HasQPX)); } else return *(TheTargetCodeGenInfo = new PPC64TargetCodeGenInfo(Types)); case llvm::Triple::ppc64le: { assert(Triple.isOSBinFormatELF() && "PPC64 LE non-ELF not supported!"); PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv2; - if (getTarget().getABI() == "elfv1") + if (getTarget().getABI() == "elfv1" || getTarget().getABI() == "elfv1-qpx") Kind = PPC64_SVR4_ABIInfo::ELFv1; + bool HasQPX = getTarget().getABI() == "elfv1-qpx"; return *(TheTargetCodeGenInfo = - new PPC64_SVR4_TargetCodeGenInfo(Types, Kind)); + new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, HasQPX)); } case llvm::Triple::nvptx: @@ -7149,8 +7184,11 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { case llvm::Triple::msp430: return *(TheTargetCodeGenInfo = new MSP430TargetCodeGenInfo(Types)); - case llvm::Triple::systemz: - return *(TheTargetCodeGenInfo = new SystemZTargetCodeGenInfo(Types)); + case llvm::Triple::systemz: { + bool HasVector = getTarget().getABI() == "vector"; + return *(TheTargetCodeGenInfo = new SystemZTargetCodeGenInfo(Types, + HasVector)); + } case llvm::Triple::tce: return *(TheTargetCodeGenInfo = new TCETargetCodeGenInfo(Types)); @@ -7162,33 +7200,32 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { bool IsWin32FloatStructABI = Triple.isOSWindows() && !Triple.isOSCygMing(); if (Triple.getOS() == llvm::Triple::Win32) { - return *(TheTargetCodeGenInfo = - new WinX86_32TargetCodeGenInfo(Types, - IsDarwinVectorABI, IsSmallStructInRegABI, - IsWin32FloatStructABI, - CodeGenOpts.NumRegisterParameters)); + return *(TheTargetCodeGenInfo = new WinX86_32TargetCodeGenInfo( + Types, IsDarwinVectorABI, IsSmallStructInRegABI, + IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters)); } else { - return *(TheTargetCodeGenInfo = - new X86_32TargetCodeGenInfo(Types, - IsDarwinVectorABI, IsSmallStructInRegABI, - IsWin32FloatStructABI, - CodeGenOpts.NumRegisterParameters)); + return *(TheTargetCodeGenInfo = new X86_32TargetCodeGenInfo( + Types, IsDarwinVectorABI, IsSmallStructInRegABI, + IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters)); } } case llvm::Triple::x86_64: { - bool HasAVX = getTarget().getABI() == "avx"; + StringRef ABI = getTarget().getABI(); + X86AVXABILevel AVXLevel = (ABI == "avx512" ? X86AVXABILevel::AVX512 : + ABI == "avx" ? X86AVXABILevel::AVX : + X86AVXABILevel::None); switch (Triple.getOS()) { case llvm::Triple::Win32: return *(TheTargetCodeGenInfo = - new WinX86_64TargetCodeGenInfo(Types, HasAVX)); - case llvm::Triple::NaCl: + new WinX86_64TargetCodeGenInfo(Types, AVXLevel)); + case llvm::Triple::PS4: return *(TheTargetCodeGenInfo = - new NaClX86_64TargetCodeGenInfo(Types, HasAVX)); + new PS4TargetCodeGenInfo(Types, AVXLevel)); default: return *(TheTargetCodeGenInfo = - new X86_64TargetCodeGenInfo(Types, HasAVX)); + new X86_64TargetCodeGenInfo(Types, AVXLevel)); } } case llvm::Triple::hexagon: diff --git a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h index cc469d6..95275d5 100644 --- a/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h +++ b/contrib/llvm/tools/clang/lib/CodeGen/TargetInfo.h @@ -53,12 +53,12 @@ public: /// getABIInfo() - Returns ABI info helper for the target. const ABIInfo &getABIInfo() const { return *Info; } - /// SetTargetAttributes - Provides a convenient hook to handle extra + /// setTargetAttributes - Provides a convenient hook to handle extra /// target-specific attributes for the given global. - virtual void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + virtual void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {} - /// EmitTargetMD - Provides a convenient hook to handle extra + /// emitTargetMD - Provides a convenient hook to handle extra /// target-specific metadata for the given global. virtual void emitTargetMD(const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {} @@ -218,13 +218,6 @@ public: virtual void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value, llvm::SmallString<32> &Opt) const {} - - /// Gets the target-specific default alignment used when an 'aligned' clause - /// is used with a 'simd' OpenMP directive without specifying a specific - /// alignment. - virtual unsigned getOpenMPSimdDefaultAlignment(QualType Type) const { - return 0; - } }; } |
